mirror of
https://github.com/rustfs/rustfs.git
synced 2026-10-27 14:27:10 +00:00
Compare commits
64 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 2d541a909a | |||
| 13a30abca4 | |||
| d33515da69 | |||
| f1f58c9619 | |||
| f4ce1d8826 | |||
| ae17a45db9 | |||
| 53b0f93a90 | |||
| f7ffcbfa3d | |||
| 583b1d6464 | |||
| 58340019af | |||
| 72e1a50c4c | |||
| 0907f1aef4 | |||
| ce47b96547 | |||
| 85593549ad | |||
| cdbca53477 | |||
| 28b3a70a70 | |||
| ef88f2cc4d | |||
| 737e4d2b61 | |||
| 8514b65e8a | |||
| 8707ad627b | |||
| c27d38f500 | |||
| 0bd811dafa | |||
| f54706e43a | |||
| 58523756ad | |||
| 740f4f0194 | |||
| ce514219bf | |||
| abf6a6b46a | |||
| ccc2b4441c | |||
| 6fb8381273 | |||
| 689c4552fe | |||
| 64cafb46f0 | |||
| 06782925ff | |||
| b210155cb4 | |||
| 94b51f096a | |||
| e3620b04f4 | |||
| fd9ea6136b | |||
| 3869fa9043 | |||
| 689970f7a7 | |||
| e671580629 | |||
| a580f8c333 | |||
| 72909b98e1 | |||
| 37c0887227 | |||
| aec1c4f0a2 | |||
| 19ad550322 | |||
| 905b2d92ad | |||
| 6586da38fc | |||
| cb78742727 | |||
| d646f85778 | |||
| 2aa49aefc7 | |||
| 2a390def87 | |||
| 9fa35154d4 | |||
| a7064fa2b9 | |||
| 34130a5b61 | |||
| f0e0880afa | |||
| adedc0c7b3 | |||
| 1fa7d72778 | |||
| 5b37dbbac6 | |||
| 6cab5cbb0b | |||
| 996016df2d | |||
| 25382f4ff7 | |||
| 2f9e3efcf8 | |||
| 47ca87f82e | |||
| 30d416fbb4 | |||
| aaba1b40a4 |
@@ -61,6 +61,7 @@ scrape_configs:
|
||||
otlp:
|
||||
promote_resource_attributes:
|
||||
- service.instance.id
|
||||
- process.pid
|
||||
- service.name
|
||||
- service.namespace
|
||||
- cloud.availability_zone
|
||||
|
||||
@@ -90,3 +90,4 @@ worktrees/*
|
||||
# insta scratch files; the accepted .snap files ARE the assertions and are committed
|
||||
*.snap.new
|
||||
*.pending-snap
|
||||
.mimosa
|
||||
Generated
+85
-82
@@ -2839,9 +2839,9 @@ checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06"
|
||||
|
||||
[[package]]
|
||||
name = "datafusion"
|
||||
version = "55.1.0"
|
||||
version = "55.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "14313430439ef858ed5f52ca9c840e9eea9844da34d3716d05a476449634da5d"
|
||||
checksum = "4cd5317db77ba6bca25c761a8091b9d294a9f4188e4f9dcd0243e6a57de862ac"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-schema",
|
||||
@@ -2888,9 +2888,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-catalog"
|
||||
version = "55.1.0"
|
||||
version = "55.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2c0756ebc770bcebae366dfd273dad50935f6103b4d212253cb17e2e53c5427f"
|
||||
checksum = "f35d36794509b7bfcbf90e11a1b5e4d047123c34610105e2e80a2048351cdb2f"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -2913,9 +2913,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-catalog-listing"
|
||||
version = "55.1.0"
|
||||
version = "55.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bbde2961698705d8402f0f382f08876fc6f1126ffde790ef17833cf0ab87cd59"
|
||||
checksum = "a6454bfb7f43b83942e2ca2576fd09342eedbd0d323e1eef72c3c8e4d57d7203"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -2937,9 +2937,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-common"
|
||||
version = "55.1.0"
|
||||
version = "55.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6e06fe711ff815c8cf3143158547aea599702fd43f76f5866c70d5a5b3693a0"
|
||||
checksum = "d26626e12b224b83351239465dcd00f72d47b02fb5e5b65a643e2dc8ed37f687"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-ipc",
|
||||
@@ -2964,9 +2964,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-common-runtime"
|
||||
version = "55.1.0"
|
||||
version = "55.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f7a5107acff4e3ed50a7732acec06bba40e74ab49df021397ed250d60f04e42a"
|
||||
checksum = "a106420cbfa980f242c22243ae9a6cce661d4d86ce96cff87dacabf7754a4470"
|
||||
dependencies = [
|
||||
"futures",
|
||||
"log",
|
||||
@@ -2975,9 +2975,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-datasource"
|
||||
version = "55.1.0"
|
||||
version = "55.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f69f45d2d700785be8df9d4b44410e1c8e4a856f10cc728b418a1066c0b594fb"
|
||||
checksum = "210aa1e9525e9454f2399dbb8b0b95510145bd8c1fdc35de74e9ccbf81bcb699"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -3005,9 +3005,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-datasource-arrow"
|
||||
version = "55.1.0"
|
||||
version = "55.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "81650af64894ef34b46d2ef818ef244b40a81d90818f3bc642bf8fbf6758f8e0"
|
||||
checksum = "65b1b041688cad707fa103c3e7f4c6d89b3134ebf67da0129070dad3381851eb"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-ipc",
|
||||
@@ -3029,9 +3029,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-datasource-csv"
|
||||
version = "55.1.0"
|
||||
version = "55.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b5e2519bed59a6907c79298b9c21fe807bbc34e4ff8414577d699b0741083593"
|
||||
checksum = "2a7b34e253c4fa5251f174977af910a2f2ec831ec39dbbb56b29ce505cff278b"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -3052,9 +3052,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-datasource-json"
|
||||
version = "55.1.0"
|
||||
version = "55.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "635e61290e171597f6e50427e0cb486eda468ab47e880fa24d15bbb0fa24ac7e"
|
||||
checksum = "1663338f43be6a5746473008e2d0fbde282577ddf0bae90d6c7386e988d71b29"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -3075,9 +3075,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-datasource-parquet"
|
||||
version = "55.1.0"
|
||||
version = "55.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "07cd518f85b3a027d913047b4ab0811111a2b734fdf84a51b78234455f10d8fa"
|
||||
checksum = "7c521ae91b6d899d7190812881f45fe9463c222c11d90149f5332a2e56555904"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-schema",
|
||||
@@ -3107,15 +3107,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-doc"
|
||||
version = "55.1.0"
|
||||
version = "55.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d44c5919b560862dd1cb20a5c75b3e48c3d9e49719c91d78da5c38eaad5495c4"
|
||||
checksum = "0c2569c2e74a34fcae3a369ab3912d61e41a062592510f11e31b1dae2725ecdc"
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-execution"
|
||||
version = "55.1.0"
|
||||
version = "55.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7620ad535af8730186bdec05d720e40d9a4584b40c7eb75f72c43aa7978e5484"
|
||||
checksum = "39cfdd5ac0769d1e74a885450e49fde8f63ca7ab41eb479a068564aaea80b837"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-buffer",
|
||||
@@ -3139,9 +3139,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-expr"
|
||||
version = "55.1.0"
|
||||
version = "55.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ea229af8b73e566fc6ebb1c8dd42ab250f8855c5304bd173f4b50ead78b64304"
|
||||
checksum = "86560fd7df4dda924f5474b588c56e7ae8e96dcf158c7327c9cfe9daff448ec5"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-schema",
|
||||
@@ -3156,15 +3156,16 @@ dependencies = [
|
||||
"indexmap 2.14.2",
|
||||
"itertools 0.15.0",
|
||||
"recursive",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sqlparser",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-expr-common"
|
||||
version = "55.1.0"
|
||||
version = "55.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6a18baa840f77760dcc9eaad281329bc57536c5c39e65e8649596f2a51216be7"
|
||||
checksum = "e82e313cf655cf6bc2dce6cbd23bcb0d20236a54a1b09c436615e4bac4d8c49f"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"datafusion-common",
|
||||
@@ -3174,9 +3175,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-functions"
|
||||
version = "55.1.0"
|
||||
version = "55.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "763bbfeec40b55a8a961d7349c0fcfcec4f3f66cc234d6295250b95371b85e34"
|
||||
checksum = "939ae2771da8a3fdc23ab4de610ae8fef9515476626b49d96d0d5c782b105771"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-buffer",
|
||||
@@ -3202,9 +3203,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-functions-aggregate"
|
||||
version = "55.1.0"
|
||||
version = "55.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "46c0c2b2e856883ecf6a4fa8b23e7c6041183af8a68d0583dc95d1d21e4eaee0"
|
||||
checksum = "fd6c3aee9f2bc65716787d651e023743a90b4efbc57a2f679f036632e16fb110"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"datafusion-common",
|
||||
@@ -3223,9 +3224,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-functions-aggregate-common"
|
||||
version = "55.1.0"
|
||||
version = "55.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e35e42c87dfbff06934ea0e99cfed3440ac614c3f3bfb8c33be35ca0587d4140"
|
||||
checksum = "988b5d3dfb4939d81c2f9fab267063610fe52a843620f2fa6cb2a1598e43369c"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"datafusion-common",
|
||||
@@ -3235,9 +3236,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-functions-nested"
|
||||
version = "55.1.0"
|
||||
version = "55.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "08079b0604cdcc43c7d6cf006cfd5d3ed9c235ade0c1ebe91085a62f875603df"
|
||||
checksum = "08149d433a97c9ea3cdf80de06480ab5c7f5453485f1e452d5b3583c81d5c0dd"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-ord",
|
||||
@@ -3260,9 +3261,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-functions-table"
|
||||
version = "55.1.0"
|
||||
version = "55.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fe2070ecf5638a93c81c8babc7858339ef52165a9f54da1411afd69adf7d85ce"
|
||||
checksum = "2c767567cce2782d4d0bdfa55fab010aaba0bdbe533509e9e0b9a6a3ee29ed59"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -3276,9 +3277,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-functions-window"
|
||||
version = "55.1.0"
|
||||
version = "55.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0e9a164ebe1998c5c3f9e15c9a0c4e6b599a0c8739b5471bfb089752441963f1"
|
||||
checksum = "0a311c1f558d1f792f1f631c3670da12ee72a20fbf5d6ca151325d66191c03c1"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"datafusion-common",
|
||||
@@ -3293,9 +3294,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-functions-window-common"
|
||||
version = "55.1.0"
|
||||
version = "55.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "98965b183f92201c16798994820ae032649c4821e089c4c80e7d0f4485d41df5"
|
||||
checksum = "fffd54b8acf6ecef284cc0677e01e1b8c71be64fec50612ba00a0ff0d9b41a52"
|
||||
dependencies = [
|
||||
"datafusion-common",
|
||||
"datafusion-physical-expr-common",
|
||||
@@ -3303,9 +3304,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-macros"
|
||||
version = "55.1.0"
|
||||
version = "55.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a711bd5a06ec85d9683e142d28a98ad0679a7220669f5c1b8589e57e9b29e46"
|
||||
checksum = "14d6df9763c8fb968b2417d797aef4c55ac2e4a895aaef38c72c16a287aa4a40"
|
||||
dependencies = [
|
||||
"datafusion-doc",
|
||||
"quote",
|
||||
@@ -3314,9 +3315,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-optimizer"
|
||||
version = "55.1.0"
|
||||
version = "55.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ff3256b62ff44616ec83c7f04741d21124970833bb377ed56d9abc5ec6f707ec"
|
||||
checksum = "acfb8cea6e05e2e75bae183340cf7d53cccb3db0449e2f1fbc07654d980fd19f"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"chrono",
|
||||
@@ -3334,9 +3335,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-physical-expr"
|
||||
version = "55.1.0"
|
||||
version = "55.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c1e37a25b5aa1f3a1db114f0ee3605bf01639b83a831dcde35119ee61f0b11d6"
|
||||
checksum = "9f002cef849e10c0597a217ec7891c2796c0f8b42d739f5d24abc24496373406"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-schema",
|
||||
@@ -3357,9 +3358,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-physical-expr-adapter"
|
||||
version = "55.1.0"
|
||||
version = "55.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a4d12787d42e88b26b0d3ec006192fa23ceb832ef2a9e87439ce820a8e038cc"
|
||||
checksum = "82761522c4536e7059d3b185cd99f10525c2bf48f1bd26a0a57e73e19eadc0fb"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"datafusion-common",
|
||||
@@ -3372,9 +3373,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-physical-expr-common"
|
||||
version = "55.1.0"
|
||||
version = "55.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d13bf7f50fd902f5afc46d50759f5185b5620c40b24054f80f7c648897d3e5a6"
|
||||
checksum = "f8c5c67aa252886f4d6b331c8114797f5b140464e774c4da36889269a1bf1bd8"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"chrono",
|
||||
@@ -3389,9 +3390,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-physical-optimizer"
|
||||
version = "55.1.0"
|
||||
version = "55.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "11fbe7e6665072957d44ed9587676e17a1f9ab3acfadfc287dac23750585f18a"
|
||||
checksum = "f1a49dd3a76e4879012682c59dc3735282e60640c97483546a4113564a60bf8f"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"datafusion-common",
|
||||
@@ -3409,9 +3410,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-physical-plan"
|
||||
version = "55.1.0"
|
||||
version = "55.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1265d58e5bce07d154e642a51ff43033b576a6ae40d50a29ac2b9f311004eb52"
|
||||
checksum = "b1451d6817e5ac0dd21819b19949d1c420ab51d748f55dd9a96ecb5418a24c97"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-data",
|
||||
@@ -3438,15 +3439,16 @@ dependencies = [
|
||||
"num-traits",
|
||||
"parking_lot",
|
||||
"pin-project-lite",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-pruning"
|
||||
version = "55.1.0"
|
||||
version = "55.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "74b06b333405c05015836ed36cea02cc29d1e2ee8fb414a77532d8346fb56492"
|
||||
checksum = "f11f0dbe2b5a642f6106e44de8bbd6654fa7220ca14f32313d4efd5514a86968"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"datafusion-common",
|
||||
@@ -3460,9 +3462,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-session"
|
||||
version = "55.1.0"
|
||||
version = "55.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e7616b17b1d2fee3e9b13c651eac91b768d153265e6ba88e06e0b4af7883f69c"
|
||||
checksum = "62dc5ce37a5ce1ff5362949776829aded2529a11dbaa13d24b0ffa6a059e106a"
|
||||
dependencies = [
|
||||
"arrow-schema",
|
||||
"async-trait",
|
||||
@@ -3475,9 +3477,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-sql"
|
||||
version = "55.1.0"
|
||||
version = "55.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5b4848be66dd008d55adbe1dbc8cbd6a2502588ad0c0b8483bab0f43fd4495d0"
|
||||
checksum = "816c224a0ef5469a32a562b5d4830544f714f97c5f50142109286d40a288c7db"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"bigdecimal",
|
||||
@@ -8343,7 +8345,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf"
|
||||
dependencies = [
|
||||
"heck 0.5.0",
|
||||
"itertools 0.14.0",
|
||||
"itertools 0.10.5",
|
||||
"log",
|
||||
"multimap",
|
||||
"once_cell",
|
||||
@@ -8363,7 +8365,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042"
|
||||
dependencies = [
|
||||
"heck 0.5.0",
|
||||
"itertools 0.14.0",
|
||||
"itertools 0.10.5",
|
||||
"log",
|
||||
"multimap",
|
||||
"petgraph 0.8.3",
|
||||
@@ -8384,7 +8386,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"itertools 0.14.0",
|
||||
"itertools 0.10.5",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
@@ -8397,7 +8399,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"itertools 0.14.0",
|
||||
"itertools 0.10.5",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
@@ -9430,9 +9432,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rust-embed"
|
||||
version = "8.12.0"
|
||||
version = "8.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e9e7760e252aaba7b09f4be00e36476cf585bdb68a53552ac954cdf504ab4bc9"
|
||||
checksum = "19afa5b4b6a611de00bd1bdae6ae6f39084c9399f0679c3f52d8469cf335cc23"
|
||||
dependencies = [
|
||||
"rust-embed-impl",
|
||||
"rust-embed-utils",
|
||||
@@ -9441,9 +9443,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rust-embed-impl"
|
||||
version = "8.12.0"
|
||||
version = "8.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3bcfc4d6f53af43755f7a723e4b6b8794fcce052a178dd8c6c1dadc5f5343097"
|
||||
checksum = "e0d8afda6374eac59e066abee06d265247ebbaf3006cf878e2879e8356e34053"
|
||||
dependencies = [
|
||||
"mime_guess",
|
||||
"proc-macro2",
|
||||
@@ -9456,9 +9458,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rust-embed-utils"
|
||||
version = "8.12.0"
|
||||
version = "8.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "42ffa149f6aa81b58a5b3011d01a857c4ed12c7a732d2c51947a4c7c692185f0"
|
||||
checksum = "0d84e8ba78bd384263e5922f084cbe1b081c3b7e69add59c8fb097b879ba968a"
|
||||
dependencies = [
|
||||
"sha2 0.11.0",
|
||||
"walkdir",
|
||||
@@ -9856,6 +9858,7 @@ dependencies = [
|
||||
"rustfs-rio",
|
||||
"rustfs-rio-v2",
|
||||
"rustfs-s3-client",
|
||||
"rustfs-s3-ops",
|
||||
"rustfs-s3-types",
|
||||
"rustfs-scanner-metrics",
|
||||
"rustfs-storage-api",
|
||||
@@ -12818,18 +12821,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "toml_datetime"
|
||||
version = "1.1.1+spec-1.1.0"
|
||||
version = "1.1.2+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7"
|
||||
checksum = "2b86d767906c6c42421dcba507eb9d203e779497710a47782a224bb871653053"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml_edit"
|
||||
version = "0.25.15+spec-1.1.0"
|
||||
version = "0.25.16+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614"
|
||||
checksum = "88f38f3af8bf71841c67a990f4e2b52143b2d0a1a994c72513ff4c14977fc722"
|
||||
dependencies = [
|
||||
"indexmap 2.14.2",
|
||||
"toml_datetime",
|
||||
@@ -12839,9 +12842,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "toml_parser"
|
||||
version = "1.1.3+spec-1.1.0"
|
||||
version = "1.1.4+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56"
|
||||
checksum = "bed810ee007d8e7f9cf20b4774e7198142fa27186e0f805871da7f3c39eaf333"
|
||||
dependencies = [
|
||||
"winnow",
|
||||
]
|
||||
@@ -14044,18 +14047,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.8.60"
|
||||
version = "0.8.62"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6400fc4bb7426f6faac3fb07566122bb18622e3022bd2c36426a92ed489b1dbe"
|
||||
checksum = "86502bf56ac7c77571a32e2647bb2a15894565e981fb2a48d7bde2d91c965a9d"
|
||||
dependencies = [
|
||||
"zerocopy-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy-derive"
|
||||
version = "0.8.60"
|
||||
version = "0.8.62"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "99c27cc7525bad4df59a45e073c45b2c2aa820efb588234cc23a547ec69a34db"
|
||||
checksum = "5457206954b06561e2608c7e19cf58b1926586d999c246eebe4502f7e2039d1a"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
||||
+4
-4
@@ -180,7 +180,7 @@ tower = { version = "0.5.3" }
|
||||
tower-http = { version = "0.7.1" }
|
||||
h3 = "0.0.8"
|
||||
h3-quinn = "0.0.10"
|
||||
quinn = { version = "0.11.11", default-features = false, features = [
|
||||
quinn = { version = "0.11.12", default-features = false, features = [
|
||||
"futures-io",
|
||||
"runtime-tokio",
|
||||
"rustls-aws-lc-rs",
|
||||
@@ -268,7 +268,7 @@ crossbeam-queue = "0.3.14"
|
||||
crossbeam-channel = "0.5.17"
|
||||
crossbeam-deque = "0.8.8"
|
||||
crossbeam-utils = "0.8.23"
|
||||
datafusion = { default-features = false, version = "55.1.0" }
|
||||
datafusion = { default-features = false, version = "55.2.0" }
|
||||
derive_builder = "0.20.2"
|
||||
ed25519-dalek = "3.0.0"
|
||||
enumset = "1.1.14"
|
||||
@@ -319,7 +319,7 @@ rumqttc = { package = "rumqttc-next", version = "0.34.0" }
|
||||
redis = { version = "1.7.1" }
|
||||
rustify = { version = "0.7", default-features = false }
|
||||
rustix = { version = "1.1.5" }
|
||||
rust-embed = { version = "8.12.0" }
|
||||
rust-embed = { version = "8.13.0" }
|
||||
rustc-hash = { version = "2.1.3" }
|
||||
s3s = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0", features = ["minio"] }
|
||||
s3s-sigv4 = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0" }
|
||||
@@ -340,7 +340,7 @@ tracing = { version = "0.1.44" }
|
||||
tracing-appender = "0.2.5"
|
||||
tracing-core = "0.1.36"
|
||||
tracing-error = "0.2.1"
|
||||
tracing-opentelemetry = { version = "0.34" }
|
||||
tracing-opentelemetry = { version = "0.34.0" }
|
||||
tracing-subscriber = { version = "0.3.23" }
|
||||
transform-stream = "0.3.1"
|
||||
url = "2.5.8"
|
||||
|
||||
@@ -51,6 +51,8 @@ unparseable = "unparseable"
|
||||
# Disaster-recovery objectives: recovery time and recovery point.
|
||||
RTO = "RTO"
|
||||
rto = "rto"
|
||||
# rustix OFlags variant (POSIX O_WRONLY equivalent).
|
||||
WRONLY = "WRONLY"
|
||||
|
||||
[files]
|
||||
extend-exclude = []
|
||||
|
||||
@@ -51,6 +51,11 @@ pub const ENV_OBS_PROFILING_EXPORT_ENABLED: &str = "RUSTFS_OBS_PROFILING_EXPORT_
|
||||
/// PUT records multiple timers and histograms when attribution is active.
|
||||
pub const ENV_OBS_PUT_STAGE_METRICS_ENABLED: &str = "RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED";
|
||||
|
||||
/// Enables sampled in-process GetObject stage summaries without OTLP histograms.
|
||||
pub const ENV_OBS_GET_STAGE_LOCAL_SUMMARY_ENABLED: &str = "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_ENABLED";
|
||||
pub const ENV_OBS_GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE: &str = "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE";
|
||||
pub const ENV_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS: &str = "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS";
|
||||
|
||||
pub const ENV_OBS_LOGGER_LEVEL: &str = "RUSTFS_OBS_LOGGER_LEVEL";
|
||||
pub const ENV_OBS_LOG_STDOUT_ENABLED: &str = "RUSTFS_OBS_LOG_STDOUT_ENABLED";
|
||||
pub const ENV_OBS_LOG_DIRECTORY: &str = "RUSTFS_OBS_LOG_DIRECTORY";
|
||||
@@ -159,6 +164,15 @@ mod tests {
|
||||
assert_eq!(ENV_OBS_LOGS_EXPORT_ENABLED, "RUSTFS_OBS_LOGS_EXPORT_ENABLED");
|
||||
assert_eq!(ENV_OBS_PROFILING_EXPORT_ENABLED, "RUSTFS_OBS_PROFILING_EXPORT_ENABLED");
|
||||
assert_eq!(ENV_OBS_PUT_STAGE_METRICS_ENABLED, "RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED");
|
||||
assert_eq!(ENV_OBS_GET_STAGE_LOCAL_SUMMARY_ENABLED, "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_ENABLED");
|
||||
assert_eq!(
|
||||
ENV_OBS_GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE,
|
||||
"RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE"
|
||||
);
|
||||
assert_eq!(
|
||||
ENV_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS,
|
||||
"RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS"
|
||||
);
|
||||
// Test log cleanup related env keys
|
||||
assert_eq!(ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES");
|
||||
assert_eq!(ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES");
|
||||
|
||||
@@ -191,6 +191,91 @@ async fn wait_for_quorum_read_admission(clients: &[Client], bucket: &str) -> Tes
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cold_degraded_object_mutations_require_only_storage_quorum() -> TestResult {
|
||||
crate::common::init_logging();
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
|
||||
cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", "EC:2");
|
||||
cluster.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
|
||||
cluster.set_env("RUSTFS_OBS_METRICS_EXPORT_ENABLED", "false");
|
||||
cluster.set_env("RUSTFS_GET_OBJECT_METADATA_CACHE_DISTRIBUTED_ENABLE", "false");
|
||||
cluster.start().await?;
|
||||
let clients = cluster
|
||||
.create_all_clients()?
|
||||
.into_iter()
|
||||
.map(|client| {
|
||||
Client::from_conf(
|
||||
client
|
||||
.config()
|
||||
.to_builder()
|
||||
.retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(1))
|
||||
.build(),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let bucket = "cold-degraded-mutations";
|
||||
clients[0].create_bucket().bucket(bucket).send().await?;
|
||||
// No object mutation may warm a process-wide configuration probe before
|
||||
// the missing peer is introduced. Three shards still satisfy EC:2 writes.
|
||||
cluster.stop_node(3)?;
|
||||
let admission_deadline = Instant::now() + Duration::from_secs(30);
|
||||
for client in &clients[..3] {
|
||||
loop {
|
||||
match client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key("never-created-readiness-probe")
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
Err(error) if error.as_service_error().and_then(|error| error.meta().code()) == Some("NoSuchKey") => break,
|
||||
Err(error)
|
||||
if error.raw_response().is_some_and(|response| response.status().as_u16() == 503)
|
||||
&& Instant::now() < admission_deadline =>
|
||||
{
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
Err(error) => return Err(error.into()),
|
||||
Ok(_) => panic!("a cold readiness probe must not create an object"),
|
||||
}
|
||||
}
|
||||
}
|
||||
let body = b"first write while one peer is offline";
|
||||
clients[0]
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("object")
|
||||
.body(Bytes::from_static(body).into())
|
||||
.send()
|
||||
.await?;
|
||||
for client in &clients[..3] {
|
||||
assert_quorum_object_body(client, bucket, "object", body).await?;
|
||||
}
|
||||
// This coordinator has not previously committed an object mutation either.
|
||||
clients[1].delete_object().bucket(bucket).key("object").send().await?;
|
||||
let error = clients[2]
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key("object")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("acknowledged deletion must be visible at another entrypoint");
|
||||
assert_eq!(error.as_service_error().and_then(|error| error.meta().code()), Some("NoSuchKey"));
|
||||
|
||||
cluster.stop_node(2)?;
|
||||
let error = clients[0]
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("below-quorum")
|
||||
.body(Bytes::from_static(body).into())
|
||||
.send()
|
||||
.await
|
||||
.expect_err("removing the cache gate must not weaken storage write quorum");
|
||||
assert_eq!(error.as_service_error().and_then(|error| error.meta().code()), Some("ServiceUnavailable"));
|
||||
cluster.stop();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_degraded_cluster_read_quorum_follows_erasure_layout() -> TestResult {
|
||||
crate::common::init_logging();
|
||||
|
||||
@@ -31,6 +31,7 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
diagnostic-read-version-caller = ["rustfs-io-metrics/diagnostic-read-version-caller"]
|
||||
gcs = ["dep:google-cloud-storage", "dep:google-cloud-auth"]
|
||||
# Compiles the controlled list-objects namespace-journal chaos injector into a
|
||||
# production binary (it is always available to tests). Off by default so the
|
||||
@@ -200,8 +201,8 @@ md-5.workspace = true
|
||||
memmap2 = { workspace = true }
|
||||
libc.workspace = true
|
||||
# "process" adds getrlimit for the io_uring fd-cache RLIMIT_NOFILE gate
|
||||
# (backlog#1178); "fs" comes from the workspace default.
|
||||
rustix = { workspace = true, features = ["process", "fs"] }
|
||||
# (backlog#1178); "thread" provides cleanup affinity; "fs" is a workspace default.
|
||||
rustix = { workspace = true, features = ["process", "fs", "thread"] }
|
||||
rustfs-madmin.workspace = true
|
||||
reqwest = { workspace = true }
|
||||
aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
|
||||
@@ -248,6 +249,7 @@ windows-sys = { workspace = true, features = ["Win32_System_Ioctl"] }
|
||||
|
||||
[dev-dependencies]
|
||||
aws-smithy-async.workspace = true
|
||||
rustfs-s3-ops.workspace = true
|
||||
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util", "fs"] }
|
||||
criterion = { workspace = true, features = ["html_reports"] }
|
||||
temp-env = { workspace = true, features = ["async_closure"] }
|
||||
|
||||
@@ -14,6 +14,15 @@
|
||||
|
||||
//! Explicit ECStore public facades for outer crate compatibility boundaries.
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
pub mod diagnostic {
|
||||
pub use crate::cluster::rpc::runtime_sources::{
|
||||
current_read_version_entry_source, observe_read_version_context_boundary, with_read_version_entry,
|
||||
with_read_version_spawn_context,
|
||||
};
|
||||
pub use rustfs_io_metrics::internode_metrics::{ReadVersionContextBoundary, ReadVersionEntrySource};
|
||||
}
|
||||
|
||||
pub mod admin {
|
||||
pub use crate::diagnostics::admin_server_info::{get_local_server_property, get_server_info};
|
||||
}
|
||||
|
||||
@@ -2451,7 +2451,13 @@ fn spawn_recovery_disposition_maintenance_once(api: Arc<ECStore>) -> Option<Join
|
||||
return None;
|
||||
}
|
||||
Some(tokio::spawn(async move {
|
||||
run_recovery_disposition_maintenance_loop(api, cancel_token).await;
|
||||
let maintenance = run_recovery_disposition_maintenance_loop(api, cancel_token);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let maintenance = crate::cluster::rpc::runtime_sources::with_read_version_entry(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::RecoveryDispositionMaintenance,
|
||||
maintenance,
|
||||
);
|
||||
maintenance.await;
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -3334,7 +3340,13 @@ fn spawn_tier_delete_journal_recovery_once(api: Arc<ECStore>) {
|
||||
return;
|
||||
}
|
||||
tokio::spawn(async move {
|
||||
run_tier_delete_journal_recovery_loop(api, cancel_token).await;
|
||||
let recovery = run_tier_delete_journal_recovery_loop(api, cancel_token);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let recovery = crate::cluster::rpc::runtime_sources::with_read_version_entry(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::LifecycleRecovery,
|
||||
recovery,
|
||||
);
|
||||
recovery.await;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -3353,7 +3365,13 @@ fn spawn_transition_transaction_recovery_once(api: Arc<ECStore>) {
|
||||
return;
|
||||
}
|
||||
tokio::spawn(async move {
|
||||
run_transition_transaction_recovery_loop(api, cancel_token).await;
|
||||
let recovery = run_transition_transaction_recovery_loop(api, cancel_token);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let recovery = crate::cluster::rpc::runtime_sources::with_read_version_entry(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::LifecycleRecovery,
|
||||
recovery,
|
||||
);
|
||||
recovery.await;
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -178,6 +178,13 @@ pub(crate) fn object_lock_config_state_from_authoritative_metadata(bm: &BucketMe
|
||||
Ok(ObjectLockConfigState::ConfirmedAbsent)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub(crate) struct CachedQuotaMetadataSnapshot {
|
||||
pub(crate) quota: Option<BucketQuota>,
|
||||
pub(crate) bucket_incarnation: Uuid,
|
||||
pub(crate) quota_revision: OffsetDateTime,
|
||||
}
|
||||
|
||||
/// Convert the persisted serving-layer configuration into the storage-level
|
||||
/// [`DefaultRetention`](crate::bucket::object_lock::types::DefaultRetention)
|
||||
/// the WORM evaluation code consumes (rustfs/backlog#1842). A rule without a
|
||||
@@ -1423,6 +1430,9 @@ fn quota_config_and_incarnation_from_authority(
|
||||
BucketMetadataAuthority::Authoritative(metadata)
|
||||
if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
|
||||
{
|
||||
if !metadata.quota_config_json.is_empty() && metadata.quota_config.is_none() {
|
||||
return Err(Error::other("persisted bucket quota configuration is invalid"));
|
||||
}
|
||||
Ok((
|
||||
metadata.quota_config.clone(),
|
||||
metadata.bucket_incarnation_id,
|
||||
@@ -1437,6 +1447,15 @@ fn quota_config_and_incarnation_from_authority(
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn get_cached_quota_config_and_incarnation_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
) -> Result<Option<CachedQuotaMetadataSnapshot>> {
|
||||
let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone();
|
||||
bucket_meta_sys.cached_quota_config_and_incarnation(bucket).await
|
||||
}
|
||||
|
||||
pub async fn get_replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
@@ -2952,6 +2971,27 @@ impl BucketMetadataSys {
|
||||
}
|
||||
}
|
||||
|
||||
async fn cached_quota_config_and_incarnation(&self, bucket: &str) -> Result<Option<CachedQuotaMetadataSnapshot>> {
|
||||
let metadata = {
|
||||
let metadata_map = self.metadata_map.read().await;
|
||||
metadata_map.get(bucket).cloned()
|
||||
};
|
||||
let Some(metadata) = metadata else {
|
||||
return Ok(None);
|
||||
};
|
||||
if !metadata.bucket_incarnation_sidecar || metadata.bucket_incarnation_id.is_nil() {
|
||||
return Ok(None);
|
||||
}
|
||||
if !metadata.quota_config_json.is_empty() && metadata.quota_config.is_none() {
|
||||
return Err(Error::other("persisted bucket quota configuration is invalid"));
|
||||
}
|
||||
Ok(Some(CachedQuotaMetadataSnapshot {
|
||||
quota: metadata.quota_config.clone(),
|
||||
bucket_incarnation: metadata.bucket_incarnation_id,
|
||||
quota_revision: metadata.quota_config_updated_at,
|
||||
}))
|
||||
}
|
||||
|
||||
pub async fn get_replication_config(&self, bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
|
||||
let (bm, _) = self.get_config(bucket).await?;
|
||||
|
||||
@@ -4856,6 +4896,43 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_admission_rejects_stale_cached_no_quota() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let sys = bucket_metadata_sys_of(&store.ctx).expect("scoped metadata system");
|
||||
let bucket = "stale-no-quota-admission";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let (stale, _) = sys.read().await.get_config(bucket).await.expect("cache initial metadata");
|
||||
store
|
||||
.update_bucket_metadata_config(bucket, "quota.json", br#"{"quota":1}"#.to_vec())
|
||||
.await
|
||||
.expect("publish quota to authoritative storage");
|
||||
// Model a peer that has not received the newer configuration snapshot.
|
||||
sys.read().await.set(bucket.to_owned(), stale).await;
|
||||
assert!(
|
||||
get_cached_quota_config_and_incarnation_in(&store.ctx, bucket)
|
||||
.await
|
||||
.expect("read stale cache")
|
||||
.expect("cached bucket identity")
|
||||
.quota
|
||||
.is_none()
|
||||
);
|
||||
|
||||
temp_env::async_with_vars([("RUSTFS_QUOTA_BEGIN_SAFE_NO_QUOTA_CACHE_FAST_PATH", Some("true"))], async {
|
||||
let error = crate::bucket::quota::reservation::begin(&store.ctx, bucket, "object", &ObjectOptions::default(), 0, 0)
|
||||
.await
|
||||
.err()
|
||||
.expect("persisted quota must reject admission without its snapshot");
|
||||
assert!(matches!(error, Error::PartMissingOrCorrupt), "{error:?}");
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn stale_config_request_cannot_mutate_a_recreated_bucket() {
|
||||
|
||||
@@ -850,6 +850,11 @@ pub(crate) async fn begin(
|
||||
let snapshot_admission = opts.quota_admission;
|
||||
let data_movement = opts.data_movement;
|
||||
if crate::bucket::utils::is_meta_bucketname(bucket) {
|
||||
let meta_bucket_fast_path_started = rustfs_io_metrics::put_stage_timer();
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_META_BUCKET_FAST_PATH,
|
||||
meta_bucket_fast_path_started,
|
||||
);
|
||||
return Ok(QuotaContext {
|
||||
store: None,
|
||||
bucket: bucket.to_string(),
|
||||
@@ -906,9 +911,63 @@ pub(crate) async fn begin(
|
||||
});
|
||||
}
|
||||
|
||||
let metadata_lock_started = rustfs_io_metrics::put_stage_timer();
|
||||
let metadata_guard = metadata_sys::acquire_bucket_metadata_transaction_read_lock_for_options_in(ctx, bucket, opts).await?;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_METADATA_LOCK,
|
||||
metadata_lock_started,
|
||||
);
|
||||
let cache_lookup_started = rustfs_io_metrics::put_stage_timer();
|
||||
let cached_quota_snapshot = if rustfs_io_metrics::put_stage_metrics_enabled() {
|
||||
match metadata_sys::get_cached_quota_config_and_incarnation_in(ctx, bucket).await {
|
||||
Ok(Some(snapshot)) => {
|
||||
rustfs_io_metrics::record_put_object_quota_cache_decision(
|
||||
"candidate",
|
||||
quota_cache_reason(snapshot.quota.as_ref()),
|
||||
);
|
||||
Some(snapshot)
|
||||
}
|
||||
Ok(None) => {
|
||||
rustfs_io_metrics::record_put_object_quota_cache_decision("miss", "no_authoritative_cache");
|
||||
None
|
||||
}
|
||||
Err(_) => {
|
||||
rustfs_io_metrics::record_put_object_quota_cache_decision("miss", "invalid_cached_quota");
|
||||
None
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CACHE_LOOKUP,
|
||||
cache_lookup_started,
|
||||
);
|
||||
let config_read_started = rustfs_io_metrics::put_stage_timer();
|
||||
let (quota, bucket_incarnation, quota_revision) =
|
||||
metadata_sys::get_quota_config_and_incarnation_from_disk_for_options_in(ctx, bucket, opts, &metadata_guard).await?;
|
||||
if let Some(cached) = cached_quota_snapshot.as_ref() {
|
||||
let reason = quota_cache_reason(quota.as_ref());
|
||||
if cached.quota == quota && cached.bucket_incarnation == bucket_incarnation && cached.quota_revision == quota_revision {
|
||||
rustfs_io_metrics::record_put_object_quota_cache_decision("authoritative_match", reason);
|
||||
} else {
|
||||
rustfs_io_metrics::record_put_object_quota_cache_decision("authoritative_mismatch", reason);
|
||||
if cached.quota != quota {
|
||||
rustfs_io_metrics::record_put_object_quota_cache_decision("authoritative_mismatch_field", "quota");
|
||||
}
|
||||
if cached.bucket_incarnation != bucket_incarnation {
|
||||
rustfs_io_metrics::record_put_object_quota_cache_decision("authoritative_mismatch_field", "bucket_incarnation");
|
||||
}
|
||||
if cached.quota_revision != quota_revision {
|
||||
rustfs_io_metrics::record_put_object_quota_cache_decision("authoritative_mismatch_field", "quota_revision");
|
||||
}
|
||||
}
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONFIG_READ,
|
||||
config_read_started,
|
||||
);
|
||||
let lock_lost_check_started = rustfs_io_metrics::put_stage_timer();
|
||||
if metadata_guard.is_lock_lost() {
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "quota_config",
|
||||
@@ -918,6 +977,11 @@ pub(crate) async fn begin(
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_LOCK_LOST_CHECK,
|
||||
lock_lost_check_started,
|
||||
);
|
||||
let policy_evaluate_started = rustfs_io_metrics::put_stage_timer();
|
||||
if quota
|
||||
.as_ref()
|
||||
.is_some_and(|quota| quota.has_unsupported_reservation_protocol())
|
||||
@@ -926,6 +990,11 @@ pub(crate) async fn begin(
|
||||
return Err(StorageError::PartMissingOrCorrupt);
|
||||
}
|
||||
let durable_quota = quota.as_ref().filter(|quota| quota.uses_durable_reservations());
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_POLICY_EVALUATE,
|
||||
policy_evaluate_started,
|
||||
);
|
||||
let capability_proof_started = rustfs_io_metrics::put_stage_timer();
|
||||
let capability_proof = if durable_quota.is_some() {
|
||||
Some(
|
||||
crate::services::notification_sys::acquire_cross_pool_fence_fleet_proof()
|
||||
@@ -934,7 +1003,12 @@ pub(crate) async fn begin(
|
||||
} else {
|
||||
None
|
||||
};
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CAPABILITY_PROOF,
|
||||
capability_proof_started,
|
||||
);
|
||||
let durable_quota_limit = durable_quota.and_then(|quota| quota.quota);
|
||||
let snapshot_admission_started = rustfs_io_metrics::put_stage_timer();
|
||||
let reservation_protocol = durable_quota.and_then(|quota| quota.reservation_protocol);
|
||||
let snapshot_admission = match quota.as_ref().filter(|quota| !quota.uses_durable_reservations()) {
|
||||
Some(quota) => match (quota.quota, snapshot_admission) {
|
||||
@@ -954,6 +1028,10 @@ pub(crate) async fn begin(
|
||||
},
|
||||
None => None,
|
||||
};
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_SNAPSHOT_ADMISSION,
|
||||
snapshot_admission_started,
|
||||
);
|
||||
let legacy_data_movement = durable_quota_limit.is_none()
|
||||
&& quota.as_ref().and_then(|quota| quota.quota).is_some()
|
||||
&& snapshot_admission.is_none()
|
||||
@@ -965,11 +1043,21 @@ pub(crate) async fn begin(
|
||||
.then(|| quota.as_ref().and_then(|quota| quota.quota))
|
||||
.flatten()
|
||||
});
|
||||
let object_store_lookup_started = rustfs_io_metrics::put_stage_timer();
|
||||
let store = if durable_quota_limit.is_some() {
|
||||
Some(metadata_sys::object_store_in(ctx).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_OBJECT_STORE_LOOKUP,
|
||||
object_store_lookup_started,
|
||||
);
|
||||
let context_build_started = rustfs_io_metrics::put_stage_timer();
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONTEXT_BUILD,
|
||||
context_build_started,
|
||||
);
|
||||
Ok(QuotaContext {
|
||||
store,
|
||||
bucket: bucket.to_string(),
|
||||
@@ -988,6 +1076,15 @@ pub(crate) async fn begin(
|
||||
})
|
||||
}
|
||||
|
||||
fn quota_cache_reason(quota: Option<&crate::bucket::quota::BucketQuota>) -> &'static str {
|
||||
match quota {
|
||||
None => "no_quota",
|
||||
Some(quota) if quota.uses_durable_reservations() => "durable_quota",
|
||||
Some(quota) if quota.quota.is_some() => "snapshot_quota",
|
||||
Some(_) => "quota_without_limit",
|
||||
}
|
||||
}
|
||||
|
||||
fn quota_capability_error(bucket: &str, object: &str) -> StorageError {
|
||||
StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "quota_capability",
|
||||
|
||||
@@ -355,6 +355,13 @@ fn apply_peer_replay_response(
|
||||
}
|
||||
}
|
||||
|
||||
fn rename_data_grpc_stage(path: &str) -> bool {
|
||||
matches!(
|
||||
path,
|
||||
"/node_service.NodeService/RenameData" | "/node_service.NodeService/RenameDataAtIncarnation"
|
||||
)
|
||||
}
|
||||
|
||||
impl<S, ReqBody, ResBody> Service<HttpRequest<ReqBody>> for ReplayScopeChannel<S>
|
||||
where
|
||||
S: Service<HttpRequest<ReqBody>, Response = HttpResponse<ResBody>>,
|
||||
@@ -372,6 +379,12 @@ where
|
||||
}
|
||||
|
||||
fn call(&mut self, mut request: HttpRequest<ReqBody>) -> Self::Future {
|
||||
let observe_rename_data = rustfs_io_metrics::put_stage_metrics_enabled() && rename_data_grpc_stage(request.uri().path());
|
||||
let request_scope_started = if observe_rename_data {
|
||||
rustfs_io_metrics::put_stage_timer()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let authenticated = self.audience.as_ref().is_some_and(|_| {
|
||||
request
|
||||
.headers()
|
||||
@@ -406,11 +419,30 @@ where
|
||||
}
|
||||
}
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_REQUEST_SCOPE,
|
||||
request_scope_started,
|
||||
);
|
||||
|
||||
let audience = self.audience.clone();
|
||||
let future = self.inner.call(request);
|
||||
Box::pin(async move {
|
||||
let response = future.await?;
|
||||
let response = if observe_rename_data {
|
||||
rustfs_io_metrics::observe_put_stage_future(
|
||||
future,
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL,
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL_POLL_PENDING_COUNT,
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL_FIRST_PENDING_TO_READY,
|
||||
)
|
||||
.await?
|
||||
} else {
|
||||
future.await?
|
||||
};
|
||||
let replay_response_started = if observe_rename_data {
|
||||
rustfs_io_metrics::put_stage_timer()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let (Some(audience), Some(challenge)) = (audience, challenge) {
|
||||
let response_state = verify_tonic_peer_replay_capabilities_response(&audience, challenge, response.headers());
|
||||
if let Err(error) = &response_state
|
||||
@@ -428,6 +460,10 @@ where
|
||||
}
|
||||
apply_peer_replay_response(audience, sent_state, response_state);
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_REPLAY_RESPONSE,
|
||||
replay_response_started,
|
||||
);
|
||||
Ok(response)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -2163,6 +2163,7 @@ impl RemoteDisk {
|
||||
self.execute_with_timeout_for_op(
|
||||
"rename_data",
|
||||
|| async {
|
||||
let prepare_started = rustfs_io_metrics::put_stage_timer();
|
||||
let file_info = compat_json(fi)?;
|
||||
let file_info_bin = encode_file_info_msgpack(fi)?;
|
||||
let mut client = self.get_client().await?;
|
||||
@@ -2190,24 +2191,67 @@ impl RemoteDisk {
|
||||
} else {
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "rename_data")?;
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_PREPARE,
|
||||
prepare_started,
|
||||
);
|
||||
|
||||
let response = if incarnation_bound {
|
||||
// Older peers return Unimplemented before mutation; never downgrade.
|
||||
client.rename_data_at_incarnation(request).await?
|
||||
} else {
|
||||
client.rename_data(request).await?
|
||||
}
|
||||
.into_inner();
|
||||
let rpc_started = rustfs_io_metrics::put_stage_timer();
|
||||
let response = rustfs_io_metrics::observe_put_stage_future(
|
||||
async {
|
||||
if incarnation_bound {
|
||||
// Older peers return Unimplemented before mutation; never downgrade.
|
||||
client.rename_data_at_incarnation(request).await
|
||||
} else {
|
||||
client.rename_data(request).await
|
||||
}
|
||||
},
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT,
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_POLL_PENDING_COUNT,
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_FIRST_PENDING_TO_READY,
|
||||
)
|
||||
.await?;
|
||||
let response_process_started = rustfs_io_metrics::put_stage_timer();
|
||||
let response_headers_started = rustfs_io_metrics::put_stage_timer();
|
||||
let _ = response.metadata().len();
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RESPONSE_HEADERS,
|
||||
response_headers_started,
|
||||
);
|
||||
let into_inner_started = rustfs_io_metrics::put_stage_timer();
|
||||
let response = response.into_inner();
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_INTO_INNER,
|
||||
into_inner_started,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC,
|
||||
rpc_started,
|
||||
);
|
||||
|
||||
let success_check_started = rustfs_io_metrics::put_stage_timer();
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_SUCCESS_CHECK,
|
||||
success_check_started,
|
||||
);
|
||||
|
||||
let decode_started = rustfs_io_metrics::put_stage_timer();
|
||||
let rename_data_resp = decode_msgpack_or_json::<RenameDataResp>(
|
||||
&response.rename_data_resp_bin,
|
||||
&response.rename_data_resp,
|
||||
"RenameDataResp",
|
||||
)?;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_DECODE,
|
||||
decode_started,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RESPONSE_PROCESS,
|
||||
response_process_started,
|
||||
);
|
||||
|
||||
Ok(rename_data_resp)
|
||||
},
|
||||
@@ -3033,7 +3077,7 @@ impl DiskAPI for RemoteDisk {
|
||||
opts_bin: opts_bin.into(),
|
||||
});
|
||||
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_request();
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_request(opts);
|
||||
if let Some(request_payload_bytes) = request_payload_bytes {
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_sent_bytes(request_payload_bytes);
|
||||
}
|
||||
@@ -3111,6 +3155,7 @@ impl DiskAPI for RemoteDisk {
|
||||
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE, encode_started);
|
||||
let request_payload_bytes = batch_read_version_attribution_enabled
|
||||
.then(|| batch_read_version_request_payload_len(&req, &batch_read_version_req, &batch_read_version_req_bin));
|
||||
let request_opts = req.opts;
|
||||
let batch_result = self
|
||||
.execute_with_timeout_for_op(
|
||||
"batch_read_version",
|
||||
@@ -3124,7 +3169,7 @@ impl DiskAPI for RemoteDisk {
|
||||
batch_read_version_req_bin: batch_read_version_req_bin.into(),
|
||||
});
|
||||
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_request();
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_request(&request_opts);
|
||||
if let Some(request_payload_bytes) = request_payload_bytes {
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_sent_bytes(
|
||||
request_payload_bytes.saturating_add(disk_len),
|
||||
@@ -4785,6 +4830,8 @@ mod tests {
|
||||
cleanup_data_dir: Some(Uuid::new_v4()),
|
||||
sign: Some(vec![0x14, 0x35]),
|
||||
old_current_size: Some(crate::disk::OldCurrentSize::Present(64 * 1024)),
|
||||
old_current_source_checked: false,
|
||||
old_current_source: None,
|
||||
};
|
||||
let json = serde_json::to_string(&response).expect("legacy rename_data JSON response should encode");
|
||||
let decode_before = rustfs_io_metrics::internode_metrics::global_internode_metrics().msgpack_json_decode_total_for_test();
|
||||
@@ -5187,6 +5234,8 @@ mod tests {
|
||||
cleanup_data_dir: Some(Uuid::new_v4()),
|
||||
sign: Some(vec![1_u8; 32]),
|
||||
old_current_size: Some(crate::disk::OldCurrentSize::Present(4096)),
|
||||
old_current_source_checked: false,
|
||||
old_current_source: None,
|
||||
};
|
||||
let json = serde_json::to_vec(&response).expect("rename data response json should encode");
|
||||
let named_msgpack = rmp_serde::encode::to_vec_named(&response).expect("rename data response named msgpack should encode");
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::disk::ReadOptions;
|
||||
use rustfs_io_metrics::internode_metrics::{
|
||||
INTERNODE_MSGPACK_CODEC_JSON, INTERNODE_MSGPACK_CODEC_MSGPACK, INTERNODE_MSGPACK_DIRECTION_RESPONSE,
|
||||
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, INTERNODE_OPERATION_GRPC_COMPARE_AND_UPDATE_FILE,
|
||||
@@ -19,8 +20,320 @@ use rustfs_io_metrics::internode_metrics::{
|
||||
INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, global_internode_metrics,
|
||||
};
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
use rustfs_io_metrics::internode_metrics::{
|
||||
ReadVersionRequestCaller, ReadVersionRequestMode, record_read_version_request_by_caller,
|
||||
};
|
||||
use std::time::Duration;
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
tokio::task_local! {
|
||||
static CURRENT_READ_VERSION_REQUEST_CALLER: std::cell::Cell<Option<ReadVersionRequestCaller>>;
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
pub(crate) fn read_version_request_caller_from_location(file: &str) -> ReadVersionRequestCaller {
|
||||
let path = file.replace('\\', "/");
|
||||
if path.ends_with("/store/peer.rs") {
|
||||
ReadVersionRequestCaller::Peer
|
||||
} else if path.ends_with("/store/init.rs") {
|
||||
ReadVersionRequestCaller::Startup
|
||||
} else if path.ends_with("/store/heal.rs") || path.ends_with("/set_disk/ops/heal.rs") {
|
||||
ReadVersionRequestCaller::Heal
|
||||
} else if path.ends_with("/set_disk/ops/heal_walk.rs") {
|
||||
ReadVersionRequestCaller::Heal
|
||||
} else if path.ends_with("/store/list_objects.rs") {
|
||||
ReadVersionRequestCaller::ListObjects
|
||||
} else if path.ends_with("/store/object.rs") || path.ends_with("/set_disk/ops/object.rs") {
|
||||
ReadVersionRequestCaller::Object
|
||||
} else if path.ends_with("/set_disk/core/metadata_read.rs") {
|
||||
ReadVersionRequestCaller::MetadataRead
|
||||
} else if path.ends_with("/set_disk/ops/multipart.rs") {
|
||||
ReadVersionRequestCaller::Multipart
|
||||
} else if path.ends_with("/disk/disk_store.rs") {
|
||||
ReadVersionRequestCaller::DiskStore
|
||||
} else if path.ends_with("/disk/mod.rs") {
|
||||
ReadVersionRequestCaller::DiskDispatch
|
||||
} else if path.ends_with("/set_disk/read.rs") {
|
||||
ReadVersionRequestCaller::SetDiskRead
|
||||
} else if path.ends_with("/set_disk/core/io_primitives.rs") {
|
||||
ReadVersionRequestCaller::SetDiskIo
|
||||
} else if path.ends_with("/services/rebalance/worker.rs") {
|
||||
ReadVersionRequestCaller::Rebalance
|
||||
} else if path.ends_with("/core/pools.rs") {
|
||||
ReadVersionRequestCaller::Decommission
|
||||
} else if path.ends_with("/tier_free_version_recovery.rs") {
|
||||
ReadVersionRequestCaller::TierRecovery
|
||||
} else if path.contains("/bucket/lifecycle/") {
|
||||
ReadVersionRequestCaller::Lifecycle
|
||||
} else if path.contains("/cache_value/") {
|
||||
ReadVersionRequestCaller::CacheValue
|
||||
} else if path.contains("/ecstore/src/") {
|
||||
ReadVersionRequestCaller::Other
|
||||
} else {
|
||||
ReadVersionRequestCaller::Unattributed
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
pub(crate) fn current_read_version_request_caller() -> Option<ReadVersionRequestCaller> {
|
||||
CURRENT_READ_VERSION_REQUEST_CALLER
|
||||
.try_with(std::cell::Cell::get)
|
||||
.ok()
|
||||
.flatten()
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
pub(crate) async fn with_read_version_request_caller<F>(caller: ReadVersionRequestCaller, future: F) -> F::Output
|
||||
where
|
||||
F: std::future::Future,
|
||||
{
|
||||
CURRENT_READ_VERSION_REQUEST_CALLER
|
||||
.scope(std::cell::Cell::new(Some(caller)), future)
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
pub(crate) async fn with_read_version_request_caller_if_absent<F>(caller: ReadVersionRequestCaller, future: F) -> F::Output
|
||||
where
|
||||
F: std::future::Future,
|
||||
{
|
||||
if current_read_version_request_caller().is_some() {
|
||||
future.await
|
||||
} else {
|
||||
with_read_version_request_caller(caller, future).await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "diagnostic-read-version-caller"))]
|
||||
pub(crate) async fn with_optional_read_version_request_caller<F>(caller: Option<ReadVersionRequestCaller>, future: F) -> F::Output
|
||||
where
|
||||
F: std::future::Future,
|
||||
{
|
||||
match caller {
|
||||
Some(caller) => with_read_version_request_caller(caller, future).await,
|
||||
None => future.await,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
use rustfs_io_metrics::internode_metrics::{ReadVersionAttribution, ReadVersionContextBoundary, ReadVersionEntrySource};
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
use rustfs_io_metrics::s3_http_metrics::{S3HttpOperationContext, current_s3_http_operation_context};
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
struct ReadVersionEntryContext {
|
||||
source: ReadVersionEntrySource,
|
||||
api_operation: Option<S3HttpOperationContext>,
|
||||
caller: Option<ReadVersionRequestCaller>,
|
||||
// Entry provenance stays fixed; the parent snapshot follows each handoff.
|
||||
parent_api_operation: Option<S3HttpOperationContext>,
|
||||
parent_caller: Option<ReadVersionRequestCaller>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
impl ReadVersionEntryContext {
|
||||
fn new(
|
||||
source: ReadVersionEntrySource,
|
||||
api_operation: Option<S3HttpOperationContext>,
|
||||
caller: Option<ReadVersionRequestCaller>,
|
||||
) -> Self {
|
||||
Self {
|
||||
source,
|
||||
api_operation,
|
||||
caller,
|
||||
parent_api_operation: api_operation,
|
||||
parent_caller: caller,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Capture values before spawning or enqueueing; absence is part of the value.
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub(crate) struct ReadVersionDiagnosticContext {
|
||||
pub(crate) api_operation: Option<S3HttpOperationContext>,
|
||||
pub(crate) caller: Option<ReadVersionRequestCaller>,
|
||||
entry: Option<ReadVersionEntryContext>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
tokio::task_local! {
|
||||
static CURRENT_READ_VERSION_ENTRY: std::cell::Cell<Option<ReadVersionEntryContext>>;
|
||||
static CURRENT_READ_VERSION_BATCH_CONTEXTS: Vec<ReadVersionDiagnosticContext>;
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
impl ReadVersionDiagnosticContext {
|
||||
pub(crate) fn capture() -> Self {
|
||||
Self {
|
||||
api_operation: current_s3_http_operation_context(),
|
||||
caller: current_read_version_request_caller(),
|
||||
entry: CURRENT_READ_VERSION_ENTRY.try_with(std::cell::Cell::get).ok().flatten(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn capture_entry(source: ReadVersionEntrySource) -> Self {
|
||||
let mut context = Self::capture();
|
||||
if context.entry.is_none() {
|
||||
context.entry = Some(ReadVersionEntryContext::new(source, context.api_operation, context.caller));
|
||||
context.observe(ReadVersionContextBoundary::Entry);
|
||||
}
|
||||
context
|
||||
}
|
||||
|
||||
pub(crate) fn observe(self, boundary: ReadVersionContextBoundary) {
|
||||
self.observe_with_mode(boundary, None);
|
||||
}
|
||||
|
||||
fn observe_with_mode(self, boundary: ReadVersionContextBoundary, mode: Option<ReadVersionRequestMode>) {
|
||||
rustfs_io_metrics::internode_metrics::record_read_version_context_transition(
|
||||
self.entry.map_or(ReadVersionEntrySource::Unclassified, |entry| entry.source),
|
||||
boundary,
|
||||
self.api_operation,
|
||||
current_s3_http_operation_context(),
|
||||
self.caller,
|
||||
current_read_version_request_caller(),
|
||||
mode,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
pub fn with_read_version_entry<F>(source: ReadVersionEntrySource, future: F) -> impl std::future::Future<Output = F::Output>
|
||||
where
|
||||
F: std::future::Future,
|
||||
{
|
||||
// Box before constructing the scope future so large storage futures do not
|
||||
// inflate every caller frame. Capture task-local values only when polled.
|
||||
let future = Box::pin(future);
|
||||
async move {
|
||||
let context = ReadVersionDiagnosticContext::capture_entry(source);
|
||||
CURRENT_READ_VERSION_ENTRY
|
||||
.scope(std::cell::Cell::new(context.entry), future)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
/// Capture on the caller before spawning, then restore when the child is polled.
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
pub fn with_read_version_spawn_context<F>(
|
||||
source: ReadVersionEntrySource,
|
||||
boundary: ReadVersionContextBoundary,
|
||||
future: F,
|
||||
) -> impl std::future::Future<Output = F::Output>
|
||||
where
|
||||
F: std::future::Future,
|
||||
{
|
||||
let context = ReadVersionDiagnosticContext::capture_entry(source);
|
||||
let future = Box::pin(future);
|
||||
with_read_version_diagnostic_context(context, boundary, future)
|
||||
}
|
||||
|
||||
/// Observe a named entry even when its scope retains an earlier source.
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
pub fn observe_read_version_context_boundary(boundary: ReadVersionContextBoundary) {
|
||||
ReadVersionDiagnosticContext::capture().observe(boundary);
|
||||
}
|
||||
|
||||
/// The earliest observed storage entry, for diagnostic consumers only.
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
pub fn current_read_version_entry_source() -> Option<ReadVersionEntrySource> {
|
||||
CURRENT_READ_VERSION_ENTRY
|
||||
.try_with(std::cell::Cell::get)
|
||||
.ok()
|
||||
.flatten()
|
||||
.map(|entry| entry.source)
|
||||
}
|
||||
|
||||
/// Restore only this item's context and observe the child after restoration.
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
pub(crate) async fn with_read_version_diagnostic_context<F>(
|
||||
context: ReadVersionDiagnosticContext,
|
||||
boundary: ReadVersionContextBoundary,
|
||||
future: F,
|
||||
) -> F::Output
|
||||
where
|
||||
F: std::future::Future,
|
||||
{
|
||||
let observed = async {
|
||||
context.observe(boundary);
|
||||
future.await
|
||||
};
|
||||
let caller = CURRENT_READ_VERSION_REQUEST_CALLER.scope(std::cell::Cell::new(context.caller), observed);
|
||||
let mut forwarded_entry = context.entry;
|
||||
if let Some(entry) = &mut forwarded_entry {
|
||||
entry.parent_api_operation = context.api_operation;
|
||||
entry.parent_caller = context.caller;
|
||||
}
|
||||
let entry = CURRENT_READ_VERSION_ENTRY.scope(std::cell::Cell::new(forwarded_entry), caller);
|
||||
rustfs_io_metrics::s3_http_metrics::with_isolated_s3_http_operation_context(context.api_operation, entry).await
|
||||
}
|
||||
|
||||
/// A batch has item contexts, not one owner's S3 task-local operation.
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
pub(crate) async fn with_read_version_batch_contexts<F>(contexts: Vec<ReadVersionDiagnosticContext>, future: F) -> F::Output
|
||||
where
|
||||
F: std::future::Future,
|
||||
{
|
||||
let batch = CURRENT_READ_VERSION_BATCH_CONTEXTS.scope(contexts, future);
|
||||
let caller = CURRENT_READ_VERSION_REQUEST_CALLER.scope(std::cell::Cell::new(None), batch);
|
||||
let entry = CURRENT_READ_VERSION_ENTRY.scope(std::cell::Cell::new(None), caller);
|
||||
rustfs_io_metrics::s3_http_metrics::with_isolated_s3_http_operation_context(None, entry).await
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
pub(crate) async fn with_batch_read_version_item_context<F>(index: usize, item_count: usize, future: F) -> F::Output
|
||||
where
|
||||
F: std::future::Future,
|
||||
{
|
||||
let context = CURRENT_READ_VERSION_BATCH_CONTEXTS
|
||||
.try_with(|contexts| (contexts.len() == item_count).then(|| contexts.get(index).copied()).flatten())
|
||||
.ok()
|
||||
.flatten();
|
||||
match context {
|
||||
Some(context) => with_read_version_diagnostic_context(context, ReadVersionContextBoundary::UnaryFallback, future).await,
|
||||
None => future.await,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
fn common_read_version_attribution<T: Copy + Eq>(values: impl Iterator<Item = Option<T>>) -> ReadVersionAttribution<T> {
|
||||
values
|
||||
.map(|value| value.map_or(ReadVersionAttribution::Absent, ReadVersionAttribution::Common))
|
||||
.reduce(|common, value| {
|
||||
if common == value {
|
||||
common
|
||||
} else {
|
||||
ReadVersionAttribution::Mixed
|
||||
}
|
||||
})
|
||||
.unwrap_or(ReadVersionAttribution::Absent)
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
fn record_read_version_entry_to_rpc_context(opts: &ReadOptions) {
|
||||
let context = ReadVersionDiagnosticContext::capture();
|
||||
let mode = Some(read_version_request_mode(opts));
|
||||
if let Some(entry) = context.entry {
|
||||
ReadVersionDiagnosticContext {
|
||||
api_operation: entry.api_operation,
|
||||
caller: entry.caller,
|
||||
..context
|
||||
}
|
||||
.observe_with_mode(ReadVersionContextBoundary::EntryToRpc, mode);
|
||||
ReadVersionDiagnosticContext {
|
||||
api_operation: entry.parent_api_operation,
|
||||
caller: entry.parent_caller,
|
||||
..context
|
||||
}
|
||||
.observe_with_mode(ReadVersionContextBoundary::RemoteRpc, mode);
|
||||
} else {
|
||||
context.observe_with_mode(ReadVersionContextBoundary::RemoteRpc, mode);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
use rustfs_io_metrics::internode_metrics::InternodeMetricsSnapshot;
|
||||
|
||||
@@ -106,7 +419,18 @@ pub(crate) fn record_remote_disk_grpc_read_all_request() {
|
||||
.record_outgoing_request_for_operation_and_backend(INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_read_version_request() {
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
fn read_version_request_mode(opts: &ReadOptions) -> ReadVersionRequestMode {
|
||||
match (opts.read_data, opts.healing, opts.incl_free_versions) {
|
||||
(false, false, false) => ReadVersionRequestMode::MetadataOnly,
|
||||
(true, false, false) => ReadVersionRequestMode::Data,
|
||||
(false, true, false) => ReadVersionRequestMode::Healing,
|
||||
(false, false, true) => ReadVersionRequestMode::IncludeFreeVersions,
|
||||
_ => ReadVersionRequestMode::Combined,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_read_version_request(_opts: &ReadOptions) {
|
||||
if !rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
@@ -114,9 +438,19 @@ pub(crate) fn record_remote_disk_grpc_read_version_request() {
|
||||
INTERNODE_OPERATION_GRPC_READ_VERSION,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let api_operation = rustfs_io_metrics::s3_http_metrics::current_s3_http_operation_context();
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
record_read_version_request_by_caller(
|
||||
api_operation,
|
||||
current_read_version_request_caller().unwrap_or_default(),
|
||||
read_version_request_mode(_opts),
|
||||
);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
record_read_version_entry_to_rpc_context(_opts);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_batch_read_version_request() {
|
||||
pub(crate) fn record_remote_disk_grpc_batch_read_version_request(_opts: &ReadOptions) {
|
||||
if !rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
@@ -124,6 +458,30 @@ pub(crate) fn record_remote_disk_grpc_batch_read_version_request() {
|
||||
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
{
|
||||
let attribution = CURRENT_READ_VERSION_BATCH_CONTEXTS.try_with(|contexts| {
|
||||
(
|
||||
common_read_version_attribution(contexts.iter().map(|context| context.api_operation)),
|
||||
common_read_version_attribution(contexts.iter().map(|context| context.caller)),
|
||||
common_read_version_attribution(contexts.iter().map(|context| context.entry.map(|entry| entry.source))),
|
||||
)
|
||||
});
|
||||
let (api, caller, source) = attribution.unwrap_or_else(|_| {
|
||||
let context = ReadVersionDiagnosticContext::capture();
|
||||
(
|
||||
common_read_version_attribution(std::iter::once(context.api_operation)),
|
||||
common_read_version_attribution(std::iter::once(context.caller)),
|
||||
common_read_version_attribution(std::iter::once(context.entry.map(|entry| entry.source))),
|
||||
)
|
||||
});
|
||||
rustfs_io_metrics::internode_metrics::record_batch_read_version_request_by_caller(
|
||||
api,
|
||||
caller,
|
||||
source,
|
||||
read_version_request_mode(_opts),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_batch_read_version_stage(stage: &'static str, duration: Duration) {
|
||||
@@ -303,3 +661,529 @@ pub(crate) fn internode_metrics_snapshot_for_test() -> InternodeMetricsSnapshot
|
||||
pub(crate) fn internode_msgpack_json_decode_error_total_for_test() -> u64 {
|
||||
global_internode_metrics().msgpack_json_decode_error_total_for_test()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
fn diagnostic_operation(operation: rustfs_s3_ops::S3Operation) -> S3HttpOperationContext {
|
||||
let mut request = rustfs_io_metrics::s3_http_metrics::S3HttpRequestGuard::new("GET");
|
||||
request.in_scope(|| {
|
||||
rustfs_io_metrics::record_s3_op(operation);
|
||||
current_s3_http_operation_context().expect("operation should be captured")
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[test]
|
||||
fn diagnostic_list_retains_background_entry_through_shared_workers() {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("test runtime");
|
||||
runtime.block_on(async {
|
||||
tokio::spawn(with_read_version_entry(ReadVersionEntrySource::RecoveryDispositionMaintenance, async {
|
||||
let entry = ReadVersionDiagnosticContext::capture();
|
||||
assert_eq!(entry.api_operation, None);
|
||||
assert_eq!(entry.caller, None);
|
||||
let expected_entry = entry.entry.expect("background entry is observed");
|
||||
assert_eq!(expected_entry.source.as_str(), "recovery_disposition_maintenance");
|
||||
with_read_version_entry(ReadVersionEntrySource::ListObjects, async {
|
||||
with_read_version_request_caller(ReadVersionRequestCaller::ListObjects, async {
|
||||
let parent = ReadVersionDiagnosticContext::capture_entry(ReadVersionEntrySource::ListObjects);
|
||||
assert_eq!(parent.entry, Some(expected_entry));
|
||||
for boundary in [
|
||||
ReadVersionContextBoundary::ListProducer,
|
||||
ReadVersionContextBoundary::ListGather,
|
||||
] {
|
||||
tokio::spawn(with_read_version_diagnostic_context(parent, boundary, async move {
|
||||
let child = ReadVersionDiagnosticContext::capture();
|
||||
assert_eq!(child.api_operation, None);
|
||||
assert_eq!(child.caller, Some(ReadVersionRequestCaller::ListObjects));
|
||||
let child_entry = child.entry.expect("worker retains background provenance");
|
||||
assert_eq!(child_entry.source, expected_entry.source);
|
||||
assert_eq!(child_entry.api_operation, None);
|
||||
assert_eq!(child_entry.caller, None);
|
||||
assert_eq!(child_entry.parent_api_operation, None);
|
||||
assert_eq!(child_entry.parent_caller, Some(ReadVersionRequestCaller::ListObjects));
|
||||
}))
|
||||
.await
|
||||
.expect("list worker completes");
|
||||
}
|
||||
})
|
||||
.await;
|
||||
})
|
||||
.await;
|
||||
assert_eq!(ReadVersionDiagnosticContext::capture(), entry);
|
||||
}))
|
||||
.await
|
||||
.expect("background task completes");
|
||||
assert_eq!(ReadVersionDiagnosticContext::capture().entry, None);
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[test]
|
||||
fn diagnostic_spawn_scope_captures_parent_and_isolates_absence() {
|
||||
struct LargeFuture([u8; 64 * 1024]);
|
||||
impl std::future::Future for LargeFuture {
|
||||
type Output = ReadVersionDiagnosticContext;
|
||||
fn poll(self: std::pin::Pin<&mut Self>, _cx: &mut std::task::Context<'_>) -> std::task::Poll<Self::Output> {
|
||||
assert_eq!(self.0[0], 7);
|
||||
std::task::Poll::Ready(ReadVersionDiagnosticContext::capture())
|
||||
}
|
||||
}
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("test runtime");
|
||||
let put = diagnostic_operation(rustfs_s3_ops::S3Operation::PutObject);
|
||||
let get = diagnostic_operation(rustfs_s3_ops::S3Operation::GetObject);
|
||||
runtime.block_on(async {
|
||||
let child = rustfs_io_metrics::s3_http_metrics::with_isolated_s3_http_operation_context(
|
||||
Some(put),
|
||||
with_read_version_request_caller(
|
||||
ReadVersionRequestCaller::Object,
|
||||
with_read_version_entry(ReadVersionEntrySource::ConfigWrite, async {
|
||||
let child = with_read_version_spawn_context(
|
||||
ReadVersionEntrySource::AppPutOwner,
|
||||
ReadVersionContextBoundary::AppPutOwner,
|
||||
LargeFuture([7; 64 * 1024]),
|
||||
);
|
||||
assert!(std::mem::size_of_val(&child) < 1024, "spawn scope must not embed the storage future");
|
||||
tokio::spawn(child).await.expect("child task")
|
||||
}),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(child.api_operation, Some(put));
|
||||
assert_eq!(child.caller, Some(ReadVersionRequestCaller::Object));
|
||||
assert_eq!(child.entry.expect("earlier source retained").source, ReadVersionEntrySource::ConfigWrite);
|
||||
let internal = with_read_version_spawn_context(
|
||||
ReadVersionEntrySource::AppPutOwner,
|
||||
ReadVersionContextBoundary::AppPutOwner,
|
||||
LargeFuture([7; 64 * 1024]),
|
||||
);
|
||||
let observed = rustfs_io_metrics::s3_http_metrics::with_isolated_s3_http_operation_context(
|
||||
Some(get),
|
||||
with_read_version_request_caller(
|
||||
ReadVersionRequestCaller::Peer,
|
||||
with_read_version_entry(ReadVersionEntrySource::ConfigRead, async {
|
||||
let observed = internal.await;
|
||||
assert_eq!(current_s3_http_operation_context(), Some(get));
|
||||
assert_eq!(current_read_version_entry_source(), Some(ReadVersionEntrySource::ConfigRead));
|
||||
observed
|
||||
}),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(observed.api_operation, None);
|
||||
assert_eq!(observed.caller, None);
|
||||
assert_eq!(
|
||||
observed.entry.expect("internal source retained").source,
|
||||
ReadVersionEntrySource::AppPutOwner
|
||||
);
|
||||
assert_eq!(current_s3_http_operation_context(), None);
|
||||
assert_eq!(current_read_version_entry_source(), None);
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[test]
|
||||
fn diagnostic_entry_keeps_large_future_off_the_scope_frame_and_captures_when_polled() {
|
||||
struct LargeFuture([u8; 64 * 1024]);
|
||||
impl std::future::Future for LargeFuture {
|
||||
type Output = ReadVersionDiagnosticContext;
|
||||
fn poll(self: std::pin::Pin<&mut Self>, _cx: &mut std::task::Context<'_>) -> std::task::Poll<Self::Output> {
|
||||
assert_eq!(self.0[0], 7);
|
||||
std::task::Poll::Ready(ReadVersionDiagnosticContext::capture())
|
||||
}
|
||||
}
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("test runtime");
|
||||
let get = diagnostic_operation(rustfs_s3_ops::S3Operation::GetObject);
|
||||
let read = with_read_version_entry(ReadVersionEntrySource::ObjectInfoRead, LargeFuture([7; 64 * 1024]));
|
||||
assert!(std::mem::size_of_val(&read) < 1024, "scope frame must not embed the storage future");
|
||||
assert_eq!(current_read_version_entry_source(), None);
|
||||
let observed = runtime.block_on(rustfs_io_metrics::s3_http_metrics::with_isolated_s3_http_operation_context(
|
||||
Some(get),
|
||||
with_read_version_entry(ReadVersionEntrySource::ConfigRead, read),
|
||||
));
|
||||
assert_eq!(observed.entry.expect("entry captured at poll").source, ReadVersionEntrySource::ConfigRead);
|
||||
assert_eq!(observed.api_operation, Some(get));
|
||||
assert_eq!(current_read_version_entry_source(), None);
|
||||
assert_eq!(current_s3_http_operation_context(), None);
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[test]
|
||||
fn diagnostic_named_entry_is_observed_when_earlier_source_is_preserved() {
|
||||
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("runtime");
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshots = recorder.snapshotter();
|
||||
rustfs_io_metrics::set_get_stage_metrics_enabled(true);
|
||||
let get = diagnostic_operation(rustfs_s3_ops::S3Operation::GetObject);
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
runtime.block_on(async {
|
||||
for earlier in [None, Some(ReadVersionEntrySource::ConfigRead)] {
|
||||
let api = earlier.map(|_| get);
|
||||
let entry = with_read_version_entry(ReadVersionEntrySource::DataUsageCacheRead, async {
|
||||
observe_read_version_context_boundary(ReadVersionContextBoundary::ScannerCacheLoad);
|
||||
assert_eq!(current_s3_http_operation_context(), api);
|
||||
});
|
||||
rustfs_io_metrics::s3_http_metrics::with_isolated_s3_http_operation_context(api, async {
|
||||
match earlier {
|
||||
Some(source) => with_read_version_entry(source, entry).await,
|
||||
None => entry.await,
|
||||
}
|
||||
})
|
||||
.await;
|
||||
assert_eq!(current_read_version_entry_source(), None);
|
||||
assert_eq!(current_s3_http_operation_context(), None);
|
||||
}
|
||||
});
|
||||
});
|
||||
let rows = snapshots.snapshot().into_vec();
|
||||
for (source, api, state) in [
|
||||
("data_usage_cache_read", "unattributed", "absent"),
|
||||
("config_read", "s3:GetObject", "preserved"),
|
||||
] {
|
||||
assert!(
|
||||
rows.iter().any(|(key, _, _, value)| {
|
||||
let labels = key
|
||||
.key()
|
||||
.labels()
|
||||
.map(|label| (label.key(), label.value()))
|
||||
.collect::<std::collections::HashMap<_, _>>();
|
||||
key.key().name() == "rustfs_storage_diagnostic_context_transitions_total"
|
||||
&& labels.get("boundary") == Some(&"scanner_cache_load")
|
||||
&& labels.get("entry_source") == Some(&source)
|
||||
&& labels.get("parent_api_operation") == Some(&api)
|
||||
&& labels.get("child_api_operation") == Some(&api)
|
||||
&& labels.get("api_context") == Some(&state)
|
||||
&& matches!(value, DebugValue::Counter(1))
|
||||
}),
|
||||
"a named cache entry remains observable with source {source}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[test]
|
||||
fn diagnostic_context_records_absent_internal_entry_and_real_child_loss() {
|
||||
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
|
||||
use rustfs_io_metrics::s3_http_metrics::{with_isolated_s3_http_operation_context, with_s3_http_operation_context};
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("test runtime");
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshots = recorder.snapshotter();
|
||||
rustfs_io_metrics::set_get_stage_metrics_enabled(true);
|
||||
let get = diagnostic_operation(rustfs_s3_ops::S3Operation::GetObject);
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
runtime.block_on(async {
|
||||
with_read_version_entry(ReadVersionEntrySource::ConfigRead, async {
|
||||
let context = ReadVersionDiagnosticContext::capture();
|
||||
tokio::spawn(with_read_version_diagnostic_context(
|
||||
context,
|
||||
ReadVersionContextBoundary::FullWait,
|
||||
async {
|
||||
assert_eq!(current_s3_http_operation_context(), None);
|
||||
record_read_version_entry_to_rpc_context(&ReadOptions::default());
|
||||
},
|
||||
))
|
||||
.await
|
||||
.expect("internal child should complete");
|
||||
})
|
||||
.await;
|
||||
with_s3_http_operation_context(
|
||||
Some(get),
|
||||
with_read_version_entry(
|
||||
ReadVersionEntrySource::ObjectRead,
|
||||
with_read_version_request_caller(ReadVersionRequestCaller::Object, async {
|
||||
let context = ReadVersionDiagnosticContext::capture();
|
||||
tokio::spawn(with_read_version_diagnostic_context(
|
||||
context,
|
||||
ReadVersionContextBoundary::EarlyStop,
|
||||
async move {
|
||||
assert_eq!(current_s3_http_operation_context(), Some(get));
|
||||
record_read_version_entry_to_rpc_context(&ReadOptions::default());
|
||||
with_isolated_s3_http_operation_context(None, async {
|
||||
record_read_version_entry_to_rpc_context(&ReadOptions::default());
|
||||
})
|
||||
.await;
|
||||
CURRENT_READ_VERSION_REQUEST_CALLER
|
||||
.scope(std::cell::Cell::new(None), async {
|
||||
record_read_version_entry_to_rpc_context(&ReadOptions::default());
|
||||
})
|
||||
.await;
|
||||
},
|
||||
))
|
||||
.await
|
||||
.expect("external child should complete");
|
||||
}),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(ReadVersionDiagnosticContext::capture().entry, None);
|
||||
record_read_version_entry_to_rpc_context(&ReadOptions::default());
|
||||
})
|
||||
});
|
||||
let rows = snapshots.snapshot().into_vec();
|
||||
let observed = |source: &str, state: &str, parent: &str, child: &str| {
|
||||
rows.iter().any(|(key, _, _, value)| {
|
||||
let labels = key
|
||||
.key()
|
||||
.labels()
|
||||
.map(|label| (label.key(), label.value()))
|
||||
.collect::<std::collections::HashMap<_, _>>();
|
||||
key.key().name() == "rustfs_storage_diagnostic_context_transitions_total"
|
||||
&& labels.get("entry_source") == Some(&source)
|
||||
&& labels.get("boundary") == Some(&"remote_rpc")
|
||||
&& labels.get("api_context") == Some(&state)
|
||||
&& labels.get("parent_api_operation") == Some(&parent)
|
||||
&& labels.get("child_api_operation") == Some(&child)
|
||||
&& labels.get("mode") == Some(&"metadata_only")
|
||||
&& matches!(value, DebugValue::Counter(1))
|
||||
})
|
||||
};
|
||||
assert!(observed("config_read", "absent", "unattributed", "unattributed"));
|
||||
assert!(observed("object_read", "preserved", "s3:GetObject", "s3:GetObject"));
|
||||
assert!(observed("object_read", "lost", "s3:GetObject", "unattributed"));
|
||||
assert!(observed("unclassified", "unobserved", "unattributed", "unattributed"));
|
||||
assert!(
|
||||
rows.iter().any(|(key, _, _, value)| {
|
||||
let labels = key
|
||||
.key()
|
||||
.labels()
|
||||
.map(|label| (label.key(), label.value()))
|
||||
.collect::<std::collections::HashMap<_, _>>();
|
||||
key.key().name() == "rustfs_storage_diagnostic_context_transitions_total"
|
||||
&& labels.get("boundary") == Some(&"remote_rpc")
|
||||
&& labels.get("parent_caller") == Some(&"object")
|
||||
&& labels.get("child_caller") == Some(&"unattributed")
|
||||
&& labels.get("caller_context") == Some(&"lost")
|
||||
&& matches!(value, DebugValue::Counter(1))
|
||||
}),
|
||||
"RPC caller must be compared with its direct parent's seed, not an earlier unseeded entry"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[tokio::test]
|
||||
async fn diagnostic_context_restores_ambient_scope_after_pending_item_is_cancelled() {
|
||||
use std::future::Future;
|
||||
let get = diagnostic_operation(rustfs_s3_ops::S3Operation::GetObject);
|
||||
let parent = ReadVersionDiagnosticContext {
|
||||
api_operation: Some(get),
|
||||
caller: Some(ReadVersionRequestCaller::Object),
|
||||
entry: Some(ReadVersionEntryContext::new(
|
||||
ReadVersionEntrySource::ObjectRead,
|
||||
Some(get),
|
||||
Some(ReadVersionRequestCaller::Object),
|
||||
)),
|
||||
};
|
||||
let internal = ReadVersionDiagnosticContext {
|
||||
api_operation: None,
|
||||
caller: None,
|
||||
entry: Some(ReadVersionEntryContext::new(ReadVersionEntrySource::ConfigRead, None, None)),
|
||||
};
|
||||
with_read_version_diagnostic_context(parent, ReadVersionContextBoundary::Entry, async {
|
||||
let mut pending = Box::pin(with_read_version_diagnostic_context(
|
||||
internal,
|
||||
ReadVersionContextBoundary::UnaryFallback,
|
||||
async {
|
||||
assert_eq!(ReadVersionDiagnosticContext::capture(), internal);
|
||||
std::future::pending::<()>().await;
|
||||
},
|
||||
));
|
||||
std::future::poll_fn(|cx| {
|
||||
assert!(pending.as_mut().poll(cx).is_pending());
|
||||
std::task::Poll::Ready(())
|
||||
})
|
||||
.await;
|
||||
assert_eq!(
|
||||
ReadVersionDiagnosticContext::capture(),
|
||||
parent,
|
||||
"a pending item must not leak into its poller's scope"
|
||||
);
|
||||
drop(pending);
|
||||
assert_eq!(
|
||||
ReadVersionDiagnosticContext::capture(),
|
||||
parent,
|
||||
"cancellation must retain the ambient request"
|
||||
);
|
||||
})
|
||||
.await;
|
||||
assert_eq!(ReadVersionDiagnosticContext::capture().entry, None);
|
||||
assert_eq!(current_s3_http_operation_context(), None);
|
||||
assert_eq!(current_read_version_request_caller(), None);
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[test]
|
||||
fn diagnostic_batch_attributes_each_dimension_and_counts_one_rpc() {
|
||||
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("test runtime");
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshots = recorder.snapshotter();
|
||||
rustfs_io_metrics::set_get_stage_metrics_enabled(true);
|
||||
let get = diagnostic_operation(rustfs_s3_ops::S3Operation::GetObject);
|
||||
let list = diagnostic_operation(rustfs_s3_ops::S3Operation::ListObjectsV2);
|
||||
let context = |api, caller, source| ReadVersionDiagnosticContext {
|
||||
api_operation: api,
|
||||
caller,
|
||||
entry: Some(ReadVersionEntryContext::new(source, api, caller)),
|
||||
};
|
||||
let external = context(Some(get), Some(ReadVersionRequestCaller::Object), ReadVersionEntrySource::ObjectRead);
|
||||
let internal = context(None, None, ReadVersionEntrySource::ConfigRead);
|
||||
let internal_object = context(None, Some(ReadVersionRequestCaller::Object), ReadVersionEntrySource::ConfigRead);
|
||||
let heal = context(Some(get), Some(ReadVersionRequestCaller::Heal), ReadVersionEntrySource::ObjectRead);
|
||||
let other_api = context(Some(list), Some(ReadVersionRequestCaller::Object), ReadVersionEntrySource::ObjectRead);
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
runtime.block_on(async {
|
||||
for contexts in [
|
||||
vec![external, external],
|
||||
vec![external, internal],
|
||||
vec![external, internal_object],
|
||||
vec![external, heal],
|
||||
vec![external, other_api],
|
||||
vec![internal, internal],
|
||||
] {
|
||||
with_read_version_batch_contexts(contexts, async {
|
||||
assert_eq!(current_s3_http_operation_context(), None);
|
||||
assert_eq!(current_read_version_request_caller(), None);
|
||||
record_remote_disk_grpc_batch_read_version_request(&ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
});
|
||||
})
|
||||
.await;
|
||||
}
|
||||
})
|
||||
});
|
||||
let labels = snapshots
|
||||
.snapshot()
|
||||
.into_vec()
|
||||
.into_iter()
|
||||
.filter(|(key, _, _, _)| {
|
||||
key.key().name() == "rustfs_system_network_internode_batch_read_version_requests_by_caller_total"
|
||||
})
|
||||
.map(|(key, _, _, value)| {
|
||||
assert!(matches!(value, DebugValue::Counter(1)), "batch must count one RPC, not its items");
|
||||
let labels = key
|
||||
.key()
|
||||
.labels()
|
||||
.map(|label| (label.key().to_owned(), label.value().to_owned()))
|
||||
.collect::<std::collections::HashMap<_, _>>();
|
||||
assert_eq!(labels["mode"], "data");
|
||||
(labels["api_operation"].clone(), labels["caller"].clone(), labels["entry_source"].clone())
|
||||
})
|
||||
.collect::<std::collections::HashSet<_>>();
|
||||
assert_eq!(labels.len(), 6);
|
||||
for (api, caller, source) in [
|
||||
("s3:GetObject", "object", "object_read"),
|
||||
("mixed", "mixed", "mixed"),
|
||||
("mixed", "object", "mixed"),
|
||||
("s3:GetObject", "mixed", "object_read"),
|
||||
("mixed", "object", "object_read"),
|
||||
("unattributed", "unattributed", "config_read"),
|
||||
] {
|
||||
assert!(labels.contains(&(api.to_owned(), caller.to_owned(), source.to_owned())));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
fn read_version_request_mode_uses_only_bounded_read_flags() {
|
||||
assert_eq!(read_version_request_mode(&ReadOptions::default()), ReadVersionRequestMode::MetadataOnly);
|
||||
assert_eq!(
|
||||
read_version_request_mode(&ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
}),
|
||||
ReadVersionRequestMode::Data
|
||||
);
|
||||
assert_eq!(
|
||||
read_version_request_mode(&ReadOptions {
|
||||
healing: true,
|
||||
..Default::default()
|
||||
}),
|
||||
ReadVersionRequestMode::Healing
|
||||
);
|
||||
assert_eq!(
|
||||
read_version_request_mode(&ReadOptions {
|
||||
incl_free_versions: true,
|
||||
..Default::default()
|
||||
}),
|
||||
ReadVersionRequestMode::IncludeFreeVersions
|
||||
);
|
||||
assert_eq!(
|
||||
read_version_request_mode(&ReadOptions {
|
||||
read_data: true,
|
||||
healing: true,
|
||||
..Default::default()
|
||||
}),
|
||||
ReadVersionRequestMode::Combined
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[test]
|
||||
fn read_version_request_caller_classification_uses_bounded_module_names() {
|
||||
assert_eq!(
|
||||
read_version_request_caller_from_location("/repo/crates/ecstore/src/store/peer.rs"),
|
||||
ReadVersionRequestCaller::Peer
|
||||
);
|
||||
assert_eq!(
|
||||
read_version_request_caller_from_location("/repo/crates/ecstore/src/set_disk/ops/heal.rs"),
|
||||
ReadVersionRequestCaller::Heal
|
||||
);
|
||||
assert_eq!(
|
||||
read_version_request_caller_from_location("/repo/crates/ecstore/src/store/object.rs"),
|
||||
ReadVersionRequestCaller::Object
|
||||
);
|
||||
assert_eq!(
|
||||
read_version_request_caller_from_location("/repo/crates/ecstore/src/set_disk/core/metadata_read.rs"),
|
||||
ReadVersionRequestCaller::MetadataRead
|
||||
);
|
||||
assert_eq!(
|
||||
read_version_request_caller_from_location("/repo/crates/ecstore/src/set_disk/ops/multipart.rs"),
|
||||
ReadVersionRequestCaller::Multipart
|
||||
);
|
||||
assert_eq!(
|
||||
read_version_request_caller_from_location("/repo/crates/ecstore/src/disk/disk_store.rs"),
|
||||
ReadVersionRequestCaller::DiskStore
|
||||
);
|
||||
assert_eq!(
|
||||
read_version_request_caller_from_location("/repo/crates/ecstore/src/other/module.rs"),
|
||||
ReadVersionRequestCaller::Other
|
||||
);
|
||||
assert_eq!(
|
||||
read_version_request_caller_from_location("/repo/external/module.rs"),
|
||||
ReadVersionRequestCaller::Unattributed
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[tokio::test]
|
||||
async fn read_version_request_caller_context_is_scoped_to_the_read_future() {
|
||||
assert_eq!(current_read_version_request_caller(), None);
|
||||
let caller =
|
||||
with_read_version_request_caller(ReadVersionRequestCaller::Peer, async { current_read_version_request_caller() })
|
||||
.await;
|
||||
assert_eq!(caller, Some(ReadVersionRequestCaller::Peer));
|
||||
assert_eq!(current_read_version_request_caller(), None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -574,17 +574,20 @@ where
|
||||
>,
|
||||
{
|
||||
let h = HeaderMap::new();
|
||||
let mut rd = api
|
||||
.get_object_reader(RUSTFS_META_BUCKET, file, None, h, opts)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
if err == Error::FileNotFound || matches!(err, Error::ObjectNotFound(_, _)) {
|
||||
Error::ConfigNotFound
|
||||
} else {
|
||||
warn!("read_config_with_metadata: err: {:?}, file: {}", err, file);
|
||||
err
|
||||
}
|
||||
})?;
|
||||
let read = api.get_object_reader(RUSTFS_META_BUCKET, file, None, h, opts);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let read = crate::cluster::rpc::runtime_sources::with_read_version_entry(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::ConfigRead,
|
||||
read,
|
||||
);
|
||||
let mut rd = read.await.map_err(|err| {
|
||||
if err == Error::FileNotFound || matches!(err, Error::ObjectNotFound(_, _)) {
|
||||
Error::ConfigNotFound
|
||||
} else {
|
||||
warn!("read_config_with_metadata: err: {:?}, file: {}", err, file);
|
||||
err
|
||||
}
|
||||
})?;
|
||||
|
||||
let data = if let Some(max_bytes) = max_bytes {
|
||||
let object_size = usize::try_from(rd.object_info.size).map_err(|_| Error::CorruptedFormat)?;
|
||||
@@ -807,7 +810,18 @@ where
|
||||
>,
|
||||
{
|
||||
let mut put_data = PutObjReader::from_vec(data);
|
||||
match api.put_object(RUSTFS_META_BUCKET, file, &mut put_data, opts).await {
|
||||
let write = api.put_object(RUSTFS_META_BUCKET, file, &mut put_data, opts);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let write = crate::cluster::rpc::runtime_sources::with_read_version_entry(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::ConfigWrite,
|
||||
async {
|
||||
crate::cluster::rpc::runtime_sources::observe_read_version_context_boundary(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionContextBoundary::ConfigWrite,
|
||||
);
|
||||
write.await
|
||||
},
|
||||
);
|
||||
match write.await {
|
||||
Ok(object_info) => Ok(object_info),
|
||||
Err(err) => {
|
||||
if log_error {
|
||||
@@ -2869,6 +2883,11 @@ mod tests {
|
||||
#[derive(Debug, Default)]
|
||||
struct MetadataWriteStore {
|
||||
error: Option<Error>,
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
observations:
|
||||
std::sync::Mutex<Vec<(Option<rustfs_io_metrics::internode_metrics::ReadVersionEntrySource>, Option<&'static str>)>>,
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
pending: bool,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
@@ -2899,7 +2918,20 @@ mod tests {
|
||||
_data: &mut Self::PutObjectReader,
|
||||
_opts: &Self::ObjectOptions,
|
||||
) -> core::result::Result<Self::ObjectInfo, Self::Error> {
|
||||
Err(self.error.clone().expect("test store error should be configured"))
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
{
|
||||
self.observations.lock().expect("observations lock").push((
|
||||
crate::cluster::rpc::runtime_sources::current_read_version_entry_source(),
|
||||
rustfs_io_metrics::s3_http_metrics::current_s3_http_operation_name(),
|
||||
));
|
||||
if self.pending {
|
||||
std::future::pending::<()>().await;
|
||||
}
|
||||
}
|
||||
match self.error.clone() {
|
||||
Some(error) => Err(error),
|
||||
None => Ok(ObjectInfo::default()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2907,6 +2939,7 @@ mod tests {
|
||||
async fn save_config_preserves_retryable_system_volume_errors() {
|
||||
let store = Arc::new(MetadataWriteStore {
|
||||
error: Some(Error::BucketNotFound(RUSTFS_META_BUCKET.to_string())),
|
||||
..Default::default()
|
||||
});
|
||||
let error =
|
||||
save_config_with_opts_inner(store, "buckets/example/.metadata.bin", Vec::new(), &ObjectOptions::default(), false)
|
||||
@@ -2918,6 +2951,59 @@ mod tests {
|
||||
Error::InsufficientWriteQuorum(RUSTFS_META_BUCKET.to_string(), "buckets/example/.metadata.bin".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[tokio::test]
|
||||
async fn diagnostic_config_write_retains_origin_and_api_for_results_and_cancellation() {
|
||||
use crate::cluster::rpc::runtime_sources::{current_read_version_entry_source, with_read_version_entry};
|
||||
use rustfs_io_metrics::internode_metrics::ReadVersionEntrySource;
|
||||
use rustfs_io_metrics::s3_http_metrics::{S3HttpRequestGuard, current_s3_http_operation_context};
|
||||
|
||||
let mut guard = S3HttpRequestGuard::new("PUT");
|
||||
let put = guard.in_scope(|| {
|
||||
rustfs_io_metrics::record_s3_op(rustfs_s3_ops::S3Operation::PutObject);
|
||||
current_s3_http_operation_context().expect("captured PutObject")
|
||||
});
|
||||
for earlier in [None, Some(ReadVersionEntrySource::TierConfigReload)] {
|
||||
for api in [None, Some(put)] {
|
||||
for case in ["success", "error", "cancelled"] {
|
||||
let store = Arc::new(MetadataWriteStore {
|
||||
error: (case == "error").then_some(Error::VolumeNotFound),
|
||||
pending: case == "cancelled",
|
||||
..Default::default()
|
||||
});
|
||||
let opts = ObjectOptions::default();
|
||||
let write = async {
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(1),
|
||||
save_config_with_opts_inner(store.clone(), "buckets/diagnostic/state", Vec::new(), &opts, false),
|
||||
)
|
||||
.await;
|
||||
match case {
|
||||
"success" => assert!(result.expect("write completes").is_ok()),
|
||||
"error" => {
|
||||
assert!(matches!(result.expect("error completes"), Err(Error::InsufficientWriteQuorum(_, _))))
|
||||
}
|
||||
_ => assert!(result.is_err(), "pending write is cancelled"),
|
||||
}
|
||||
};
|
||||
rustfs_io_metrics::s3_http_metrics::with_isolated_s3_http_operation_context(api, async {
|
||||
match earlier {
|
||||
Some(source) => with_read_version_entry(source, write).await,
|
||||
None => write.await,
|
||||
}
|
||||
})
|
||||
.await;
|
||||
assert_eq!(
|
||||
*store.observations.lock().expect("observations lock"),
|
||||
vec![(Some(earlier.unwrap_or(ReadVersionEntrySource::ConfigWrite)), api.map(|_| "s3:PutObject"))]
|
||||
);
|
||||
assert_eq!(current_read_version_entry_source(), None, "write scope restores: {case}");
|
||||
assert_eq!(current_s3_http_operation_context(), None, "API scope restores: {case}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
use rustfs_lock::client::LockClient;
|
||||
use rustfs_lock::client::local::LocalClient;
|
||||
use rustfs_lock::{LockError, LockInfo, LockResponse, LockStats};
|
||||
|
||||
@@ -110,19 +110,17 @@ async fn read_replica(set: Arc<SetDisks>) -> Result<ScannerPauseBacklogRetiremen
|
||||
set_index: set.set_index,
|
||||
data: None,
|
||||
};
|
||||
let reader = match set
|
||||
.get_object_reader(
|
||||
RUSTFS_META_BUCKET,
|
||||
SCANNER_PAUSE_BACKLOG_PATH,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
let read = set.get_object_reader(RUSTFS_META_BUCKET, SCANNER_PAUSE_BACKLOG_PATH, None, HeaderMap::new(), &opts);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let read = crate::cluster::rpc::runtime_sources::with_read_version_entry(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::ScannerBacklogRead,
|
||||
read,
|
||||
);
|
||||
let reader = match read.await {
|
||||
Ok(reader) => reader,
|
||||
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {
|
||||
return Ok(ScannerPauseBacklogRetirementRead { replica, etag: None });
|
||||
|
||||
@@ -2529,19 +2529,17 @@ where
|
||||
use crate::object_api::ObjectOptions;
|
||||
use http::HeaderMap;
|
||||
|
||||
match store
|
||||
.get_object_reader(
|
||||
RUSTFS_META_BUCKET,
|
||||
key,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
let read = store.get_object_reader(RUSTFS_META_BUCKET, key, None, HeaderMap::new(), &opts);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let read = crate::cluster::rpc::runtime_sources::with_read_version_entry(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::DataUsageCacheRead,
|
||||
read,
|
||||
);
|
||||
match read.await {
|
||||
// A cache object that fails to decode is treated as absent rather than
|
||||
// as an error: a corrupt cache should not stall the caller, and the
|
||||
// next scanner pass rewrites it.
|
||||
@@ -3064,6 +3062,8 @@ mod tests {
|
||||
transient_failures: Mutex<usize>,
|
||||
reads: Mutex<Vec<String>>,
|
||||
terminal_error: Mutex<Option<Error>>,
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
record_context: bool,
|
||||
}
|
||||
|
||||
impl UsageCacheReadStore {
|
||||
@@ -3072,6 +3072,8 @@ mod tests {
|
||||
transient_failures: Mutex::new(n),
|
||||
reads: Mutex::new(Vec::new()),
|
||||
terminal_error: Mutex::new(None),
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
record_context: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3080,6 +3082,8 @@ mod tests {
|
||||
transient_failures: Mutex::new(0),
|
||||
reads: Mutex::new(Vec::new()),
|
||||
terminal_error: Mutex::new(Some(error)),
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
record_context: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3106,6 +3110,12 @@ mod tests {
|
||||
_h: Self::HeaderMap,
|
||||
_opts: &Self::ObjectOptions,
|
||||
) -> Result<Self::GetObjectReader, Self::Error> {
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
if self.record_context {
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_request(
|
||||
&crate::disk::ReadOptions::default(),
|
||||
);
|
||||
}
|
||||
self.reads.lock().await.push(object.to_string());
|
||||
if let Some(error) = self.terminal_error.lock().await.clone() {
|
||||
return Err(error);
|
||||
@@ -3205,6 +3215,89 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[test]
|
||||
fn diagnostic_usage_cache_read_preserves_api_and_earlier_entry() {
|
||||
use crate::cluster::rpc::runtime_sources::with_read_version_entry;
|
||||
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
|
||||
use rustfs_io_metrics::internode_metrics::ReadVersionEntrySource;
|
||||
use rustfs_io_metrics::s3_http_metrics::{
|
||||
S3HttpRequestGuard, current_s3_http_operation_context, with_s3_http_operation_context,
|
||||
};
|
||||
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("diagnostic usage cache runtime");
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshots = recorder.snapshotter();
|
||||
rustfs_io_metrics::set_get_stage_metrics_enabled(true);
|
||||
let mut request = S3HttpRequestGuard::new("GET");
|
||||
let get = request.in_scope(|| {
|
||||
rustfs_io_metrics::record_s3_op(rustfs_s3_ops::S3Operation::GetObject);
|
||||
current_s3_http_operation_context().expect("dispatched GET operation")
|
||||
});
|
||||
request.response(200);
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
runtime.block_on(async {
|
||||
let ambient = crate::cluster::rpc::runtime_sources::ReadVersionDiagnosticContext::capture();
|
||||
for (operation, source) in [
|
||||
(None, None),
|
||||
(Some(get), None),
|
||||
(None, Some(ReadVersionEntrySource::ServerInfoPools)),
|
||||
(Some(get), Some(ReadVersionEntrySource::StoreObjectReader)),
|
||||
] {
|
||||
let store = UsageCacheReadStore {
|
||||
record_context: true,
|
||||
..Default::default()
|
||||
};
|
||||
with_s3_http_operation_context(operation, async {
|
||||
let load = load_data_usage_cache(&store, "usage-cache");
|
||||
let cache = match source {
|
||||
Some(source) => with_read_version_entry(source, load).await,
|
||||
None => load.await,
|
||||
}
|
||||
.expect("missing diagnostic cache is harmless");
|
||||
assert!(cache.cache.is_empty());
|
||||
assert_eq!(store.read_keys().await.len(), 2, "both prefixed and legacy cache reads are observed");
|
||||
})
|
||||
.await;
|
||||
assert_eq!(
|
||||
crate::cluster::rpc::runtime_sources::ReadVersionDiagnosticContext::capture(),
|
||||
ambient,
|
||||
"cache reads must restore the caller's complete diagnostic scope"
|
||||
);
|
||||
}
|
||||
assert_eq!(current_s3_http_operation_context(), None);
|
||||
});
|
||||
});
|
||||
let rows = snapshots.snapshot().into_vec();
|
||||
for (source, operation, state) in [
|
||||
("data_usage_cache_read", "unattributed", "absent"),
|
||||
("data_usage_cache_read", "s3:GetObject", "preserved"),
|
||||
("server_info_pools", "unattributed", "absent"),
|
||||
("store_object_reader", "s3:GetObject", "preserved"),
|
||||
] {
|
||||
assert!(
|
||||
rows.iter().any(|(key, _, _, value)| {
|
||||
let labels = key
|
||||
.key()
|
||||
.labels()
|
||||
.map(|label| (label.key(), label.value()))
|
||||
.collect::<std::collections::HashMap<_, _>>();
|
||||
key.key().name() == "rustfs_storage_diagnostic_context_transitions_total"
|
||||
&& labels.get("entry_source") == Some(&source)
|
||||
&& labels.get("boundary") == Some(&"remote_rpc")
|
||||
&& labels.get("parent_api_operation") == Some(&operation)
|
||||
&& labels.get("child_api_operation") == Some(&operation)
|
||||
&& labels.get("api_context") == Some(&state)
|
||||
&& matches!(value, DebugValue::Counter(2))
|
||||
}),
|
||||
"cache reads must retain API and the earliest observed entry: {source}/{operation}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn data_usage_removal_surfaces_missing_system_volume_as_read_quorum() {
|
||||
for cause in [Error::BucketNotFound(RUSTFS_META_BUCKET.to_string()), Error::VolumeNotFound] {
|
||||
|
||||
@@ -42,7 +42,11 @@ use std::{
|
||||
use time::OffsetDateTime;
|
||||
use tokio::time::timeout;
|
||||
use tonic::Request;
|
||||
use tracing::warn;
|
||||
use tracing::{error, trace, warn};
|
||||
|
||||
const EVENT_SERVER_PING: &str = "server_ping";
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_DIAGNOSTICS: &str = "diagnostics";
|
||||
|
||||
use shadow_rs::shadow;
|
||||
|
||||
@@ -157,11 +161,22 @@ async fn is_server_resolvable(endpoint: &Endpoint) -> Result<()> {
|
||||
|
||||
let response: PingResponse = client.ping(request).await?.into_inner();
|
||||
|
||||
let ping_response_body = flatbuffers::root::<PingBody>(&response.body);
|
||||
if let Err(e) = ping_response_body {
|
||||
eprintln!("{e}");
|
||||
if flatbuffers::root::<PingBody>(&response.body).is_err() {
|
||||
error!(
|
||||
event = EVENT_SERVER_PING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DIAGNOSTICS,
|
||||
result = "invalid_response",
|
||||
"Server ping response decoding failed"
|
||||
);
|
||||
} else {
|
||||
println!("ping_resp:body(flatbuffer): {ping_response_body:?}");
|
||||
trace!(
|
||||
event = EVENT_SERVER_PING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DIAGNOSTICS,
|
||||
result = "success",
|
||||
"Server ping completed"
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -678,12 +693,16 @@ async fn get_pools_info(all_disks: &[Disk]) -> Result<HashMap<i32, HashMap<i32,
|
||||
|
||||
if erasure_set.id == 0 {
|
||||
erasure_set.id = d.set_index;
|
||||
match load_data_usage_cache(
|
||||
let load = load_data_usage_cache(
|
||||
store.pools[d.pool_index as usize].disk_set[d.set_index as usize].as_ref(),
|
||||
DATA_USAGE_CACHE_NAME,
|
||||
)
|
||||
.await
|
||||
{
|
||||
);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let load = crate::cluster::rpc::runtime_sources::with_read_version_entry(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::ServerInfoPools,
|
||||
load,
|
||||
);
|
||||
match load.await {
|
||||
Ok(cache) => apply_erasure_set_usage(&cache, erasure_set),
|
||||
Err(_) => erasure_set.usage_error = Some(DATA_USAGE_UNAVAILABLE_ERROR.to_string()),
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::disk::cleanup_runtime;
|
||||
use crate::disk::io_admission::{
|
||||
AdmissionReader, BoxedChunkReader, DiskIoAdmission, DiskIoClass, DiskIoContext, current_disk_io_context,
|
||||
with_disk_io_context, with_disk_io_permit,
|
||||
@@ -485,10 +486,10 @@ impl LocalDiskWrapper {
|
||||
opts: DeleteOptions,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
self.track_disk_health_mutation(
|
||||
self.track_cleanup_mutation(
|
||||
"delete",
|
||||
DiskMetricMutation::Delete,
|
||||
|| async { Box::pin(self.disk.delete_with_namespace_owner(volume, path, opts, namespace_owner)).await },
|
||||
|| async { Box::pin(self.cleanup_delete(volume, path, opts, namespace_owner)).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
@@ -1476,6 +1477,28 @@ impl LocalDiskWrapper {
|
||||
)
|
||||
}
|
||||
|
||||
async fn cleanup_delete(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
options: DeleteOptions,
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
if !cleanup_runtime::enabled()? {
|
||||
return self.disk.delete_with_namespace_owner(volume, path, options, owner).await;
|
||||
}
|
||||
let disk = self.disk.clone();
|
||||
let volume = volume.to_owned();
|
||||
let path = path.to_owned();
|
||||
let guarded = owner.is_some();
|
||||
let operation = async move { disk.delete_with_namespace_owner(&volume, &path, options, owner).await };
|
||||
if guarded {
|
||||
cleanup_runtime::guarded_disk_operation(operation).await
|
||||
} else {
|
||||
cleanup_runtime::disk_operation(operation).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Run a delete under an owned coordinator task when a publication guard
|
||||
/// is present. This keeps the guard alive if the RPC waiter is cancelled
|
||||
/// while the local namespace mutation is still in progress.
|
||||
@@ -1494,12 +1517,13 @@ impl LocalDiskWrapper {
|
||||
} else {
|
||||
get_max_timeout_duration()
|
||||
};
|
||||
let cleanup_owner = external_guard.clone();
|
||||
run_owned_mutation(external_guard, move || async move {
|
||||
operation
|
||||
.track_disk_health_mutation(
|
||||
.track_cleanup_mutation(
|
||||
"delete",
|
||||
DiskMetricMutation::Delete,
|
||||
|| async { operation.disk.delete(&volume, &path, options).await },
|
||||
|| async { operation.cleanup_delete(&volume, &path, options, cleanup_owner).await },
|
||||
timeout_duration,
|
||||
)
|
||||
.await
|
||||
@@ -2015,6 +2039,31 @@ impl LocalDiskWrapper {
|
||||
self.track_disk_health_with_op("unknown", operation, timeout_duration).await
|
||||
}
|
||||
|
||||
/// Cleanup latency includes waiting behind a node-wide execution budget,
|
||||
/// so a waiter deadline alone is not evidence that this disk is faulty.
|
||||
/// The active disk health probe retains its original timeout policy.
|
||||
async fn track_cleanup_mutation<T, F, Fut>(
|
||||
&self,
|
||||
op: &'static str,
|
||||
mutation: DiskMetricMutation,
|
||||
operation: F,
|
||||
timeout_duration: Duration,
|
||||
) -> Result<T>
|
||||
where
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: std::future::Future<Output = Result<T>>,
|
||||
{
|
||||
// Runtime initialization must not prevent control-plane release of
|
||||
// reader tokens. Disk I/O entries still propagate configuration errors.
|
||||
let action = if cleanup_runtime::enabled().unwrap_or(false) {
|
||||
TimeoutHealthAction::IgnoreFailure
|
||||
} else {
|
||||
TimeoutHealthAction::MarkFailure
|
||||
};
|
||||
self.track_disk_health_with_op_timeout_action_and_mutation(op, operation, timeout_duration, action, mutation)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn track_disk_health_mutation<T, F, Fut>(
|
||||
&self,
|
||||
op: &'static str,
|
||||
@@ -2470,9 +2519,37 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
}
|
||||
|
||||
async fn release_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> Result<()> {
|
||||
self.track_disk_health_with_op(
|
||||
// Quota fence release is control-plane work and may wait for in-flight
|
||||
// mutations. Keep its original waiter lifecycle instead of creating
|
||||
// detached cleanup tasks for repeated fence-release RPCs.
|
||||
if volume == super::RUSTFS_META_BUCKET && super::is_quota_mutation_fence_path(path) {
|
||||
return self
|
||||
.track_disk_health_with_op(
|
||||
"release_snapshot_lease",
|
||||
|| async { self.disk.release_snapshot_lease(volume, path, token).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
self.track_cleanup_mutation(
|
||||
"release_snapshot_lease",
|
||||
|| async { self.disk.release_snapshot_lease(volume, path, token).await },
|
||||
DiskMetricMutation::None,
|
||||
|| async {
|
||||
if self.disk.release_snapshot_lease_without_cleanup(volume, path, token).await {
|
||||
return Ok(());
|
||||
}
|
||||
if !cleanup_runtime::enabled().unwrap_or(false) {
|
||||
// The raw release removes the token before trying the
|
||||
// reclaim budget. Initialization failure therefore keeps
|
||||
// the cleanup intent and data, without leaking a reader
|
||||
// or falling back to unbudgeted filesystem deletion.
|
||||
return self.disk.release_snapshot_lease(volume, path, token).await;
|
||||
}
|
||||
let disk = self.disk.clone();
|
||||
let volume = volume.to_owned();
|
||||
let path = path.to_owned();
|
||||
cleanup_runtime::release_lease(async move { disk.release_snapshot_lease(&volume, &path, token).await }).await
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
@@ -2497,10 +2574,22 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
} else {
|
||||
get_max_timeout_duration()
|
||||
};
|
||||
self.track_disk_health_mutation(
|
||||
self.track_cleanup_mutation(
|
||||
"delete_data_dir",
|
||||
DiskMetricMutation::Delete,
|
||||
|| async { self.disk.delete_data_dir(volume, path, opts).await },
|
||||
|| async {
|
||||
if !cleanup_runtime::enabled()? {
|
||||
return self.disk.delete_data_dir(volume, path, opts).await;
|
||||
}
|
||||
let disk = self.disk.clone();
|
||||
let volume = volume.to_owned();
|
||||
let path = path.to_owned();
|
||||
cleanup_runtime::disk_operation(async move {
|
||||
let _scope = scope;
|
||||
disk.delete_data_dir(&volume, &path, opts).await
|
||||
})
|
||||
.await
|
||||
},
|
||||
timeout,
|
||||
)
|
||||
.await
|
||||
@@ -2535,6 +2624,7 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
.await
|
||||
}
|
||||
|
||||
#[track_caller]
|
||||
async fn read_version(
|
||||
&self,
|
||||
org_volume: &str,
|
||||
@@ -2543,7 +2633,7 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
version_id: &str,
|
||||
opts: &ReadOptions,
|
||||
) -> Result<FileInfo> {
|
||||
self.track_disk_health_with_op_timeout_action_and_mutation_with_io_permit(
|
||||
let operation = self.track_disk_health_with_op_timeout_action_and_mutation_with_io_permit(
|
||||
"read_version",
|
||||
|io_permit| async move {
|
||||
with_disk_io_permit(io_permit, self.disk.read_version(org_volume, volume, path, version_id, opts)).await
|
||||
@@ -2551,8 +2641,20 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
get_max_timeout_duration(),
|
||||
TimeoutHealthAction::MarkFailure,
|
||||
DiskMetricMutation::None,
|
||||
)
|
||||
.await
|
||||
);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
{
|
||||
if crate::cluster::rpc::runtime_sources::current_read_version_request_caller().is_some() {
|
||||
operation.await
|
||||
} else {
|
||||
let caller = crate::cluster::rpc::runtime_sources::read_version_request_caller_from_location(
|
||||
std::panic::Location::caller().file(),
|
||||
);
|
||||
crate::cluster::rpc::runtime_sources::with_read_version_request_caller(caller, operation).await
|
||||
}
|
||||
}
|
||||
#[cfg(not(feature = "diagnostic-read-version-caller"))]
|
||||
operation.await
|
||||
}
|
||||
|
||||
async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo> {
|
||||
@@ -2747,10 +2849,10 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
}
|
||||
|
||||
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
|
||||
self.track_disk_health_mutation(
|
||||
self.track_cleanup_mutation(
|
||||
"delete",
|
||||
DiskMetricMutation::Delete,
|
||||
|| async { self.disk.delete(volume, path, opt).await },
|
||||
|| async { self.cleanup_delete(volume, path, opt, None).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
@@ -2783,11 +2885,25 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
}
|
||||
|
||||
async fn write_all(&self, volume: &str, path: &str, data: Bytes) -> Result<()> {
|
||||
self.track_disk_health_mutation(
|
||||
let cleanup = path.rsplit('/').next() == Some(cleanup_runtime::OLD_DATA_CLEANUP_RECEIPT_FILE);
|
||||
self.track_disk_health_with_op_timeout_action_and_mutation(
|
||||
"write_all",
|
||||
DiskMetricMutation::Write,
|
||||
|| async { self.disk.write_all(volume, path, data).await },
|
||||
|| async {
|
||||
if !cleanup || !cleanup_runtime::enabled()? {
|
||||
return self.disk.write_all(volume, path, data).await;
|
||||
}
|
||||
let disk = self.disk.clone();
|
||||
let volume = volume.to_owned();
|
||||
let path = path.to_owned();
|
||||
cleanup_runtime::disk_operation(async move { disk.write_all(&volume, &path, data).await }).await
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
if cleanup && cleanup_runtime::enabled()? {
|
||||
TimeoutHealthAction::IgnoreFailure
|
||||
} else {
|
||||
TimeoutHealthAction::MarkFailure
|
||||
},
|
||||
DiskMetricMutation::Write,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -3180,6 +3296,256 @@ mod tests {
|
||||
let _ = walk.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn invalid_cleanup_config_does_not_leak_snapshot_readers() {
|
||||
const CHILD: &str = "RUSTFS_CLEANUP_INVALID_LEASE_TEST_CHILD";
|
||||
if std::env::var_os(CHILD).is_none() {
|
||||
let output = tokio::task::spawn_blocking(|| {
|
||||
std::process::Command::new(std::env::current_exe().expect("test executable"))
|
||||
.args([
|
||||
"--exact",
|
||||
"disk::disk_store::tests::invalid_cleanup_config_does_not_leak_snapshot_readers",
|
||||
"--nocapture",
|
||||
])
|
||||
.env(CHILD, "1")
|
||||
.env("RUSTFS_CLEANUP_ISOLATE_ENABLE", "true")
|
||||
.env("RUSTFS_CLEANUP_ASYNC_THREADS", "0")
|
||||
.output()
|
||||
.expect("run with invalid cleanup configuration")
|
||||
})
|
||||
.await
|
||||
.expect("child process task");
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"{}\n{}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
assert!(String::from_utf8_lossy(&output.stdout).contains("1 passed"));
|
||||
return;
|
||||
}
|
||||
let dir = tempfile::tempdir().expect("temporary disk");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk"));
|
||||
let wrapper = LocalDiskWrapper::new(disk.clone(), false);
|
||||
let volume = "invalid-cleanup-config";
|
||||
let path = "object/data";
|
||||
wrapper.make_volume(volume).await.expect("bucket");
|
||||
wrapper
|
||||
.write_all(volume, "object/data/part.1", Bytes::from_static(b"live shard"))
|
||||
.await
|
||||
.expect("write shard");
|
||||
let token = wrapper.acquire_snapshot_lease(volume, path).await.expect("GET lease");
|
||||
wrapper
|
||||
.release_snapshot_lease(volume, path, token)
|
||||
.await
|
||||
.expect("control-only release must not depend on cleanup initialization");
|
||||
assert!(matches!(
|
||||
wrapper.renew_snapshot_lease(volume, path, token).await,
|
||||
Err(DiskError::FileNotFound)
|
||||
));
|
||||
|
||||
let token = wrapper
|
||||
.acquire_snapshot_lease(volume, path)
|
||||
.await
|
||||
.expect("reader of old data");
|
||||
// Register reclaim intent directly; this only updates the lease
|
||||
// registry while a reader is present and performs no filesystem I/O.
|
||||
assert_eq!(
|
||||
disk.delete_data_dir(
|
||||
volume,
|
||||
path,
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
}
|
||||
)
|
||||
.await
|
||||
.expect("defer reclaim"),
|
||||
DataDirDeleteStatus::Deferred
|
||||
);
|
||||
assert!(
|
||||
wrapper.release_snapshot_lease(volume, path, token).await.is_err(),
|
||||
"invalid configuration must still reject reclamation"
|
||||
);
|
||||
assert!(
|
||||
matches!(wrapper.renew_snapshot_lease(volume, path, token).await, Err(DiskError::FileNotFound)),
|
||||
"failed reclamation must not strand a reader token"
|
||||
);
|
||||
assert_eq!(
|
||||
wrapper.read_all(volume, "object/data/part.1").await.expect("retained data"),
|
||||
Bytes::from_static(b"live shard"),
|
||||
"configuration failure must not fall back to unbudgeted deletion"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rpc_cleanup_entries_share_disk_budget() {
|
||||
const CHILD: &str = "RUSTFS_CLEANUP_RPC_TEST_CHILD";
|
||||
if std::env::var_os(CHILD).is_none() {
|
||||
let output = tokio::task::spawn_blocking(|| {
|
||||
std::process::Command::new(std::env::current_exe().unwrap())
|
||||
.args([
|
||||
"--exact",
|
||||
"disk::disk_store::tests::rpc_cleanup_entries_share_disk_budget",
|
||||
"--nocapture",
|
||||
])
|
||||
.env(CHILD, "1")
|
||||
.env("RUSTFS_CLEANUP_ISOLATE_ENABLE", "true")
|
||||
.env("RUSTFS_CLEANUP_ASYNC_THREADS", "1")
|
||||
.env("RUSTFS_CLEANUP_DISK_MAX_PENDING", "1")
|
||||
.env("RUSTFS_CLEANUP_DISK_WORKERS", "1")
|
||||
.env("RUSTFS_CLEANUP_GC_WORKERS", "1")
|
||||
.env("RUSTFS_CLEANUP_CPUS", "none")
|
||||
.env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP")
|
||||
.env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER")
|
||||
.env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_ZERO_TARGET_TMP_DELETE_SKIP")
|
||||
.output()
|
||||
.unwrap()
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"{}\n{}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
assert!(String::from_utf8_lossy(&output.stdout).contains("1 passed"));
|
||||
return;
|
||||
}
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().unwrap()).unwrap();
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.unwrap());
|
||||
let wrapper = LocalDiskWrapper::new(disk, false);
|
||||
let volume = "cleanup-receiver";
|
||||
let path = "object/old-data";
|
||||
let body_path = format!("{path}/part.1");
|
||||
let receipt_path = format!("{path}/{}", cleanup_runtime::OLD_DATA_CLEANUP_RECEIPT_FILE);
|
||||
wrapper.make_volume(volume).await.unwrap();
|
||||
wrapper
|
||||
.write_all(volume, &body_path, Bytes::from_static(b"old body"))
|
||||
.await
|
||||
.unwrap();
|
||||
let read_lease = wrapper
|
||||
.acquire_snapshot_lease(volume, path)
|
||||
.await
|
||||
.expect("ordinary GET lease");
|
||||
let (entered, entry) = tokio::sync::oneshot::channel();
|
||||
let (release, released) = std::sync::mpsc::channel();
|
||||
let occupied = tokio::spawn(cleanup_runtime::disk_operation(async move {
|
||||
entered.send(()).unwrap();
|
||||
// Deliberately stall the only cleanup scheduler to prove that
|
||||
// ordinary GET completion does not dispatch to it.
|
||||
released
|
||||
.recv_timeout(Duration::from_secs(30))
|
||||
.expect("release stalled scheduler");
|
||||
Ok(())
|
||||
}));
|
||||
tokio::time::timeout(Duration::from_secs(20), entry).await.unwrap().unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(5), wrapper.release_snapshot_lease(volume, path, read_lease))
|
||||
.await
|
||||
.expect("ordinary GET release must progress while the cleanup scheduler is stalled")
|
||||
.expect("release without reclamation");
|
||||
let fence_path = super::super::quota_mutation_fence_path(volume, "object");
|
||||
let fence = wrapper
|
||||
.acquire_snapshot_lease(super::super::RUSTFS_META_BUCKET, &fence_path)
|
||||
.await
|
||||
.unwrap();
|
||||
wrapper
|
||||
.release_snapshot_lease(super::super::RUSTFS_META_BUCKET, &fence_path, fence)
|
||||
.await
|
||||
.unwrap();
|
||||
let timeout = wrapper
|
||||
.track_cleanup_mutation(
|
||||
"delete",
|
||||
DiskMetricMutation::Delete,
|
||||
|| std::future::pending::<Result<()>>(),
|
||||
Duration::from_millis(10),
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(timeout, Err(DiskError::Timeout)));
|
||||
assert!(!wrapper.health.is_faulty(), "cleanup queue delay must not mark a healthy disk faulty");
|
||||
let options = DeleteOptions {
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
};
|
||||
let drops = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
// This is the same entry point used after RPC authentication and
|
||||
// scanner publication-lease validation by handle_delete.
|
||||
let result = wrapper
|
||||
.delete_with_publication_guard(volume, path, options.clone(), Some(Arc::new(DropProbe(drops.clone()))))
|
||||
.await;
|
||||
assert!(matches!(result, Err(DiskError::Io(err)) if err.kind() == io::ErrorKind::WouldBlock));
|
||||
assert_eq!(drops.load(Ordering::SeqCst), 1);
|
||||
assert!(matches!(wrapper.delete_data_dir(volume, path, options.clone()).await,
|
||||
Err(DiskError::Io(err)) if err.kind() == io::ErrorKind::WouldBlock));
|
||||
assert!(matches!(wrapper.write_all(volume, &receipt_path, Bytes::from_static(b"receipt")).await,
|
||||
Err(DiskError::Io(err)) if err.kind() == io::ErrorKind::WouldBlock));
|
||||
assert_eq!(wrapper.read_all(volume, &body_path).await.unwrap(), Bytes::from_static(b"old body"));
|
||||
release.send(()).unwrap();
|
||||
occupied.await.unwrap().unwrap();
|
||||
wrapper
|
||||
.write_all(volume, &receipt_path, Bytes::from_static(b"receipt"))
|
||||
.await
|
||||
.unwrap();
|
||||
wrapper
|
||||
.delete_with_publication_guard(volume, path, options, None)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(matches!(wrapper.read_all(volume, &body_path).await, Err(DiskError::FileNotFound)));
|
||||
// Snapshot readers defer old-data cleanup. Saturation must release
|
||||
// their token while retaining reclaim intent, rather than leaking a
|
||||
// live token or performing an unbudgeted delete on the RPC runtime.
|
||||
wrapper
|
||||
.write_all(volume, &body_path, Bytes::from_static(b"leased body"))
|
||||
.await
|
||||
.unwrap();
|
||||
let lease = wrapper.acquire_snapshot_lease(volume, path).await.unwrap();
|
||||
assert_eq!(
|
||||
wrapper
|
||||
.delete_data_dir(
|
||||
volume,
|
||||
path,
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
}
|
||||
)
|
||||
.await
|
||||
.unwrap(),
|
||||
DataDirDeleteStatus::Deferred
|
||||
);
|
||||
let (entered, entry) = tokio::sync::oneshot::channel();
|
||||
let (release, released) = tokio::sync::oneshot::channel();
|
||||
let occupied = tokio::spawn(cleanup_runtime::disk_operation(async move {
|
||||
entered.send(()).unwrap();
|
||||
released.await.unwrap();
|
||||
Ok(())
|
||||
}));
|
||||
tokio::time::timeout(Duration::from_secs(20), entry).await.unwrap().unwrap();
|
||||
assert!(matches!(wrapper.release_snapshot_lease(volume, path, lease).await,
|
||||
Err(DiskError::Io(err)) if err.kind() == io::ErrorKind::WouldBlock));
|
||||
assert_eq!(wrapper.read_all(volume, &body_path).await.unwrap(), Bytes::from_static(b"leased body"));
|
||||
release.send(()).unwrap();
|
||||
occupied.await.unwrap().unwrap();
|
||||
assert_eq!(
|
||||
wrapper
|
||||
.delete_data_dir(
|
||||
volume,
|
||||
path,
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
}
|
||||
)
|
||||
.await
|
||||
.unwrap(),
|
||||
DataDirDeleteStatus::Deleted
|
||||
);
|
||||
assert!(matches!(wrapper.read_all(volume, &body_path).await, Err(DiskError::FileNotFound)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn owned_mutation_keeps_publication_guard_after_waiter_cancellation() {
|
||||
let drops = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
|
||||
@@ -223,12 +223,6 @@ pub fn rename_std(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()
|
||||
std::fs::rename(from, to)
|
||||
}
|
||||
|
||||
#[cfg(any(not(windows), test))]
|
||||
#[tracing::instrument(level = "debug", skip_all)]
|
||||
pub async fn read_file(path: impl AsRef<Path>) -> io::Result<Vec<u8>> {
|
||||
fs::read(path.as_ref()).await
|
||||
}
|
||||
|
||||
// Bucket existence cache - reduces statx syscalls for repeated bucket checks
|
||||
|
||||
/// Cache for bucket directory existence checks.
|
||||
@@ -614,28 +608,6 @@ mod tests {
|
||||
assert_eq!(content, "test content");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_read_file() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let file_path = temp_dir.path().join("test_read.txt");
|
||||
|
||||
let test_content = b"This is test content for reading";
|
||||
tokio::fs::write(&file_path, test_content).await.unwrap();
|
||||
|
||||
// Read file
|
||||
let read_content = read_file(&file_path).await.unwrap();
|
||||
assert_eq!(read_content, test_content);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_read_file_nonexistent() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let file_path = temp_dir.path().join("nonexistent.txt");
|
||||
|
||||
// Should fail for non-existent file
|
||||
assert!(read_file(&file_path).await.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_same_file() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
|
||||
+344
-177
@@ -203,8 +203,10 @@ struct ListingMetadataRead {
|
||||
has_namespace_child_candidate: bool,
|
||||
}
|
||||
|
||||
fn read_all_data_std(path: &Path) -> core::result::Result<(Vec<u8>, Option<OffsetDateTime>), ReadAllError> {
|
||||
let mut file = std::fs::File::open(path).map_err(ReadAllError::Open)?;
|
||||
fn read_all_data_std(
|
||||
opened: std::io::Result<std::fs::File>,
|
||||
) -> core::result::Result<(Vec<u8>, Option<OffsetDateTime>), ReadAllError> {
|
||||
let mut file = opened.map_err(ReadAllError::Open)?;
|
||||
let metadata = file.metadata().map_err(|err| ReadAllError::Disk(to_file_error(err).into()))?;
|
||||
|
||||
if metadata.is_dir() {
|
||||
@@ -5173,7 +5175,7 @@ pub struct LocalDisk {
|
||||
/// pathname is later covered by another mount.
|
||||
io_root: PathBuf,
|
||||
#[cfg(target_os = "linux")]
|
||||
mount_lease: std::fs::File,
|
||||
mount_lease: Arc<std::fs::File>,
|
||||
#[cfg(target_os = "linux")]
|
||||
mount_lease_mount_id: Option<u64>,
|
||||
/// Public path for callers that need the configured disk layout. Internal
|
||||
@@ -5430,24 +5432,6 @@ fn mount_id_from_mountinfo_contents(mountinfo: &str, path: &Path) -> Option<u64>
|
||||
}
|
||||
|
||||
impl LocalDisk {
|
||||
#[cfg(target_os = "linux")]
|
||||
fn open_mount_lease(root: &Path) -> Result<(std::fs::File, PathBuf, Option<u64>)> {
|
||||
use rustix::fs::{Mode, OFlags, open};
|
||||
use std::os::fd::AsRawFd as _;
|
||||
|
||||
let fd = open(
|
||||
root,
|
||||
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
|
||||
Mode::empty(),
|
||||
)
|
||||
.map_err(std::io::Error::from)
|
||||
.map_err(DiskError::from)?;
|
||||
let lease = std::fs::File::from(fd);
|
||||
let io_root = PathBuf::from(format!("/proc/self/fd/{}/.", lease.as_raw_fd()));
|
||||
let mount_id = mount_id_for_fd(&lease);
|
||||
Ok((lease, io_root, mount_id))
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
fn open_mount_lease(root: &Path) -> Result<PathBuf> {
|
||||
Ok(root.to_path_buf())
|
||||
@@ -5531,7 +5515,11 @@ impl LocalDisk {
|
||||
let root = publication_root.path().to_path_buf();
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
let (mount_lease, io_root, mount_lease_mount_id) = Self::open_mount_lease(&root)?;
|
||||
let (mount_lease, io_root, mount_lease_mount_id) = (
|
||||
publication_root.directory().clone(),
|
||||
publication_root.io_path().to_path_buf(),
|
||||
mount_id_for_fd(publication_root.directory()),
|
||||
);
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let io_root = Self::open_mount_lease(&root)?;
|
||||
|
||||
@@ -5579,9 +5567,11 @@ impl LocalDisk {
|
||||
state = "format_path_resolved",
|
||||
"Local disk format path resolved"
|
||||
);
|
||||
let (format_data, format_meta) = read_file_exists(&io_format_path).await.inspect_err(|err| {
|
||||
log_startup_disk_error("read_format_json", &io_format_path, err);
|
||||
})?;
|
||||
let (format_data, format_meta) = read_file_exists(&io_format_path, &publication_root)
|
||||
.await
|
||||
.inspect_err(|err| {
|
||||
log_startup_disk_error("read_format_json", &io_format_path, err);
|
||||
})?;
|
||||
|
||||
let mut id = None;
|
||||
// let mut format_legacy = false;
|
||||
@@ -5727,7 +5717,13 @@ impl LocalDisk {
|
||||
|
||||
let io_root = disk.io_root.clone();
|
||||
let publication_root = disk.publication_root.clone();
|
||||
tokio::spawn(Self::cleanup_deleted_objects_loop(io_root, publication_root, exit_rx));
|
||||
tokio::spawn(Self::cleanup_deleted_objects_loop(
|
||||
io_root,
|
||||
publication_root,
|
||||
#[cfg(target_os = "linux")]
|
||||
disk.mount_lease.clone(),
|
||||
exit_rx,
|
||||
));
|
||||
debug!(
|
||||
event = EVENT_DISK_LOCAL_STARTUP_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -5743,30 +5739,53 @@ impl LocalDisk {
|
||||
async fn cleanup_deleted_objects_loop(
|
||||
root: PathBuf,
|
||||
publication_root: os::PublicationRoot,
|
||||
#[cfg(target_os = "linux")] mount_lease: Arc<std::fs::File>,
|
||||
mut exit_rx: tokio::sync::broadcast::Receiver<()>,
|
||||
) {
|
||||
let start_at = Instant::now() + DELETED_OBJECTS_CLEANUP_INTERVAL;
|
||||
let mut interval = interval_at(start_at, DELETED_OBJECTS_CLEANUP_INTERVAL);
|
||||
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = interval.tick() => {
|
||||
if let Err(err) = Self::cleanup_deleted_objects(root.clone()).await {
|
||||
let root = root.clone();
|
||||
let publication_root = publication_root.clone();
|
||||
#[cfg(target_os = "linux")]
|
||||
let mount_lease = mount_lease.clone();
|
||||
if let Err(err) = super::cleanup_runtime::run_gc(move |budget| async move {
|
||||
// Queued/detached scans must keep /proc/self/fd paths
|
||||
// attached to the original mount after LocalDisk drops.
|
||||
#[cfg(target_os = "linux")]
|
||||
let _mount_lease = mount_lease;
|
||||
if let Err(err) = Self::cleanup_deleted_objects(root.clone(), &budget).await {
|
||||
error!(
|
||||
event = EVENT_DISK_LOCAL_BACKGROUND_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
task = "deleted_objects",
|
||||
state = "failed",
|
||||
error = ?err,
|
||||
"Disk local background cleanup failed"
|
||||
);
|
||||
}
|
||||
if let Err(err) = Self::cleanup_stale_tmp_objects(root.clone(), &publication_root, &budget).await {
|
||||
error!(
|
||||
event = EVENT_DISK_LOCAL_BACKGROUND_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
task = "stale_tmp_objects",
|
||||
state = "failed",
|
||||
error = ?err,
|
||||
"Disk local background cleanup failed"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}).await {
|
||||
error!(
|
||||
event = EVENT_DISK_LOCAL_BACKGROUND_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
task = "deleted_objects",
|
||||
state = "failed",
|
||||
error = ?err,
|
||||
"Disk local background cleanup failed"
|
||||
);
|
||||
}
|
||||
if let Err(err) = Self::cleanup_stale_tmp_objects(root.clone(), &publication_root).await {
|
||||
error!(
|
||||
event = EVENT_DISK_LOCAL_BACKGROUND_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
task = "stale_tmp_objects",
|
||||
task = "cleanup_dispatch",
|
||||
state = "failed",
|
||||
error = ?err,
|
||||
"Disk local background cleanup failed"
|
||||
@@ -5857,95 +5876,123 @@ impl LocalDisk {
|
||||
}
|
||||
}
|
||||
|
||||
async fn cleanup_stale_tmp_objects(root: PathBuf, publication_root: &os::PublicationRoot) -> Result<()> {
|
||||
Self::cleanup_stale_tmp_objects_with_expiry(root, publication_root, STALE_TMP_OBJECT_EXPIRY).await
|
||||
async fn cleanup_stale_tmp_objects(
|
||||
root: PathBuf,
|
||||
publication_root: &os::PublicationRoot,
|
||||
budget: &super::cleanup_runtime::GcBudget,
|
||||
) -> Result<()> {
|
||||
Self::cleanup_stale_tmp_objects_with_expiry(root, publication_root, STALE_TMP_OBJECT_EXPIRY, budget).await
|
||||
}
|
||||
|
||||
async fn cleanup_stale_tmp_objects_with_expiry(
|
||||
root: PathBuf,
|
||||
publication_root: &os::PublicationRoot,
|
||||
expiry: Duration,
|
||||
budget: &super::cleanup_runtime::GcBudget,
|
||||
) -> Result<()> {
|
||||
let tmp_path = Self::meta_path(&root, RUSTFS_META_TMP_BUCKET);
|
||||
let mut entries = match fs::read_dir(&tmp_path).await {
|
||||
let mut entries = match budget.step(async { Ok(fs::read_dir(&tmp_path).await?) }).await {
|
||||
Ok(entries) => entries,
|
||||
Err(e) => {
|
||||
if e.kind() == ErrorKind::NotFound {
|
||||
if matches!(&e, DiskError::Io(err) if err.kind() == ErrorKind::NotFound) {
|
||||
return Ok(());
|
||||
}
|
||||
return Err(e.into());
|
||||
return Err(e);
|
||||
}
|
||||
};
|
||||
|
||||
while let Some(entry) = entries.next_entry().await? {
|
||||
let name = entry.file_name().to_string_lossy().to_string();
|
||||
if name.is_empty() || name == "." || name == ".." || name == ".trash" {
|
||||
continue;
|
||||
loop {
|
||||
let more = budget
|
||||
.step(async {
|
||||
let Some(entry) = entries.next_entry().await? else {
|
||||
return Ok(false);
|
||||
};
|
||||
Self::cleanup_stale_tmp_entry(entry, &root, publication_root, expiry).await?;
|
||||
Ok(true)
|
||||
})
|
||||
.await?;
|
||||
if !more {
|
||||
break;
|
||||
}
|
||||
|
||||
let file_type = entry.file_type().await?;
|
||||
if !file_type.is_dir() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let Some(age) = entry
|
||||
.metadata()
|
||||
.await?
|
||||
.modified()
|
||||
.ok()
|
||||
.and_then(|modified| modified.elapsed().ok())
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
if age <= expiry {
|
||||
continue;
|
||||
}
|
||||
|
||||
let target_path = Self::meta_path(&root, RUSTFS_META_TMP_DELETED_BUCKET).join(Uuid::new_v4().to_string());
|
||||
rename_all(entry.path(), target_path, Self::meta_path(&root, RUSTFS_META_BUCKET), publication_root).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn cleanup_deleted_objects(root: PathBuf) -> Result<()> {
|
||||
async fn cleanup_stale_tmp_entry(
|
||||
entry: fs::DirEntry,
|
||||
root: &Path,
|
||||
publication_root: &os::PublicationRoot,
|
||||
expiry: Duration,
|
||||
) -> Result<()> {
|
||||
let name = entry.file_name().to_string_lossy().to_string();
|
||||
if name.is_empty() || name == "." || name == ".." || name == ".trash" || !entry.file_type().await?.is_dir() {
|
||||
return Ok(());
|
||||
}
|
||||
let Some(age) = entry
|
||||
.metadata()
|
||||
.await?
|
||||
.modified()
|
||||
.ok()
|
||||
.and_then(|modified| modified.elapsed().ok())
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
if age <= expiry {
|
||||
return Ok(());
|
||||
}
|
||||
let target_path = Self::meta_path(root, RUSTFS_META_TMP_DELETED_BUCKET).join(Uuid::new_v4().to_string());
|
||||
rename_all(entry.path(), target_path, Self::meta_path(root, RUSTFS_META_BUCKET), publication_root).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn cleanup_deleted_objects(root: PathBuf, budget: &super::cleanup_runtime::GcBudget) -> Result<()> {
|
||||
let trash = Self::meta_path(&root, RUSTFS_META_TMP_DELETED_BUCKET);
|
||||
let mut entries = match fs::read_dir(&trash).await {
|
||||
let mut entries = match budget.step(async { Ok(fs::read_dir(&trash).await?) }).await {
|
||||
Ok(entries) => entries,
|
||||
Err(e) => {
|
||||
if e.kind() == ErrorKind::NotFound {
|
||||
if matches!(&e, DiskError::Io(err) if err.kind() == ErrorKind::NotFound) {
|
||||
return Ok(());
|
||||
}
|
||||
return Err(e.into());
|
||||
return Err(e);
|
||||
}
|
||||
};
|
||||
|
||||
while let Some(entry) = entries.next_entry().await? {
|
||||
let name = entry.file_name().to_string_lossy().to_string();
|
||||
if name.is_empty() || name == "." || name == ".." {
|
||||
continue;
|
||||
}
|
||||
|
||||
let file_type = entry.file_type().await?;
|
||||
|
||||
let path = trash.join(name);
|
||||
|
||||
if file_type.is_dir() {
|
||||
if let Err(e) = tokio::fs::remove_dir_all(path).await
|
||||
&& e.kind() != ErrorKind::NotFound
|
||||
{
|
||||
return Err(e.into());
|
||||
}
|
||||
} else if let Err(e) = tokio::fs::remove_file(path).await
|
||||
&& e.kind() != ErrorKind::NotFound
|
||||
{
|
||||
return Err(e.into());
|
||||
loop {
|
||||
let more = budget
|
||||
.step(async {
|
||||
let Some(entry) = entries.next_entry().await? else {
|
||||
return Ok(false);
|
||||
};
|
||||
Self::cleanup_trash_entry(entry, &trash).await?;
|
||||
Ok(true)
|
||||
})
|
||||
.await?;
|
||||
if !more {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn cleanup_trash_entry(entry: fs::DirEntry, trash: &Path) -> Result<()> {
|
||||
let name = entry.file_name().to_string_lossy().to_string();
|
||||
if name.is_empty() || name == "." || name == ".." {
|
||||
return Ok(());
|
||||
}
|
||||
let path = trash.join(name);
|
||||
let result = if entry.file_type().await?.is_dir() {
|
||||
fs::remove_dir_all(path).await
|
||||
} else {
|
||||
fs::remove_file(path).await
|
||||
};
|
||||
match result {
|
||||
Err(err) if err.kind() != ErrorKind::NotFound => Err(err.into()),
|
||||
_ => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_valid_volname(volname: &str) -> bool {
|
||||
if volname.len() < 3 {
|
||||
return false;
|
||||
@@ -6072,6 +6119,19 @@ impl LocalDisk {
|
||||
self.local_disk_object_path(self.io_root(), bucket, key)
|
||||
}
|
||||
|
||||
/// Only for operations that open through PublicationRoot or a commit guard:
|
||||
/// containment and symlink checks happen with the actual open, off the runtime.
|
||||
fn io_get_object_open_path(&self, bucket: &str, key: &str) -> Result<PathBuf> {
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
let (bucket_path, path) = build_local_disk_object_path(self.io_root(), bucket, key);
|
||||
check_local_disk_object_path_components(self.io_root(), &bucket_path, &path)?;
|
||||
Ok(path)
|
||||
}
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
self.io_get_object_path(bucket, key)
|
||||
}
|
||||
|
||||
fn io_get_bucket_path(&self, bucket: &str) -> Result<PathBuf> {
|
||||
self.local_disk_bucket_path(self.io_root(), bucket)
|
||||
}
|
||||
@@ -6230,7 +6290,7 @@ impl LocalDisk {
|
||||
|
||||
let mut meta = FileMeta::new();
|
||||
if !fi.fresh {
|
||||
let (buf, _) = read_file_exists(&p).await?;
|
||||
let (buf, _) = read_file_exists(&p, &self.publication_root).await?;
|
||||
if !buf.is_empty() {
|
||||
let _ = meta.unmarshal_msg(&buf).map_err(|_| {
|
||||
meta = FileMeta::new();
|
||||
@@ -6256,6 +6316,36 @@ impl LocalDisk {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Keep the common GET completion on its caller. Only a last-reader
|
||||
/// release with pending reclamation needs an owned cleanup task. Leave
|
||||
/// that token untouched so the slow path owns the complete transition.
|
||||
pub(in crate::disk) async fn release_snapshot_lease_without_cleanup(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
token: SnapshotLeaseToken,
|
||||
) -> bool {
|
||||
let key = SnapshotLeaseKey {
|
||||
volume: volume.to_owned(),
|
||||
path: path.to_owned(),
|
||||
};
|
||||
let mut registry = self.snapshot_leases.lock();
|
||||
let Some(entry) = registry.entries.get_mut(&key) else {
|
||||
return true;
|
||||
};
|
||||
if !entry.deleting
|
||||
&& entry.pending_delete.is_some()
|
||||
&& (entry.tokens.is_empty() || (entry.tokens.len() == 1 && entry.tokens.contains(&token)))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
entry.tokens.remove(&token);
|
||||
if entry.tokens.is_empty() && !entry.deleting {
|
||||
registry.entries.remove(&key);
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
async fn delete_data_dir_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -7042,10 +7132,11 @@ impl LocalDisk {
|
||||
// - metadata failure -> to_file_error
|
||||
// - parse failure -> propagated verbatim from read_xl_meta_no_data_sync (`?`)
|
||||
let path = file_path.as_ref().to_path_buf();
|
||||
let root = self.publication_root.clone();
|
||||
let (data, modtime) =
|
||||
spawn_blocking_with_disk_io_permit(io_permit, move || -> Result<(Vec<u8>, Option<OffsetDateTime>)> {
|
||||
// Read-only open, equivalent to O_RDONLY (get_readonly_options only sets read(true)).
|
||||
let mut f = std::fs::File::open(&path).map_err(to_file_error)?;
|
||||
let mut f = root.open_readonly(&path).map_err(to_file_error)?;
|
||||
|
||||
let meta = f.metadata().map_err(to_file_error)?;
|
||||
|
||||
@@ -7099,6 +7190,7 @@ impl LocalDisk {
|
||||
let metadata_path = self.io_get_object_path(bucket, path.to_string_lossy().as_ref());
|
||||
// A part's existence check, metadata read and decode share one dispatch.
|
||||
// Keep open errors unmapped for the existing missing-volume fallback.
|
||||
let root = self.publication_root.clone();
|
||||
let result = spawn_blocking_with_disk_io_permit(io_permit, move || -> Result<_> {
|
||||
let part_error = |error: String| ObjectPartInfo {
|
||||
number: num,
|
||||
@@ -7111,7 +7203,7 @@ impl LocalDisk {
|
||||
// Invalid metadata paths remain request errors, but missing data wins
|
||||
// first, as it does in the serial reader.
|
||||
let metadata_path = metadata_path?;
|
||||
Ok(read_all_data_std(&metadata_path)
|
||||
Ok(read_all_data_std(root.open_readonly(&metadata_path))
|
||||
.map(|(data, _)| ObjectPartInfo::unmarshal(&data).unwrap_or_else(|err| part_error(err.to_string()))))
|
||||
})
|
||||
.await;
|
||||
@@ -7131,8 +7223,9 @@ impl LocalDisk {
|
||||
let object_dir = self.io_get_object_path(volume, object_name)?;
|
||||
let metadata_path = object_dir.join(STORAGE_FORMAT_FILE);
|
||||
let volume_dir = self.io_get_bucket_path(volume)?;
|
||||
let root = self.publication_root.clone();
|
||||
let result = spawn_blocking_with_disk_io_permit(io_permit, move || {
|
||||
let (bytes, _) = read_all_data_std(&metadata_path)?;
|
||||
let (bytes, _) = read_all_data_std(root.open_readonly(&metadata_path))?;
|
||||
let file_meta = FileMeta::load(&bytes).ok();
|
||||
let data_dirs: HashSet<String> = file_meta
|
||||
.as_ref()
|
||||
@@ -7230,7 +7323,8 @@ impl LocalDisk {
|
||||
// gating the volume fallback on the open error alone is equivalent to
|
||||
// the original code, where the fallback lived solely in the open match arm.
|
||||
let path = file_path.as_ref().to_path_buf();
|
||||
let res = spawn_blocking_with_disk_io_permit(io_permit, move || read_all_data_std(&path))
|
||||
let root = self.publication_root.clone();
|
||||
let res = spawn_blocking_with_disk_io_permit(io_permit, move || read_all_data_std(root.open_readonly(&path)))
|
||||
.await
|
||||
.map_err(DiskError::from)?;
|
||||
|
||||
@@ -7967,16 +8061,16 @@ impl LocalDisk {
|
||||
os::make_dir_all(parent, skip_parent).await?;
|
||||
}
|
||||
|
||||
let root = self.publication_root.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
#[cfg(test)]
|
||||
run_owned_file_write_before_open(&path);
|
||||
|
||||
let mut file = std::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.write(true)
|
||||
.truncate(true)
|
||||
.open(&path)
|
||||
.map_err(to_file_error)?;
|
||||
let (mut file, parent_handle) = if sync == SyncMode::FileAndDir {
|
||||
root.create_truncate_with_parent(&path).map_err(to_file_error)?
|
||||
} else {
|
||||
(root.create_truncate(&path).map_err(to_file_error)?, None)
|
||||
};
|
||||
std::io::Write::write_all(&mut file, buf.as_ref()).map_err(to_file_error)?;
|
||||
if sync != SyncMode::None {
|
||||
file.sync_data().map_err(to_file_error)?;
|
||||
@@ -7985,7 +8079,7 @@ impl LocalDisk {
|
||||
if sync == SyncMode::FileAndDir
|
||||
&& let Some(parent) = path.parent()
|
||||
{
|
||||
os::fsync_dir_std(parent).map_err(to_file_error)?;
|
||||
os::fsync_directory_handle_std(parent, parent_handle.as_ref()).map_err(to_file_error)?;
|
||||
}
|
||||
}
|
||||
Ok::<_, std::io::Error>(())
|
||||
@@ -8010,7 +8104,18 @@ impl LocalDisk {
|
||||
os::make_dir_all(parent, skip_parent).await?;
|
||||
}
|
||||
|
||||
let f = super::fs::open_file(path.as_ref(), mode).await.map_err(to_file_error)?;
|
||||
let f = if mode == O_CREATE | O_WRONLY | O_TRUNC {
|
||||
let path = path.as_ref().to_path_buf();
|
||||
let root = self.publication_root.clone();
|
||||
File::from_std(
|
||||
tokio::task::spawn_blocking(move || root.create_truncate(&path))
|
||||
.await
|
||||
.map_err(DiskError::from)?
|
||||
.map_err(to_file_error)?,
|
||||
)
|
||||
} else {
|
||||
super::fs::open_file(path.as_ref(), mode).await.map_err(to_file_error)?
|
||||
};
|
||||
|
||||
Ok(f)
|
||||
}
|
||||
@@ -8886,9 +8991,9 @@ fn file_meta_counts_toward_limit(meta: &FileMeta) -> bool {
|
||||
}
|
||||
|
||||
// Filter std::io::ErrorKind::NotFound
|
||||
async fn read_file_exists(path: impl AsRef<Path>) -> Result<(Bytes, Option<Metadata>)> {
|
||||
async fn read_file_exists(path: impl AsRef<Path>, root: &os::PublicationRoot) -> Result<(Bytes, Option<Metadata>)> {
|
||||
let p = path.as_ref();
|
||||
let (data, meta) = match read_file_all(&p).await {
|
||||
let (data, meta) = match read_file_all(&p, root).await {
|
||||
Ok((data, meta)) => (data, Some(meta)),
|
||||
Err(e) => {
|
||||
if e == Error::FileNotFound {
|
||||
@@ -8907,32 +9012,28 @@ async fn read_file_exists(path: impl AsRef<Path>) -> Result<(Bytes, Option<Metad
|
||||
Ok((data, meta))
|
||||
}
|
||||
|
||||
async fn read_file_all(path: impl AsRef<Path>) -> Result<(Bytes, Metadata)> {
|
||||
let p = path.as_ref();
|
||||
let meta = read_file_metadata(&path).await?;
|
||||
|
||||
let data = fs::read(&p)
|
||||
.await
|
||||
.inspect_err(|err| {
|
||||
log_startup_disk_io_error("read_file_all", p, err);
|
||||
})
|
||||
.map_err(to_file_error)?;
|
||||
|
||||
Ok((data.into(), meta))
|
||||
}
|
||||
|
||||
async fn read_file_metadata(p: impl AsRef<Path>) -> Result<Metadata> {
|
||||
let path = p.as_ref();
|
||||
let meta = fs::metadata(path)
|
||||
.await
|
||||
.inspect_err(|err| {
|
||||
if err.kind() != ErrorKind::NotFound {
|
||||
log_startup_disk_io_error("read_file_metadata", path, err);
|
||||
}
|
||||
})
|
||||
.map_err(to_file_error)?;
|
||||
|
||||
Ok(meta)
|
||||
async fn read_file_all(path: impl AsRef<Path>, root: &os::PublicationRoot) -> Result<(Bytes, Metadata)> {
|
||||
let path = path.as_ref().to_path_buf();
|
||||
let root = root.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let result = (|| {
|
||||
let mut file = root.open_readonly(&path)?;
|
||||
let metadata = file.metadata()?;
|
||||
let mut data = Vec::new();
|
||||
std::io::Read::read_to_end(&mut file, &mut data)?;
|
||||
Ok::<_, std::io::Error>((Bytes::from(data), metadata))
|
||||
})();
|
||||
result
|
||||
.inspect_err(|err| {
|
||||
if err.kind() != ErrorKind::NotFound {
|
||||
log_startup_disk_io_error("read_file_all", &path, err);
|
||||
}
|
||||
})
|
||||
.map_err(to_file_error)
|
||||
.map_err(DiskError::from)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?
|
||||
}
|
||||
|
||||
fn skip_access_checks(p: impl AsRef<str>) -> bool {
|
||||
@@ -9014,13 +9115,19 @@ fn check_local_disk_valid_path(root: &Path, path: impl AsRef<Path>) -> Result<()
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn check_local_disk_valid_object_path_at(root: &Path, root_fd: &std::fs::File, bucket_path: &Path, path: &Path) -> Result<()> {
|
||||
check_local_disk_object_path_components(root, bucket_path, path)?;
|
||||
reject_local_disk_symlink_components_at(root, root_fd, &normalize_path_components(path))
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn check_local_disk_object_path_components(root: &Path, bucket_path: &Path, path: &Path) -> Result<()> {
|
||||
let bucket_path = normalize_path_components(bucket_path);
|
||||
let path = normalize_path_components(path);
|
||||
if !bucket_path.starts_with(root) || !path.starts_with(&bucket_path) {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
|
||||
reject_local_disk_symlink_components_at(root, root_fd, &path)
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
@@ -9710,9 +9817,9 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
}
|
||||
|
||||
let p = self.io_get_object_path(volume, path)?;
|
||||
let p = self.io_get_object_open_path(volume, path)?;
|
||||
|
||||
let (data, _) = read_file_all(&p).await?;
|
||||
let (data, _) = read_file_all(&p, &self.publication_root).await?;
|
||||
|
||||
Ok(data)
|
||||
}
|
||||
@@ -11004,8 +11111,17 @@ impl DiskAPI for LocalDisk {
|
||||
entry.deleting = true;
|
||||
opts
|
||||
};
|
||||
// Release the reader token even under saturation. A rejected reclaim
|
||||
// retains pending_delete (and its on-disk receipt) for a later retry.
|
||||
// Do not wait for execution while a releasing reader may own locks.
|
||||
let claim = SnapshotLeaseDeleteClaim::new(&self.snapshot_leases, key);
|
||||
let result = self.delete_unleased(volume, path, &opts).await;
|
||||
let result = match super::cleanup_runtime::try_disk_execution() {
|
||||
Ok(permit) => {
|
||||
let _permit = permit;
|
||||
self.delete_unleased(volume, path, &opts).await
|
||||
}
|
||||
Err(err) => Err(err.into()),
|
||||
};
|
||||
claim.finish(result.is_ok());
|
||||
result
|
||||
}
|
||||
@@ -11295,7 +11411,7 @@ impl DiskAPI for LocalDisk {
|
||||
};
|
||||
|
||||
// if req.metadata_only {}
|
||||
match read_file_all(&fpath).await {
|
||||
match read_file_all(&fpath, &self.publication_root).await {
|
||||
Ok((data, meta)) => {
|
||||
found += 1;
|
||||
|
||||
@@ -12891,10 +13007,15 @@ mod test {
|
||||
disk.wait_for_startup_cleanup().await;
|
||||
assert_eq!(disk.startup_cleanup_ready.load(Ordering::Acquire), 1);
|
||||
|
||||
LocalDisk::cleanup_stale_tmp_objects_with_expiry(dir.path().join("missing-root"), &publication_root, Duration::ZERO)
|
||||
.await
|
||||
.expect("missing tmp path should be a cleanup no-op");
|
||||
LocalDisk::cleanup_deleted_objects(dir.path().join("missing-root"))
|
||||
LocalDisk::cleanup_stale_tmp_objects_with_expiry(
|
||||
dir.path().join("missing-root"),
|
||||
&publication_root,
|
||||
Duration::ZERO,
|
||||
&crate::disk::cleanup_runtime::GcBudget::default(),
|
||||
)
|
||||
.await
|
||||
.expect("missing tmp path should be a cleanup no-op");
|
||||
LocalDisk::cleanup_deleted_objects(dir.path().join("missing-root"), &crate::disk::cleanup_runtime::GcBudget::default())
|
||||
.await
|
||||
.expect("missing trash path should be a cleanup no-op");
|
||||
|
||||
@@ -12907,9 +13028,14 @@ mod test {
|
||||
fs::create_dir_all(&trash_root).await.expect("trash dir should be created");
|
||||
backdate_mtime(&stale_dir, Duration::from_secs(10));
|
||||
|
||||
LocalDisk::cleanup_stale_tmp_objects_with_expiry(dir.path().to_path_buf(), &publication_root, Duration::ZERO)
|
||||
.await
|
||||
.expect("stale tmp directory should move to trash");
|
||||
LocalDisk::cleanup_stale_tmp_objects_with_expiry(
|
||||
dir.path().to_path_buf(),
|
||||
&publication_root,
|
||||
Duration::ZERO,
|
||||
&crate::disk::cleanup_runtime::GcBudget::default(),
|
||||
)
|
||||
.await
|
||||
.expect("stale tmp directory should move to trash");
|
||||
assert!(!stale_dir.exists(), "stale tmp directory should be moved away");
|
||||
assert!(live_file.exists(), "plain tmp files should be ignored by stale dir cleanup");
|
||||
|
||||
@@ -12919,7 +13045,7 @@ mod test {
|
||||
fs::create_dir_all(trash_root.join("trash-dir"))
|
||||
.await
|
||||
.expect("trash dir should be created");
|
||||
LocalDisk::cleanup_deleted_objects(dir.path().to_path_buf())
|
||||
LocalDisk::cleanup_deleted_objects(dir.path().to_path_buf(), &crate::disk::cleanup_runtime::GcBudget::default())
|
||||
.await
|
||||
.expect("trash cleanup should remove files and directories");
|
||||
assert!(
|
||||
@@ -19048,9 +19174,14 @@ mod test {
|
||||
// Backdate after the write above: creating stale/data refreshes the
|
||||
// scanned tmp/stale directory's mtime.
|
||||
backdate_mtime(&tmp.join("stale"), Duration::from_secs(10));
|
||||
LocalDisk::cleanup_stale_tmp_objects_with_expiry(dir.path().to_path_buf(), &publication_root, Duration::ZERO)
|
||||
.await
|
||||
.expect("operation should succeed");
|
||||
LocalDisk::cleanup_stale_tmp_objects_with_expiry(
|
||||
dir.path().to_path_buf(),
|
||||
&publication_root,
|
||||
Duration::ZERO,
|
||||
&crate::disk::cleanup_runtime::GcBudget::default(),
|
||||
)
|
||||
.await
|
||||
.expect("operation should succeed");
|
||||
|
||||
assert!(!tmp.join("stale").exists());
|
||||
assert!(trash.exists());
|
||||
@@ -19077,9 +19208,14 @@ mod test {
|
||||
fs::write(&fresh_dir, b"temporary").await.expect("operation should succeed");
|
||||
fs::write(®ular_file, b"keep").await.expect("operation should succeed");
|
||||
|
||||
LocalDisk::cleanup_stale_tmp_objects_with_expiry(dir.path().to_path_buf(), &publication_root, Duration::from_secs(60))
|
||||
.await
|
||||
.expect("operation should succeed");
|
||||
LocalDisk::cleanup_stale_tmp_objects_with_expiry(
|
||||
dir.path().to_path_buf(),
|
||||
&publication_root,
|
||||
Duration::from_secs(60),
|
||||
&crate::disk::cleanup_runtime::GcBudget::default(),
|
||||
)
|
||||
.await
|
||||
.expect("operation should succeed");
|
||||
|
||||
assert!(tmp.join("fresh").exists());
|
||||
assert!(regular_file.exists());
|
||||
@@ -19088,6 +19224,44 @@ mod test {
|
||||
assert!(entries.next_entry().await.expect("operation should succeed").is_none());
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test]
|
||||
async fn background_cleanup_keeps_mount_lease_after_disk_drop() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().unwrap()).unwrap();
|
||||
let disk = LocalDisk::new(&endpoint, false).await.unwrap();
|
||||
let root = disk.io_root.clone();
|
||||
let trash_entry = LocalDisk::meta_path(&root, RUSTFS_META_TMP_DELETED_BUCKET).join("detached-gc");
|
||||
fs::create_dir_all(&trash_entry).await.unwrap();
|
||||
fs::write(trash_entry.join("part.1"), b"garbage").await.unwrap();
|
||||
let live_file = root.join("keep");
|
||||
fs::write(&live_file, b"live").await.unwrap();
|
||||
let (exit, exited) = tokio::sync::broadcast::channel(1);
|
||||
tokio::time::pause();
|
||||
let cleanup = tokio::spawn(LocalDisk::cleanup_deleted_objects_loop(
|
||||
root.clone(),
|
||||
disk.publication_root.clone(),
|
||||
disk.mount_lease.clone(),
|
||||
exited,
|
||||
));
|
||||
tokio::task::yield_now().await;
|
||||
drop(disk);
|
||||
tokio::task::yield_now().await;
|
||||
assert!(live_file.exists(), "the GC owner must retain the original /proc/self/fd root");
|
||||
tokio::time::advance(DELETED_OBJECTS_CLEANUP_INTERVAL + Duration::from_secs(1)).await;
|
||||
tokio::time::resume();
|
||||
timeout(Duration::from_secs(20), async {
|
||||
while trash_entry.exists() {
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("detached cleanup must finish against the pinned mount");
|
||||
assert_eq!(fs::read(&live_file).await.unwrap(), b"live");
|
||||
exit.send(()).unwrap();
|
||||
timeout(Duration::from_secs(20), cleanup).await.unwrap().unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn cleanup_loop_interval_does_not_tick_immediately() {
|
||||
let start_at = Instant::now() + DELETED_OBJECTS_CLEANUP_INTERVAL;
|
||||
@@ -22228,10 +22402,12 @@ mod test {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_read_file_exists() {
|
||||
let test_file = "./test_read_exists.txt";
|
||||
let directory = tempfile::tempdir().expect("test directory");
|
||||
let root = os::PublicationRoot::new(directory.path()).expect("publication root");
|
||||
let test_file = &directory.path().join("read_exists");
|
||||
|
||||
// Test non-existent file
|
||||
let (data, metadata) = read_file_exists(test_file).await.expect("operation should succeed");
|
||||
let (data, metadata) = read_file_exists(test_file, &root).await.expect("operation should succeed");
|
||||
assert!(data.is_empty());
|
||||
assert!(metadata.is_none());
|
||||
|
||||
@@ -22239,7 +22415,7 @@ mod test {
|
||||
fs::write(test_file, b"test content").await.expect("operation should succeed");
|
||||
|
||||
// Test existing file
|
||||
let (data, metadata) = read_file_exists(test_file).await.expect("operation should succeed");
|
||||
let (data, metadata) = read_file_exists(test_file, &root).await.expect("operation should succeed");
|
||||
assert_eq!(data.as_ref(), b"test content");
|
||||
assert!(metadata.is_some());
|
||||
|
||||
@@ -22249,14 +22425,16 @@ mod test {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_read_file_all() {
|
||||
let test_file = "./test_read_all.txt";
|
||||
let directory = tempfile::tempdir().expect("test directory");
|
||||
let root = os::PublicationRoot::new(directory.path()).expect("publication root");
|
||||
let test_file = &directory.path().join("read_all");
|
||||
let test_content = b"test content for read_all";
|
||||
|
||||
// Create test file
|
||||
fs::write(test_file, test_content).await.expect("operation should succeed");
|
||||
|
||||
// Test reading file
|
||||
let (data, metadata) = read_file_all(test_file).await.expect("operation should succeed");
|
||||
let (data, metadata) = read_file_all(test_file, &root).await.expect("operation should succeed");
|
||||
assert_eq!(data.as_ref(), test_content);
|
||||
assert!(metadata.is_file());
|
||||
assert_eq!(metadata.len(), test_content.len() as u64);
|
||||
@@ -22265,22 +22443,6 @@ mod test {
|
||||
let _ = fs::remove_file(test_file).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_read_file_metadata() {
|
||||
let test_file = "./test_metadata.txt";
|
||||
|
||||
// Create test file
|
||||
fs::write(test_file, b"test").await.expect("operation should succeed");
|
||||
|
||||
// Test reading metadata
|
||||
let metadata = read_file_metadata(test_file).await.expect("operation should succeed");
|
||||
assert!(metadata.is_file());
|
||||
assert_eq!(metadata.len(), 4); // "test" is 4 bytes
|
||||
|
||||
// Clean up
|
||||
let _ = fs::remove_file(test_file).await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_root_path() {
|
||||
// Unix root path
|
||||
@@ -22869,9 +23031,14 @@ mod test {
|
||||
Bytes::from_static(b"later")
|
||||
);
|
||||
|
||||
disk.release_snapshot_lease(volume, &data_dir, renewed)
|
||||
.await
|
||||
.expect("renewed lease release should succeed");
|
||||
assert!(
|
||||
disk.release_snapshot_lease_without_cleanup(volume, &data_dir, renewed).await,
|
||||
"a non-final reader release must not dispatch cleanup"
|
||||
);
|
||||
assert!(
|
||||
!disk.release_snapshot_lease_without_cleanup(volume, &data_dir, second).await,
|
||||
"the final reader must retain its token for owned deferred reclamation"
|
||||
);
|
||||
assert!(
|
||||
disk.read_all(volume, &first_part).await.is_ok(),
|
||||
"one remaining lease must keep the data directory"
|
||||
|
||||
@@ -47,6 +47,33 @@ use tokio::fs;
|
||||
use tracing::{info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
fn transitioned_old_current_source(
|
||||
dst_meta_existed: bool,
|
||||
dst_meta_unparsable: bool,
|
||||
xlmeta: &FileMeta,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
version_id: Uuid,
|
||||
) -> (bool, Option<FileInfo>) {
|
||||
if dst_meta_unparsable {
|
||||
return (false, None);
|
||||
}
|
||||
if !dst_meta_existed {
|
||||
return (true, None);
|
||||
}
|
||||
let Ok(old_current) = xlmeta.into_fileinfo(volume, path, "", false, false, true) else {
|
||||
return (false, None);
|
||||
};
|
||||
if old_current.version_id.unwrap_or_default() != version_id {
|
||||
return (true, None);
|
||||
}
|
||||
if old_current.transition_status == rustfs_filemeta::TRANSITION_COMPLETE {
|
||||
(true, Some(old_current))
|
||||
} else {
|
||||
(true, None)
|
||||
}
|
||||
}
|
||||
|
||||
/// Hold later repair publications after admitting one baseline object. The
|
||||
/// fixture arms this on one replacement disk before rejoining the cluster.
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
@@ -214,7 +241,7 @@ pub(super) async fn lock_rename_commit_directories(
|
||||
publication_root: &os::PublicationRoot,
|
||||
mutation_lease: Arc<os::NamespaceMutationLease>,
|
||||
) -> Result<os::RenameCommitGuard> {
|
||||
#[cfg(windows)]
|
||||
#[cfg(any(windows, target_os = "linux"))]
|
||||
let result = {
|
||||
let source_parent = source_parent.to_path_buf();
|
||||
let destination_parent = destination_parent.to_path_buf();
|
||||
@@ -222,7 +249,7 @@ pub(super) async fn lock_rename_commit_directories(
|
||||
let publication_root = publication_root.clone();
|
||||
os::run_blocking_namespace_operation(mutation_lease, move || {
|
||||
let result = os::prepare_rename_commit_guard(&source_parent, &destination_parent, &base_dir, &publication_root);
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, windows))]
|
||||
if result.is_ok() {
|
||||
run_destination_commit_directory_preparation(&destination_parent);
|
||||
}
|
||||
@@ -230,7 +257,7 @@ pub(super) async fn lock_rename_commit_directories(
|
||||
})
|
||||
.await
|
||||
};
|
||||
#[cfg(not(windows))]
|
||||
#[cfg(not(any(windows, target_os = "linux")))]
|
||||
let result = {
|
||||
let _ = mutation_lease;
|
||||
os::prepare_rename_commit_guard(source_parent, destination_parent, base_dir, publication_root)
|
||||
@@ -244,12 +271,12 @@ pub(super) async fn lock_rename_commit_directories(
|
||||
result.map_err(to_file_error).map_err(DiskError::from)
|
||||
}
|
||||
|
||||
async fn read_rename_destination_metadata(
|
||||
async fn prepare_rename_destination_metadata(
|
||||
file_path: &Path,
|
||||
rename_commit_guard: &os::RenameCommitGuard,
|
||||
mutation_lease: Arc<os::NamespaceMutationLease>,
|
||||
) -> Result<Option<Bytes>> {
|
||||
#[cfg(windows)]
|
||||
#[cfg(any(windows, target_os = "linux"))]
|
||||
let result = {
|
||||
let file_path = file_path.to_path_buf();
|
||||
let rename_commit_guard = rename_commit_guard.clone();
|
||||
@@ -258,10 +285,10 @@ async fn read_rename_destination_metadata(
|
||||
})
|
||||
.await
|
||||
};
|
||||
#[cfg(not(windows))]
|
||||
#[cfg(not(any(windows, target_os = "linux")))]
|
||||
let _ = (rename_commit_guard, mutation_lease);
|
||||
#[cfg(not(windows))]
|
||||
let result = match super::super::fs::read_file(file_path).await {
|
||||
#[cfg(not(any(windows, target_os = "linux")))]
|
||||
let result = match fs::read(file_path).await {
|
||||
Ok(data) => Ok(Some(data)),
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => Ok(None),
|
||||
Err(err) => Err(err),
|
||||
@@ -398,7 +425,7 @@ impl LocalDisk {
|
||||
|
||||
// xl.meta path
|
||||
let src_file_path = self.io_get_object_path(src_volume, format!("{}/{}", src_path, STORAGE_FORMAT_FILE).as_str())?;
|
||||
let dst_file_path = self.io_get_object_path(dst_volume, format!("{}/{}", dst_path, STORAGE_FORMAT_FILE).as_str())?;
|
||||
let dst_file_path = self.io_get_object_open_path(dst_volume, format!("{}/{}", dst_path, STORAGE_FORMAT_FILE).as_str())?;
|
||||
|
||||
// data_dir path
|
||||
let has_data_dir_path = {
|
||||
@@ -453,8 +480,7 @@ impl LocalDisk {
|
||||
fs::create_dir_all(src_file_parent).await.map_err(to_file_error)?;
|
||||
}
|
||||
// Acquire the common trees before reading destination metadata. On
|
||||
// Windows this pins the object directory identity across metadata
|
||||
// preparation, data publication, rollback backup, and final commit.
|
||||
// Windows this pins the object directory identity across publication.
|
||||
let rename_commit_guard = lock_rename_commit_directories(
|
||||
src_file_parent,
|
||||
dst_file_parent,
|
||||
@@ -472,9 +498,12 @@ impl LocalDisk {
|
||||
source_volume_dir: &src_volume_dir,
|
||||
data_paths: has_data_dir_path.as_ref(),
|
||||
};
|
||||
let has_dst_buf =
|
||||
read_rename_destination_metadata(commit.destination_metadata, &commit.directory_guard, commit.mutation_lease.clone())
|
||||
.await?;
|
||||
let has_dst_buf = prepare_rename_destination_metadata(
|
||||
commit.destination_metadata,
|
||||
&commit.directory_guard,
|
||||
commit.mutation_lease.clone(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
if no_inline {
|
||||
// Non-inline: read xl.meta, parse, write, rename data dir, rename xl.meta
|
||||
@@ -508,6 +537,14 @@ impl LocalDisk {
|
||||
}
|
||||
|
||||
let version_id = fi.version_id.unwrap_or_default();
|
||||
let (old_current_source_checked, old_current_source) = transitioned_old_current_source(
|
||||
has_dst_buf.is_some(),
|
||||
dst_meta_unparsable,
|
||||
&xlmeta,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
version_id,
|
||||
);
|
||||
let has_old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id));
|
||||
let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok();
|
||||
let rollback_data_dir = has_old_data_dir.or_else(|| {
|
||||
@@ -798,7 +835,9 @@ impl LocalDisk {
|
||||
&& let Some(parent) = dst_file_path.parent()
|
||||
{
|
||||
let fsync_started = rustfs_io_metrics::put_stage_timer();
|
||||
if let Err(err) = os::fsync_dst_dir_group_commit(parent, Some(commit.mutation_lease.clone())).await {
|
||||
if let Err(err) =
|
||||
os::fsync_commit_directory(parent, &commit.directory_guard, commit.mutation_lease.clone(), None).await
|
||||
{
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC,
|
||||
fsync_started,
|
||||
@@ -897,6 +936,8 @@ impl LocalDisk {
|
||||
cleanup_data_dir: has_old_data_dir,
|
||||
sign: version_signature,
|
||||
old_current_size,
|
||||
old_current_source_checked,
|
||||
old_current_source,
|
||||
})
|
||||
} else {
|
||||
// Inline metadata preparation is blocking. The transaction lease is
|
||||
@@ -910,6 +951,8 @@ impl LocalDisk {
|
||||
None
|
||||
};
|
||||
let dst_path_for_failpoint = dst_path.to_string();
|
||||
let dst_volume_for_source = dst_volume.to_string();
|
||||
let dst_path_for_source = dst_path.to_string();
|
||||
#[cfg(windows)]
|
||||
let source_parent = src_file_parent.to_path_buf();
|
||||
let rename_commit_guard_for_preparation = commit.directory_guard.clone();
|
||||
@@ -954,6 +997,14 @@ impl LocalDisk {
|
||||
};
|
||||
|
||||
let version_id = fi.version_id.unwrap_or_default();
|
||||
let (old_current_source_checked, old_current_source) = transitioned_old_current_source(
|
||||
has_dst_buf.is_some(),
|
||||
dst_meta_unparsable,
|
||||
&xlmeta,
|
||||
&dst_volume_for_source,
|
||||
&dst_path_for_source,
|
||||
version_id,
|
||||
);
|
||||
let old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id));
|
||||
let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok();
|
||||
let rollback_data_dir = old_data_dir.or_else(|| {
|
||||
@@ -1003,6 +1054,8 @@ impl LocalDisk {
|
||||
old_data_dir,
|
||||
version_signature,
|
||||
old_current_size,
|
||||
old_current_source_checked,
|
||||
old_current_source,
|
||||
staged_rollback_path,
|
||||
has_dst_buf.is_none(),
|
||||
prepared_metadata_source,
|
||||
@@ -1022,6 +1075,8 @@ impl LocalDisk {
|
||||
cleanup_data_dir,
|
||||
version_signature,
|
||||
old_current_size,
|
||||
old_current_source_checked,
|
||||
old_current_source,
|
||||
mut local_rollback_path,
|
||||
destination_was_absent,
|
||||
prepared_metadata_source,
|
||||
@@ -1147,10 +1202,11 @@ impl LocalDisk {
|
||||
&& let Some(dst_parent) = dst_file_path.parent()
|
||||
{
|
||||
let fsync_started = rustfs_io_metrics::put_stage_timer();
|
||||
if let Err(err) = os::fsync_dst_dir_group_commit_or_namespace_file_sync_limit(
|
||||
if let Err(err) = os::fsync_commit_directory(
|
||||
dst_parent,
|
||||
&commit.directory_guard,
|
||||
commit.mutation_lease.clone(),
|
||||
admission,
|
||||
Some(admission),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -1262,6 +1318,8 @@ impl LocalDisk {
|
||||
cleanup_data_dir,
|
||||
sign: version_signature,
|
||||
old_current_size,
|
||||
old_current_source_checked,
|
||||
old_current_source,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
// #730: disk abstractions still carry staged health and direct-I/O migration paths.
|
||||
|
||||
pub(crate) mod cleanup_runtime;
|
||||
pub mod disk_store;
|
||||
pub mod endpoint;
|
||||
pub mod error;
|
||||
@@ -458,6 +459,7 @@ impl DiskAPI for Disk {
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
#[track_caller]
|
||||
async fn read_version(
|
||||
&self,
|
||||
_org_volume: &str,
|
||||
@@ -466,10 +468,20 @@ impl DiskAPI for Disk {
|
||||
version_id: &str,
|
||||
opts: &ReadOptions,
|
||||
) -> Result<FileInfo> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => local_disk.read_version(_org_volume, volume, path, version_id, opts).await,
|
||||
Disk::Remote(remote_disk) => remote_disk.read_version(_org_volume, volume, path, version_id, opts).await,
|
||||
let operation = async {
|
||||
match self {
|
||||
Disk::Local(local_disk) => local_disk.read_version(_org_volume, volume, path, version_id, opts).await,
|
||||
Disk::Remote(remote_disk) => remote_disk.read_version(_org_volume, volume, path, version_id, opts).await,
|
||||
}
|
||||
};
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
if crate::cluster::rpc::runtime_sources::current_read_version_request_caller().is_none() {
|
||||
let caller = crate::cluster::rpc::runtime_sources::read_version_request_caller_from_location(
|
||||
std::panic::Location::caller().file(),
|
||||
);
|
||||
return crate::cluster::rpc::runtime_sources::with_read_version_request_caller(caller, operation).await;
|
||||
}
|
||||
operation.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
@@ -1161,6 +1173,7 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
|
||||
}
|
||||
async fn write_metadata(&self, org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()>;
|
||||
async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()>;
|
||||
#[track_caller]
|
||||
async fn read_version(
|
||||
&self,
|
||||
org_volume: &str,
|
||||
@@ -1284,13 +1297,15 @@ where
|
||||
{
|
||||
validate_batch_read_version_item_count(req.items.len())?;
|
||||
let parallelism = batch_read_version_server_parallelism();
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let item_count = req.items.len();
|
||||
|
||||
let mut responses = stream::iter(req.items.into_iter().enumerate())
|
||||
.map(|(index, item)| async move {
|
||||
match disk
|
||||
.read_version(&item.org_volume, &item.volume, &item.path, &item.version_id, &req.opts)
|
||||
.await
|
||||
{
|
||||
let read = disk.read_version(&item.org_volume, &item.volume, &item.path, &item.version_id, &req.opts);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let read = crate::cluster::rpc::runtime_sources::with_batch_read_version_item_context(index, item_count, read);
|
||||
match read.await {
|
||||
Ok(file_info) => BatchReadVersionResp {
|
||||
index,
|
||||
path: item.path,
|
||||
@@ -1568,6 +1583,14 @@ pub struct RenameDataResp {
|
||||
/// "cannot vote", never as `Absent`.
|
||||
#[serde(default)]
|
||||
pub old_current_size: Option<OldCurrentSize>,
|
||||
/// Whether this disk decoded enough of the overwritten current version to
|
||||
/// decide if a transitioned source has to be returned.
|
||||
#[serde(default)]
|
||||
pub old_current_source_checked: bool,
|
||||
/// Transitioned old-current source used by PUT free-version cleanup receipt.
|
||||
/// Empty when the old current is not transitioned or no source was known.
|
||||
#[serde(default)]
|
||||
pub old_current_source: Option<FileInfo>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
@@ -2044,6 +2067,8 @@ mod tests {
|
||||
cleanup_data_dir: Some(uuid),
|
||||
sign: Some(signature.clone()),
|
||||
old_current_size: Some(OldCurrentSize::Present(42)),
|
||||
old_current_source_checked: false,
|
||||
old_current_source: None,
|
||||
};
|
||||
|
||||
assert_eq!(resp.old_data_dir, Some(uuid));
|
||||
@@ -2064,6 +2089,8 @@ mod tests {
|
||||
cleanup_data_dir: Some(Uuid::new_v4()),
|
||||
sign: Some(vec![0x01, 0x02, 0x03]),
|
||||
old_current_size,
|
||||
old_current_source_checked: false,
|
||||
old_current_source: None,
|
||||
};
|
||||
|
||||
let encoded = rmp_serde::encode::to_vec_named(&resp).expect("named msgpack should encode");
|
||||
@@ -2101,6 +2128,8 @@ mod tests {
|
||||
assert_eq!(decoded.cleanup_data_dir, None);
|
||||
assert_eq!(decoded.sign, legacy.sign);
|
||||
assert_eq!(decoded.old_current_size, None);
|
||||
assert!(!decoded.old_current_source_checked);
|
||||
assert!(decoded.old_current_source.is_none());
|
||||
}
|
||||
|
||||
/// Test constants
|
||||
|
||||
+834
-71
File diff suppressed because it is too large
Load Diff
@@ -4715,6 +4715,23 @@ impl TierConfigMgr {
|
||||
}
|
||||
|
||||
async fn reload_handle_with<S>(handle: &Arc<RwLock<Self>>, api: Arc<S>) -> io::Result<()>
|
||||
where
|
||||
S: EcstoreObjectIO
|
||||
+ EcstoreObjectOperations
|
||||
+ NamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>
|
||||
+ ListOperations<Error = Error, ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>>,
|
||||
{
|
||||
// Periodic and committed-mutation refreshes call this shared entry directly.
|
||||
let reload = Self::reload_handle_inner(handle, api);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let reload = crate::cluster::rpc::runtime_sources::with_read_version_entry(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::TierConfigReload,
|
||||
reload,
|
||||
);
|
||||
reload.await
|
||||
}
|
||||
|
||||
async fn reload_handle_inner<S>(handle: &Arc<RwLock<Self>>, api: Arc<S>) -> io::Result<()>
|
||||
where
|
||||
S: EcstoreObjectIO
|
||||
+ EcstoreObjectOperations
|
||||
@@ -12533,6 +12550,51 @@ mod tests {
|
||||
assert!(reloaded.iter().any(|intent| intent.mutation_id == prepared_id));
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[tokio::test]
|
||||
async fn diagnostic_reload_shared_entry_preserves_source_and_api() {
|
||||
use crate::api::diagnostic::{ReadVersionEntrySource, current_read_version_entry_source, with_read_version_entry};
|
||||
use rustfs_io_metrics::s3_http_metrics::{
|
||||
S3HttpRequestGuard, current_s3_http_operation_context, with_isolated_s3_http_operation_context,
|
||||
};
|
||||
|
||||
let store = Arc::new(CasConfigStore::default());
|
||||
empty_mgr()
|
||||
.save_tiering_config_if_current(store.clone(), None)
|
||||
.await
|
||||
.expect("empty tier config fixture should persist");
|
||||
let mut request = S3HttpRequestGuard::new("GET");
|
||||
let get = request.in_scope(|| {
|
||||
rustfs_io_metrics::record_s3_op(rustfs_s3_ops::S3Operation::GetObject);
|
||||
current_s3_http_operation_context().expect("GetObject context should be captured")
|
||||
});
|
||||
|
||||
for (earlier, api) in [(None, None), (Some(ReadVersionEntrySource::ConfigRead), Some(get))] {
|
||||
store.reload_contexts.lock().expect("context recorder").clear();
|
||||
let handle = TierConfigMgr::new();
|
||||
with_isolated_s3_http_operation_context(api, async {
|
||||
let reload = TierConfigMgr::reload_handle_with(&handle, store.clone());
|
||||
if let Some(source) = earlier {
|
||||
with_read_version_entry(source, reload).await
|
||||
} else {
|
||||
reload.await
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("shared reload entry should load config and scan intents");
|
||||
|
||||
let observed = store.reload_contexts.lock().expect("context recorder");
|
||||
assert!(observed.iter().any(|(boundary, _, _)| *boundary == "reader"));
|
||||
assert!(observed.iter().any(|(boundary, _, _)| *boundary == "list"));
|
||||
let expected = earlier.unwrap_or(ReadVersionEntrySource::TierConfigReload);
|
||||
assert!(observed.iter().all(|(_, source, operation)| {
|
||||
*source == Some(expected) && *operation == api.map(|context| context.operation_name())
|
||||
}));
|
||||
assert_eq!(current_read_version_entry_source(), None);
|
||||
assert_eq!(current_s3_http_operation_context(), None);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reload_handle_keeps_committed_block_until_publish_swaps_generation() {
|
||||
let store = Arc::new(CasConfigStore::default());
|
||||
@@ -14725,8 +14787,17 @@ mod tests {
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
type ReloadDiagnosticObservation = (
|
||||
&'static str,
|
||||
Option<rustfs_io_metrics::internode_metrics::ReadVersionEntrySource>,
|
||||
Option<&'static str>,
|
||||
);
|
||||
|
||||
#[derive(Debug)]
|
||||
struct CasConfigStore {
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
reload_contexts: Mutex<Vec<ReloadDiagnosticObservation>>,
|
||||
objects: tokio::sync::Mutex<HashMap<String, (Vec<u8>, String)>>,
|
||||
legacy_state: tokio::sync::Mutex<Option<Vec<u8>>>,
|
||||
if_match_race_rewrite: tokio::sync::Mutex<std::collections::VecDeque<(String, Vec<u8>)>>,
|
||||
@@ -14750,6 +14821,8 @@ mod tests {
|
||||
impl Default for CasConfigStore {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
reload_contexts: Mutex::new(Vec::new()),
|
||||
objects: tokio::sync::Mutex::new(HashMap::new()),
|
||||
legacy_state: tokio::sync::Mutex::new(None),
|
||||
if_match_race_rewrite: tokio::sync::Mutex::new(std::collections::VecDeque::new()),
|
||||
@@ -14904,6 +14977,12 @@ mod tests {
|
||||
_headers: Self::HeaderMap,
|
||||
_opts: &Self::ObjectOptions,
|
||||
) -> Result<Self::GetObjectReader> {
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
self.reload_contexts.lock().expect("context recorder").push((
|
||||
"reader",
|
||||
crate::api::diagnostic::current_read_version_entry_source(),
|
||||
rustfs_io_metrics::s3_http_metrics::current_s3_http_operation_name(),
|
||||
));
|
||||
let legacy_config_path = tier_config_path(TIER_CONFIG_LEGACY_FILE);
|
||||
let (data, etag) = if object == legacy_config_path {
|
||||
let state = self.legacy_state.lock().await;
|
||||
@@ -15259,6 +15338,12 @@ mod tests {
|
||||
_start_after: Option<String>,
|
||||
_incl_deleted: bool,
|
||||
) -> Result<Self::ListObjectsV2Info> {
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
self.reload_contexts.lock().expect("context recorder").push((
|
||||
"list",
|
||||
crate::api::diagnostic::current_read_version_entry_source(),
|
||||
rustfs_io_metrics::s3_http_metrics::current_s3_http_operation_name(),
|
||||
));
|
||||
if matches!(
|
||||
prefix,
|
||||
crate::services::tier::tier_mutation_intent::TIER_MUTATION_INTENT_RECORD_PREFIX
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -170,6 +170,23 @@ fn read_version_coalescing_delay() -> Duration {
|
||||
struct CoalescedReadVersionRequest {
|
||||
item: BatchReadVersionItem,
|
||||
tx: oneshot::Sender<disk::error::Result<FileInfo>>,
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
context: crate::cluster::rpc::runtime_sources::ReadVersionDiagnosticContext,
|
||||
}
|
||||
|
||||
impl CoalescedReadVersionRequest {
|
||||
fn new(item: BatchReadVersionItem, tx: oneshot::Sender<disk::error::Result<FileInfo>>) -> Self {
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let context = crate::cluster::rpc::runtime_sources::ReadVersionDiagnosticContext::capture();
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
context.observe(rustfs_io_metrics::internode_metrics::ReadVersionContextBoundary::CoalescerEnqueue);
|
||||
Self {
|
||||
item,
|
||||
tx,
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
context,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
@@ -245,26 +262,49 @@ async fn read_version_via_coalescer(
|
||||
allow_coalescing: bool,
|
||||
) -> disk::error::Result<FileInfo> {
|
||||
if !allow_coalescing || !read_version_coalescing_enabled() {
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
return crate::cluster::rpc::runtime_sources::with_read_version_request_caller_if_absent(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionRequestCaller::MetadataRead,
|
||||
disk.read_version(org_bucket, bucket, object, version_id, opts),
|
||||
)
|
||||
.await;
|
||||
#[cfg(not(feature = "diagnostic-read-version-caller"))]
|
||||
return disk.read_version(org_bucket, bucket, object, version_id, opts).await;
|
||||
}
|
||||
if !matches!(disk.as_ref(), Disk::Remote(_)) {
|
||||
record_read_version_coalescer_event("bypass_non_remote", 1);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
return crate::cluster::rpc::runtime_sources::with_read_version_request_caller_if_absent(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionRequestCaller::MetadataRead,
|
||||
disk.read_version(org_bucket, bucket, object, version_id, opts),
|
||||
)
|
||||
.await;
|
||||
#[cfg(not(feature = "diagnostic-read-version-caller"))]
|
||||
return disk.read_version(org_bucket, bucket, object, version_id, opts).await;
|
||||
}
|
||||
|
||||
let (tx, rx) = oneshot::channel();
|
||||
let item = BatchReadVersionItem {
|
||||
org_volume: org_bucket.to_string(),
|
||||
volume: bucket.to_string(),
|
||||
path: object.to_string(),
|
||||
version_id: version_id.to_string(),
|
||||
};
|
||||
enqueue_coalesced_read_version(disk, item, opts).await
|
||||
}
|
||||
|
||||
pub(super) async fn enqueue_coalesced_read_version(
|
||||
disk: DiskStore,
|
||||
item: BatchReadVersionItem,
|
||||
opts: &ReadOptions,
|
||||
) -> disk::error::Result<FileInfo> {
|
||||
let (tx, rx) = oneshot::channel();
|
||||
let lane_key = ReadVersionCoalescerKey::new(&disk, opts);
|
||||
let request = CoalescedReadVersionRequest::new(item, tx);
|
||||
let pending = {
|
||||
let mut coalescer = read_version_coalescer().lock().await;
|
||||
let lane = coalescer.lanes.entry(lane_key).or_default();
|
||||
let schedule_delayed_flush = lane.is_empty();
|
||||
lane.push(CoalescedReadVersionRequest { item, tx });
|
||||
lane.push(request);
|
||||
if lane.len() >= BATCH_READ_VERSION_MAX_ITEMS {
|
||||
coalescer.lanes.remove(&lane_key)
|
||||
} else if schedule_delayed_flush {
|
||||
@@ -325,20 +365,24 @@ async fn flush_read_version_coalescer_pending(
|
||||
|
||||
let mut senders = Vec::with_capacity(pending.len());
|
||||
let mut items = Vec::with_capacity(pending.len());
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let mut contexts = Vec::with_capacity(pending.len());
|
||||
for request in pending {
|
||||
senders.push(request.tx);
|
||||
items.push(request.item);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
contexts.push(request.context);
|
||||
}
|
||||
|
||||
let expected_items = items.iter().map(ExpectedBatchReadVersionItem::from).collect::<Vec<_>>();
|
||||
record_read_version_coalescer_event("attempted_batch", items.len());
|
||||
let result =
|
||||
match tokio::time::timeout(get_drive_metadata_timeout(), disk.batch_read_version(BatchReadVersionReq { items, opts }))
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(_) => Err(DiskError::Timeout),
|
||||
};
|
||||
let batch = disk.batch_read_version(BatchReadVersionReq { items, opts });
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let batch = crate::cluster::rpc::runtime_sources::with_read_version_batch_contexts(contexts, batch);
|
||||
let result = match tokio::time::timeout(get_drive_metadata_timeout(), batch).await {
|
||||
Ok(result) => result,
|
||||
Err(_) => Err(DiskError::Timeout),
|
||||
};
|
||||
match result {
|
||||
Ok(responses) => {
|
||||
let map_started = batch_read_version_stage_timer();
|
||||
@@ -575,6 +619,7 @@ impl MetadataFanoutDiagnostics {
|
||||
|
||||
pub(in crate::set_disk) fn record_quorum_candidate_latency(&self, path: &'static str, read_quorum: usize) {
|
||||
if let Some(latency) = self.quorum_candidate_latency(read_quorum) {
|
||||
rustfs_io_metrics::record_get_object_metadata_quorum_result(path, "reached");
|
||||
rustfs_io_metrics::record_get_object_quorum_reached_latency(path, latency.as_secs_f64());
|
||||
}
|
||||
}
|
||||
@@ -1012,6 +1057,10 @@ impl SetDisks {
|
||||
let object: Arc<str> = Arc::from(object);
|
||||
let version_id: Arc<str> = Arc::from(version_id);
|
||||
let slowtail_fault = get_metadata_slowtail_fault_request(bucket.as_ref(), object.as_ref(), read_data);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let diagnostic_context = crate::cluster::rpc::runtime_sources::ReadVersionDiagnosticContext::capture_entry(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::MetadataRead,
|
||||
);
|
||||
let futures = disks.iter().enumerate().map(|(disk_index, disk)| {
|
||||
let disk = disk.clone();
|
||||
let task_opts = opts;
|
||||
@@ -1020,21 +1069,34 @@ impl SetDisks {
|
||||
let object = object.clone();
|
||||
let version_id = version_id.clone();
|
||||
let slowtail_fault = slowtail_fault.clone();
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let diagnostic_context = diagnostic_context;
|
||||
AbortOnDropJoinHandle(tokio::spawn(async move {
|
||||
let response_start = observe.then(Instant::now);
|
||||
let result = if let Some(disk) = disk {
|
||||
Self::record_read_version_call(&object, disk_index);
|
||||
if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(disk_index)) {
|
||||
Self::record_metadata_slowtail_fault(&object, disk_index);
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
read_version_via_coalescer(disk, &org_bucket, &bucket, &object, &version_id, &task_opts, allow_coalescing)
|
||||
.await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
let read = async move {
|
||||
let response_start = observe.then(Instant::now);
|
||||
let result = if let Some(disk) = disk {
|
||||
Self::record_read_version_call(&object, disk_index);
|
||||
if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(disk_index)) {
|
||||
Self::record_metadata_slowtail_fault(&object, disk_index);
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
read_version_via_coalescer(disk, &org_bucket, &bucket, &object, &version_id, &task_opts, allow_coalescing)
|
||||
.await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
};
|
||||
let elapsed = response_start.map(|start| start.elapsed());
|
||||
(result, elapsed)
|
||||
};
|
||||
let elapsed = response_start.map(|start| start.elapsed());
|
||||
(result, elapsed)
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
return crate::cluster::rpc::runtime_sources::with_read_version_diagnostic_context(
|
||||
diagnostic_context,
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionContextBoundary::FullWait,
|
||||
read,
|
||||
)
|
||||
.await;
|
||||
#[cfg(not(feature = "diagnostic-read-version-caller"))]
|
||||
read.await
|
||||
}))
|
||||
});
|
||||
|
||||
@@ -1125,6 +1187,10 @@ impl SetDisks {
|
||||
let mut non_inline_candidate_eligible = None;
|
||||
let mut single_pending_hedge_deadline = None;
|
||||
let slowtail_fault = get_metadata_slowtail_fault_request(bucket.as_ref(), object.as_ref(), read_data);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let diagnostic_context = crate::cluster::rpc::runtime_sources::ReadVersionDiagnosticContext::capture_entry(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::MetadataRead,
|
||||
);
|
||||
let spawn_read_version = |join_set: &mut JoinSet<(usize, disk::error::Result<FileInfo>, Duration)>,
|
||||
task_ids: &mut [Option<tokio::task::Id>],
|
||||
index: usize,
|
||||
@@ -1135,27 +1201,44 @@ impl SetDisks {
|
||||
let object = object.clone();
|
||||
let version_id = version_id.clone();
|
||||
let slowtail_fault = slowtail_fault.clone();
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let diagnostic_context = diagnostic_context;
|
||||
let task = join_set.spawn(async move {
|
||||
let response_start = Instant::now();
|
||||
let result = if let Some(disk) = disk {
|
||||
#[cfg(test)]
|
||||
let _fanout_task_guard = rename_fanout_barrier::task_guard(&object);
|
||||
Self::record_read_version_call(&object, index);
|
||||
#[cfg(test)]
|
||||
Self::read_version_fanout_barrier(&object, index).await;
|
||||
if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(index)) {
|
||||
Self::record_metadata_slowtail_fault(&object, index);
|
||||
let read = async move {
|
||||
let response_start = Instant::now();
|
||||
let result = if let Some(disk) = disk {
|
||||
#[cfg(test)]
|
||||
rename_fanout_barrier::checkpoint(&object, index, rename_fanout_barrier::PHASE_METADATA_SLOWTAIL_FAULT)
|
||||
let _fanout_task_guard = rename_fanout_barrier::task_guard(&object);
|
||||
Self::record_read_version_call(&object, index);
|
||||
#[cfg(test)]
|
||||
Self::read_version_fanout_barrier(&object, index).await;
|
||||
if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(index)) {
|
||||
Self::record_metadata_slowtail_fault(&object, index);
|
||||
#[cfg(test)]
|
||||
rename_fanout_barrier::checkpoint(
|
||||
&object,
|
||||
index,
|
||||
rename_fanout_barrier::PHASE_METADATA_SLOWTAIL_FAULT,
|
||||
)
|
||||
.await;
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
read_version_via_coalescer(disk, &org_bucket, &bucket, &object, &version_id, &task_opts, allow_coalescing)
|
||||
.await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
read_version_via_coalescer(disk, &org_bucket, &bucket, &object, &version_id, &task_opts, allow_coalescing)
|
||||
.await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
};
|
||||
(index, result, response_start.elapsed())
|
||||
};
|
||||
(index, result, response_start.elapsed())
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
return crate::cluster::rpc::runtime_sources::with_read_version_diagnostic_context(
|
||||
diagnostic_context,
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionContextBoundary::EarlyStop,
|
||||
read,
|
||||
)
|
||||
.await;
|
||||
#[cfg(not(feature = "diagnostic-read-version-caller"))]
|
||||
read.await
|
||||
});
|
||||
task_ids[index] = Some(task.id());
|
||||
};
|
||||
|
||||
@@ -166,7 +166,7 @@ use sha2::Sha256;
|
||||
use std::hash::{BuildHasher, Hash, Hasher};
|
||||
use std::mem::{self};
|
||||
use std::pin::Pin;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use std::task::{Context, Poll};
|
||||
use std::time::{Instant, SystemTime, UNIX_EPOCH};
|
||||
@@ -410,6 +410,12 @@ impl Drop for ObjectLockDiagGuard {
|
||||
}
|
||||
|
||||
let hold = self.acquired_at.elapsed();
|
||||
if self.op == "put_object_commit" && self.mode == "write" && rustfs_io_metrics::put_stage_metrics_enabled() {
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD,
|
||||
hold.as_secs_f64() * 1000.0,
|
||||
);
|
||||
}
|
||||
record_object_lock_diag_hold_duration(self.op, self.mode, hold);
|
||||
let threshold = get_object_lock_diag_slow_hold_threshold();
|
||||
if hold >= threshold {
|
||||
@@ -4083,6 +4089,9 @@ pub struct SetDisks {
|
||||
/// writes skip the global registry mutex (backlog#1315). `Arc` so clones of
|
||||
/// a set share one generation marker.
|
||||
capacity_dirty_generation: Arc<AtomicU64>,
|
||||
/// Per-set proof that successful rename_data peers return the old-current
|
||||
/// source capability needed before PUT may skip its pre-rename lookup.
|
||||
tier_free_version_rename_data_source_capable: Arc<AtomicBool>,
|
||||
/// Orphan prefixes whose last purge scan met data that can never be
|
||||
/// purged by listing (residue without a committed marker, an in-flight
|
||||
/// write), keyed by `bucket/prefix` with the time of that scan. Empty
|
||||
@@ -4338,6 +4347,14 @@ struct GetObjectMetadataCacheKey {
|
||||
hash: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
struct GetObjectMetadataCacheLookupKey<'a> {
|
||||
bucket: &'a str,
|
||||
object: &'a str,
|
||||
generation: u64,
|
||||
hash: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
struct GetObjectMetadataCacheGeneration {
|
||||
index: usize,
|
||||
@@ -4426,6 +4443,33 @@ impl GetObjectMetadataCacheKey {
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> GetObjectMetadataCacheLookupKey<'a> {
|
||||
fn new(bucket: &'a str, object: &'a str, generation: GetObjectMetadataCacheGeneration) -> Self {
|
||||
Self {
|
||||
bucket,
|
||||
object,
|
||||
generation: generation.value,
|
||||
hash: generation.hash,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Hash for GetObjectMetadataCacheLookupKey<'_> {
|
||||
fn hash<H: Hasher>(&self, state: &mut H) {
|
||||
self.hash.hash(state);
|
||||
self.generation.hash(state);
|
||||
}
|
||||
}
|
||||
|
||||
impl moka::Equivalent<GetObjectMetadataCacheKey> for GetObjectMetadataCacheLookupKey<'_> {
|
||||
fn equivalent(&self, key: &GetObjectMetadataCacheKey) -> bool {
|
||||
self.bucket == key.bucket.as_ref()
|
||||
&& self.object == key.object.as_ref()
|
||||
&& self.generation == key.generation
|
||||
&& self.hash == key.hash
|
||||
}
|
||||
}
|
||||
|
||||
impl Hash for GetObjectMetadataCacheKey {
|
||||
fn hash<H: Hasher>(&self, state: &mut H) {
|
||||
self.hash.hash(state);
|
||||
@@ -4971,6 +5015,7 @@ impl SetDisks {
|
||||
ctx,
|
||||
capacity_scope_cache: Arc::new(std::sync::RwLock::new(CapacityScopeCache::default())),
|
||||
capacity_dirty_generation: Arc::new(AtomicU64::new(u64::MAX)),
|
||||
tier_free_version_rename_data_source_capable: Arc::new(AtomicBool::new(false)),
|
||||
orphan_purge_backoff: Arc::new(std::sync::Mutex::new(HashMap::new())),
|
||||
#[cfg(test)]
|
||||
storage_class_config_override: Arc::new(std::sync::RwLock::new(None)),
|
||||
@@ -4985,6 +5030,15 @@ impl SetDisks {
|
||||
&self.ctx
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn tier_free_version_rename_data_source_capable(&self) -> bool {
|
||||
self.tier_free_version_rename_data_source_capable.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn observe_tier_free_version_rename_data_source_capability(&self, capable: bool) {
|
||||
self.tier_free_version_rename_data_source_capable
|
||||
.store(capable, Ordering::Release);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn set_instance_ctx_for_test(&mut self, ctx: Arc<InstanceContext>) {
|
||||
self.local_lock_manager = ctx.lock_manager();
|
||||
|
||||
@@ -746,8 +746,15 @@ impl SetDisks {
|
||||
false,
|
||||
true,
|
||||
self.default_parity_count,
|
||||
);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let metadata_read = crate::cluster::rpc::runtime_sources::with_read_version_request_caller_if_absent(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionRequestCaller::Heal,
|
||||
metadata_read,
|
||||
)
|
||||
.await?;
|
||||
#[cfg(not(feature = "diagnostic-read-version-caller"))]
|
||||
let metadata_read = metadata_read.await?;
|
||||
let (parts_metadata, errs, _) = metadata_read.into_legacy();
|
||||
let (read_quorum, _) = Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count)?;
|
||||
let read_quorum = usize::try_from(read_quorum).map_err(|_| DiskError::ErasureReadQuorum)?;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -15,12 +15,13 @@
|
||||
use super::{
|
||||
Arc, Bytes, DiskError, DiskOption, DiskStore, Endpoint, ErasureCache, Error, FileInfo, GetCodecStreamingFallbackReason,
|
||||
GetObjectFileInfo, GetObjectMetadataCacheEntry, GetObjectMetadataCacheGeneration, GetObjectMetadataCacheKey,
|
||||
GetObjectReadPolicy, HashAlgorithm, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_SET_DISK, OBJECT_OP_IGNORED_ERRS, ObjectInfo,
|
||||
ObjectOptions, RUSTFS_META_BUCKET, ReadOptions, Result, SetDisks, StorageError, adaptive_duplex_buffer_size,
|
||||
build_get_codec_streaming_decode_engine, build_inline_bitrot_readers_from_refs, collect_inline_data_shard_fileinfos_by_index,
|
||||
debug, error, get_codec_streaming_metrics_path, get_codec_streaming_multipart_max_parts, get_object_read_policy,
|
||||
is_codec_streaming_multipart_enabled, is_multipart_reader_setup_prefetch_enabled, object_fits_single_block,
|
||||
reduce_read_quorum_errs, to_object_err, try_read_inline_data_shards_direct, warn,
|
||||
GetObjectMetadataCacheLookupKey, GetObjectReadPolicy, HashAlgorithm, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_SET_DISK,
|
||||
OBJECT_OP_IGNORED_ERRS, ObjectInfo, ObjectOptions, RUSTFS_META_BUCKET, ReadOptions, Result, SetDisks, StorageError,
|
||||
adaptive_duplex_buffer_size, build_get_codec_streaming_decode_engine, build_inline_bitrot_readers_from_refs,
|
||||
collect_inline_data_shard_fileinfos_by_index, debug, error, get_codec_streaming_metrics_path,
|
||||
get_codec_streaming_multipart_max_parts, get_object_read_policy, is_codec_streaming_multipart_enabled,
|
||||
is_multipart_reader_setup_prefetch_enabled, object_fits_single_block, reduce_read_quorum_errs, to_object_err,
|
||||
try_read_inline_data_shards_direct, warn,
|
||||
};
|
||||
use crate::diagnostics::get::{
|
||||
GET_DIRECT_MEMORY_SUBPATH_DISK_DATA_BLOCKS, GET_DIRECT_MEMORY_SUBPATH_INLINE_BUFFERED, GET_METADATA_CACHE_DECISION_HIT,
|
||||
@@ -206,6 +207,16 @@ use tokio::time::timeout;
|
||||
#[cfg(test)]
|
||||
use uuid::Uuid;
|
||||
|
||||
fn metadata_quorum_outcome(error: &StorageError) -> &'static str {
|
||||
if error.is_not_found() {
|
||||
"not_found"
|
||||
} else if error.is_quorum_error() {
|
||||
"not_reached"
|
||||
} else {
|
||||
"unavailable"
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct GetObjectDownstreamWriter<W> {
|
||||
inner: W,
|
||||
}
|
||||
@@ -392,7 +403,7 @@ impl SetDisks {
|
||||
let Some(generation) = self.get_object_metadata_cache_generation(bucket, object) else {
|
||||
return MetadataCacheLookup::Miss;
|
||||
};
|
||||
let key = GetObjectMetadataCacheKey::new(bucket, object, generation);
|
||||
let key = GetObjectMetadataCacheLookupKey::new(bucket, object, generation);
|
||||
// moka handles TTL expiry automatically; no is_fresh() check needed
|
||||
let Some(entry) = self.get_object_metadata_cache.get(&key).await else {
|
||||
return MetadataCacheLookup::Miss;
|
||||
@@ -481,6 +492,10 @@ impl SetDisks {
|
||||
let object: Arc<str> = Arc::from(object);
|
||||
let version_id: Arc<str> = Arc::from(version_id);
|
||||
let opts = *opts;
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let api_operation = rustfs_io_metrics::get_stage_metrics_enabled()
|
||||
.then(rustfs_io_metrics::s3_http_metrics::current_s3_http_operation_context)
|
||||
.flatten();
|
||||
|
||||
let processor = runtime_sources::batch_processors().read_processor();
|
||||
let tasks: Vec<_> = disks
|
||||
@@ -492,8 +507,27 @@ impl SetDisks {
|
||||
let object = object.clone();
|
||||
let version_id = version_id.clone();
|
||||
let task_opts = opts;
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let api_operation = api_operation;
|
||||
|
||||
async move { disk.read_version(&bucket, &bucket, &object, &version_id, &task_opts).await }
|
||||
async move {
|
||||
let read_version =
|
||||
async move { disk.read_version(&bucket, &bucket, &object, &version_id, &task_opts).await };
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let read_version = crate::cluster::rpc::runtime_sources::with_read_version_request_caller_if_absent(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionRequestCaller::SetDiskRead,
|
||||
read_version,
|
||||
);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
if let Some(api_operation) = api_operation {
|
||||
rustfs_io_metrics::s3_http_metrics::with_s3_http_operation_context(Some(api_operation), read_version)
|
||||
.await
|
||||
} else {
|
||||
read_version.await
|
||||
}
|
||||
#[cfg(not(feature = "diagnostic-read-version-caller"))]
|
||||
read_version.await
|
||||
}
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
@@ -569,6 +603,11 @@ impl SetDisks {
|
||||
allow_read_version_coalescing: bool,
|
||||
) -> Result<GetObjectFileInfo> {
|
||||
let vid = opts.version_id.clone().unwrap_or_default();
|
||||
let metadata_metrics_path = if crate::bucket::utils::is_meta_bucketname(bucket) {
|
||||
GET_OBJECT_PATH_INTERNAL_META
|
||||
} else {
|
||||
GET_OBJECT_PATH_LEGACY_DUPLEX
|
||||
};
|
||||
let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
|
||||
|
||||
let metadata_cache_lookup_start = get_stage_timer_if_enabled(stage_metrics_enabled);
|
||||
@@ -583,6 +622,7 @@ impl SetDisks {
|
||||
} else if vid.is_empty() {
|
||||
match self.lookup_cached_get_object_fileinfo(bucket, object).await {
|
||||
MetadataCacheLookup::Hit(cached) => {
|
||||
rustfs_io_metrics::record_get_object_metadata_quorum_result(metadata_metrics_path, "reached");
|
||||
rustfs_io_metrics::record_get_object_metadata_cache_decision(
|
||||
GET_OBJECT_PATH_SET_DISK,
|
||||
GET_METADATA_CACHE_DECISION_HIT,
|
||||
@@ -631,41 +671,59 @@ impl SetDisks {
|
||||
// read_metadata_observed (see read_all_fileinfo_early_stop in
|
||||
// core/metadata_read.rs); unsafe requests and callers that opt out
|
||||
// (allow_early_stop=false) fall back to full-wait.
|
||||
let metadata_read = if allow_read_version_coalescing {
|
||||
Self::read_metadata_for_get_object(
|
||||
&disks,
|
||||
"",
|
||||
bucket,
|
||||
object,
|
||||
vid.as_str(),
|
||||
read_data,
|
||||
opts.incl_free_versions,
|
||||
allow_early_stop,
|
||||
self.default_parity_count,
|
||||
)
|
||||
.await?
|
||||
} else {
|
||||
Self::read_metadata_observed(
|
||||
&disks,
|
||||
"",
|
||||
bucket,
|
||||
object,
|
||||
vid.as_str(),
|
||||
read_data,
|
||||
false,
|
||||
opts.incl_free_versions,
|
||||
allow_early_stop,
|
||||
self.default_parity_count,
|
||||
)
|
||||
.await?
|
||||
let metadata_read = async {
|
||||
if allow_read_version_coalescing {
|
||||
Self::read_metadata_for_get_object(
|
||||
&disks,
|
||||
"",
|
||||
bucket,
|
||||
object,
|
||||
vid.as_str(),
|
||||
read_data,
|
||||
opts.incl_free_versions,
|
||||
allow_early_stop,
|
||||
self.default_parity_count,
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
Self::read_metadata_observed(
|
||||
&disks,
|
||||
"",
|
||||
bucket,
|
||||
object,
|
||||
vid.as_str(),
|
||||
read_data,
|
||||
false,
|
||||
opts.incl_free_versions,
|
||||
allow_early_stop,
|
||||
self.default_parity_count,
|
||||
)
|
||||
.await
|
||||
}
|
||||
};
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let metadata_read = crate::cluster::rpc::runtime_sources::with_read_version_request_caller_if_absent(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionRequestCaller::Object,
|
||||
metadata_read,
|
||||
);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let metadata_read_result = crate::cluster::rpc::runtime_sources::with_read_version_entry(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::ObjectRead,
|
||||
metadata_read,
|
||||
)
|
||||
.await;
|
||||
#[cfg(not(feature = "diagnostic-read-version-caller"))]
|
||||
let metadata_read_result = metadata_read.await;
|
||||
let metadata_read = match metadata_read_result {
|
||||
Ok(metadata_read) => metadata_read,
|
||||
Err(err) => {
|
||||
let err = StorageError::from(err);
|
||||
rustfs_io_metrics::record_get_object_metadata_quorum_result(metadata_metrics_path, metadata_quorum_outcome(&err));
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
let metadata_fanout_complete = metadata_read.is_complete();
|
||||
let (mut parts_metadata, errs, metadata_fanout_diagnostics) = metadata_read.into_legacy();
|
||||
let metadata_metrics_path = if crate::bucket::utils::is_meta_bucketname(bucket) {
|
||||
GET_OBJECT_PATH_INTERNAL_META
|
||||
} else {
|
||||
GET_OBJECT_PATH_LEGACY_DUPLEX
|
||||
};
|
||||
metadata_fanout_diagnostics.record(metadata_metrics_path);
|
||||
// warn!("get_object_fileinfo parts_metadata {:?}", &parts_metadata);
|
||||
// warn!("get_object_fileinfo {}/{} errs {:?}", bucket, object, &errs);
|
||||
@@ -678,6 +736,7 @@ impl SetDisks {
|
||||
{
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
rustfs_io_metrics::record_get_object_metadata_quorum_result(metadata_metrics_path, metadata_quorum_outcome(&e));
|
||||
// error!("Self::object_quorum_from_meta: {:?}, bucket: {}, object: {}", &e, bucket, object);
|
||||
record_get_stage_duration_if_enabled(
|
||||
GET_OBJECT_PATH_SET_DISK,
|
||||
@@ -692,6 +751,7 @@ impl SetDisks {
|
||||
let write_quorum = usize::try_from(write_quorum)
|
||||
.map_err(|_| to_object_err(DiskError::ErasureWriteQuorum.into(), vec![bucket, object]))?;
|
||||
if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
|
||||
rustfs_io_metrics::record_get_object_metadata_quorum_result(metadata_metrics_path, "not_reached");
|
||||
error!("reduce_read_quorum_errs: {:?}, bucket: {}, object: {}", &err, bucket, object);
|
||||
record_get_stage_duration_if_enabled(
|
||||
GET_OBJECT_PATH_SET_DISK,
|
||||
@@ -702,7 +762,17 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
let (op_online_disks, mut fi, fileinfo_selection_quorum) =
|
||||
Self::select_valid_fileinfo(&disks, &parts_metadata, &errs, vid.as_str(), read_quorum, write_quorum)?;
|
||||
match Self::select_valid_fileinfo(&disks, &parts_metadata, &errs, vid.as_str(), read_quorum, write_quorum) {
|
||||
Ok(selection) => selection,
|
||||
Err(err) => {
|
||||
let err = StorageError::from(err);
|
||||
rustfs_io_metrics::record_get_object_metadata_quorum_result(
|
||||
metadata_metrics_path,
|
||||
metadata_quorum_outcome(&err),
|
||||
);
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
let include_part_checksums =
|
||||
opts.include_part_checksums || opts.part_number.is_some() || opts.data_movement || opts.raw_data_movement_read;
|
||||
if include_part_checksums {
|
||||
@@ -725,7 +795,15 @@ impl SetDisks {
|
||||
"metadata_read_error",
|
||||
)
|
||||
.await;
|
||||
} else if use_metadata_cache && metadata_fanout_complete {
|
||||
} else if use_metadata_cache
|
||||
&& (metadata_fanout_complete
|
||||
|| (read_data
|
||||
&& allow_early_stop
|
||||
&& metadata_fanout_diagnostics.valid_responses() >= read_quorum
|
||||
&& errs.iter().all(Option::is_none)
|
||||
&& fileinfo_selection_quorum >= read_quorum
|
||||
&& op_online_disks.iter().filter(|disk| disk.is_some()).count() >= read_quorum))
|
||||
{
|
||||
#[cfg(test)]
|
||||
metadata_cache_tests::wait_before_metadata_cache_publish(bucket, object).await;
|
||||
self.cache_get_object_fileinfo(
|
||||
@@ -2581,6 +2659,7 @@ fn is_get_object_metadata_cache_request_eligible(bucket: &str, opts: &ObjectOpti
|
||||
#[cfg(test)]
|
||||
mod metadata_cache_tests {
|
||||
use super::*;
|
||||
use crate::set_disk::core::io_primitives::disk_call_counters;
|
||||
use rustfs_heal_contracts::heal_channel::HealAdmissionDropReason;
|
||||
use serial_test::serial;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
@@ -3897,7 +3976,7 @@ mod metadata_cache_tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(metadata_cache_publish_barrier)]
|
||||
#[serial(metadata_cache_early_stop)]
|
||||
async fn metadata_cache_production_fanout_cannot_publish_after_invalidation() {
|
||||
// Isolated context: an ambient DistErasure window (another test's
|
||||
// SetupTypeGuard) would bypass metadata-cache publication entirely
|
||||
@@ -3951,6 +4030,143 @@ mod metadata_cache_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial(metadata_cache_early_stop)]
|
||||
async fn metadata_cache_publishes_quorum_selected_early_stop_result() {
|
||||
let isolated_ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
isolated_ctx
|
||||
.update_erasure_type(crate::layout::endpoints::SetupType::Erasure)
|
||||
.await;
|
||||
let (_dirs, set) = crate::ecstore_validation_blackbox::make_local_set_disks_with_ctx(4, 2, isolated_ctx).await;
|
||||
let bucket = "metadata-cache-early-stop";
|
||||
let object = "objects/repeated-key";
|
||||
let disks = set.disks.read().await.clone();
|
||||
for disk in disks.iter().flatten() {
|
||||
disk.make_volume(bucket)
|
||||
.await
|
||||
.expect("test bucket should be created on each disk");
|
||||
}
|
||||
let expected_body = vec![0x5a; 1024 * 1024];
|
||||
set.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(expected_body.clone()),
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("test object should be written before enabling the slow-tail fault");
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("false")),
|
||||
("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS", Some("500")),
|
||||
("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS", Some("3")),
|
||||
("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET", Some(bucket)),
|
||||
("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX", Some("objects/")),
|
||||
],
|
||||
async {
|
||||
let calls = disk_call_counters::observe(object);
|
||||
let first = tokio::time::timeout(
|
||||
Duration::from_millis(400),
|
||||
set.get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, true),
|
||||
)
|
||||
.await
|
||||
.expect("first metadata read should stop before the delayed peer")
|
||||
.expect("read quorum should provide valid object metadata");
|
||||
assert!(first.owned.is_some(), "the first lookup should resolve through disk metadata");
|
||||
assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), 4);
|
||||
assert_eq!(calls.total(disk_call_counters::KIND_METADATA_SLOWTAIL_FAULT), 1);
|
||||
|
||||
let second = set
|
||||
.get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, true)
|
||||
.await
|
||||
.expect("second lookup should use the quorum-published metadata cache entry");
|
||||
assert!(second.shared_entry().is_some(), "the repeated GET should hit the published cache entry");
|
||||
assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), 4);
|
||||
|
||||
let cached = second.shared_entry().expect("second lookup should share cached metadata");
|
||||
assert_eq!(cached.fi.name, object);
|
||||
assert!(cached.online_disks.iter().filter(|disk| disk.is_some()).count() >= cached.read_quorum);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial(metadata_cache_early_stop)]
|
||||
async fn metadata_cache_early_stop_publication_is_fenced_by_invalidation() {
|
||||
let isolated_ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
isolated_ctx
|
||||
.update_erasure_type(crate::layout::endpoints::SetupType::Erasure)
|
||||
.await;
|
||||
let (_dirs, set) = crate::ecstore_validation_blackbox::make_local_set_disks_with_ctx(4, 2, isolated_ctx).await;
|
||||
let bucket = "metadata-cache-early-stop-fence";
|
||||
let object = "objects/repeated-key";
|
||||
let disks = set.disks.read().await.clone();
|
||||
for disk in disks.iter().flatten() {
|
||||
disk.make_volume(bucket)
|
||||
.await
|
||||
.expect("test bucket should be created on each disk");
|
||||
}
|
||||
set.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![0x5a; 1024 * 1024]),
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("test object should be written before enabling the slow-tail fault");
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("false")),
|
||||
("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS", Some("500")),
|
||||
("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS", Some("3")),
|
||||
("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET", Some(bucket)),
|
||||
("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX", Some("objects/")),
|
||||
],
|
||||
async {
|
||||
let barrier = MetadataCachePublishBarrier::install(bucket, object);
|
||||
let stale_generation = set
|
||||
.get_object_metadata_cache_generation(bucket, object)
|
||||
.expect("metadata cache generation should be active");
|
||||
let reader_set = Arc::clone(&set);
|
||||
let read = tokio::spawn(async move {
|
||||
reader_set
|
||||
.get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, true)
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
set.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||
barrier.release();
|
||||
|
||||
let snapshot = read
|
||||
.await
|
||||
.expect("early-stop metadata read should not panic")
|
||||
.expect("read quorum should resolve the selected FileInfo");
|
||||
assert!(snapshot.owned.is_some());
|
||||
assert!(
|
||||
set.get_object_metadata_cache
|
||||
.get(&GetObjectMetadataCacheKey::new(bucket, object, stale_generation))
|
||||
.await
|
||||
.is_none(),
|
||||
"early-stop publication must not retain an entry under the retired generation"
|
||||
);
|
||||
assert!(
|
||||
set.cached_get_object_fileinfo(bucket, object).await.is_none(),
|
||||
"invalidation must reject the quorum-selected early-stop publication"
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_cache_lookup_rechecks_generation_after_get() {
|
||||
let set = new_metadata_cache_test_set().await;
|
||||
@@ -4082,12 +4298,15 @@ mod metadata_cache_tests {
|
||||
.get_object_metadata_cache_generation("bucket-a", "object-a")
|
||||
.expect("metadata cache generation should be active");
|
||||
let first_key = GetObjectMetadataCacheKey::new("bucket-a", "object-a", generation);
|
||||
let first_lookup = GetObjectMetadataCacheLookupKey::new("bucket-a", "object-a", generation);
|
||||
let second_key = GetObjectMetadataCacheKey {
|
||||
bucket: Arc::from("bucket-b"),
|
||||
object: Arc::from("object-b"),
|
||||
generation: generation.value,
|
||||
hash: generation.hash,
|
||||
};
|
||||
let second_lookup = GetObjectMetadataCacheLookupKey::new("bucket-b", "object-b", generation);
|
||||
let unrelated_lookup = GetObjectMetadataCacheLookupKey::new("bucket-c", "object-c", generation);
|
||||
let first_fi = valid_test_fileinfo("object-a");
|
||||
let second_fi = valid_test_fileinfo("object-b");
|
||||
let entry = |fi: FileInfo| {
|
||||
@@ -4107,7 +4326,7 @@ mod metadata_cache_tests {
|
||||
|
||||
assert_eq!(
|
||||
set.get_object_metadata_cache
|
||||
.get(&first_key)
|
||||
.get(&first_lookup)
|
||||
.await
|
||||
.expect("first colliding entry should remain addressable")
|
||||
.fi
|
||||
@@ -4116,13 +4335,14 @@ mod metadata_cache_tests {
|
||||
);
|
||||
assert_eq!(
|
||||
set.get_object_metadata_cache
|
||||
.get(&second_key)
|
||||
.get(&second_lookup)
|
||||
.await
|
||||
.expect("second colliding entry should remain addressable")
|
||||
.fi
|
||||
.name,
|
||||
"object-b"
|
||||
);
|
||||
assert!(set.get_object_metadata_cache.get(&unrelated_lookup).await.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -4254,6 +4474,16 @@ mod tests {
|
||||
const CODEC_STREAMING_TEST_OBJECT: &str = "object";
|
||||
static CAPTURED_READ_REPAIR_CALLS: AtomicUsize = AtomicUsize::new(0);
|
||||
|
||||
#[test]
|
||||
fn metadata_quorum_outcome_uses_bounded_error_categories() {
|
||||
assert_eq!(metadata_quorum_outcome(&StorageError::ErasureReadQuorum), "not_reached");
|
||||
assert_eq!(metadata_quorum_outcome(&StorageError::FileNotFound), "not_found");
|
||||
assert_eq!(
|
||||
metadata_quorum_outcome(&StorageError::Io(std::io::Error::other("disk unavailable"))),
|
||||
"unavailable"
|
||||
);
|
||||
}
|
||||
|
||||
fn capture_read_repair_submitter(
|
||||
_request: rustfs_heal_contracts::heal_channel::HealChannelRequest,
|
||||
) -> ReadRepairAdmissionFuture {
|
||||
|
||||
@@ -56,7 +56,7 @@ use rustfs_filemeta::{
|
||||
};
|
||||
use rustfs_io_metrics::{
|
||||
LIST_OBJECTS_GATHER_OUTCOME_INPUT_CLOSED, LIST_OBJECTS_GATHER_OUTCOME_LIMIT_REACHED, LIST_OBJECTS_SOURCE_WALKER,
|
||||
ListObjectsGatherObservation, ListObjectsIndexPageObservation,
|
||||
ListObjectsGatherCaller, ListObjectsGatherObservation, ListObjectsIndexPageObservation,
|
||||
};
|
||||
use rustfs_utils::path::{self, SLASH_SEPARATOR, base_dir_from_prefix};
|
||||
use std::cmp::Reverse;
|
||||
@@ -109,6 +109,7 @@ fn recursive_walk_list_path_options(bucket: String, prefix: &str, opts: &WalkOpt
|
||||
incl_deleted: true,
|
||||
recursive: true,
|
||||
versioned: true,
|
||||
caller: ListObjectsGatherCaller::S3ListVersions,
|
||||
walkdir_timeout: opts.walkdir_timeout,
|
||||
walkdir_stall_timeout: opts.walkdir_stall_timeout,
|
||||
..Default::default()
|
||||
@@ -274,6 +275,7 @@ fn log_list_path_finished(
|
||||
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct ListPathOptions {
|
||||
pub caller: ListObjectsGatherCaller,
|
||||
pub id: Option<String>,
|
||||
|
||||
// Bucket of the listing.
|
||||
@@ -4116,6 +4118,7 @@ impl ECStore {
|
||||
let effective_max_keys = if max_keys <= 0 { 0 } else { max_keys_plus_one(max_keys, true) };
|
||||
let mut opts = ListPathOptions {
|
||||
bucket: bucket.to_owned(),
|
||||
caller: ListObjectsGatherCaller::S3List,
|
||||
prefix: prefix.to_owned(),
|
||||
separator: delimiter.clone(),
|
||||
// Always request max_keys + 1 to detect if there are more results
|
||||
@@ -4289,6 +4292,11 @@ impl ECStore {
|
||||
let effective_max_keys = list_versions_scan_limit(max_keys, has_version_marker);
|
||||
let mut opts = ListPathOptions {
|
||||
bucket: bucket.to_owned(),
|
||||
caller: if include_version_purge {
|
||||
ListObjectsGatherCaller::Lifecycle
|
||||
} else {
|
||||
ListObjectsGatherCaller::S3ListVersions
|
||||
},
|
||||
prefix: prefix.to_owned(),
|
||||
separator: delimiter.clone(),
|
||||
limit: effective_max_keys,
|
||||
@@ -4993,11 +5001,45 @@ where
|
||||
let _cancel_guard = cancel.clone().drop_guard();
|
||||
let (result_tx, mut result_rx) = mpsc::channel(1);
|
||||
let mut workers = JoinSet::new();
|
||||
workers.spawn(async move { producer.await.map(|()| None) }.instrument(tracing::Span::current()));
|
||||
let gather_cancel = cancel.clone();
|
||||
workers.spawn(
|
||||
async move { gather_results(gather_cancel, opts, recv, result_tx).await.map(Some) }.instrument(tracing::Span::current()),
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let mut diagnostic_context = crate::cluster::rpc::runtime_sources::ReadVersionDiagnosticContext::capture_entry(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::ListObjects,
|
||||
);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let request_caller = diagnostic_context.caller.or_else(|| {
|
||||
use rustfs_io_metrics::ListObjectsGatherCaller;
|
||||
use rustfs_io_metrics::internode_metrics::ReadVersionRequestCaller;
|
||||
|
||||
match opts.caller {
|
||||
ListObjectsGatherCaller::S3List | ListObjectsGatherCaller::S3ListVersions => {
|
||||
Some(ReadVersionRequestCaller::ListObjects)
|
||||
}
|
||||
ListObjectsGatherCaller::Lifecycle => Some(ReadVersionRequestCaller::Lifecycle),
|
||||
ListObjectsGatherCaller::Other => Some(ReadVersionRequestCaller::Other),
|
||||
ListObjectsGatherCaller::Unattributed => None,
|
||||
}
|
||||
});
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
{
|
||||
diagnostic_context.caller = request_caller;
|
||||
}
|
||||
let producer = async move { producer.await.map(|()| None) };
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let producer = crate::cluster::rpc::runtime_sources::with_read_version_diagnostic_context(
|
||||
diagnostic_context,
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionContextBoundary::ListProducer,
|
||||
producer,
|
||||
);
|
||||
workers.spawn(producer.instrument(tracing::Span::current()));
|
||||
let gather_cancel = cancel.clone();
|
||||
let gather = async move { gather_results(gather_cancel, opts, recv, result_tx).await.map(Some) };
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let gather = crate::cluster::rpc::runtime_sources::with_read_version_diagnostic_context(
|
||||
diagnostic_context,
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionContextBoundary::ListGather,
|
||||
gather,
|
||||
);
|
||||
workers.spawn(gather.instrument(tracing::Span::current()));
|
||||
|
||||
let mut failure = None;
|
||||
let mut stopping = false;
|
||||
@@ -5133,6 +5175,7 @@ async fn gather_results(
|
||||
let duration_ms = started.elapsed().as_secs_f64() * 1000.0;
|
||||
rustfs_io_metrics::record_list_objects_gather(ListObjectsGatherObservation {
|
||||
source: LIST_OBJECTS_SOURCE_WALKER,
|
||||
caller: opts.caller,
|
||||
outcome: LIST_OBJECTS_GATHER_OUTCOME_LIMIT_REACHED,
|
||||
limit: opts.limit,
|
||||
scanned_entries,
|
||||
@@ -5195,6 +5238,7 @@ async fn gather_results(
|
||||
let duration_ms = started.elapsed().as_secs_f64() * 1000.0;
|
||||
rustfs_io_metrics::record_list_objects_gather(ListObjectsGatherObservation {
|
||||
source: LIST_OBJECTS_SOURCE_WALKER,
|
||||
caller: opts.caller,
|
||||
outcome: LIST_OBJECTS_GATHER_OUTCOME_INPUT_CLOSED,
|
||||
limit: opts.limit,
|
||||
scanned_entries,
|
||||
@@ -12294,6 +12338,44 @@ mod test {
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[tokio::test]
|
||||
async fn list_path_producer_worker_preserves_s3_and_caller_context() {
|
||||
let mut request = rustfs_io_metrics::s3_http_metrics::S3HttpRequestGuard::new("GET");
|
||||
let api_operation = request
|
||||
.in_scope(|| {
|
||||
rustfs_io_metrics::record_s3_op(rustfs_s3_ops::S3Operation::ListObjectsV2);
|
||||
rustfs_io_metrics::s3_http_metrics::current_s3_http_operation_context()
|
||||
})
|
||||
.expect("dispatched list operation should be captured");
|
||||
request.response(200);
|
||||
|
||||
let (sender, recv) = mpsc::channel(1);
|
||||
let opts = ListPathOptions {
|
||||
caller: rustfs_io_metrics::ListObjectsGatherCaller::S3List,
|
||||
..Default::default()
|
||||
};
|
||||
let result = rustfs_io_metrics::s3_http_metrics::with_s3_http_operation_context(Some(api_operation), async move {
|
||||
collect_list_path_results(CancellationToken::new(), opts, recv, async move {
|
||||
assert_eq!(
|
||||
rustfs_io_metrics::s3_http_metrics::current_s3_http_operation_name(),
|
||||
Some("s3:ListObjectsV2")
|
||||
);
|
||||
assert_eq!(
|
||||
crate::cluster::rpc::runtime_sources::current_read_version_request_caller(),
|
||||
Some(rustfs_io_metrics::internode_metrics::ReadVersionRequestCaller::ListObjects)
|
||||
);
|
||||
drop(sender);
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
})
|
||||
.await;
|
||||
|
||||
assert!(result.err.is_some(), "the empty page channel remains an error");
|
||||
assert_eq!(rustfs_io_metrics::s3_http_metrics::current_s3_http_operation_name(), None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_workers_preserve_error_after_input_closes() {
|
||||
let (sender, recv) = mpsc::channel(1);
|
||||
|
||||
@@ -1363,7 +1363,13 @@ impl crate::storage_api_contracts::object::ObjectIO for ECStore {
|
||||
h: HeaderMap,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<GetObjectReader> {
|
||||
self.handle_get_object_reader(bucket, object, range, h, opts).await
|
||||
let read = self.handle_get_object_reader(bucket, object, range, h, opts);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let read = crate::cluster::rpc::runtime_sources::with_read_version_entry(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::StoreObjectReader,
|
||||
read,
|
||||
);
|
||||
read.await
|
||||
}
|
||||
#[instrument(level = "debug", skip(self, data))]
|
||||
async fn put_object(&self, bucket: &str, object: &str, data: &mut PutObjReader, opts: &ObjectOptions) -> Result<ObjectInfo> {
|
||||
|
||||
@@ -18,6 +18,11 @@ use crate::disk::error::{DiskError, Result as DiskResult};
|
||||
use crate::disk::{DeleteOptions, Disk, RenameDataGuards, RenameDataResp};
|
||||
use crate::runtime::instance::{InstanceContext, NamespaceCommitGuard};
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use rustfs_io_metrics::{
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_ADMIT, PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOCAL_RENAME,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOOKUP, PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_OVERHEAD,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_TOTAL,
|
||||
};
|
||||
use tracing::{debug, error};
|
||||
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
@@ -331,15 +336,38 @@ async fn rename_local_data_with_ctx(
|
||||
destination: (&str, &str),
|
||||
mut guards: RenameDataGuards,
|
||||
) -> DiskResult<RenameDataResp> {
|
||||
let handler_started = rustfs_io_metrics::put_stage_timer();
|
||||
let lookup_started = rustfs_io_metrics::put_stage_timer();
|
||||
let (disk, disk_id) = local_disk_candidate(ctx, disk_ref).await?;
|
||||
let lookup_ms = lookup_started.map(|started| started.elapsed().as_secs_f64() * 1000.0);
|
||||
if let Some(duration_ms) = lookup_ms {
|
||||
rustfs_io_metrics::record_put_object_stage_duration(PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOOKUP, duration_ms);
|
||||
}
|
||||
let mutates_namespace = !is_meta_bucketname(source.0) || !is_meta_bucketname(destination.0);
|
||||
let admit_started = rustfs_io_metrics::put_stage_timer();
|
||||
let owner = admit_local_disk(ctx, &disk, disk_id, mutates_namespace).await?;
|
||||
let admit_ms = admit_started.map(|started| started.elapsed().as_secs_f64() * 1000.0);
|
||||
if let Some(duration_ms) = admit_ms {
|
||||
rustfs_io_metrics::record_put_object_stage_duration(PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_ADMIT, duration_ms);
|
||||
}
|
||||
guards.namespace_owner = owner.as_ref().map(|owner| owner.clone() as Arc<dyn Send + Sync>);
|
||||
let local_rename_started = rustfs_io_metrics::put_stage_timer();
|
||||
let result = disk
|
||||
.rename_data_borrowed_with_fence_observed(source.0, source.1, fi, destination.0, destination.1, guards)
|
||||
.await
|
||||
.result;
|
||||
let local_rename_ms = local_rename_started.map(|started| started.elapsed().as_secs_f64() * 1000.0);
|
||||
if let Some(duration_ms) = local_rename_ms {
|
||||
rustfs_io_metrics::record_put_object_stage_duration(PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOCAL_RENAME, duration_ms);
|
||||
}
|
||||
drop(owner);
|
||||
if let Some(handler_started) = handler_started {
|
||||
let handler_ms = handler_started.elapsed().as_secs_f64() * 1000.0;
|
||||
rustfs_io_metrics::record_put_object_stage_duration(PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_TOTAL, handler_ms);
|
||||
let child_ms = lookup_ms.unwrap_or(0.0) + admit_ms.unwrap_or(0.0) + local_rename_ms.unwrap_or(0.0);
|
||||
let overhead_ms = (handler_ms - child_ms).max(0.0);
|
||||
rustfs_io_metrics::record_put_object_stage_duration(PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_OVERHEAD, overhead_ms);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#![cfg(target_os = "linux")]
|
||||
|
||||
use bytes::Bytes;
|
||||
mod storage_api;
|
||||
|
||||
use std::time::Duration;
|
||||
use storage_api::cleanup_isolation::{DeleteOptions, DiskAPI, DiskError, DiskOption, Endpoint, new_disk};
|
||||
|
||||
#[tokio::test]
|
||||
async fn cleanup_disk_and_periodic_gc_use_affinity_without_changing_foreground() {
|
||||
const CHILD: &str = "RUSTFS_CLEANUP_LINUX_TEST_CHILD";
|
||||
let foreground = rustix::thread::sched_getaffinity(None).unwrap();
|
||||
let cpu = (0..rustix::thread::CpuSet::MAX_CPU)
|
||||
.find(|cpu| foreground.is_set(*cpu))
|
||||
.expect("the test process must have an allowed CPU");
|
||||
if std::env::var_os(CHILD).is_none() {
|
||||
for mode in ["custom", "default", "none", "restricted"] {
|
||||
if mode == "default" && (!foreground.is_set(1) || !foreground.is_set(2)) {
|
||||
continue;
|
||||
}
|
||||
let output = tokio::task::spawn_blocking(move || {
|
||||
let mut command = std::process::Command::new(std::env::current_exe().unwrap());
|
||||
command
|
||||
.args([
|
||||
"--exact",
|
||||
"cleanup_disk_and_periodic_gc_use_affinity_without_changing_foreground",
|
||||
"--nocapture",
|
||||
])
|
||||
.env(CHILD, mode)
|
||||
.env("RUSTFS_CLEANUP_ISOLATE_ENABLE", "true")
|
||||
.env("RUSTFS_PUT_RENAME_TAIL_CLEANUP_MAX_PENDING", "4")
|
||||
.env("RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_WORKERS", "1")
|
||||
.env("RUSTFS_CLEANUP_DISK_MAX_PENDING", "4")
|
||||
.env("RUSTFS_CLEANUP_DISK_WORKERS", "1")
|
||||
.env("RUSTFS_CLEANUP_GC_WORKERS", "1")
|
||||
.env("RUSTFS_CLEANUP_BLOCKING_THREADS", "1")
|
||||
.env("RUSTFS_CLEANUP_ASYNC_THREADS", "1")
|
||||
.env_remove("RUSTFS_CLEANUP_CPUS")
|
||||
.env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP")
|
||||
.env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER")
|
||||
.env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_ZERO_TARGET_TMP_DELETE_SKIP");
|
||||
match mode {
|
||||
"custom" => {
|
||||
command.env("RUSTFS_CLEANUP_CPUS", cpu.to_string());
|
||||
}
|
||||
"none" => {
|
||||
command.env("RUSTFS_CLEANUP_CPUS", "none");
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
command.output().unwrap()
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"{}\n{}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
assert!(String::from_utf8_lossy(&output.stdout).contains("1 passed"));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
let mode = std::env::var(CHILD).unwrap();
|
||||
if mode == "restricted" {
|
||||
let mut restricted = rustix::thread::CpuSet::new();
|
||||
restricted.set(cpu);
|
||||
rustix::thread::sched_setaffinity(None, &restricted).expect("restrict initialization to one allowed CPU");
|
||||
}
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let disk = new_disk(&Endpoint::try_from(dir.path().to_str().unwrap()).unwrap(), &DiskOption::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let bucket = "cleanup-affinity";
|
||||
disk.make_volume(bucket).await.unwrap();
|
||||
disk.write_all(bucket, "old/part.1", Bytes::from_static(b"garbage"))
|
||||
.await
|
||||
.unwrap();
|
||||
let receipt = disk
|
||||
.write_all(bucket, "old/.rustfs-old-data-cleanup-receipt.json", Bytes::from_static(b"receipt"))
|
||||
.await;
|
||||
if mode == "restricted" {
|
||||
assert!(
|
||||
receipt
|
||||
.expect_err("unavailable default CPUs must reject cleanup")
|
||||
.to_string()
|
||||
.contains("allowed CPU set")
|
||||
);
|
||||
assert_eq!(disk.read_all(bucket, "old/part.1").await.unwrap(), Bytes::from_static(b"garbage"));
|
||||
return;
|
||||
}
|
||||
receipt.unwrap();
|
||||
let expected_mask = match mode.as_str() {
|
||||
"default" => "1-2".to_owned(),
|
||||
"none" => std::fs::read_to_string("/proc/thread-self/status")
|
||||
.unwrap()
|
||||
.lines()
|
||||
.find_map(|line| line.strip_prefix("Cpus_allowed_list:"))
|
||||
.unwrap()
|
||||
.trim()
|
||||
.to_owned(),
|
||||
_ => cpu.to_string(),
|
||||
};
|
||||
disk.write_all(bucket, "live/part.1", Bytes::from_static(b"keep"))
|
||||
.await
|
||||
.unwrap();
|
||||
disk.delete(
|
||||
bucket,
|
||||
"old",
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(matches!(disk.read_all(bucket, "old/part.1").await, Err(DiskError::FileNotFound)));
|
||||
|
||||
// One async worker and one lazily started blocking worker. Inspect the
|
||||
// real process threads, including the threads executing filesystem work.
|
||||
let mut cleanup_threads = 0;
|
||||
for entry in std::fs::read_dir("/proc/self/task").unwrap() {
|
||||
let task = entry.unwrap().path();
|
||||
let Ok(name) = std::fs::read_to_string(task.join("comm")) else {
|
||||
continue;
|
||||
};
|
||||
if name.trim() != "rustfs-cleanup" {
|
||||
continue;
|
||||
}
|
||||
cleanup_threads += 1;
|
||||
let status = std::fs::read_to_string(task.join("status")).unwrap();
|
||||
let mask = status
|
||||
.lines()
|
||||
.find_map(|line| line.strip_prefix("Cpus_allowed_list:"))
|
||||
.unwrap()
|
||||
.trim();
|
||||
assert_eq!(mask, expected_mask);
|
||||
}
|
||||
assert_eq!(cleanup_threads, 2, "async and blocking workers must both use the configured runtime");
|
||||
assert_eq!(rustix::thread::sched_getaffinity(None).unwrap(), foreground);
|
||||
|
||||
let trash = dir.path().join(".rustfs.sys/tmp/.trash");
|
||||
assert!(std::fs::read_dir(&trash).unwrap().next().is_some());
|
||||
// Advance the foreground timer that dispatches the five-minute periodic
|
||||
// scan; the dedicated runtime then performs the actual physical removal.
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(301)).await;
|
||||
tokio::time::resume();
|
||||
tokio::time::timeout(Duration::from_secs(20), async {
|
||||
while std::fs::read_dir(&trash).unwrap().next().is_some() {
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("periodic GC must physically remove trash");
|
||||
assert_eq!(disk.read_all(bucket, "live/part.1").await.unwrap(), Bytes::from_static(b"keep"));
|
||||
assert_eq!(rustix::thread::sched_getaffinity(None).unwrap(), foreground);
|
||||
}
|
||||
@@ -0,0 +1,283 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#![cfg(all(target_os = "linux", feature = "test-util"))]
|
||||
|
||||
mod storage_api;
|
||||
|
||||
use bytes::Bytes;
|
||||
use rustfs_filemeta::{FileInfo, ObjectPartInfo};
|
||||
use std::{path::Path, time::Duration};
|
||||
use storage_api::fd_relative::{
|
||||
DiskAPI, DiskError, DiskOption, DiskStore, Endpoint, LocalPublicationPause, LocalPublicationStage, ReadOptions, new_disk,
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
|
||||
const RUSTFS_META_TMP_BUCKET: &str = ".rustfs.sys/tmp";
|
||||
|
||||
async fn disk_at(path: &Path) -> DiskStore {
|
||||
new_disk(
|
||||
&Endpoint::try_from(path.to_str().expect("UTF-8 test path")).expect("endpoint"),
|
||||
&DiskOption::default(),
|
||||
)
|
||||
.await
|
||||
.expect("local disk")
|
||||
}
|
||||
|
||||
fn metadata(data: Bytes, inline: bool, directory: Uuid) -> FileInfo {
|
||||
let mut fi = FileInfo::new("object", 1, 0);
|
||||
fi.erasure.index = 1;
|
||||
fi.version_id = Some(Uuid::nil());
|
||||
fi.data_dir = Some(directory);
|
||||
fi.size = i64::try_from(data.len()).expect("test data size");
|
||||
fi.parts = vec![ObjectPartInfo {
|
||||
number: 1,
|
||||
size: data.len(),
|
||||
actual_size: fi.size,
|
||||
..Default::default()
|
||||
}];
|
||||
fi.mod_time = Some(OffsetDateTime::now_utc());
|
||||
fi.data = inline.then_some(data);
|
||||
if inline {
|
||||
fi.set_inline_data();
|
||||
}
|
||||
fi
|
||||
}
|
||||
|
||||
async fn put(disk: &DiskStore, staging: &str, data: Bytes, inline: bool) -> Uuid {
|
||||
let directory = Uuid::new_v4();
|
||||
if !inline {
|
||||
disk.write_all(RUSTFS_META_TMP_BUCKET, &format!("{staging}/{directory}/part.1"), data.clone())
|
||||
.await
|
||||
.expect("stage shard");
|
||||
}
|
||||
disk.rename_data(RUSTFS_META_TMP_BUCKET, staging, metadata(data, inline, directory), "bucket", "object")
|
||||
.await
|
||||
.expect("commit object");
|
||||
directory
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pinned_root_reads_and_writes_original_disk_after_path_replacement() {
|
||||
let temp = tempfile::tempdir().expect("test directory");
|
||||
let root = temp.path().join("disk");
|
||||
std::fs::create_dir(&root).expect("disk root");
|
||||
let disk = disk_at(&root).await;
|
||||
disk.make_volume("bucket").await.expect("bucket");
|
||||
disk.write_all("bucket", "metadata", Bytes::from_static(b"original"))
|
||||
.await
|
||||
.expect("initial write");
|
||||
let moved = temp.path().join("original");
|
||||
std::fs::rename(&root, &moved).expect("move mount path");
|
||||
std::fs::create_dir_all(root.join("bucket")).expect("replacement disk");
|
||||
std::fs::write(root.join("bucket/metadata"), b"replacement").expect("replacement sentinel");
|
||||
assert_eq!(
|
||||
disk.read_all("bucket", "metadata").await.expect("pinned read"),
|
||||
Bytes::from_static(b"original")
|
||||
);
|
||||
disk.write_all("bucket", "metadata", Bytes::from_static(b"updated"))
|
||||
.await
|
||||
.expect("pinned write");
|
||||
assert_eq!(std::fs::read(moved.join("bucket/metadata")).expect("original disk"), b"updated");
|
||||
assert_eq!(std::fs::read(root.join("bucket/metadata")).expect("replacement disk"), b"replacement");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn relative_reads_and_writes_reject_symlink_escape() {
|
||||
let temp = tempfile::tempdir().expect("test directory");
|
||||
let disk = disk_at(temp.path()).await;
|
||||
disk.make_volume("bucket").await.expect("bucket");
|
||||
let outside = tempfile::tempdir().expect("outside directory");
|
||||
std::fs::write(outside.path().join("sentinel"), b"preserved").expect("sentinel");
|
||||
std::os::unix::fs::symlink(outside.path(), temp.path().join("bucket/link")).expect("directory link");
|
||||
std::os::unix::fs::symlink(outside.path().join("sentinel"), temp.path().join("bucket/file-link")).expect("file link");
|
||||
for path in ["link/sentinel", "file-link"] {
|
||||
assert!(matches!(disk.read_all("bucket", path).await, Err(DiskError::InvalidPath)));
|
||||
assert!(matches!(
|
||||
disk.write_all("bucket", path, Bytes::from_static(b"unsafe")).await,
|
||||
Err(DiskError::InvalidPath)
|
||||
));
|
||||
}
|
||||
assert_eq!(std::fs::read(outside.path().join("sentinel")).expect("outside bytes"), b"preserved");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn inline_and_non_inline_overwrites_remain_readable_after_reopen() {
|
||||
for inline in [true, false] {
|
||||
let temp = tempfile::tempdir().expect("test directory");
|
||||
let disk = disk_at(temp.path()).await;
|
||||
disk.make_volume("bucket").await.expect("bucket");
|
||||
put(&disk, "first", Bytes::from_static(b"old bytes"), inline).await;
|
||||
let directory = put(&disk, "second", Bytes::from_static(b"new bytes"), inline).await;
|
||||
drop(disk);
|
||||
let disk = disk_at(temp.path()).await;
|
||||
let fi = disk
|
||||
.read_version(
|
||||
"bucket",
|
||||
"bucket",
|
||||
"object",
|
||||
"",
|
||||
&ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("committed metadata after reopen");
|
||||
assert_eq!(fi.data_dir, Some(directory));
|
||||
if inline {
|
||||
assert_eq!(fi.data, Some(Bytes::from_static(b"new bytes")));
|
||||
} else {
|
||||
assert_eq!(
|
||||
disk.read_all("bucket", &format!("object/{directory}/part.1"))
|
||||
.await
|
||||
.expect("shard bytes"),
|
||||
Bytes::from_static(b"new bytes")
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rename_rejects_symlinked_destination_metadata() {
|
||||
for inline in [true, false] {
|
||||
let temp = tempfile::tempdir().expect("disk directory");
|
||||
let outside = tempfile::tempdir().expect("outside directory");
|
||||
let disk = disk_at(temp.path()).await;
|
||||
disk.make_volume("bucket").await.expect("bucket");
|
||||
put(&disk, "seed", Bytes::from_static(b"old object"), inline).await;
|
||||
let destination = temp.path().join("bucket/object/xl.meta");
|
||||
let original = std::fs::read(&destination).expect("valid existing metadata");
|
||||
let external_metadata = outside.path().join("xl.meta");
|
||||
std::fs::write(&external_metadata, &original).expect("external valid metadata");
|
||||
std::fs::remove_file(&destination).expect("replace destination metadata");
|
||||
std::os::unix::fs::symlink(&external_metadata, &destination).expect("metadata escape link");
|
||||
|
||||
let directory = Uuid::new_v4();
|
||||
let data = Bytes::from_static(b"replacement");
|
||||
if !inline {
|
||||
disk.write_all(RUSTFS_META_TMP_BUCKET, &format!("replacement/{directory}/part.1"), data.clone())
|
||||
.await
|
||||
.expect("stage non-inline data");
|
||||
}
|
||||
let result = disk
|
||||
.rename_data(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"replacement",
|
||||
metadata(data, inline, directory),
|
||||
"bucket",
|
||||
"object",
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
matches!(result, Err(DiskError::InvalidPath)),
|
||||
"symlinked metadata must be rejected: {result:?}"
|
||||
);
|
||||
assert_eq!(std::fs::read_link(&destination).expect("link must remain"), external_metadata);
|
||||
assert_eq!(std::fs::read(&external_metadata).expect("external metadata must survive"), original);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn prepared_publication_rejects_source_or_directory_replacement() {
|
||||
for replace_directory in [false, true] {
|
||||
let temp = tempfile::tempdir().expect("test directory");
|
||||
let disk = disk_at(temp.path()).await;
|
||||
disk.make_volume("bucket").await.expect("bucket");
|
||||
put(&disk, "first", Bytes::from_static(b"old bytes"), true).await;
|
||||
let old = std::fs::read(temp.path().join("bucket/object/xl.meta")).expect("old metadata");
|
||||
let mut pause = LocalPublicationPause::install(&disk, "bucket", "object/xl.meta", LocalPublicationStage::PreparedRename)
|
||||
.expect("publication pause");
|
||||
let writer = disk.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
writer
|
||||
.rename_data(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"second",
|
||||
metadata(Bytes::from_static(b"new bytes"), true, Uuid::new_v4()),
|
||||
"bucket",
|
||||
"object",
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), pause.entered())
|
||||
.await
|
||||
.expect("publication reached pause")
|
||||
.expect("pause entered");
|
||||
if replace_directory {
|
||||
std::fs::rename(temp.path().join("bucket/object"), temp.path().join("detached")).expect("detach object");
|
||||
std::fs::create_dir(temp.path().join("bucket/object")).expect("replacement directory");
|
||||
std::fs::write(temp.path().join("bucket/object/xl.meta"), &old).expect("replacement metadata");
|
||||
} else {
|
||||
let src = temp.path().join(RUSTFS_META_TMP_BUCKET).join("second/xl.meta");
|
||||
std::fs::rename(&src, src.with_extension("detached")).expect("detach staged metadata");
|
||||
std::fs::write(src, b"unrelated").expect("replacement staged file");
|
||||
}
|
||||
drop(pause);
|
||||
assert!(task.await.expect("writer task").is_err());
|
||||
assert_eq!(
|
||||
std::fs::read(temp.path().join("bucket/object/xl.meta")).expect("old metadata survives"),
|
||||
old
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn cancelled_waiter_keeps_publication_serialized_until_physical_completion() {
|
||||
let temp = tempfile::tempdir().expect("test directory");
|
||||
let disk = disk_at(temp.path()).await;
|
||||
disk.make_volume("bucket").await.expect("bucket");
|
||||
put(&disk, "first", Bytes::from_static(b"old bytes"), true).await;
|
||||
let mut pause = LocalPublicationPause::install(&disk, "bucket", "object/xl.meta", LocalPublicationStage::PreparedRename)
|
||||
.expect("publication pause");
|
||||
let writer = disk.clone();
|
||||
let pending = tokio::spawn(async move { put(&writer, "shared-stage", Bytes::from_static(b"cancelled waiter"), true).await });
|
||||
tokio::time::timeout(Duration::from_secs(30), pause.entered())
|
||||
.await
|
||||
.expect("publication reached pause")
|
||||
.expect("pause entered");
|
||||
pending.abort();
|
||||
assert!(pending.await.expect_err("waiter aborted").is_cancelled());
|
||||
let writer = disk.clone();
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let mut retry = tokio::spawn(async move {
|
||||
let _ = started_tx.send(());
|
||||
put(&writer, "shared-stage", Bytes::from_static(b"latest bytes"), true).await
|
||||
});
|
||||
started_rx.await.expect("retry started");
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(200), &mut retry).await.is_err(),
|
||||
"a retry must not publish while the cancelled writer still owns physical IO"
|
||||
);
|
||||
drop(pause);
|
||||
tokio::time::timeout(Duration::from_secs(30), retry)
|
||||
.await
|
||||
.expect("retry completed")
|
||||
.expect("retry task");
|
||||
let fi = disk
|
||||
.read_version(
|
||||
"bucket",
|
||||
"bucket",
|
||||
"object",
|
||||
"",
|
||||
&ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("latest metadata");
|
||||
assert_eq!(fi.data, Some(Bytes::from_static(b"latest bytes")));
|
||||
}
|
||||
@@ -18,6 +18,7 @@ mod storage_api;
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use storage_api::contract_compat::ECStore;
|
||||
use storage_api::metadata_lock::{
|
||||
BucketOperations, CompletePart, Error, MakeBucketOptions, MultipartOperations, NamespaceLocking, ObjectIO, ObjectOperations,
|
||||
ObjectOptions, PutObjReader, PutObjectCommitBarrier, PutObjectCommitPause, init_bucket_metadata_sys,
|
||||
@@ -170,6 +171,100 @@ async fn reused_metadata_lock_still_enforces_quota() {
|
||||
.expect("acquire writer");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn authoritative_quota_read_does_not_ignore_later_quota() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "authoritative-quota-quota-set";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
|
||||
let mut first = PutObjReader::from_vec(b"cache no quota".to_vec());
|
||||
store
|
||||
.put_object(bucket, "warm", &mut first, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("warm no-quota metadata cache");
|
||||
store
|
||||
.update_bucket_metadata_config(bucket, "quota.json", br#"{"quota":1}"#.to_vec())
|
||||
.await
|
||||
.expect("set quota after no-quota cache was observed");
|
||||
|
||||
let mut opts = quota_snapshot_options(&store, bucket).await;
|
||||
assert!(opts.set_quota_admission(0, 1));
|
||||
let mut data = PutObjReader::from_vec(b"too large".to_vec());
|
||||
let error = store
|
||||
.put_object(bucket, "object", &mut data, &opts)
|
||||
.await
|
||||
.expect_err("quota set after a no-quota cache hit must still reject growth");
|
||||
assert!(matches!(error, Error::QuotaExceeded { limit: 1, .. }), "{error:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn authoritative_quota_read_allows_write_after_quota_is_cleared() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "authoritative-quota-quota-cleared";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
store
|
||||
.update_bucket_metadata_config(bucket, "quota.json", br#"{"quota":1}"#.to_vec())
|
||||
.await
|
||||
.expect("set quota");
|
||||
store
|
||||
.update_bucket_metadata_config(bucket, "quota.json", Vec::new())
|
||||
.await
|
||||
.expect("clear quota");
|
||||
|
||||
let mut data = PutObjReader::from_vec(b"allowed after clear".to_vec());
|
||||
store
|
||||
.put_object(bucket, "object", &mut data, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("authoritatively cleared quota should allow writes");
|
||||
|
||||
let mut reader = store
|
||||
.get_object_reader(bucket, "object", None, Default::default(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read object written after quota clear");
|
||||
let mut bytes = Vec::new();
|
||||
reader.stream.read_to_end(&mut bytes).await.expect("read body");
|
||||
assert_eq!(bytes, b"allowed after clear");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn authoritative_quota_read_fails_closed_on_invalid_quota_json() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "authoritative-quota-invalid-quota";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
store
|
||||
.update_bucket_metadata_config(bucket, "quota.json", b"{not-json".to_vec())
|
||||
.await
|
||||
.expect("persist invalid quota bytes for fail-closed read");
|
||||
|
||||
let mut data = PutObjReader::from_vec(b"must fail".to_vec());
|
||||
store
|
||||
.put_object(bucket, "object", &mut data, &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("invalid quota JSON must not degrade to no-quota");
|
||||
}
|
||||
|
||||
async fn quota_snapshot_options(store: &Arc<ECStore>, bucket: &str) -> ObjectOptions {
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(Uuid::new_v4().to_string()),
|
||||
expected_bucket_incarnation_id: Some(store.bucket_incarnation_id(bucket).await.expect("load incarnation")),
|
||||
object_lock_config_snapshot: Some(store.object_lock_config_snapshot(bucket).await.expect("capture snapshot")),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_put_releases_shared_metadata_guard() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
|
||||
@@ -61,6 +61,17 @@ pub(crate) mod minio_generated_read {
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod cleanup_isolation {
|
||||
pub(crate) use super::{DiskAPI, DiskOption, Endpoint, new_disk};
|
||||
pub(crate) use rustfs_ecstore::api::disk::{DeleteOptions, DiskError};
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "test-util", target_os = "linux"))]
|
||||
pub(crate) mod fd_relative {
|
||||
pub(crate) use super::{DiskAPI, DiskOption, DiskStore, Endpoint, new_disk};
|
||||
pub(crate) use rustfs_ecstore::api::disk::{DiskError, LocalPublicationPause, LocalPublicationStage, ReadOptions};
|
||||
}
|
||||
|
||||
pub(crate) mod pinned_erasure_fixtures {
|
||||
pub(crate) use super::storage_contracts::{BucketOperations, HTTPRangeSpec, MakeBucketOptions, ObjectIO};
|
||||
pub(crate) use super::{ECStore, Endpoint, Erasure, Error, ObjectOptions};
|
||||
|
||||
@@ -30,6 +30,7 @@ harness = false
|
||||
|
||||
[features]
|
||||
default = []
|
||||
diagnostic-read-version-caller = []
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
"hotpath/tokio",
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
use criterion::{Criterion, criterion_group, criterion_main};
|
||||
use rustfs_io_metrics::{MetricsCollector, PerformanceMetrics, record_get_object_request_started};
|
||||
use rustfs_io_metrics::{MetricsCollector, PerformanceMetrics, record_get_object_timeout_operation_start};
|
||||
use rustfs_io_metrics::{record_s3_op, s3_http_metrics::S3HttpRequestGuard};
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use std::hint::black_box;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
fn bench_record_get_object_request_started(c: &mut Criterion) {
|
||||
c.bench_function("record_get_object_request_started", |b| b.iter(record_get_object_request_started));
|
||||
fn bench_record_get_object_timeout_operation_start(c: &mut Criterion) {
|
||||
c.bench_function("record_get_object_timeout_operation_start", |b| {
|
||||
b.iter(record_get_object_timeout_operation_start)
|
||||
});
|
||||
}
|
||||
|
||||
fn bench_s3_http_outcomes(c: &mut Criterion) {
|
||||
@@ -49,7 +51,7 @@ fn bench_metrics_collector_record_io_operation(c: &mut Criterion) {
|
||||
|
||||
criterion_group!(
|
||||
benches,
|
||||
bench_record_get_object_request_started,
|
||||
bench_record_get_object_timeout_operation_start,
|
||||
bench_s3_http_outcomes,
|
||||
bench_update_concurrent_requests,
|
||||
bench_metrics_collector_record_io_operation
|
||||
|
||||
@@ -12,7 +12,10 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::s3_http_metrics::S3HttpOperationContext;
|
||||
use metrics::{counter, gauge};
|
||||
#[cfg(not(test))]
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{
|
||||
Arc, LazyLock, OnceLock, RwLock,
|
||||
@@ -65,6 +68,340 @@ pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE: &str = "batch_read
|
||||
pub const INTERNODE_STAGE_BATCH_READ_VERSION_COALESCER_WAIT: &str = "batch_read_version_coalescer_wait";
|
||||
pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MAP: &str = "batch_read_version_response_map";
|
||||
|
||||
const INTERNODE_READ_VERSION_REQUESTS_BY_CALLER_TOTAL: &str =
|
||||
"rustfs_system_network_internode_read_version_requests_by_caller_total";
|
||||
|
||||
/// An attribution dimension is absent, common to all items, or heterogeneous.
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum ReadVersionAttribution<T> {
|
||||
Absent,
|
||||
Common(T),
|
||||
Mixed,
|
||||
}
|
||||
|
||||
/// Storage entry names describe call paths, not proof of an external request.
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum ReadVersionEntrySource {
|
||||
Unclassified,
|
||||
ConfigRead,
|
||||
ConfigWrite,
|
||||
ScannerConfigWrite,
|
||||
WritePreconditionRead,
|
||||
ObjectInfoRead,
|
||||
AppPutOwner,
|
||||
PutCommitOwner,
|
||||
LifecycleRecovery,
|
||||
RecoveryDispositionMaintenance,
|
||||
StoreObjectReader,
|
||||
DataUsageCacheRead,
|
||||
ServerInfoPools,
|
||||
ScannerBacklogRead,
|
||||
ScannerRevisionRead,
|
||||
ScannerCycleStateRead,
|
||||
TierConfigReload,
|
||||
ObjectRead,
|
||||
ListObjects,
|
||||
MetadataRead,
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
impl ReadVersionEntrySource {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Unclassified => "unclassified",
|
||||
Self::ConfigRead => "config_read",
|
||||
Self::ConfigWrite => "config_write",
|
||||
Self::ScannerConfigWrite => "scanner_config_write",
|
||||
Self::WritePreconditionRead => "write_precondition_read",
|
||||
Self::ObjectInfoRead => "object_info_read",
|
||||
Self::AppPutOwner => "app_put_owner",
|
||||
Self::PutCommitOwner => "put_commit_owner",
|
||||
Self::LifecycleRecovery => "lifecycle_recovery",
|
||||
Self::RecoveryDispositionMaintenance => "recovery_disposition_maintenance",
|
||||
Self::StoreObjectReader => "store_object_reader",
|
||||
Self::DataUsageCacheRead => "data_usage_cache_read",
|
||||
Self::ServerInfoPools => "server_info_pools",
|
||||
Self::ScannerBacklogRead => "scanner_backlog_read",
|
||||
Self::ScannerRevisionRead => "scanner_revision_read",
|
||||
Self::ScannerCycleStateRead => "scanner_cycle_state_read",
|
||||
Self::TierConfigReload => "tier_config_reload",
|
||||
Self::ObjectRead => "object_read",
|
||||
Self::ListObjects => "list_objects",
|
||||
Self::MetadataRead => "metadata_read",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub enum ReadVersionContextBoundary {
|
||||
Entry,
|
||||
FullWait,
|
||||
EarlyStop,
|
||||
ListProducer,
|
||||
ListGather,
|
||||
CoalescerEnqueue,
|
||||
UnaryFallback,
|
||||
EntryToRpc,
|
||||
RemoteRpc,
|
||||
ScannerCacheLoad,
|
||||
ScannerCycleStateLoad,
|
||||
ConfigWrite,
|
||||
ScannerConfigWrite,
|
||||
WritePreconditionLookup,
|
||||
ObjectInfoRead,
|
||||
AppPutOwner,
|
||||
PutCommitOwner,
|
||||
TierFreeVersionSourceLookup,
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
impl ReadVersionContextBoundary {
|
||||
const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Entry => "entry",
|
||||
Self::FullWait => "full_wait",
|
||||
Self::EarlyStop => "early_stop",
|
||||
Self::ListProducer => "list_producer",
|
||||
Self::ListGather => "list_gather",
|
||||
Self::CoalescerEnqueue => "coalescer_enqueue",
|
||||
Self::UnaryFallback => "unary_fallback",
|
||||
Self::EntryToRpc => "entry_to_rpc",
|
||||
Self::RemoteRpc => "remote_rpc",
|
||||
Self::ScannerCacheLoad => "scanner_cache_load",
|
||||
Self::ScannerCycleStateLoad => "scanner_cycle_state_load",
|
||||
Self::ConfigWrite => "config_write",
|
||||
Self::ScannerConfigWrite => "scanner_config_write",
|
||||
Self::WritePreconditionLookup => "write_precondition_lookup",
|
||||
Self::ObjectInfoRead => "object_info_read",
|
||||
Self::AppPutOwner => "app_put_owner",
|
||||
Self::PutCommitOwner => "put_commit_owner",
|
||||
Self::TierFreeVersionSourceLookup => "tier_free_version_source_lookup",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
fn context_transition<T: Eq>(parent: Option<T>, child: Option<T>) -> &'static str {
|
||||
match (parent, child) {
|
||||
(None, None) => "absent",
|
||||
(Some(_), None) => "lost",
|
||||
(None, Some(_)) => "gained",
|
||||
(Some(parent), Some(child)) if parent == child => "preserved",
|
||||
(Some(_), Some(_)) => "changed",
|
||||
}
|
||||
}
|
||||
|
||||
/// Count observations separately from RPCs, using only bounded context labels.
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
pub fn record_read_version_context_transition(
|
||||
source: ReadVersionEntrySource,
|
||||
boundary: ReadVersionContextBoundary,
|
||||
parent_api: Option<S3HttpOperationContext>,
|
||||
child_api: Option<S3HttpOperationContext>,
|
||||
parent_caller: Option<ReadVersionRequestCaller>,
|
||||
child_caller: Option<ReadVersionRequestCaller>,
|
||||
mode: Option<ReadVersionRequestMode>,
|
||||
) {
|
||||
if !crate::get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
counter!(
|
||||
"rustfs_storage_diagnostic_context_transitions_total",
|
||||
"entry_source" => source.as_str(),
|
||||
"boundary" => boundary.as_str(),
|
||||
"parent_api_operation" => parent_api.map(S3HttpOperationContext::operation_name).unwrap_or("unattributed"),
|
||||
"child_api_operation" => child_api.map(S3HttpOperationContext::operation_name).unwrap_or("unattributed"),
|
||||
"parent_caller" => parent_caller.map(ReadVersionRequestCaller::as_str).unwrap_or("unattributed"),
|
||||
"child_caller" => child_caller.map(ReadVersionRequestCaller::as_str).unwrap_or("unattributed"),
|
||||
"mode" => mode.map(ReadVersionRequestMode::as_str).unwrap_or("not_applicable"),
|
||||
"api_context" => if source == ReadVersionEntrySource::Unclassified && matches!(boundary, ReadVersionContextBoundary::RemoteRpc) {
|
||||
"unobserved"
|
||||
} else { context_transition(parent_api, child_api) },
|
||||
"caller_context" => if source == ReadVersionEntrySource::Unclassified && matches!(boundary, ReadVersionContextBoundary::RemoteRpc) {
|
||||
"unobserved"
|
||||
} else { context_transition(parent_caller, child_caller) },
|
||||
)
|
||||
.increment(1);
|
||||
}
|
||||
|
||||
/// Count a native batch RPC once; item observations and unary fallback are separate.
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
pub fn record_batch_read_version_request_by_caller(
|
||||
operation: ReadVersionAttribution<S3HttpOperationContext>,
|
||||
caller: ReadVersionAttribution<ReadVersionRequestCaller>,
|
||||
source: ReadVersionAttribution<ReadVersionEntrySource>,
|
||||
mode: ReadVersionRequestMode,
|
||||
) {
|
||||
let api_label = match operation {
|
||||
ReadVersionAttribution::Absent => "unattributed",
|
||||
ReadVersionAttribution::Common(operation) => operation.operation_name(),
|
||||
ReadVersionAttribution::Mixed => "mixed",
|
||||
};
|
||||
let caller_label = match caller {
|
||||
ReadVersionAttribution::Absent => "unattributed",
|
||||
ReadVersionAttribution::Common(caller) => caller.as_str(),
|
||||
ReadVersionAttribution::Mixed => "mixed",
|
||||
};
|
||||
let source_label = match source {
|
||||
ReadVersionAttribution::Absent => "unclassified",
|
||||
ReadVersionAttribution::Common(source) => source.as_str(),
|
||||
ReadVersionAttribution::Mixed => "mixed",
|
||||
};
|
||||
counter!(
|
||||
"rustfs_system_network_internode_batch_read_version_requests_by_caller_total",
|
||||
"api_operation" => api_label,
|
||||
"caller" => caller_label,
|
||||
"entry_source" => source_label,
|
||||
"mode" => mode.as_str(),
|
||||
)
|
||||
.increment(1);
|
||||
}
|
||||
#[cfg(not(test))]
|
||||
const READ_VERSION_REQUEST_MODE_COUNT: usize = 5;
|
||||
const READ_VERSION_REQUEST_CALLER_COUNT: usize = 18;
|
||||
|
||||
#[cfg(not(test))]
|
||||
static READ_VERSION_CALLER_COUNTERS: LazyLock<Box<[OnceLock<metrics::Counter>]>> = LazyLock::new(|| {
|
||||
std::iter::repeat_with(OnceLock::new)
|
||||
.take((S3Operation::ALL.len() + 1) * READ_VERSION_REQUEST_MODE_COUNT * READ_VERSION_REQUEST_CALLER_COUNT)
|
||||
.collect::<Vec<_>>()
|
||||
.into_boxed_slice()
|
||||
});
|
||||
|
||||
/// Coarse read mode for diagnostic attribution of remote metadata reads.
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum ReadVersionRequestMode {
|
||||
MetadataOnly,
|
||||
Data,
|
||||
Healing,
|
||||
IncludeFreeVersions,
|
||||
Combined,
|
||||
}
|
||||
|
||||
/// Bounded internal call-site class for an outgoing ReadVersion RPC.
|
||||
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
|
||||
#[repr(usize)]
|
||||
pub enum ReadVersionRequestCaller {
|
||||
#[default]
|
||||
Unattributed,
|
||||
Peer,
|
||||
Startup,
|
||||
Heal,
|
||||
ListObjects,
|
||||
Object,
|
||||
MetadataRead,
|
||||
Multipart,
|
||||
DiskStore,
|
||||
DiskDispatch,
|
||||
SetDiskRead,
|
||||
SetDiskIo,
|
||||
Rebalance,
|
||||
Decommission,
|
||||
Lifecycle,
|
||||
TierRecovery,
|
||||
CacheValue,
|
||||
Other,
|
||||
}
|
||||
|
||||
impl ReadVersionRequestCaller {
|
||||
pub const ALL: [Self; READ_VERSION_REQUEST_CALLER_COUNT] = [
|
||||
Self::Unattributed,
|
||||
Self::Peer,
|
||||
Self::Startup,
|
||||
Self::Heal,
|
||||
Self::ListObjects,
|
||||
Self::Object,
|
||||
Self::MetadataRead,
|
||||
Self::Multipart,
|
||||
Self::DiskStore,
|
||||
Self::DiskDispatch,
|
||||
Self::SetDiskRead,
|
||||
Self::SetDiskIo,
|
||||
Self::Rebalance,
|
||||
Self::Decommission,
|
||||
Self::Lifecycle,
|
||||
Self::TierRecovery,
|
||||
Self::CacheValue,
|
||||
Self::Other,
|
||||
];
|
||||
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Unattributed => "unattributed",
|
||||
Self::Peer => "peer",
|
||||
Self::Startup => "startup",
|
||||
Self::Heal => "heal",
|
||||
Self::ListObjects => "list_objects",
|
||||
Self::Object => "object",
|
||||
Self::MetadataRead => "metadata_read",
|
||||
Self::Multipart => "multipart",
|
||||
Self::DiskStore => "disk_store",
|
||||
Self::DiskDispatch => "disk_dispatch",
|
||||
Self::SetDiskRead => "set_disk_read",
|
||||
Self::SetDiskIo => "set_disk_io",
|
||||
Self::Rebalance => "rebalance",
|
||||
Self::Decommission => "decommission",
|
||||
Self::Lifecycle => "lifecycle",
|
||||
Self::TierRecovery => "tier_recovery",
|
||||
Self::CacheValue => "cache_value",
|
||||
Self::Other => "other",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ReadVersionRequestMode {
|
||||
const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::MetadataOnly => "metadata_only",
|
||||
Self::Data => "data",
|
||||
Self::Healing => "healing",
|
||||
Self::IncludeFreeVersions => "include_free_versions",
|
||||
Self::Combined => "combined",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Record a bounded diagnostic caller class for one outgoing ReadVersion RPC.
|
||||
/// The caller label is either a fixed S3 operation name or `unattributed`.
|
||||
pub fn record_read_version_request_by_caller(
|
||||
operation: Option<S3HttpOperationContext>,
|
||||
caller: ReadVersionRequestCaller,
|
||||
mode: ReadVersionRequestMode,
|
||||
) {
|
||||
let api_operation = operation
|
||||
.map(S3HttpOperationContext::operation_name)
|
||||
.unwrap_or("unattributed");
|
||||
|
||||
#[cfg(test)]
|
||||
counter!(
|
||||
INTERNODE_READ_VERSION_REQUESTS_BY_CALLER_TOTAL,
|
||||
"api_operation" => api_operation,
|
||||
"caller" => caller.as_str(),
|
||||
"mode" => mode.as_str(),
|
||||
)
|
||||
.increment(1);
|
||||
|
||||
#[cfg(not(test))]
|
||||
{
|
||||
let caller_index = operation.map_or(S3Operation::ALL.len(), |context| context.operation_index());
|
||||
let counter_index = (caller_index * READ_VERSION_REQUEST_MODE_COUNT + mode as usize) * READ_VERSION_REQUEST_CALLER_COUNT
|
||||
+ caller as usize;
|
||||
READ_VERSION_CALLER_COUNTERS[counter_index]
|
||||
.get_or_init(|| {
|
||||
counter!(
|
||||
INTERNODE_READ_VERSION_REQUESTS_BY_CALLER_TOTAL,
|
||||
"api_operation" => api_operation,
|
||||
"caller" => caller.as_str(),
|
||||
"mode" => mode.as_str(),
|
||||
)
|
||||
})
|
||||
.increment(1);
|
||||
}
|
||||
}
|
||||
|
||||
const OPERATION_LABEL: &str = "operation";
|
||||
const BACKEND_LABEL: &str = "backend";
|
||||
const SERVER_LABEL: &str = "server";
|
||||
@@ -1231,6 +1568,61 @@ mod tests {
|
||||
assert_eq!(snapshot.replay_cache_evictions_total, 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_version_caller_metric_uses_only_bounded_labels() {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
with_local_recorder(&recorder, || {
|
||||
let mut request = crate::s3_http_metrics::S3HttpRequestGuard::new("GET");
|
||||
let operation = request
|
||||
.in_scope(|| {
|
||||
crate::s3_http_metrics::observe_s3_http_operation(rustfs_s3_ops::S3Operation::GetObject);
|
||||
crate::s3_http_metrics::current_s3_http_operation_context()
|
||||
})
|
||||
.expect("S3 request should expose its dispatched operation");
|
||||
record_read_version_request_by_caller(
|
||||
Some(operation),
|
||||
ReadVersionRequestCaller::SetDiskRead,
|
||||
ReadVersionRequestMode::Data,
|
||||
);
|
||||
record_read_version_request_by_caller(None, ReadVersionRequestCaller::Peer, ReadVersionRequestMode::MetadataOnly);
|
||||
});
|
||||
|
||||
let rows = snapshotter
|
||||
.snapshot()
|
||||
.into_vec()
|
||||
.into_iter()
|
||||
.filter(|(composite, _, _, _)| composite.key().name() == INTERNODE_READ_VERSION_REQUESTS_BY_CALLER_TOTAL)
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(rows.len(), 2);
|
||||
let labels = rows
|
||||
.iter()
|
||||
.map(|(composite, _, _, _)| {
|
||||
composite
|
||||
.key()
|
||||
.labels()
|
||||
.map(|label| (label.key().to_string(), label.value().to_string()))
|
||||
.collect::<HashMap<_, _>>()
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
assert!(labels.iter().any(|labels| {
|
||||
labels.get("api_operation").map(String::as_str) == Some("s3:GetObject")
|
||||
&& labels.get("caller").map(String::as_str) == Some("set_disk_read")
|
||||
&& labels.get("mode").map(String::as_str) == Some("data")
|
||||
}));
|
||||
assert!(labels.iter().any(|labels| {
|
||||
labels.get("api_operation").map(String::as_str) == Some("unattributed")
|
||||
&& labels.get("mode").map(String::as_str) == Some("metadata_only")
|
||||
&& labels.get("caller").map(String::as_str) == Some("peer")
|
||||
}));
|
||||
assert!(labels.iter().all(|labels| {
|
||||
labels.len() == 3
|
||||
&& labels
|
||||
.keys()
|
||||
.all(|key| key == "api_operation" || key == "caller" || key == "mode")
|
||||
}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn operation_stage_duration_records_low_cardinality_stage_labels() {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
|
||||
@@ -49,8 +49,9 @@
|
||||
#[macro_use]
|
||||
extern crate metrics;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{
|
||||
Mutex,
|
||||
LazyLock, Mutex,
|
||||
atomic::{AtomicBool, AtomicU64, Ordering},
|
||||
};
|
||||
|
||||
@@ -61,6 +62,25 @@ use std::sync::{
|
||||
/// Enabled only through an explicit runtime opt-in.
|
||||
static PUT_STAGE_METRICS_ENABLED: AtomicBool = AtomicBool::new(false);
|
||||
static GET_STAGE_METRICS_ENABLED: AtomicBool = AtomicBool::new(false);
|
||||
static GET_STAGE_LOCAL_SUMMARY_ENABLED: AtomicBool = AtomicBool::new(false);
|
||||
static GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE: AtomicU64 = AtomicU64::new(64);
|
||||
static GET_STAGE_LOCAL_SUMMARY_SEQUENCE: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
type GetStageLocalSummaryKey = (&'static str, &'static str, &'static str, &'static str);
|
||||
|
||||
/// Sampled local aggregate for one bounded GetObject stage label set.
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct GetStageLocalSummary {
|
||||
pub path: &'static str,
|
||||
pub stage: &'static str,
|
||||
pub object_class: &'static str,
|
||||
pub size_bucket: &'static str,
|
||||
pub sampled_count: u64,
|
||||
pub duration_nanoseconds: u64,
|
||||
}
|
||||
|
||||
static GET_STAGE_LOCAL_SUMMARY: LazyLock<Mutex<HashMap<GetStageLocalSummaryKey, GetStageLocalSummary>>> =
|
||||
LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
/// Global switch for all remaining (non GET/PUT-stage) metric emission in this
|
||||
/// crate's free `record_*` functions — I/O scheduler, bytes-pool, zero-copy,
|
||||
@@ -87,6 +107,70 @@ pub fn set_get_stage_metrics_enabled(enabled: bool) {
|
||||
GET_STAGE_METRICS_ENABLED.store(enabled, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn set_get_stage_local_summary_enabled(enabled: bool) {
|
||||
GET_STAGE_LOCAL_SUMMARY_ENABLED.store(enabled, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn set_get_stage_local_summary_sample_rate(sample_rate: u64) {
|
||||
GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE.store(sample_rate.max(1), Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn get_stage_local_summary_enabled() -> bool {
|
||||
GET_STAGE_LOCAL_SUMMARY_ENABLED.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn get_stage_local_summary_sample_rate() -> u64 {
|
||||
GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn take_get_stage_local_summary() -> Vec<GetStageLocalSummary> {
|
||||
let Ok(mut summary) = GET_STAGE_LOCAL_SUMMARY.lock() else {
|
||||
return Vec::new();
|
||||
};
|
||||
let mut rows = summary.drain().map(|(_, row)| row).collect::<Vec<_>>();
|
||||
rows.sort_unstable_by_key(|row| (row.path, row.stage, row.object_class, row.size_bucket));
|
||||
rows
|
||||
}
|
||||
|
||||
fn record_get_stage_local_summary(
|
||||
path: &'static str,
|
||||
stage: &'static str,
|
||||
object_class: &'static str,
|
||||
size_bucket: &'static str,
|
||||
duration_secs: f64,
|
||||
) {
|
||||
let sample_rate = get_stage_local_summary_sample_rate();
|
||||
if sample_rate > 1
|
||||
&& !GET_STAGE_LOCAL_SUMMARY_SEQUENCE
|
||||
.fetch_add(1, Ordering::Relaxed)
|
||||
.is_multiple_of(sample_rate)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
let Ok(duration) = std::time::Duration::try_from_secs_f64(duration_secs) else {
|
||||
return;
|
||||
};
|
||||
let Ok(duration_nanoseconds) = u64::try_from(duration.as_nanos()) else {
|
||||
return;
|
||||
};
|
||||
|
||||
let key = (path, stage, object_class, size_bucket);
|
||||
let Ok(mut summary) = GET_STAGE_LOCAL_SUMMARY.lock() else {
|
||||
return;
|
||||
};
|
||||
let row = summary.entry(key).or_insert_with(|| GetStageLocalSummary {
|
||||
path,
|
||||
stage,
|
||||
object_class,
|
||||
size_bucket,
|
||||
sampled_count: 0,
|
||||
duration_nanoseconds: 0,
|
||||
});
|
||||
row.sampled_count = row.sampled_count.saturating_add(1);
|
||||
row.duration_nanoseconds = row.duration_nanoseconds.saturating_add(duration_nanoseconds);
|
||||
}
|
||||
|
||||
/// Enable or disable general (non GET/PUT-stage) metric emission.
|
||||
///
|
||||
/// Called once during startup, typically gated by `rustfs_obs::observability_metric_enabled()`.
|
||||
@@ -110,8 +194,210 @@ pub fn put_stage_timer() -> Option<std::time::Instant> {
|
||||
}
|
||||
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT: &str = "put_object_commit_namespace_lock_wait";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD: &str = "put_object_commit_namespace_lock_held";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_RENAME: &str = "put_object_commit_namespace_lock_held_rename";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_PRE_RENAME_PREPARE: &str =
|
||||
"put_object_commit_namespace_lock_held_pre_rename_prepare";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_RENAME_RESULT_TO_TAIL_HANDOFF: &str =
|
||||
"put_object_commit_namespace_lock_held_rename_result_to_tail_handoff";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_TAIL_HANDOFF: &str =
|
||||
"put_object_commit_namespace_lock_held_tail_handoff";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_POST_RENAME: &str = "put_object_commit_namespace_lock_held_post_rename";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_GUARD_RELEASE: &str =
|
||||
"put_object_commit_namespace_lock_held_guard_release";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE: &str =
|
||||
"put_object_commit_namespace_lock_held_quota_fence_release";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_COMMIT: &str =
|
||||
"put_object_commit_namespace_lock_held_quota_commit";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT: &str = "put_object_commit_namespace_lock_held_heal_submit";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT: &str =
|
||||
"put_object_commit_namespace_lock_held_cleanup_receipt";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_METADATA_INVALIDATE: &str =
|
||||
"put_object_commit_namespace_lock_held_metadata_invalidate";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_OBJECT_INFO_BUILD: &str =
|
||||
"put_object_commit_namespace_lock_held_object_info_build";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_POST_OBJECT_INFO_TO_RETURN: &str =
|
||||
"put_object_commit_namespace_lock_held_post_object_info_to_return";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_TASK_WAIT: &str = "put_object_commit_task_wait";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_FUTURE_TOTAL: &str = "put_object_commit_future_total";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_OPERATION_SELECT_WAIT: &str = "put_object_commit_operation_select_wait";
|
||||
pub const PUT_STAGE_PUT_OBJECT_OPERATION_AWAIT_TOTAL: &str = "put_object_operation_await_total";
|
||||
pub const PUT_STAGE_PUT_OBJECT_OPERATION_BODY_TO_COMMIT_SUBMIT: &str = "put_object_operation_body_to_commit_submit";
|
||||
pub const PUT_STAGE_PUT_OBJECT_OPERATION_RESULT_RETURN: &str = "put_object_operation_result_return";
|
||||
pub const PUT_STAGE_PUT_OBJECT_OPERATION_FIRST_POLL_DELAY: &str = "put_object_operation_first_poll_delay";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_ACQUIRED_TO_COMMIT_READY: &str =
|
||||
"put_object_commit_namespace_lock_acquired_to_commit_ready";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_PRECONDITION_AND_TIMESTAMP: &str =
|
||||
"put_object_commit_ready_precondition_and_timestamp";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_EXPECTED_VERSION_LOOKUP: &str = "put_object_commit_ready_expected_version_lookup";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_EXPLICIT_VERSION_LOOKUP: &str = "put_object_commit_ready_explicit_version_lookup";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_RESTORE_VERIFY: &str = "put_object_commit_ready_restore_verify";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_FENCE_VERIFY: &str = "put_object_commit_ready_fence_verify";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_DECOMMISSION_CAPACITY_FENCE: &str =
|
||||
"put_object_commit_ready_decommission_capacity_fence";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_TRANSACTION_EPOCH_FENCE_READ: &str =
|
||||
"put_object_commit_ready_transaction_epoch_fence_read";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_BEGIN: &str = "put_object_commit_ready_quota_begin";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_METADATA_ADJUST: &str = "put_object_commit_ready_metadata_adjust";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_RESERVE: &str = "put_object_commit_ready_quota_reserve";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_MUTATION_FENCE_PREPARE: &str =
|
||||
"put_object_commit_ready_quota_mutation_fence_prepare";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_CONTEXT_PREPARE: &str = "put_object_commit_ready_context_prepare";
|
||||
pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_META_BUCKET_FAST_PATH: &str = "put_object_quota_begin_meta_bucket_fast_path";
|
||||
pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_METADATA_LOCK: &str = "put_object_quota_begin_metadata_lock";
|
||||
pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CACHE_LOOKUP: &str = "put_object_quota_begin_cache_lookup";
|
||||
pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONFIG_READ: &str = "put_object_quota_begin_config_read";
|
||||
pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_LOCK_LOST_CHECK: &str = "put_object_quota_begin_lock_lost_check";
|
||||
pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_POLICY_EVALUATE: &str = "put_object_quota_begin_policy_evaluate";
|
||||
pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CAPABILITY_PROOF: &str = "put_object_quota_begin_capability_proof";
|
||||
pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_SNAPSHOT_ADMISSION: &str = "put_object_quota_begin_snapshot_admission";
|
||||
pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_OBJECT_STORE_LOOKUP: &str = "put_object_quota_begin_object_store_lookup";
|
||||
pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONTEXT_BUILD: &str = "put_object_quota_begin_context_build";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_SUBMIT_TO_CLOSURE_ENTER: &str = "put_object_commit_submit_to_closure_enter";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_CLOSURE_ENTER_TO_FIRST_AWAIT: &str = "put_object_commit_closure_enter_to_first_await";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_CLOSURE_TOTAL: &str = "put_object_commit_closure_total";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_BEFORE_RENAME: &str = "put_object_commit_before_rename";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_AWAIT_TOTAL: &str = "put_object_commit_pre_rename_await_total";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_QUOTA_MARK_COMMIT_STARTED: &str =
|
||||
"put_object_commit_pre_rename_quota_mark_commit_started";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_FIRST_FENCE_CHECK: &str = "put_object_commit_pre_rename_first_fence_check";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_RESTORE_VERIFY: &str = "put_object_commit_pre_rename_restore_verify";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_TRANSACTION_FENCE_VERIFY: &str =
|
||||
"put_object_commit_pre_rename_transaction_fence_verify";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_SECOND_FENCE_CHECK: &str = "put_object_commit_pre_rename_second_fence_check";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_SCANNER_SCOPE_CHECK: &str = "put_object_commit_pre_rename_scanner_scope_check";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_TIER_FREE_VERSION_SOURCE_LOOKUP: &str =
|
||||
"put_object_commit_pre_rename_tier_free_version_source_lookup";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_ASSIGN_RENAME_INDEXES: &str =
|
||||
"put_object_commit_pre_rename_assign_rename_indexes";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_POST_AWAIT_TO_RENAME: &str = "put_object_commit_pre_rename_post_await_to_rename";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_AFTER_RENAME_TO_RETURN: &str = "put_object_commit_after_rename_to_return";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_QUORUM_WAIT: &str = "set_disk_rename_quorum_wait";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_SYNC_FULL_FANOUT_WAIT: &str = "set_disk_rename_sync_full_fanout_wait";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_SPAWN_TO_QUORUM_WAIT: &str = "set_disk_rename_early_ack_spawn_to_quorum_wait";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_ATTACH_TAIL_OWNER: &str = "set_disk_rename_early_ack_attach_tail_owner";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_AFTER_QUORUM_WAIT: &str = "set_disk_rename_tail_after_quorum_wait";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_ASYNC_WAIT: &str = "set_disk_rename_tail_async_wait";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_GUARD_RELEASE_WAIT: &str = "set_disk_rename_tail_guard_release_wait";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_FINALIZE: &str = "set_disk_rename_tail_finalize";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP: &str = "set_disk_rename_tail_cleanup";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_SUBMIT: &str = "set_disk_rename_tail_cleanup_submit";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP: &str = "set_disk_rename_tail_cleanup_counterfactual_skip";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_ENQUEUED: &str = "set_disk_rename_tail_cleanup_defer_enqueued";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_FALLBACK_SYNC: &str = "set_disk_rename_tail_cleanup_defer_fallback_sync";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_PUSH: &str = "set_disk_rename_tail_cleanup_defer_queue_push";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH: &str = "set_disk_rename_tail_cleanup_defer_queue_depth";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH_MAX_SAMPLE: &str =
|
||||
"set_disk_rename_tail_cleanup_defer_queue_depth_max_sample";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE: &str = "set_disk_rename_tail_cleanup_defer_oldest_age";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE_MAX_SAMPLE: &str =
|
||||
"set_disk_rename_tail_cleanup_defer_oldest_age_max_sample";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_TARGETS: &str = "set_disk_rename_tail_cleanup_defer_targets";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_START_DELAY: &str = "set_disk_rename_tail_cleanup_defer_start_delay";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_COMPLETED_LAG: &str = "set_disk_rename_tail_cleanup_defer_completed_lag";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER: &str = "set_disk_rename_tail_cleanup_defer_hold_worker";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_DELAY: &str = "set_disk_rename_tail_cleanup_defer_worker_delay";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_PER_DISK_PERMIT_WAIT: &str = "set_disk_rename_tail_cleanup_per_disk_permit_wait";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_PER_DISK_INFLIGHT: &str = "set_disk_rename_tail_cleanup_per_disk_inflight";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_FOREGROUND_YIELD: &str = "set_disk_rename_tail_cleanup_foreground_yield";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFERRED_FOREGROUND_ACTIVE: &str =
|
||||
"set_disk_rename_tail_cleanup_deferred_foreground_active";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_FOREGROUND_ACTIVE: &str = "set_disk_rename_foreground_active";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BATCH_WINDOW: &str = "set_disk_rename_tail_cleanup_defer_batch_window";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BATCH_SIZE: &str = "set_disk_rename_tail_cleanup_defer_batch_size";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BURST_BATCH_SIZE: &str =
|
||||
"set_disk_rename_tail_cleanup_defer_burst_batch_size";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_ACTIVE: &str = "set_disk_rename_tail_cleanup_defer_worker_active";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_POLL_WAIT: &str =
|
||||
"set_disk_rename_tail_cleanup_defer_worker_poll_wait";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_IDLE_WAIT: &str =
|
||||
"set_disk_rename_tail_cleanup_defer_worker_idle_wait";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_JOB_WAIT: &str =
|
||||
"set_disk_rename_tail_cleanup_defer_worker_job_wait";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_COMPLETED: &str = "set_disk_rename_tail_cleanup_defer_completed";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_DURATION: &str =
|
||||
"set_disk_rename_tail_cleanup_defer_worker_duration";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_RECEIPT: &str = "set_disk_rename_tail_cleanup_target_receipt";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_COMMIT_CAPACITY: &str =
|
||||
"set_disk_rename_tail_cleanup_target_commit_capacity";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_REPORT: &str = "set_disk_rename_tail_cleanup_target_report";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_COUNT: &str = "set_disk_rename_tail_cleanup_target_count";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_ZERO_TARGET: &str = "set_disk_rename_tail_cleanup_zero_target";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TMP_DELETE: &str = "set_disk_rename_tail_cleanup_tmp_delete";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TMP_DELETE_COUNTERFACTUAL_SKIP: &str =
|
||||
"set_disk_rename_tail_cleanup_tmp_delete_counterfactual_skip";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DECOMMISSION_GUARD_DROP: &str =
|
||||
"set_disk_rename_tail_cleanup_decommission_guard_drop";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_DEPTH: &str = "set_disk_rename_tail_cleanup_lane_queue_depth";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_PUSH: &str = "set_disk_rename_tail_cleanup_lane_queue_push";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_DEPTH_MAX_SAMPLE: &str =
|
||||
"set_disk_rename_tail_cleanup_lane_queue_depth_max_sample";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_OLDEST_AGE: &str = "set_disk_rename_tail_cleanup_lane_oldest_age";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_OLDEST_AGE_MAX_SAMPLE: &str =
|
||||
"set_disk_rename_tail_cleanup_lane_oldest_age_max_sample";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_BATCH_SIZE: &str = "set_disk_rename_tail_cleanup_lane_batch_size";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_BURST_BATCH_SIZE: &str =
|
||||
"set_disk_rename_tail_cleanup_lane_burst_batch_size";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_ACTIVE: &str = "set_disk_rename_tail_cleanup_lane_worker_active";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_POLL_WAIT: &str =
|
||||
"set_disk_rename_tail_cleanup_lane_worker_poll_wait";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_IDLE_WAIT: &str =
|
||||
"set_disk_rename_tail_cleanup_lane_worker_idle_wait";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_JOB_WAIT: &str = "set_disk_rename_tail_cleanup_lane_worker_job_wait";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_SERVICE_TIME: &str = "set_disk_rename_tail_cleanup_lane_service_time";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_REQUEUE: &str = "set_disk_rename_tail_cleanup_lane_requeue";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_FOREGROUND_ACTIVE_HIT: &str =
|
||||
"set_disk_rename_tail_cleanup_lane_foreground_active_hit";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_HEAL_SUBMIT: &str = "set_disk_rename_tail_heal_submit";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_DISK_WAIT: &str = "set_disk_rename_disk_wait";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_LOCAL: &str = "set_disk_rename_disk_wait_local";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_REMOTE: &str = "set_disk_rename_disk_wait_remote";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_PREPARE: &str = "set_disk_rename_remote_client_prepare";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC: &str = "set_disk_rename_remote_client_rpc";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT: &str = "set_disk_rename_remote_client_rpc_await";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_POLL_PENDING_COUNT: &str =
|
||||
"set_disk_rename_remote_client_rpc_await_poll_pending_count";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_FIRST_PENDING_TO_READY: &str =
|
||||
"set_disk_rename_remote_client_rpc_await_first_pending_to_ready";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_REQUEST_SCOPE: &str = "set_disk_rename_remote_client_request_scope";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL: &str = "set_disk_rename_remote_client_transport_call";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL_POLL_PENDING_COUNT: &str =
|
||||
"set_disk_rename_remote_client_transport_call_poll_pending_count";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL_FIRST_PENDING_TO_READY: &str =
|
||||
"set_disk_rename_remote_client_transport_call_first_pending_to_ready";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_REPLAY_RESPONSE: &str = "set_disk_rename_remote_client_replay_response";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_INTO_INNER: &str = "set_disk_rename_remote_client_into_inner";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_SUCCESS_CHECK: &str = "set_disk_rename_remote_client_success_check";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_DECODE: &str = "set_disk_rename_remote_client_decode";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RESPONSE_PROCESS: &str = "set_disk_rename_remote_client_response_process";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RESPONSE_HEADERS: &str = "set_disk_rename_remote_client_response_headers";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_ENCODE: &str = "set_disk_rename_remote_service_response_encode";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE_TRIGGERED: &str =
|
||||
"put_object_commit_namespace_lock_held_quota_fence_release_triggered";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE_SKIPPED: &str =
|
||||
"put_object_commit_namespace_lock_held_quota_fence_release_skipped";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT_TRIGGERED: &str =
|
||||
"put_object_commit_namespace_lock_held_heal_submit_triggered";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT_SKIPPED: &str =
|
||||
"put_object_commit_namespace_lock_held_heal_submit_skipped";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT_TRIGGERED: &str =
|
||||
"put_object_commit_namespace_lock_held_cleanup_receipt_triggered";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT_SKIPPED: &str =
|
||||
"put_object_commit_namespace_lock_held_cleanup_receipt_skipped";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_TOTAL: &str = "set_disk_rename_remote_handler_total";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_OVERHEAD: &str = "set_disk_rename_remote_handler_overhead";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOOKUP: &str = "set_disk_rename_remote_handler_lookup";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_ADMIT: &str = "set_disk_rename_remote_handler_admit";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOCAL_RENAME: &str = "set_disk_rename_remote_handler_local_rename";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_ENTRY: &str = "set_disk_rename_remote_service_entry";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_BEFORE_HANDLER: &str = "set_disk_rename_remote_service_before_handler";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_HANDLER_AWAIT: &str = "set_disk_rename_remote_service_handler_await";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_AFTER_HANDLER: &str = "set_disk_rename_remote_service_after_handler";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_BUILD: &str = "set_disk_rename_remote_service_response_build";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN: &str = "set_disk_rename_remote_service_response_return";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_SERVICE_FUTURE: &str = "set_disk_rename_remote_transport_service_future";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_FIRST_FRAME: &str = "set_disk_rename_remote_transport_first_frame";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_BODY_COMPLETE: &str = "set_disk_rename_remote_transport_body_complete";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_FILE_SYNC_PERMIT_WAIT: &str = "set_disk_rename_file_sync_permit_wait";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_GLOBAL_FILE_SYNC_PERMIT_WAIT: &str = "set_disk_rename_global_file_sync_permit_wait";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_FILE_FDATASYNC: &str = "set_disk_rename_file_fdatasync";
|
||||
@@ -260,8 +546,8 @@ pub use io_metrics::{
|
||||
pub use list_objects_metrics::{
|
||||
LIST_OBJECTS_GATHER_OUTCOME_INPUT_CLOSED, LIST_OBJECTS_GATHER_OUTCOME_LIMIT_REACHED,
|
||||
LIST_OBJECTS_LOCAL_READ_DIR_OUTCOME_ERROR, LIST_OBJECTS_LOCAL_READ_DIR_OUTCOME_OK, LIST_OBJECTS_SOURCE_WALKER,
|
||||
ListObjectsGatherObservation, ListObjectsIndexPageObservation, ListObjectsLocalReadDirObservation, init_list_objects_metrics,
|
||||
record_list_objects_gather, record_list_objects_index_attempt, record_list_objects_index_fallback,
|
||||
ListObjectsGatherCaller, ListObjectsGatherObservation, ListObjectsIndexPageObservation, ListObjectsLocalReadDirObservation,
|
||||
init_list_objects_metrics, record_list_objects_gather, record_list_objects_index_attempt, record_list_objects_index_fallback,
|
||||
record_list_objects_index_live_verify_failure, record_list_objects_index_served, record_list_objects_local_read_dir,
|
||||
record_list_objects_merge,
|
||||
};
|
||||
@@ -460,7 +746,16 @@ pub fn record_get_object_request_start(concurrent_requests: usize) {
|
||||
gauge!("rustfs_io_get_object_concurrent_requests").set(concurrent_requests as f64);
|
||||
}
|
||||
|
||||
/// Record GetObject request start without concurrency context.
|
||||
/// Record one timeout-wrapped GetObject storage operation start.
|
||||
#[inline(always)]
|
||||
pub fn record_get_object_timeout_operation_start() {
|
||||
if !get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
counter!("rustfs_io_get_object_timeout_operations_total").increment(1);
|
||||
}
|
||||
|
||||
/// Record a GetObject request start without concurrency context.
|
||||
#[inline(always)]
|
||||
pub fn record_get_object_request_started() {
|
||||
if !get_stage_metrics_enabled() {
|
||||
@@ -479,6 +774,16 @@ pub fn record_get_object_request_result(status: &str, duration_secs: f64) {
|
||||
histogram!("rustfs_io_get_object_request_duration_seconds", "status" => status.to_string()).record(duration_secs);
|
||||
}
|
||||
|
||||
/// Record the result of one timeout-wrapped GetObject storage operation.
|
||||
#[inline(always)]
|
||||
pub fn record_get_object_timeout_operation_result(status: &str, duration_secs: f64) {
|
||||
if !get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
counter!("rustfs_io_get_object_timeout_operation_results_total", "status" => status.to_string()).increment(1);
|
||||
histogram!("rustfs_io_get_object_timeout_operation_duration_seconds", "status" => status.to_string()).record(duration_secs);
|
||||
}
|
||||
|
||||
/// Record PutObject request start.
|
||||
#[inline(always)]
|
||||
pub fn record_put_object_request_start(concurrent_requests: usize) {
|
||||
@@ -760,6 +1065,10 @@ pub fn record_get_object_io_state(
|
||||
/// Record GetObject phase duration for the current read path.
|
||||
#[inline(always)]
|
||||
pub fn record_get_object_stage_duration(path: &'static str, stage: &'static str, duration_secs: f64) {
|
||||
if get_stage_local_summary_enabled() {
|
||||
record_get_stage_local_summary(path, stage, "", "", duration_secs);
|
||||
return;
|
||||
}
|
||||
if !get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
@@ -775,6 +1084,10 @@ pub fn record_get_object_stage_duration_by_size(
|
||||
size_bucket: &'static str,
|
||||
duration_secs: f64,
|
||||
) {
|
||||
if get_stage_local_summary_enabled() {
|
||||
record_get_stage_local_summary(path, stage, object_class, size_bucket, duration_secs);
|
||||
return;
|
||||
}
|
||||
if !get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
@@ -827,6 +1140,15 @@ pub fn record_get_object_metadata_response(path: &'static str, outcome: &'static
|
||||
counter!("rustfs_io_get_object_metadata_response_total", "path" => path, "outcome" => outcome).increment(1);
|
||||
}
|
||||
|
||||
/// Record whether metadata resolution selected an object identity at read quorum.
|
||||
#[inline(always)]
|
||||
pub fn record_get_object_metadata_quorum_result(path: &'static str, outcome: &'static str) {
|
||||
if !get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
counter!("rustfs_io_get_object_metadata_quorum_results_total", "path" => path, "outcome" => outcome).increment(1);
|
||||
}
|
||||
|
||||
/// Record one bounded metadata cache decision.
|
||||
#[inline(always)]
|
||||
pub fn record_get_object_metadata_cache_decision(path: &'static str, decision: &'static str, reason: &'static str) {
|
||||
@@ -1668,6 +1990,10 @@ pub fn record_get_object_metadata_phase_duration(duration_secs: f64) {
|
||||
/// Record metadata phase duration with early-stop state label.
|
||||
#[inline(always)]
|
||||
pub fn record_get_object_metadata_phase_duration_with_early_stop(duration_secs: f64, early_stop_active: &'static str) {
|
||||
if get_stage_local_summary_enabled() {
|
||||
record_get_stage_local_summary("legacy_duplex", "metadata", early_stop_active, "", duration_secs);
|
||||
return;
|
||||
}
|
||||
if !get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
@@ -2073,6 +2399,57 @@ pub fn record_put_object_stage_duration_from(stage: &'static str, started_at: Op
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_put_object_stage_count(stage: &'static str, count: u64) {
|
||||
if !put_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
counter!("rustfs_s3_put_object_stage_count", "stage" => stage).increment(count);
|
||||
}
|
||||
|
||||
/// Observe a future's latency and wakeup behavior only while detailed PUT
|
||||
/// attribution is enabled. The disabled path awaits the original future
|
||||
/// directly, so normal remote mutation RPCs do not gain a polling wrapper.
|
||||
pub async fn observe_put_stage_future<F>(
|
||||
future: F,
|
||||
duration_stage: &'static str,
|
||||
pending_count_stage: &'static str,
|
||||
first_pending_to_ready_stage: &'static str,
|
||||
) -> F::Output
|
||||
where
|
||||
F: std::future::Future,
|
||||
{
|
||||
if !put_stage_metrics_enabled() {
|
||||
return future.await;
|
||||
}
|
||||
|
||||
let duration_started = put_stage_timer();
|
||||
let mut first_pending_at = None;
|
||||
let mut pending_count = 0_u64;
|
||||
let mut future = std::pin::pin!(future);
|
||||
let output = std::future::poll_fn(|cx| match future.as_mut().poll(cx) {
|
||||
std::task::Poll::Ready(output) => std::task::Poll::Ready(output),
|
||||
std::task::Poll::Pending => {
|
||||
pending_count = pending_count.saturating_add(1);
|
||||
first_pending_at.get_or_insert_with(std::time::Instant::now);
|
||||
std::task::Poll::Pending
|
||||
}
|
||||
})
|
||||
.await;
|
||||
record_put_object_stage_duration_from(duration_stage, duration_started);
|
||||
record_put_object_stage_count(pending_count_stage, pending_count);
|
||||
record_put_object_stage_duration_from(first_pending_to_ready_stage, first_pending_at);
|
||||
output
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_put_object_quota_cache_decision(decision: &'static str, reason: &'static str) {
|
||||
if !put_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
counter!("rustfs_s3_put_object_quota_cache_total", "decision" => decision, "reason" => reason).increment(1);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_put_object_commit_lock_admission(budget: &'static str, outcome: &'static str) {
|
||||
if !put_stage_metrics_enabled() {
|
||||
@@ -3219,8 +3596,162 @@ mod tests {
|
||||
assert_eq!(PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT, "put_object_commit_namespace_lock_wait");
|
||||
let stages = [
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_RENAME,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_PRE_RENAME_PREPARE,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_RENAME_RESULT_TO_TAIL_HANDOFF,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_TAIL_HANDOFF,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_POST_RENAME,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_GUARD_RELEASE,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_COMMIT,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_METADATA_INVALIDATE,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_OBJECT_INFO_BUILD,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_POST_OBJECT_INFO_TO_RETURN,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_TASK_WAIT,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_FUTURE_TOTAL,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_OPERATION_SELECT_WAIT,
|
||||
PUT_STAGE_PUT_OBJECT_OPERATION_AWAIT_TOTAL,
|
||||
PUT_STAGE_PUT_OBJECT_OPERATION_BODY_TO_COMMIT_SUBMIT,
|
||||
PUT_STAGE_PUT_OBJECT_OPERATION_RESULT_RETURN,
|
||||
PUT_STAGE_PUT_OBJECT_OPERATION_FIRST_POLL_DELAY,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_ACQUIRED_TO_COMMIT_READY,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_READY_PRECONDITION_AND_TIMESTAMP,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_READY_EXPECTED_VERSION_LOOKUP,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_READY_EXPLICIT_VERSION_LOOKUP,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_READY_RESTORE_VERIFY,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_READY_FENCE_VERIFY,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_READY_DECOMMISSION_CAPACITY_FENCE,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_READY_TRANSACTION_EPOCH_FENCE_READ,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_BEGIN,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_READY_METADATA_ADJUST,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_RESERVE,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_MUTATION_FENCE_PREPARE,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_READY_CONTEXT_PREPARE,
|
||||
PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_META_BUCKET_FAST_PATH,
|
||||
PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_METADATA_LOCK,
|
||||
PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CACHE_LOOKUP,
|
||||
PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONFIG_READ,
|
||||
PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_LOCK_LOST_CHECK,
|
||||
PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_POLICY_EVALUATE,
|
||||
PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CAPABILITY_PROOF,
|
||||
PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_SNAPSHOT_ADMISSION,
|
||||
PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_OBJECT_STORE_LOOKUP,
|
||||
PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONTEXT_BUILD,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_SUBMIT_TO_CLOSURE_ENTER,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_CLOSURE_ENTER_TO_FIRST_AWAIT,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_CLOSURE_TOTAL,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_BEFORE_RENAME,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_AWAIT_TOTAL,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_QUOTA_MARK_COMMIT_STARTED,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_FIRST_FENCE_CHECK,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_RESTORE_VERIFY,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_TRANSACTION_FENCE_VERIFY,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_SECOND_FENCE_CHECK,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_SCANNER_SCOPE_CHECK,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_TIER_FREE_VERSION_SOURCE_LOOKUP,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_ASSIGN_RENAME_INDEXES,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_POST_AWAIT_TO_RENAME,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_AFTER_RENAME_TO_RETURN,
|
||||
PUT_STAGE_SET_DISK_RENAME_QUORUM_WAIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_SYNC_FULL_FANOUT_WAIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_SPAWN_TO_QUORUM_WAIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_ATTACH_TAIL_OWNER,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_AFTER_QUORUM_WAIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_ASYNC_WAIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_GUARD_RELEASE_WAIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_FINALIZE,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_SUBMIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_ENQUEUED,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_FALLBACK_SYNC,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_PUSH,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH_MAX_SAMPLE,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE_MAX_SAMPLE,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_TARGETS,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_START_DELAY,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_COMPLETED_LAG,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_DELAY,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_PER_DISK_PERMIT_WAIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_PER_DISK_INFLIGHT,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_FOREGROUND_YIELD,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFERRED_FOREGROUND_ACTIVE,
|
||||
PUT_STAGE_SET_DISK_RENAME_FOREGROUND_ACTIVE,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BATCH_WINDOW,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BATCH_SIZE,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BURST_BATCH_SIZE,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_ACTIVE,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_POLL_WAIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_IDLE_WAIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_JOB_WAIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_COMPLETED,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_DURATION,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_RECEIPT,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_COMMIT_CAPACITY,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_REPORT,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_COUNT,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_ZERO_TARGET,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TMP_DELETE,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TMP_DELETE_COUNTERFACTUAL_SKIP,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DECOMMISSION_GUARD_DROP,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_DEPTH,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_PUSH,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_DEPTH_MAX_SAMPLE,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_OLDEST_AGE,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_OLDEST_AGE_MAX_SAMPLE,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_BATCH_SIZE,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_BURST_BATCH_SIZE,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_ACTIVE,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_POLL_WAIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_IDLE_WAIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_JOB_WAIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_SERVICE_TIME,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_REQUEUE,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_FOREGROUND_ACTIVE_HIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_HEAL_SUBMIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_DISK_WAIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_LOCAL,
|
||||
PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_REMOTE,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_PREPARE,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_FIRST_PENDING_TO_READY,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_REQUEST_SCOPE,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL_FIRST_PENDING_TO_READY,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_REPLAY_RESPONSE,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_INTO_INNER,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_SUCCESS_CHECK,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_DECODE,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RESPONSE_PROCESS,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RESPONSE_HEADERS,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_ENCODE,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE_TRIGGERED,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE_SKIPPED,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT_TRIGGERED,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT_SKIPPED,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT_TRIGGERED,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT_SKIPPED,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_TOTAL,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_OVERHEAD,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOOKUP,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_ADMIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOCAL_RENAME,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_ENTRY,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_BEFORE_HANDLER,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_HANDLER_AWAIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_AFTER_HANDLER,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_BUILD,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_SERVICE_FUTURE,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_FIRST_FRAME,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_BODY_COMPLETE,
|
||||
PUT_STAGE_SET_DISK_RENAME_FILE_SYNC_PERMIT_WAIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_GLOBAL_FILE_SYNC_PERMIT_WAIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_FILE_FDATASYNC,
|
||||
@@ -3233,11 +3764,36 @@ mod tests {
|
||||
let unique = stages.iter().copied().collect::<HashSet<_>>();
|
||||
assert_eq!(unique.len(), stages.len());
|
||||
assert!(stages.iter().all(|stage| {
|
||||
(stage.starts_with("set_disk_rename_") || *stage == PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT)
|
||||
(stage.starts_with("set_disk_rename_")
|
||||
|| *stage == PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT
|
||||
|| *stage == PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD
|
||||
|| *stage == PUT_STAGE_PUT_OBJECT_COMMIT_TASK_WAIT
|
||||
|| *stage == PUT_STAGE_PUT_OBJECT_COMMIT_FUTURE_TOTAL
|
||||
|| *stage == PUT_STAGE_PUT_OBJECT_COMMIT_OPERATION_SELECT_WAIT
|
||||
|| *stage == PUT_STAGE_PUT_OBJECT_OPERATION_AWAIT_TOTAL
|
||||
|| *stage == PUT_STAGE_PUT_OBJECT_OPERATION_BODY_TO_COMMIT_SUBMIT
|
||||
|| *stage == PUT_STAGE_PUT_OBJECT_OPERATION_RESULT_RETURN
|
||||
|| *stage == PUT_STAGE_PUT_OBJECT_OPERATION_FIRST_POLL_DELAY
|
||||
|| *stage == PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_ACQUIRED_TO_COMMIT_READY
|
||||
|| stage.starts_with("put_object_commit_ready_")
|
||||
|| stage.starts_with("put_object_quota_begin_")
|
||||
|| *stage == PUT_STAGE_PUT_OBJECT_COMMIT_SUBMIT_TO_CLOSURE_ENTER
|
||||
|| *stage == PUT_STAGE_PUT_OBJECT_COMMIT_CLOSURE_ENTER_TO_FIRST_AWAIT
|
||||
|| *stage == PUT_STAGE_PUT_OBJECT_COMMIT_CLOSURE_TOTAL
|
||||
|| *stage == PUT_STAGE_PUT_OBJECT_COMMIT_BEFORE_RENAME
|
||||
|| stage.starts_with("put_object_commit_pre_rename_")
|
||||
|| *stage == PUT_STAGE_PUT_OBJECT_COMMIT_AFTER_RENAME_TO_RETURN
|
||||
|| stage.starts_with("put_object_commit_namespace_lock_held_"))
|
||||
&& !stage.contains('/')
|
||||
&& !stage.contains('{')
|
||||
}));
|
||||
|
||||
let count_stages = [
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_POLL_PENDING_COUNT,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL_POLL_PENDING_COUNT,
|
||||
];
|
||||
assert_eq!(count_stages.iter().copied().collect::<HashSet<_>>().len(), count_stages.len());
|
||||
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
@@ -3245,16 +3801,25 @@ mod tests {
|
||||
for stage in stages {
|
||||
record_put_object_stage_duration(stage, 1.0);
|
||||
}
|
||||
for stage in count_stages {
|
||||
record_put_object_stage_count(stage, 1);
|
||||
}
|
||||
set_put_stage_metrics_enabled(true);
|
||||
for stage in stages {
|
||||
record_put_object_stage_duration(stage, 1.0);
|
||||
}
|
||||
for stage in count_stages {
|
||||
record_put_object_stage_count(stage, 1);
|
||||
}
|
||||
set_put_stage_metrics_enabled(false);
|
||||
});
|
||||
|
||||
let recorded = snapshotter
|
||||
.snapshot()
|
||||
.into_vec()
|
||||
let rows = snapshotter.snapshot().into_vec();
|
||||
assert_eq!(
|
||||
counter_total(&rows, "rustfs_s3_put_object_stage_count"),
|
||||
Some(u64::try_from(count_stages.len()).expect("count stage length fits u64")),
|
||||
);
|
||||
let recorded = rows
|
||||
.into_iter()
|
||||
.filter(|(composite, _, _, _)| {
|
||||
composite.kind() == MetricKind::Histogram && composite.key().name() == "rustfs_s3_put_object_stage_duration_ms"
|
||||
@@ -3349,6 +3914,63 @@ mod tests {
|
||||
])));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_stage_future_observer_preserves_outputs_and_counts_pending() {
|
||||
let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.build()
|
||||
.expect("build current-thread runtime for observer test");
|
||||
let duration_stage = PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT;
|
||||
let count_stage = PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_POLL_PENDING_COUNT;
|
||||
let pending_stage = PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_FIRST_PENDING_TO_READY;
|
||||
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
set_put_stage_metrics_enabled(false);
|
||||
let output = runtime.block_on(observe_put_stage_future(
|
||||
std::future::ready(42),
|
||||
duration_stage,
|
||||
count_stage,
|
||||
pending_stage,
|
||||
));
|
||||
assert_eq!(output, 42, "disabled observer must preserve the future output");
|
||||
});
|
||||
assert!(snapshotter.snapshot().into_vec().is_empty(), "disabled observer must emit no metrics");
|
||||
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
set_put_stage_metrics_enabled(true);
|
||||
let mut pending = 2;
|
||||
let future = std::future::poll_fn(|cx| {
|
||||
if pending > 0 {
|
||||
pending -= 1;
|
||||
cx.waker().wake_by_ref();
|
||||
std::task::Poll::Pending
|
||||
} else {
|
||||
std::task::Poll::Ready(Err::<(), _>("original error"))
|
||||
}
|
||||
});
|
||||
let output = runtime.block_on(observe_put_stage_future(future, duration_stage, count_stage, pending_stage));
|
||||
set_put_stage_metrics_enabled(false);
|
||||
assert_eq!(output, Err("original error"), "enabled observer must preserve errors");
|
||||
});
|
||||
|
||||
let rows = snapshotter.snapshot().into_vec();
|
||||
assert_eq!(counter_total(&rows, "rustfs_s3_put_object_stage_count"), Some(2));
|
||||
let count_labels = rows
|
||||
.iter()
|
||||
.filter(|(composite, _, _, _)| composite.key().name() == "rustfs_s3_put_object_stage_count")
|
||||
.flat_map(|(composite, _, _, _)| composite.key().labels().map(|label| label.value().to_string()))
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(count_labels, vec![count_stage.to_string()]);
|
||||
let durations = rows
|
||||
.iter()
|
||||
.filter(|(composite, _, _, _)| composite.key().name() == "rustfs_s3_put_object_stage_duration_ms")
|
||||
.flat_map(|(composite, _, _, _)| composite.key().labels().map(|label| label.value().to_string()))
|
||||
.collect::<HashSet<_>>();
|
||||
assert_eq!(durations, HashSet::from([duration_stage.to_string(), pending_stage.to_string()]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_rename_code_level_metrics_are_static_and_gated() {
|
||||
let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||||
@@ -3578,6 +4200,35 @@ mod tests {
|
||||
set_put_stage_metrics_enabled(false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn get_timeout_operation_and_metadata_quorum_metrics_have_separate_scopes() {
|
||||
let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
set_get_stage_metrics_enabled(true);
|
||||
record_get_object_request_start(1);
|
||||
record_get_object_request_result("ok", 0.01);
|
||||
record_get_object_timeout_operation_start();
|
||||
record_get_object_timeout_operation_result("success", 0.005);
|
||||
record_get_object_metadata_quorum_result("legacy_duplex", "reached");
|
||||
record_get_object_metadata_quorum_result("legacy_duplex", "not_reached");
|
||||
set_get_stage_metrics_enabled(false);
|
||||
});
|
||||
|
||||
let rows = snapshotter.snapshot().into_vec();
|
||||
assert_eq!(counter_total(&rows, "rustfs_io_get_object_requests_total"), Some(1));
|
||||
assert_eq!(counter_total(&rows, "rustfs_io_get_object_request_results_total"), Some(1));
|
||||
assert_eq!(counter_total(&rows, "rustfs_io_get_object_timeout_operations_total"), Some(1));
|
||||
assert_eq!(counter_total(&rows, "rustfs_io_get_object_timeout_operation_results_total"), Some(1));
|
||||
assert_eq!(counter_total(&rows, "rustfs_io_get_object_metadata_quorum_results_total"), Some(2));
|
||||
assert_eq!(
|
||||
histogram_samples(&rows, "rustfs_io_get_object_timeout_operation_duration_seconds"),
|
||||
vec![0.005]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_record_get_object_path_and_stage() {
|
||||
let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||||
|
||||
@@ -22,7 +22,31 @@ pub const LIST_OBJECTS_GATHER_OUTCOME_LIMIT_REACHED: &str = "limit_reached";
|
||||
pub const LIST_OBJECTS_GATHER_OUTCOME_INPUT_CLOSED: &str = "input_closed";
|
||||
pub const LIST_OBJECTS_MERGE_OUTCOME_STARTED: &str = "started";
|
||||
|
||||
/// Bounded caller class for a ListObjects walker gather.
|
||||
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
|
||||
pub enum ListObjectsGatherCaller {
|
||||
#[default]
|
||||
Unattributed,
|
||||
S3List,
|
||||
S3ListVersions,
|
||||
Lifecycle,
|
||||
Other,
|
||||
}
|
||||
|
||||
impl ListObjectsGatherCaller {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Unattributed => "unattributed",
|
||||
Self::S3List => "s3_list",
|
||||
Self::S3ListVersions => "s3_list_versions",
|
||||
Self::Lifecycle => "lifecycle",
|
||||
Self::Other => "other",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const LIST_OBJECTS_GATHER_TOTAL: &str = "rustfs_s3_list_objects_gather_total";
|
||||
const LIST_OBJECTS_GATHER_CALLER_TOTAL: &str = "rustfs_s3_list_objects_gather_caller_total";
|
||||
const LIST_OBJECTS_GATHER_DURATION_MS: &str = "rustfs_s3_list_objects_gather_duration_ms";
|
||||
const LIST_OBJECTS_GATHER_SCANNED_ENTRIES: &str = "rustfs_s3_list_objects_gather_scanned_entries";
|
||||
const LIST_OBJECTS_GATHER_RETURNED_ENTRIES: &str = "rustfs_s3_list_objects_gather_returned_entries";
|
||||
@@ -53,6 +77,7 @@ pub const LIST_OBJECTS_LOCAL_READ_DIR_OUTCOME_ERROR: &str = "error";
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct ListObjectsGatherObservation {
|
||||
pub source: &'static str,
|
||||
pub caller: ListObjectsGatherCaller,
|
||||
pub outcome: &'static str,
|
||||
pub limit: i32,
|
||||
pub scanned_entries: usize,
|
||||
@@ -213,6 +238,7 @@ pub fn record_list_objects_gather(observation: ListObjectsGatherObservation) {
|
||||
} else {
|
||||
count_as_f64(observation.scanned_entries) / count_as_f64(observation.returned_entries)
|
||||
};
|
||||
let api_operation = crate::s3_http_metrics::current_s3_http_operation_name().unwrap_or("unattributed");
|
||||
|
||||
counter!(
|
||||
LIST_OBJECTS_GATHER_TOTAL,
|
||||
@@ -223,6 +249,14 @@ pub fn record_list_objects_gather(observation: ListObjectsGatherObservation) {
|
||||
"has_marker" => bool_label(observation.has_marker),
|
||||
)
|
||||
.increment(1);
|
||||
counter!(
|
||||
LIST_OBJECTS_GATHER_CALLER_TOTAL,
|
||||
"api_operation" => api_operation,
|
||||
"caller" => observation.caller.as_str(),
|
||||
"source" => observation.source,
|
||||
"outcome" => observation.outcome,
|
||||
)
|
||||
.increment(1);
|
||||
histogram!(
|
||||
LIST_OBJECTS_GATHER_DURATION_MS,
|
||||
"source" => observation.source,
|
||||
@@ -432,6 +466,7 @@ mod tests {
|
||||
let rows = recorded(|| {
|
||||
record_list_objects_gather(ListObjectsGatherObservation {
|
||||
source: LIST_OBJECTS_SOURCE_WALKER,
|
||||
caller: ListObjectsGatherCaller::Unattributed,
|
||||
outcome: LIST_OBJECTS_GATHER_OUTCOME_INPUT_CLOSED,
|
||||
limit: 1001,
|
||||
scanned_entries: 42,
|
||||
@@ -444,6 +479,17 @@ mod tests {
|
||||
});
|
||||
|
||||
assert_eq!(counter_total(&rows, LIST_OBJECTS_GATHER_TOTAL), Some(1));
|
||||
assert_eq!(counter_total(&rows, LIST_OBJECTS_GATHER_CALLER_TOTAL), Some(1));
|
||||
let caller_labels = rows
|
||||
.iter()
|
||||
.find(|(composite, _, _, _)| composite.key().name() == LIST_OBJECTS_GATHER_CALLER_TOTAL)
|
||||
.expect("internal caller counter should be emitted")
|
||||
.0
|
||||
.key()
|
||||
.labels()
|
||||
.map(|label| (label.key().to_string(), label.value().to_string()))
|
||||
.collect::<std::collections::HashMap<_, _>>();
|
||||
assert_eq!(caller_labels.get("api_operation").map(String::as_str), Some("unattributed"));
|
||||
// Zero returned entries must not divide: the amplification reports the
|
||||
// scan count itself rather than an infinity or a NaN.
|
||||
assert_eq!(histogram_samples(&rows, LIST_OBJECTS_GATHER_SCAN_AMPLIFICATION), vec![42.0]);
|
||||
@@ -452,6 +498,44 @@ mod tests {
|
||||
assert_eq!(histogram_samples(&rows, LIST_OBJECTS_GATHER_DURATION_MS), vec![3.5]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gather_caller_metric_attributes_the_s3_api_operation() {
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
|
||||
let rows = recorded(|| {
|
||||
let mut request = crate::s3_http_metrics::S3HttpRequestGuard::new("GET");
|
||||
request.in_scope(|| {
|
||||
crate::s3_http_metrics::observe_s3_http_operation(S3Operation::ListObjectsV2);
|
||||
record_list_objects_gather(ListObjectsGatherObservation {
|
||||
source: LIST_OBJECTS_SOURCE_WALKER,
|
||||
caller: ListObjectsGatherCaller::S3List,
|
||||
outcome: LIST_OBJECTS_GATHER_OUTCOME_INPUT_CLOSED,
|
||||
limit: 1000,
|
||||
scanned_entries: 1,
|
||||
returned_entries: 1,
|
||||
duration_ms: 1.0,
|
||||
has_prefix: false,
|
||||
has_delimiter: false,
|
||||
has_marker: false,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
assert_eq!(counter_total(&rows, LIST_OBJECTS_GATHER_CALLER_TOTAL), Some(1));
|
||||
let labels = rows
|
||||
.iter()
|
||||
.find(|(composite, _, _, _)| composite.key().name() == LIST_OBJECTS_GATHER_CALLER_TOTAL)
|
||||
.expect("caller counter should be emitted")
|
||||
.0
|
||||
.key()
|
||||
.labels()
|
||||
.map(|label| (label.key().to_string(), label.value().to_string()))
|
||||
.collect::<std::collections::HashMap<_, _>>();
|
||||
assert_eq!(labels.get("api_operation").map(String::as_str), Some("s3:ListObjectsV2"));
|
||||
assert_eq!(labels.get("caller").map(String::as_str), Some("s3_list"));
|
||||
assert_eq!(labels.get("source").map(String::as_str), Some(LIST_OBJECTS_SOURCE_WALKER));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_records_a_zero_read_quorum_rather_than_skipping_it() {
|
||||
let rows = recorded(|| record_list_objects_merge(LIST_OBJECTS_SOURCE_WALKER, 4, 0));
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use std::cell::Cell;
|
||||
use std::future::Future;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{LazyLock, OnceLock};
|
||||
use std::time::{Duration, Instant};
|
||||
@@ -38,6 +39,81 @@ const UNKNOWN_OPERATION: usize = S3Operation::ALL.len();
|
||||
static COUNTERS: LazyLock<HttpOutcomeCounters> = LazyLock::new(HttpOutcomeCounters::new);
|
||||
static COMPLETION_OBSERVER: OnceLock<(S3HttpCompletionObserverEnabled, S3HttpCompletionObserver)> = OnceLock::new();
|
||||
|
||||
/// A bounded S3 operation captured from the current external-request context.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct S3HttpOperationContext(S3Operation);
|
||||
|
||||
impl S3HttpOperationContext {
|
||||
/// Return the stable operation index used for bounded metric-handle lookup.
|
||||
pub fn operation_index(self) -> usize {
|
||||
self.0.metric_index()
|
||||
}
|
||||
|
||||
/// Return the stable S3 operation label associated with this context.
|
||||
pub fn operation_name(self) -> &'static str {
|
||||
self.0.as_str()
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the current S3 operation, when the task is inside a dispatched S3 request.
|
||||
pub fn current_s3_http_operation_context() -> Option<S3HttpOperationContext> {
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
if let Ok(index) = ISOLATED_OPERATION.try_with(Cell::get) {
|
||||
return S3Operation::ALL.get(index).copied().map(S3HttpOperationContext);
|
||||
}
|
||||
CURRENT_OPERATION
|
||||
.try_with(|current| {
|
||||
let index = current.get();
|
||||
S3Operation::ALL.get(index).copied().map(S3HttpOperationContext)
|
||||
})
|
||||
.ok()
|
||||
.flatten()
|
||||
}
|
||||
|
||||
/// Return the current S3 operation label, or `None` for internal/non-S3 work.
|
||||
pub fn current_s3_http_operation_name() -> Option<&'static str> {
|
||||
current_s3_http_operation_context().map(S3HttpOperationContext::operation_name)
|
||||
}
|
||||
|
||||
/// Reapply a captured S3 operation while polling work on a child Tokio task.
|
||||
pub async fn with_s3_http_operation_context<F>(context: Option<S3HttpOperationContext>, future: F) -> F::Output
|
||||
where
|
||||
F: Future,
|
||||
{
|
||||
match context {
|
||||
Some(context) => {
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
return with_isolated_s3_http_operation_context(Some(context), future).await;
|
||||
#[cfg(not(feature = "diagnostic-read-version-caller"))]
|
||||
CURRENT_OPERATION.scope(Cell::new(context.operation_index()), future).await
|
||||
}
|
||||
None => future.await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Restore an item's operation, including explicit absence, without changing
|
||||
/// the ambient HTTP request guard or borrowing the request that flushed a batch.
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
pub async fn with_isolated_s3_http_operation_context<F>(context: Option<S3HttpOperationContext>, future: F) -> F::Output
|
||||
where
|
||||
F: Future,
|
||||
{
|
||||
let index = context.map_or(UNKNOWN_OPERATION, S3HttpOperationContext::operation_index);
|
||||
ISOLATED_OPERATION
|
||||
.scope(Cell::new(index), async {
|
||||
match context {
|
||||
Some(context) => CURRENT_OPERATION.scope(Cell::new(context.operation_index()), future).await,
|
||||
None => future.await,
|
||||
}
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
tokio::task_local! {
|
||||
static ISOLATED_OPERATION: Cell<usize>;
|
||||
}
|
||||
|
||||
tokio::task_local! {
|
||||
static CURRENT_OPERATION: Cell<usize>;
|
||||
}
|
||||
@@ -240,6 +316,25 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn captured_operation_context_can_be_reapplied_to_child_task() {
|
||||
let mut request = S3HttpRequestGuard::new("GET");
|
||||
let operation = request
|
||||
.in_scope(|| {
|
||||
observe_s3_http_operation(S3Operation::GetObject);
|
||||
current_s3_http_operation_context()
|
||||
})
|
||||
.expect("dispatched S3 operation should be captured");
|
||||
|
||||
let child_operation = tokio::spawn(with_s3_http_operation_context(Some(operation), async {
|
||||
current_s3_http_operation_name()
|
||||
}))
|
||||
.await
|
||||
.expect("child task should finish");
|
||||
|
||||
assert_eq!(child_operation, Some("s3:GetObject"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn outcome_counters_distinguish_partial_and_complete_write_failure() {
|
||||
let counters = HttpOutcomeCounters::new();
|
||||
@@ -279,6 +374,39 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[tokio::test]
|
||||
async fn isolated_absent_operation_does_not_inherit_the_flush_request() {
|
||||
let mut request = S3HttpRequestGuard::new("GET");
|
||||
let operation = request.in_scope(|| {
|
||||
observe_s3_http_operation(S3Operation::GetObject);
|
||||
current_s3_http_operation_context().expect("GetObject operation should be captured")
|
||||
});
|
||||
with_s3_http_operation_context(Some(operation), async {
|
||||
assert!(S3HttpRequestGuard::is_active());
|
||||
with_isolated_s3_http_operation_context(None, async {
|
||||
assert_eq!(current_s3_http_operation_context(), None);
|
||||
assert!(
|
||||
S3HttpRequestGuard::is_active(),
|
||||
"isolating an item must leave ambient HTTP accounting intact"
|
||||
);
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(current_s3_http_operation_context(), None);
|
||||
})
|
||||
.await;
|
||||
assert_eq!(current_s3_http_operation_context(), Some(operation));
|
||||
})
|
||||
.await;
|
||||
assert_eq!(current_s3_http_operation_context(), None);
|
||||
with_isolated_s3_http_operation_context(None, async {
|
||||
assert!(
|
||||
!S3HttpRequestGuard::is_active(),
|
||||
"absent operation must not fabricate an HTTP request guard"
|
||||
);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_guard_preserves_operation_across_polls_and_finishes_once() {
|
||||
let totals = || {
|
||||
|
||||
@@ -16,9 +16,9 @@
|
||||
//!
|
||||
//! A `Resource` describes the entity producing telemetry data. The resource
|
||||
//! built here includes the service name, service version, deployment
|
||||
//! environment, the stable RustFS node identity, and the local machine IP
|
||||
//! address so that data can be correlated across services in a distributed
|
||||
//! system.
|
||||
//! environment, the stable RustFS node identity, the process ID, and the local
|
||||
//! machine IP address so telemetry series can be correlated across service
|
||||
//! restarts and nodes.
|
||||
|
||||
use crate::config::OtelConfig;
|
||||
use crate::node_identity::{RUSTFS_NODE_ATTRIBUTE, local_node_identity};
|
||||
@@ -27,7 +27,7 @@ use opentelemetry_sdk::Resource;
|
||||
use opentelemetry_semantic_conventions::{
|
||||
SCHEMA_URL,
|
||||
attribute::{
|
||||
DEPLOYMENT_ENVIRONMENT_NAME, NETWORK_LOCAL_ADDRESS, SERVICE_INSTANCE_ID as OTEL_SERVICE_INSTANCE_ID,
|
||||
DEPLOYMENT_ENVIRONMENT_NAME, NETWORK_LOCAL_ADDRESS, PROCESS_PID, SERVICE_INSTANCE_ID as OTEL_SERVICE_INSTANCE_ID,
|
||||
SERVICE_VERSION as OTEL_SERVICE_VERSION,
|
||||
},
|
||||
};
|
||||
@@ -48,6 +48,8 @@ use std::borrow::Cow;
|
||||
/// - `network.local.address` — the primary local IP of the current host,
|
||||
/// useful as an operational fallback when the stable node name is not yet
|
||||
/// initialized.
|
||||
/// - `process.pid` — distinguishes telemetry emitted by different processes
|
||||
/// running on the same node.
|
||||
///
|
||||
/// All attributes are attached to the resource using the semantic conventions
|
||||
/// schema URL to ensure compatibility with standard OTLP backends.
|
||||
@@ -70,6 +72,7 @@ pub(super) fn build_resource(config: &OtelConfig) -> Resource {
|
||||
KeyValue::new(RUSTFS_NODE_ATTRIBUTE, node_identity.clone()),
|
||||
KeyValue::new(OTEL_SERVICE_INSTANCE_ID, node_identity),
|
||||
KeyValue::new(NETWORK_LOCAL_ADDRESS, local_ip),
|
||||
KeyValue::new(PROCESS_PID, i64::from(std::process::id())),
|
||||
],
|
||||
SCHEMA_URL,
|
||||
)
|
||||
@@ -117,6 +120,12 @@ mod tests {
|
||||
.map(|value| value.to_string()),
|
||||
Some("node1:9000".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
resource
|
||||
.get(&Key::from_static_str(PROCESS_PID))
|
||||
.map(|value| value.to_string()),
|
||||
Some(std::process::id().to_string())
|
||||
);
|
||||
|
||||
rustfs_common::set_global_local_node_name(&previous).await;
|
||||
}
|
||||
|
||||
@@ -31,6 +31,7 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
diagnostic-read-version-caller = ["rustfs-ecstore/diagnostic-read-version-caller"]
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
"hotpath/tokio",
|
||||
|
||||
@@ -100,19 +100,17 @@ pub(crate) async fn read_config_with_revision<S: ScannerObjectIO>(
|
||||
store: Arc<S>,
|
||||
path: &str,
|
||||
) -> StorageResult<(Option<Vec<u8>>, DataUsageCacheRevision)> {
|
||||
match store
|
||||
.get_object_reader(
|
||||
RUSTFS_META_BUCKET,
|
||||
path,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
let read = store.get_object_reader(RUSTFS_META_BUCKET, path, None, HeaderMap::new(), &opts);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let read = rustfs_ecstore::api::diagnostic::with_read_version_entry(
|
||||
rustfs_ecstore::api::diagnostic::ReadVersionEntrySource::ScannerRevisionRead,
|
||||
read,
|
||||
);
|
||||
match read.await {
|
||||
Ok(mut reader) => {
|
||||
let revision = reader
|
||||
.object_info
|
||||
@@ -170,19 +168,17 @@ fn error_chain_has_usage_floor_corruption_signature(error: &(dyn std::error::Err
|
||||
|
||||
/// Read only the object revision without materializing its body.
|
||||
pub(crate) async fn read_config_revision<S: ScannerObjectIO>(store: Arc<S>, path: &str) -> StorageResult<DataUsageCacheRevision> {
|
||||
match store
|
||||
.get_object_reader(
|
||||
RUSTFS_META_BUCKET,
|
||||
path,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
let read = store.get_object_reader(RUSTFS_META_BUCKET, path, None, HeaderMap::new(), &opts);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let read = rustfs_ecstore::api::diagnostic::with_read_version_entry(
|
||||
rustfs_ecstore::api::diagnostic::ReadVersionEntrySource::ScannerRevisionRead,
|
||||
read,
|
||||
);
|
||||
match read.await {
|
||||
Ok(reader) => reader
|
||||
.object_info
|
||||
.etag
|
||||
|
||||
@@ -370,6 +370,17 @@ impl DataUsageCache {
|
||||
}
|
||||
};
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let load_fut = rustfs_ecstore::api::diagnostic::with_read_version_entry(
|
||||
rustfs_ecstore::api::diagnostic::ReadVersionEntrySource::DataUsageCacheRead,
|
||||
async {
|
||||
rustfs_ecstore::api::diagnostic::observe_read_version_context_boundary(
|
||||
rustfs_ecstore::api::diagnostic::ReadVersionContextBoundary::ScannerCacheLoad,
|
||||
);
|
||||
load_fut.await
|
||||
},
|
||||
);
|
||||
|
||||
match timeout(timeout_duration, load_fut).await {
|
||||
Ok(Ok((Some(cache), revision, _))) => Ok(DataUsageCacheLoadAttempt::Loaded {
|
||||
cache: Box::new(cache),
|
||||
|
||||
@@ -113,6 +113,18 @@ impl BackupFallbackStore {
|
||||
enum CacheReadBody {
|
||||
Bytes(Vec<u8>),
|
||||
PrefixThenError(Vec<u8>),
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
Pending,
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
struct PendingCacheReader;
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
impl AsyncRead for PendingCacheReader {
|
||||
fn poll_read(self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
|
||||
Poll::Pending
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -138,7 +150,10 @@ impl AsyncRead for PrefixThenErrorReader {
|
||||
struct CacheReadStore {
|
||||
main: CacheReadBody,
|
||||
backup: Option<Vec<u8>>,
|
||||
legacy: Option<Vec<u8>>,
|
||||
puts: AtomicUsize,
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
entry_sources: Mutex<Vec<Option<rustfs_ecstore::api::diagnostic::ReadVersionEntrySource>>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
@@ -152,7 +167,10 @@ impl CacheReadStore {
|
||||
Self {
|
||||
main,
|
||||
backup,
|
||||
legacy: None,
|
||||
puts: AtomicUsize::new(0),
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
entry_sources: Mutex::new(Vec::new()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -163,6 +181,8 @@ impl CacheReadStore {
|
||||
prefix: Cursor::new(prefix),
|
||||
failed: false,
|
||||
}),
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
CacheReadBody::Pending => Box::new(PendingCacheReader),
|
||||
};
|
||||
ScannerGetObjectReader {
|
||||
stream,
|
||||
@@ -194,6 +214,18 @@ impl ObjectIO for CacheReadStore {
|
||||
_h: Self::HeaderMap,
|
||||
_opts: &Self::ObjectOptions,
|
||||
) -> StorageResult<Self::GetObjectReader> {
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
self.entry_sources
|
||||
.lock()
|
||||
.await
|
||||
.push(rustfs_ecstore::api::diagnostic::current_read_version_entry_source());
|
||||
if bucket == DATA_USAGE_BUCKET.as_str() {
|
||||
return self
|
||||
.legacy
|
||||
.clone()
|
||||
.map(|data| Self::reader(CacheReadBody::Bytes(data), "legacy-etag"))
|
||||
.ok_or(Error::FileNotFound);
|
||||
}
|
||||
if bucket != RUSTFS_META_BUCKET {
|
||||
return Err(Error::FileNotFound);
|
||||
}
|
||||
@@ -400,6 +432,51 @@ fn cache_revisions_map_to_compare_and_swap_preconditions() {
|
||||
assert!(existing.if_none_match_value().is_none());
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[tokio::test]
|
||||
async fn diagnostic_revision_readers_keep_scanner_or_earlier_entry() {
|
||||
use rustfs_ecstore::api::diagnostic::{ReadVersionEntrySource, current_read_version_entry_source, with_read_version_entry};
|
||||
let path = path_join_buf(&[BUCKET_META_PREFIX, DATA_USAGE_CACHE_NAME]);
|
||||
let payload = b"scanner revision diagnostic".to_vec();
|
||||
for earlier in [None, Some(ReadVersionEntrySource::TierConfigReload)] {
|
||||
let store = Arc::new(CacheReadStore::new(CacheReadBody::Bytes(payload.clone()), None));
|
||||
let read = async {
|
||||
let (body, revision) = read_config_with_revision(store.clone(), &path)
|
||||
.await
|
||||
.expect("revision and body read");
|
||||
assert_eq!(body, Some(payload.clone()));
|
||||
assert_eq!(revision, DataUsageCacheRevision::Etag("main-etag".to_string()));
|
||||
assert_eq!(
|
||||
read_config_revision(store.clone(), &path).await.expect("revision-only read"),
|
||||
DataUsageCacheRevision::Etag("main-etag".to_string())
|
||||
);
|
||||
let missing = format!("{path}.missing");
|
||||
assert_eq!(
|
||||
read_config_with_revision(store.clone(), &missing)
|
||||
.await
|
||||
.expect("missing body is harmless"),
|
||||
(None, DataUsageCacheRevision::Missing)
|
||||
);
|
||||
assert_eq!(
|
||||
read_config_revision(store.clone(), &missing)
|
||||
.await
|
||||
.expect("missing revision is harmless"),
|
||||
DataUsageCacheRevision::Missing
|
||||
);
|
||||
};
|
||||
match earlier {
|
||||
Some(source) => with_read_version_entry(source, read).await,
|
||||
None => read.await,
|
||||
}
|
||||
assert_eq!(
|
||||
*store.entry_sources.lock().await,
|
||||
vec![Some(earlier.unwrap_or(ReadVersionEntrySource::ScannerRevisionRead)); 4],
|
||||
"both readers and missing paths preserve the earliest diagnostic source"
|
||||
);
|
||||
assert_eq!(current_read_version_entry_source(), None, "reader scopes must not leak to the caller");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backup_cache_load_recovers_main_revision_before_cas_save() {
|
||||
let mut expected = DataUsageCache::default();
|
||||
@@ -497,6 +574,69 @@ async fn cache_save_reconciles_an_ambiguous_committed_write() {
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[tokio::test]
|
||||
async fn diagnostic_cache_load_attempts_preserve_source_through_fallback_and_timeout() {
|
||||
use rustfs_ecstore::api::diagnostic::{ReadVersionEntrySource, current_read_version_entry_source, with_read_version_entry};
|
||||
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.info.name = "diagnostic-cache".to_string();
|
||||
let bytes = cache.marshal_msg().expect("cache fixture should serialize");
|
||||
for earlier in [None, Some(ReadVersionEntrySource::TierConfigReload)] {
|
||||
for case in ["primary", "legacy", "missing", "body_error", "timeout"] {
|
||||
let body = match case {
|
||||
"body_error" => CacheReadBody::PrefixThenError(vec![0x81, 0xa4, b'n', b'a']),
|
||||
"timeout" => CacheReadBody::Pending,
|
||||
_ => CacheReadBody::Bytes(bytes.clone()),
|
||||
};
|
||||
let mut fixture = CacheReadStore::new(body, None);
|
||||
if case == "legacy" {
|
||||
fixture.legacy = Some(bytes.clone());
|
||||
}
|
||||
let store = Arc::new(fixture);
|
||||
let name = if matches!(case, "legacy" | "missing") {
|
||||
"diagnostic-legacy-cache"
|
||||
} else {
|
||||
DATA_USAGE_CACHE_NAME
|
||||
};
|
||||
let timeout = if case == "timeout" {
|
||||
Duration::from_millis(1)
|
||||
} else {
|
||||
Duration::from_secs(1)
|
||||
};
|
||||
let load = DataUsageCache::try_load_inner(store.clone(), name, timeout);
|
||||
let attempt = match earlier {
|
||||
Some(source) => with_read_version_entry(source, load).await,
|
||||
None => load.await,
|
||||
}
|
||||
.expect("diagnostic wrapper must preserve load-attempt results");
|
||||
match (case, attempt) {
|
||||
("primary", DataUsageCacheLoadAttempt::Loaded { cache, revision }) => {
|
||||
assert_eq!(cache.info.name, "diagnostic-cache");
|
||||
assert_eq!(revision, Some(DataUsageCacheRevision::Etag("main-etag".to_string())));
|
||||
}
|
||||
("legacy", DataUsageCacheLoadAttempt::Loaded { cache, revision }) => {
|
||||
assert_eq!(cache.info.name, "diagnostic-cache");
|
||||
assert_eq!(revision, Some(DataUsageCacheRevision::Missing));
|
||||
}
|
||||
("missing", DataUsageCacheLoadAttempt::Missing { revision }) => {
|
||||
assert_eq!(revision, Some(DataUsageCacheRevision::Missing));
|
||||
}
|
||||
("body_error" | "timeout", DataUsageCacheLoadAttempt::Retryable(_)) => {}
|
||||
_ => panic!("unexpected load-attempt outcome for {case}"),
|
||||
}
|
||||
let calls = if matches!(case, "legacy" | "missing") { 2 } else { 1 };
|
||||
assert_eq!(
|
||||
*store.entry_sources.lock().await,
|
||||
vec![Some(earlier.unwrap_or(ReadVersionEntrySource::DataUsageCacheRead)); calls],
|
||||
"primary and legacy reads must retain this attempt's entry source: {case}"
|
||||
);
|
||||
assert_eq!(store.puts.load(Ordering::SeqCst), 0, "load attempts must not save: {case}");
|
||||
assert_eq!(current_read_version_entry_source(), None, "scope must restore even after timeout: {case}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backup_cache_load_fails_closed_without_main_revision_quorum() {
|
||||
let backup = DataUsageCache::default().marshal_msg().expect("serialize backup cache");
|
||||
|
||||
+34
-22
@@ -734,17 +734,23 @@ where
|
||||
S: ScannerObjectIO,
|
||||
{
|
||||
let mut reader = ScannerPutObjReader::from_vec(data);
|
||||
api.put_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
file,
|
||||
&mut reader,
|
||||
&ScannerObjectOptions {
|
||||
max_parity: true,
|
||||
http_preconditions: Some(preconditions),
|
||||
..Default::default()
|
||||
let opts = ScannerObjectOptions {
|
||||
max_parity: true,
|
||||
http_preconditions: Some(preconditions),
|
||||
..Default::default()
|
||||
};
|
||||
let write = api.put_object(RUSTFS_META_BUCKET, file, &mut reader, &opts);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let write = rustfs_ecstore::api::diagnostic::with_read_version_entry(
|
||||
rustfs_ecstore::api::diagnostic::ReadVersionEntrySource::ScannerConfigWrite,
|
||||
async {
|
||||
rustfs_ecstore::api::diagnostic::observe_read_version_context_boundary(
|
||||
rustfs_ecstore::api::diagnostic::ReadVersionContextBoundary::ScannerConfigWrite,
|
||||
);
|
||||
write.await
|
||||
},
|
||||
)
|
||||
.await
|
||||
);
|
||||
write.await
|
||||
}
|
||||
|
||||
pub(crate) async fn save_config_with_publication_admission_for_epoch<S>(
|
||||
@@ -859,19 +865,25 @@ where
|
||||
fence.to_string(),
|
||||
);
|
||||
}
|
||||
api.put_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
file,
|
||||
&mut reader,
|
||||
&ScannerObjectOptions {
|
||||
max_parity: true,
|
||||
http_preconditions: Some(preconditions),
|
||||
scanner_publication_commit_scope,
|
||||
user_defined,
|
||||
..Default::default()
|
||||
let opts = ScannerObjectOptions {
|
||||
max_parity: true,
|
||||
http_preconditions: Some(preconditions),
|
||||
scanner_publication_commit_scope,
|
||||
user_defined,
|
||||
..Default::default()
|
||||
};
|
||||
let write = api.put_object(RUSTFS_META_BUCKET, file, &mut reader, &opts);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let write = rustfs_ecstore::api::diagnostic::with_read_version_entry(
|
||||
rustfs_ecstore::api::diagnostic::ReadVersionEntrySource::ScannerConfigWrite,
|
||||
async {
|
||||
rustfs_ecstore::api::diagnostic::observe_read_version_context_boundary(
|
||||
rustfs_ecstore::api::diagnostic::ReadVersionContextBoundary::ScannerConfigWrite,
|
||||
);
|
||||
write.await
|
||||
},
|
||||
)
|
||||
.await
|
||||
);
|
||||
write.await
|
||||
}
|
||||
|
||||
pub(crate) async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> std::result::Result<(), DiskError> {
|
||||
|
||||
@@ -1461,6 +1461,23 @@ async fn mark_cycle_recovery_cleanup_pending(
|
||||
|
||||
pub(crate) async fn load_scanner_cycle_state_for_startup(
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
) -> ScannerCycleStateStartup {
|
||||
let load = load_scanner_cycle_state_inner(storeapi);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let load = rustfs_ecstore::api::diagnostic::with_read_version_entry(
|
||||
rustfs_ecstore::api::diagnostic::ReadVersionEntrySource::ScannerCycleStateRead,
|
||||
async {
|
||||
rustfs_ecstore::api::diagnostic::observe_read_version_context_boundary(
|
||||
rustfs_ecstore::api::diagnostic::ReadVersionContextBoundary::ScannerCycleStateLoad,
|
||||
);
|
||||
load.await
|
||||
},
|
||||
);
|
||||
load.await
|
||||
}
|
||||
|
||||
async fn load_scanner_cycle_state_inner(
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
) -> ScannerCycleStateStartup {
|
||||
let marker = match read_cycle_recovery_marker_bytes(storeapi.clone()).await {
|
||||
Ok((None, _)) => None,
|
||||
|
||||
@@ -883,6 +883,12 @@ impl Drop for ScannerDefaultCycleGuard {
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct MemoryConfigStore {
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
entry_sources: Mutex<Vec<Option<rustfs_ecstore::api::diagnostic::ReadVersionEntrySource>>>,
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
write_entry_sources: Mutex<Vec<Option<rustfs_ecstore::api::diagnostic::ReadVersionEntrySource>>>,
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
pending_diagnostic_write: AtomicBool,
|
||||
objects: Mutex<HashMap<String, Vec<u8>>>,
|
||||
revisions: Mutex<HashMap<String, u64>>,
|
||||
insert_after_gets: Mutex<HashMap<String, Vec<u8>>>,
|
||||
@@ -937,6 +943,11 @@ impl crate::storage_api::scanner_io::ObjectIO for MemoryConfigStore {
|
||||
_h: http::HeaderMap,
|
||||
_opts: &ObjectOptions,
|
||||
) -> EcstoreResult<GetObjectReader> {
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
self.entry_sources
|
||||
.lock()
|
||||
.await
|
||||
.push(rustfs_ecstore::api::diagnostic::current_read_version_entry_source());
|
||||
let key = memory_config_key(bucket, object);
|
||||
if let Some(error) = self.read_errors.lock().await.get(&key).cloned() {
|
||||
return Err(error);
|
||||
@@ -983,6 +994,16 @@ impl crate::storage_api::scanner_io::ObjectIO for MemoryConfigStore {
|
||||
data: &mut PutObjReader,
|
||||
opts: &ObjectOptions,
|
||||
) -> EcstoreResult<ObjectInfo> {
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
{
|
||||
self.write_entry_sources
|
||||
.lock()
|
||||
.await
|
||||
.push(rustfs_ecstore::api::diagnostic::current_read_version_entry_source());
|
||||
if self.pending_diagnostic_write.load(Ordering::Relaxed) {
|
||||
std::future::pending::<()>().await;
|
||||
}
|
||||
}
|
||||
let mut buf = Vec::new();
|
||||
data.stream.read_to_end(&mut buf).await?;
|
||||
let key = memory_config_key(bucket, object);
|
||||
@@ -1803,6 +1824,136 @@ fn scanner_startup_fails_closed_on_nonempty_corrupt_cycle_state() {
|
||||
assert!(encode_scanner_cycle_state(&exhausted, 7).is_err());
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn diagnostic_scanner_config_writes_preserve_origin_for_both_helpers_and_cancellation() {
|
||||
use rustfs_ecstore::api::diagnostic::{ReadVersionEntrySource, current_read_version_entry_source, with_read_version_entry};
|
||||
|
||||
for earlier in [None, Some(ReadVersionEntrySource::TierConfigReload)] {
|
||||
for shared in [false, true] {
|
||||
for case in ["success", "error", "cancelled"] {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let path = "buckets/diagnostic/write-state";
|
||||
let key = memory_config_key(RUSTFS_META_BUCKET, path);
|
||||
if case == "error" {
|
||||
store.fail_put_number.lock().await.insert(key.clone(), 1);
|
||||
}
|
||||
store.pending_diagnostic_write.store(case == "cancelled", Ordering::Relaxed);
|
||||
let write = async {
|
||||
let operation = async {
|
||||
let preconditions = crate::HTTPPreconditions {
|
||||
if_none_match: Some("*".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
if shared {
|
||||
crate::save_config_shared_with_preconditions_and_lease_fence_and_scope(
|
||||
store.clone(),
|
||||
path,
|
||||
bytes::Bytes::from_static(b"state"),
|
||||
None,
|
||||
preconditions,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
crate::save_config_with_preconditions(store.clone(), path, b"state".to_vec(), preconditions).await
|
||||
}
|
||||
};
|
||||
let result = tokio::time::timeout(Duration::from_secs(1), operation).await;
|
||||
match case {
|
||||
"success" => assert!(result.expect("write completes").is_ok()),
|
||||
"error" => assert!(result.expect("error completes").is_err()),
|
||||
_ => assert!(result.is_err(), "pending write is cancelled"),
|
||||
}
|
||||
};
|
||||
match earlier {
|
||||
Some(source) => with_read_version_entry(source, write).await,
|
||||
None => write.await,
|
||||
}
|
||||
assert_eq!(
|
||||
*store.write_entry_sources.lock().await,
|
||||
vec![Some(earlier.unwrap_or(ReadVersionEntrySource::ScannerConfigWrite))]
|
||||
);
|
||||
assert_eq!(current_read_version_entry_source(), None, "scope restores: {case}");
|
||||
let stored = store.objects.lock().await.get(&key).cloned();
|
||||
assert_eq!(stored, (case == "success").then(|| b"state".to_vec()), "write result preserved: {case}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn diagnostic_cycle_state_load_preserves_entry_for_results_and_cancellation() {
|
||||
use rustfs_ecstore::api::diagnostic::{ReadVersionEntrySource, current_read_version_entry_source, with_read_version_entry};
|
||||
|
||||
for earlier in [None, Some(ReadVersionEntrySource::TierConfigReload)] {
|
||||
for case in ["missing", "valid", "error", "cancelled"] {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let marker = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str());
|
||||
let state = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
|
||||
if case == "valid" {
|
||||
let cycle = CurrentCycle {
|
||||
next: 9,
|
||||
..Default::default()
|
||||
};
|
||||
store
|
||||
.objects
|
||||
.lock()
|
||||
.await
|
||||
.insert(state.clone(), encode_scanner_cycle_state(&cycle, 3).expect("cycle fixture"));
|
||||
store.revisions.lock().await.insert(state, 7);
|
||||
} else if case == "error" {
|
||||
store
|
||||
.read_errors
|
||||
.lock()
|
||||
.await
|
||||
.insert(marker.clone(), EcstoreError::ErasureReadQuorum);
|
||||
} else if case == "cancelled" {
|
||||
store.delayed_gets.lock().await.insert(marker, Duration::from_secs(1));
|
||||
}
|
||||
let load = async {
|
||||
if case == "cancelled" {
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(1), load_scanner_cycle_state_for_startup(store.clone()))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
return;
|
||||
}
|
||||
let result = load_scanner_cycle_state_for_startup(store.clone()).await;
|
||||
match (case, result) {
|
||||
("missing", ScannerCycleStateStartup::Ready { .. }) => {}
|
||||
(
|
||||
"valid",
|
||||
ScannerCycleStateStartup::Ready {
|
||||
cycle, leader_epoch: 3, ..
|
||||
},
|
||||
) => assert_eq!(cycle.next, 9),
|
||||
("error", ScannerCycleStateStartup::Transient(_)) => {}
|
||||
_ => panic!("unexpected cycle load outcome: {case}"),
|
||||
}
|
||||
};
|
||||
match earlier {
|
||||
Some(source) => with_read_version_entry(source, load).await,
|
||||
None => load.await,
|
||||
}
|
||||
let observed = store.entry_sources.lock().await;
|
||||
assert!(!observed.is_empty(), "load must enter a reader: {case}");
|
||||
assert!(
|
||||
observed
|
||||
.iter()
|
||||
.all(|source| *source == Some(earlier.unwrap_or(ReadVersionEntrySource::ScannerCycleStateRead))),
|
||||
"all readers preserve the original source: {case}"
|
||||
);
|
||||
assert_eq!(current_read_version_entry_source(), None, "scope restores for all outcomes: {case}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn corrupt_cycle_state_is_quarantined_once() {
|
||||
|
||||
@@ -19,9 +19,6 @@ no-default-features = false
|
||||
version = 2
|
||||
yanked = "deny"
|
||||
ignore = [
|
||||
# `paste 1.0.15` — unmaintained. No direct dependency.
|
||||
# owner: rustfs-maintainers review: 2026-07
|
||||
{ id = "RUSTSEC-2024-0436", reason = "paste unmaintained; transitive only; tracked for upgrade" },
|
||||
# `rsa` Marvin timing sidechannel (RUSTSEC-2023-0071). Pulled in via
|
||||
# `openidconnect` (transitive) and historically used directly. No upstream
|
||||
# fix is available yet. Tracked separately for follow-up; remove this
|
||||
|
||||
@@ -8,6 +8,8 @@ collection:
|
||||
|
||||
## Operations
|
||||
|
||||
For PUT cleanup admission, blocking-pool isolation, and optional CPU affinity,
|
||||
see [PUT cleanup isolation](operations/put-cleanup-isolation.md).
|
||||
- [Architecture guard troubleshooting](operations/architecture-guard-troubleshooting.md) — diagnose repository guard failures.
|
||||
- [Logging governance](operations/logging-governance.md) — broad event audits, migrations, and guardrail changes.
|
||||
|
||||
|
||||
@@ -31,6 +31,8 @@ A previously published gauge can remain visible after a statistic becomes unavai
|
||||
|
||||
When live requested bytes are unavailable, capture a supported heap profile or allocation/free trace. Preserve attribution to allocation stacks and lifetime; committed bytes and RSS alone cannot establish a leak. Other raw allocator statistics, including page accounting, depend on the allocator build and are not interchangeable with live application bytes.
|
||||
|
||||
For Linux diagnostics, build with `--features diagnostic-allocator-live` to export `rustfs_memory_allocator_rust_global_requested_live_bytes` and its availability gauge. Use the byte gauge only when `rustfs_memory_allocator_rust_global_requested_live_bytes_available` is `1`; it remains unavailable on Windows and with `hotpath-alloc`. The feature wraps MiMalloc and updates an atomic counter for successful Rust `GlobalAlloc` allocations (including zeroed allocations), reallocations, and deallocations. It excludes direct native allocator calls and adds per-allocation overhead, so use it to observe memory behavior, not to compare throughput. Enabling `--cfg tokio_unstable` for Tokio blocking-pool metrics does not enable this allocator wrapper; enable both explicitly when both measurements are needed.
|
||||
|
||||
### Idle reclaim
|
||||
|
||||
The reclaim loop waits for request, delete-tail, scanner, heal, EC and GET-buffer activity to become idle. Continuous traffic can skip reclaim. Check the activity gauges, skipped reasons and idle streak rather than assuming that waiting a fixed period guarantees collection.
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
# PUT and disk cleanup isolation
|
||||
|
||||
Cleanup can use process-wide admission budgets and a dedicated Tokio runtime.
|
||||
This is opt-in. It preserves cleanup receipts, eligible-data moves to trash,
|
||||
PUT temporary workspace cleanup, and periodic physical trash removal. It does
|
||||
not disable healing or skip cleanup.
|
||||
|
||||
## Configuration
|
||||
|
||||
| Variable | Default | Meaning when isolation is enabled |
|
||||
| --- | --- | --- |
|
||||
| `RUSTFS_CLEANUP_ISOLATE_ENABLE` | `false` | Enable admission and execution isolation on this node. |
|
||||
| `RUSTFS_PUT_RENAME_TAIL_CLEANUP_MAX_PENDING` | `1024` | Maximum admitted PUT ownership groups, including surviving commit and cleanup owners. |
|
||||
| `RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_WORKERS` | `2` | Maximum active PUT cleanup continuations. Also controls existing deferred queue workers; does not determine OS thread count. |
|
||||
| `RUSTFS_CLEANUP_ASYNC_THREADS` | up to `2` | Dedicated async worker threads, limited by available parallelism and an explicit CPU mask by default. Independent of logical job concurrency. |
|
||||
| `RUSTFS_CLEANUP_DISK_MAX_PENDING` | `16 × disk workers` (`64`) | Maximum admitted local disk cleanup operations, including queued and running operations. Shared by local requests, incoming RPC cleanup and periodic GC steps. |
|
||||
| `RUSTFS_CLEANUP_DISK_WORKERS` | blocking threads (`4`) | Maximum active disk cleanup operations within that budget. |
|
||||
| `RUSTFS_CLEANUP_GC_WORKERS` | `1` | Maximum periodic disk scans producing GC steps. |
|
||||
| `RUSTFS_CLEANUP_BLOCKING_THREADS` | `4` | Maximum additional blocking threads on the cleanup runtime. |
|
||||
| `RUSTFS_CLEANUP_CPUS` | `1-2` on Linux; unbound elsewhere | CPU affinity for cleanup async and blocking threads. Override with a CPU list such as `4-5,8`, or `none` to disable affinity while keeping isolation. |
|
||||
| `RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ENABLE` | `false` | Use the existing bounded deferred queue before coordinator execution. Isolation also works without this queue. |
|
||||
| `RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_QUEUE_CAPACITY` | `1024` | Capacity of that existing queue; this is not the total admission budget. |
|
||||
|
||||
Counts must be positive. PUT workers must not exceed PUT pending capacity;
|
||||
GC workers must not exceed disk workers, which must not exceed disk pending
|
||||
capacity. Async and blocking thread counts must not exceed 1024. CPU IDs must be supported by the affinity implementation and belong
|
||||
to the initiating thread's allowed CPU set. Non-Linux hosts reject explicit CPU
|
||||
masks. Configuration is read once at first use, including disk cleanup or GC;
|
||||
changes require a restart. Invalid configuration rejects new cleanup admission.
|
||||
PUT checks configuration before staging data.
|
||||
|
||||
Isolation remains disabled by default. When enabling it on Linux, the default
|
||||
mask requires logical CPU IDs 1 and 2 to be available inside the process/container
|
||||
CPU set. On smaller or restricted CPU sets, configure permitted CPU IDs or `none`
|
||||
before enabling isolation. An unavailable default mask is an error; it does not
|
||||
silently select different CPUs. Configure each node separately.
|
||||
|
||||
Do not combine isolation with cleanup counterfactuals
|
||||
`RUSTFS_PUT_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP`,
|
||||
`RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER`, or
|
||||
`RUSTFS_PUT_RENAME_TAIL_CLEANUP_ZERO_TARGET_TMP_DELETE_SKIP`. These combinations
|
||||
are rejected because they deliberately suppress cleanup or stop queue progress.
|
||||
|
||||
Do not enable `RUSTFS_PUT_RENAME_TAIL_CLEANUP_LANE_ENABLE` together with
|
||||
`RUSTFS_CLEANUP_ISOLATE_ENABLE`. The legacy per-disk lane adds a second queue
|
||||
inside the isolated coordinator budget, so a slow lane can hold coordinator
|
||||
capacity while unrelated cleanup waits. This combination is rejected before
|
||||
PUT admission; either scheduler can still be enabled on its own.
|
||||
|
||||
## Ownership and execution
|
||||
|
||||
PUT attempts admission before staging. Full admission returns `SlowDown`, so
|
||||
clients should retry with backoff. Admission does not wait: COPY readers and
|
||||
data-movement callers can already hold locks needed by admitted PUTs. No body
|
||||
is ingested or tmp data staged by a rejected PUT.
|
||||
|
||||
The request, detached commit owner, and cleanup continuations share one
|
||||
reservation. An early ACK or cancelled waiter cannot release it while an owner
|
||||
still needs cleanup. After the existing rename-convergence and guard-release
|
||||
handoff, deferred workers, queue overflow and direct continuations all acquire
|
||||
the same coordinator execution semaphore. Queue overflow cannot bypass that
|
||||
limit. Awaited failure cleanup and no-tail tmp cleanup use the same budget.
|
||||
Full-tail receipt persistence remains awaited at its existing handoff point;
|
||||
old-data reclamation remains after the existing guard release. Incomplete
|
||||
rollback still preserves staging for recovery.
|
||||
|
||||
Each local disk operation has separate pending and active permits. The wrapper
|
||||
below RPC authentication uses this gate for low-level `delete`, `delete_data_dir`
|
||||
and old-data cleanup receipt `write_all`. Local PUT fan-out and incoming peer
|
||||
RPCs therefore share the same node-local disk budget. Receipt classification
|
||||
uses the internal receipt filename; ordinary writes retain their current path.
|
||||
RPC schemas and authentication are unchanged. Enable isolation on every node
|
||||
to isolate peer execution; settings and CPU masks are not transmitted by RPC.
|
||||
|
||||
Coordinator and receiver permits are deliberately distinct: a coordinator can
|
||||
wait for peer RPCs without occupying the slots those peers need to service disk
|
||||
cleanup. Full disk admission returns an I/O `WouldBlock` error through the
|
||||
existing disk error path. A mutation carrying an external publication or
|
||||
namespace owner also refuses a full active budget instead of waiting while it
|
||||
holds a lock. Existing cleanup error handling and receipt recovery still apply.
|
||||
|
||||
Ordinary snapshot-lease release stays on the caller and only updates the in-memory
|
||||
registry. It does not allocate a cleanup task or wait for the cleanup scheduler.
|
||||
Invalid cleanup configuration also must not strand a reader token: release still
|
||||
updates the registry, while any deferred reclamation reports the initialization
|
||||
error and keeps both its intent and data. It does not fall back to unbudgeted I/O.
|
||||
If the last release triggers deferred old-data
|
||||
reclamation, that reclamation tries the same pending and active disk permits.
|
||||
Only this case dispatches an owned task to the cleanup runtime.
|
||||
Saturation leaves the token released and the cleanup intent/receipt available
|
||||
for retry; it does not strand a live reader token or bypass the budget. Quota
|
||||
fence release keeps its original receiver lifecycle and does not acquire disk
|
||||
execution permits, since it can wait for mutations that already own them.
|
||||
|
||||
An admitted request waits for disk execution capacity in the caller's existing
|
||||
future. Cancellation or timeout discards this unstarted operation and releases
|
||||
pending capacity; it cannot later start deleting a reused path. After execution
|
||||
capacity is acquired, the worker also checks whether its result receiver closed
|
||||
before its first poll. A job cancelled while waiting for the runtime to schedule
|
||||
it is discarded when the worker polls it, releasing its permits without starting
|
||||
I/O. Once the worker starts the operation, the owned job retains its permits and
|
||||
disk reference until completion. Cancelling or timing out its waiter does not
|
||||
free disk capacity while that started job still runs.
|
||||
This also retains the disk's mount ownership during detached execution. Cleanup
|
||||
waiter timeouts include queue delay and do not by themselves mark a disk faulty;
|
||||
the existing active disk-health probe retains its timeout policy.
|
||||
|
||||
Periodic trash and stale-tmp scans have a separate scan limit. Each scan acquires
|
||||
the shared disk budget to open its directory and then for each entry, releasing
|
||||
it between entries. The steps borrow the scan's iterator and paths and run in
|
||||
the same owned task, without a spawn/join or path clones per entry. A scan cannot occupy a disk execution slot while waiting
|
||||
for another scan slot. GC waits for admission without holding object locks;
|
||||
request admission is bounded without creating waiting producer tasks. A single
|
||||
trash subtree is still removed as one operation, so a very large subtree can
|
||||
hold a slot for its entire removal. Queued scans retain the original mount
|
||||
handle even when the corresponding disk object is dropped.
|
||||
Missed periodic ticks are skipped so a slow scan does not cause a burst of
|
||||
catch-up scans.
|
||||
|
||||
Cleanup runs on `rustfs-cleanup` threads. Nested Tokio tasks, Tokio filesystem
|
||||
work and ordinary `spawn_blocking` use that runtime. Explicit fsync dispatch
|
||||
also stays on its blocking pool. File/directory group-commit keys separate
|
||||
cleanup and foreground producers, preventing batches from moving into the
|
||||
other class's runtime. Fsync durability settings remain unchanged.
|
||||
|
||||
The blocking limit bounds simultaneous blocking execution, not Tokio's internal
|
||||
queue or kernel I/O. Disk admission bounds selected operations, not bytes,
|
||||
syscall count or device IOPS. Existing synchronous path probes and disk-full
|
||||
delete fallbacks can still run on the dedicated async workers, under the disk
|
||||
operation budget. All classes still share the filesystem, journal,
|
||||
page cache and devices with foreground work.
|
||||
|
||||
If CPU binding fails at thread startup, new admissions fail and the failure is
|
||||
logged; already-admitted cleanup can finish. Affinity does not reserve exclusive
|
||||
cores, move foreground threads elsewhere, change cgroups, or control kernel
|
||||
workers and interrupt affinity.
|
||||
|
||||
## Scope and recovery
|
||||
|
||||
The limits are per process, not cluster-wide quotas. This is not a durable work
|
||||
queue. Existing cleanup receipts and tmp/trash recovery retain their role after
|
||||
process exit. The option does not bound all storage mutations: metadata/version
|
||||
DELETE transactions, heal/scanner coordination and
|
||||
multipart orchestration retain their scheduling. Calls from those paths that
|
||||
reach the selected disk wrapper entries do share the disk budget; internal
|
||||
LocalDisk operations that bypass those entries do not.
|
||||
|
||||
## Measurement and tuning
|
||||
|
||||
On Linux, `cargo test -p rustfs-ecstore --test cleanup_isolation_test` exercises
|
||||
the public disk facade, actual thread affinity, receipt/delete execution and
|
||||
periodic physical trash removal with temporary disks. It does not benchmark
|
||||
throughput.
|
||||
|
||||
With PUT stage metrics enabled, observe:
|
||||
|
||||
- `rustfs_cleanup_admitted_puts` and `rustfs_cleanup_admission_rejected_total`;
|
||||
- `rustfs_cleanup_execution_wait_seconds`;
|
||||
- `rustfs_cleanup_disk_pending`, `rustfs_cleanup_disk_active`, and
|
||||
`rustfs_cleanup_disk_admission_rejected_total`;
|
||||
- `rustfs_cleanup_disk_execution_wait_seconds` and `rustfs_cleanup_gc_active`;
|
||||
- existing deferred queue depth, age, overflow, completion and service time.
|
||||
|
||||
Compare isolation disabled, budget/pool isolation without CPU affinity, and
|
||||
isolation with affinity. Keep durability, object sizes, overwrite ratio,
|
||||
concurrency and cluster placement fixed. Measure PUT/GET throughput and p95/p99
|
||||
latency, rejections, queue age, receipt/trash growth, per-disk I/O latency, CPU
|
||||
usage and context switches. Include a long steady-state interval covering
|
||||
physical GC, and verify backlog drains afterward.
|
||||
|
||||
Start with small worker and blocking-thread limits, then increase only when
|
||||
cleanup cannot keep up without harming foreground latency. Too few workers or
|
||||
CPUs can increase rejections and lower PUT throughput. Larger queues do not
|
||||
increase sustainable device throughput. Affinity helps only when CPU scheduling
|
||||
contention matters to the measured workload; no throughput gain is guaranteed.
|
||||
|
||||
The defaults protect a node; they are not a claim of optimal production throughput.
|
||||
The default disk queue permits sixteen service waves, rather than 256 waves with
|
||||
1024 pending operations and four workers. For illustration, at a uniform 100 ms
|
||||
per operation, these bounds correspond to roughly 1.6 s versus 25.6 s to drain a
|
||||
full queue with no new arrivals. Real service times, stalls and disk fan-out vary;
|
||||
there is no queue latency guarantee. The ordinary disk waiter timeout defaults to
|
||||
30 s and does not cancel already-admitted work.
|
||||
|
||||
Size pending capacity for measured arrival bursts and local/peer fan-out, not
|
||||
only for PUT request concurrency. For example, two coordinator jobs each touching
|
||||
16 local disks can submit 32 disk operations before incoming peer requests and
|
||||
GC are counted. Sixty-four slots need not absorb every distributed burst. Increase
|
||||
the explicit limit when measured rejection rates require it, while keeping queue
|
||||
age comfortably below waiter deadlines. On many-disk nodes, measure whether four
|
||||
active operations underutilize devices before increasing disk/blocking workers.
|
||||
The 1024 PUT admission limit includes uploading requests and detached owners;
|
||||
it is an object-count bound, not a memory or byte budget. The older deferred queue
|
||||
remains optional and disabled by default; its overflow path cannot bypass the
|
||||
new coordinator or disk limits. No unbounded foreground fallback is introduced.
|
||||
@@ -77,6 +77,10 @@ rio-v2 = ["rustfs-ecstore/rio-v2"]
|
||||
pyroscope = ["rustfs-obs/pyroscope", "dep:pyroscope", "dep:object", "dep:symbolic-demangle"]
|
||||
# Tokio runtime telemetry. Requires `--cfg tokio_unstable`; use `make build-profiling`.
|
||||
dial9 = ["rustfs-obs/dial9"]
|
||||
# Diagnostic-only Rust GlobalAlloc live-byte accounting; adds per-allocation atomic updates.
|
||||
diagnostic-allocator-live = []
|
||||
# Diagnostic-only S3-operation attribution for internode ReadVersion and list-walker metrics.
|
||||
diagnostic-read-version-caller = ["rustfs-ecstore/diagnostic-read-version-caller", "rustfs-io-metrics/diagnostic-read-version-caller", "rustfs-scanner/diagnostic-read-version-caller"]
|
||||
http3 = ["dep:h3", "dep:h3-quinn", "dep:quinn"]
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
fn main() -> shadow_rs::SdResult<()> {
|
||||
println!("cargo:rustc-check-cfg=cfg(tokio_unstable)");
|
||||
println!("cargo:rerun-if-env-changed=RUSTFS_BUILD_VERSION");
|
||||
if let Ok(version) = std::env::var("RUSTFS_BUILD_VERSION")
|
||||
&& !version.is_empty()
|
||||
|
||||
@@ -1708,6 +1708,11 @@ async fn compensation_driven_copy_still_completes_transition() {
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let transition_state = get_global_transition_state();
|
||||
let missed_before = transition_state.missed_immediate_tasks();
|
||||
let compensation_before = transition_state.compensation_scheduled_tasks();
|
||||
let put_barrier = backend.arm_put_barrier().await;
|
||||
|
||||
with_forced_immediate_enqueue_timeout(&backend, || async {
|
||||
Box::pin(usecase.execute_copy_object(build_request(copy_input, Method::PUT)))
|
||||
.await
|
||||
@@ -1715,6 +1720,25 @@ async fn compensation_driven_copy_still_completes_transition() {
|
||||
})
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
transition_state.missed_immediate_tasks() > missed_before,
|
||||
"copy should exercise the forced immediate enqueue failure"
|
||||
);
|
||||
assert!(
|
||||
transition_state.compensation_scheduled_tasks() > compensation_before,
|
||||
"copy should schedule automatic compensation backfill"
|
||||
);
|
||||
|
||||
// Observe automatic compensation reaching the tier before timing its final
|
||||
// metadata commit; discovery and queue latency are separate from that commit.
|
||||
put_barrier.wait_until_paused().await;
|
||||
assert_eq!(
|
||||
backend.object_count().await,
|
||||
1,
|
||||
"compensation should store the copied body before committing"
|
||||
);
|
||||
put_barrier.release();
|
||||
|
||||
let info = wait_for_transition(&ecstore, dst_bucket.as_str(), dst_object, TRANSITION_WAIT_TIMEOUT)
|
||||
.await
|
||||
.expect("copied object should eventually transition after compensation backfill");
|
||||
|
||||
@@ -221,6 +221,16 @@ const GET_OBJECT_STAGE_LIFECYCLE_EXPIRATION: &str = "lifecycle_expiration";
|
||||
|
||||
const GET_OBJECT_STAGE_METADATA_FILTER: &str = "metadata_filter";
|
||||
|
||||
const GET_OBJECT_STAGE_BODY_CACHE_HOOK_HIT: &str = "body_cache_hook_hit";
|
||||
const GET_OBJECT_STAGE_BODY_CACHE_LOOKUP: &str = "body_cache_lookup";
|
||||
const GET_OBJECT_STAGE_BODY_CACHE_FILL_HANDOFF: &str = "body_cache_fill_handoff";
|
||||
const GET_OBJECT_STAGE_BODY_BUFFERED_HANDOFF: &str = "body_buffered_handoff";
|
||||
const GET_OBJECT_STAGE_OUTPUT_STRUCT_BUILD: &str = "output_struct_build";
|
||||
const GET_OBJECT_STAGE_PREPARE_READ_EXECUTION: &str = "prepare_read_execution";
|
||||
const GET_OBJECT_STAGE_RESPONSE_CORS_WRAP: &str = "response_cors_wrap";
|
||||
const GET_OBJECT_STAGE_RESPONSE_HEADER_INJECT: &str = "response_header_inject";
|
||||
const GET_OBJECT_STAGE_RESPONSE_EVENT_COMPLETE: &str = "response_event_complete";
|
||||
|
||||
const GET_OBJECT_STREAM_WARN_THRESHOLD: Duration = Duration::from_secs(5);
|
||||
|
||||
static GET_OBJECT_BUFFER_THRESHOLD_WARNED: AtomicBool = AtomicBool::new(false);
|
||||
@@ -499,10 +509,10 @@ fn get_object_stream_failure_reason(error_class: &'static str) -> &'static str {
|
||||
}
|
||||
}
|
||||
|
||||
/// A mid-stream reopen can recover only when another representation of the
|
||||
/// committed object may exist. A single-disk, no-parity object has no alternate
|
||||
/// shard after a bitrot/read-quorum failure, so retrying the same read only
|
||||
/// repeats the failure and adds latency.
|
||||
/// A mid-stream reopen can only recover when another representation of the
|
||||
/// committed object may exist. Single-disk, no-parity objects have no alternate
|
||||
/// shard to discover after a bitrot/read-quorum failure, so arming the normal
|
||||
/// three-attempt resume loop only repeats the same failing read and adds delay.
|
||||
fn should_attach_get_object_resume(info: &ObjectInfo) -> bool {
|
||||
info.parity_blocks > 0 || info.data_blocks > 1 || !info.transitioned_object.tier.is_empty()
|
||||
}
|
||||
@@ -3341,12 +3351,15 @@ impl DefaultObjectUsecase {
|
||||
// already supplied this body, the request-level plan was built before
|
||||
// the authoritative lookup. Serve it without planning a second time.
|
||||
if cache_hook_served && let Some(bytes) = buffered_body.take() {
|
||||
return Ok(Self::build_memory_bytes_blob(
|
||||
let hook_hit_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
|
||||
let body = Self::build_memory_bytes_blob(
|
||||
bytes,
|
||||
response_content_length,
|
||||
GET_MEMORY_BODY_SOURCE_OBJECT_DATA_CACHE,
|
||||
lifecycle,
|
||||
));
|
||||
);
|
||||
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_CACHE_HOOK_HIT, hook_hit_start);
|
||||
return Ok(body);
|
||||
}
|
||||
|
||||
if !cache_fill_allowed {
|
||||
@@ -3387,20 +3400,27 @@ impl DefaultObjectUsecase {
|
||||
// authoritative because it ran after fresh metadata resolution, so the
|
||||
// app layer skips its own lookup and only uses the plan to fill.
|
||||
if !cache_hook_probed {
|
||||
match lookup_get_object_body_cache_hit(cache_adapter, &cache_plan).await {
|
||||
let cache_lookup_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
|
||||
let cache_lookup = lookup_get_object_body_cache_hit(cache_adapter, &cache_plan).await;
|
||||
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_CACHE_LOOKUP, cache_lookup_start);
|
||||
match cache_lookup {
|
||||
GetObjectBodyCacheLookup::Hit(bytes) => {
|
||||
return Ok(Self::build_memory_bytes_blob(
|
||||
let cache_hit_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
|
||||
let body = Self::build_memory_bytes_blob(
|
||||
bytes,
|
||||
response_content_length,
|
||||
GET_MEMORY_BODY_SOURCE_OBJECT_DATA_CACHE,
|
||||
lifecycle,
|
||||
));
|
||||
);
|
||||
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_CACHE_HOOK_HIT, cache_hit_start);
|
||||
return Ok(body);
|
||||
}
|
||||
GetObjectBodyCacheLookup::Disabled | GetObjectBodyCacheLookup::Skip | GetObjectBodyCacheLookup::Miss => {}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(buffered_body) = buffered_body {
|
||||
let cache_fill_handoff_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
|
||||
// ODC-15: the body is already fully in hand, so keep the fill off the
|
||||
// response's critical path. For a cacheable plan, run the fill in a
|
||||
// detached task (Bytes is a cheap clone) and return immediately. For
|
||||
@@ -3416,13 +3436,17 @@ impl DefaultObjectUsecase {
|
||||
} else if cache_fill_allowed {
|
||||
let _ = fill_get_object_body_cache_from_buffered_body(cache_adapter, &cache_plan, &buffered_body).await;
|
||||
}
|
||||
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_CACHE_FILL_HANDOFF, cache_fill_handoff_start);
|
||||
|
||||
return Ok(Self::build_memory_bytes_blob(
|
||||
let buffered_handoff_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
|
||||
let body = Self::build_memory_bytes_blob(
|
||||
buffered_body,
|
||||
response_content_length,
|
||||
GET_MEMORY_BODY_SOURCE_BUFFERED_BODY,
|
||||
lifecycle,
|
||||
));
|
||||
);
|
||||
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_BUFFERED_HANDOFF, buffered_handoff_start);
|
||||
return Ok(body);
|
||||
}
|
||||
|
||||
let should_materialize_for_cache = cache_adapter.materialize_fill_enabled()
|
||||
@@ -3476,6 +3500,7 @@ impl DefaultObjectUsecase {
|
||||
{
|
||||
Ok(buf) => {
|
||||
let bytes = Bytes::from(buf);
|
||||
let cache_fill_handoff_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
|
||||
// ODC-15: fill off the response's critical path (see the
|
||||
// buffered-body branch above).
|
||||
let cache_adapter = cache_adapter.clone();
|
||||
@@ -3484,6 +3509,10 @@ impl DefaultObjectUsecase {
|
||||
tokio::spawn(async move {
|
||||
let _ = fill_get_object_body_cache_from_materialized_body(&cache_adapter, &cache_plan, &fill_bytes).await;
|
||||
});
|
||||
record_get_object_s3_handler_stage_duration(
|
||||
GET_OBJECT_STAGE_BODY_CACHE_FILL_HANDOFF,
|
||||
cache_fill_handoff_start,
|
||||
);
|
||||
|
||||
return Ok(Self::build_memory_bytes_blob(
|
||||
bytes,
|
||||
@@ -3622,13 +3651,19 @@ impl DefaultObjectUsecase {
|
||||
None => helper,
|
||||
};
|
||||
let helper = helper.version_id(version_id_for_event);
|
||||
let cors_wrap_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
|
||||
let mut response = wrap_response_with_cors(bucket, method, headers, output).await;
|
||||
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_RESPONSE_CORS_WRAP, cors_wrap_start);
|
||||
let header_inject_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
|
||||
inject_accept_ranges_header(&mut response.headers);
|
||||
// Emit XXHash3/64/128 and SHA-512 checksums that s3s GetObjectOutput cannot
|
||||
// carry (#1257). This is the download-side integrity path AWS SDKs verify.
|
||||
inject_additional_checksum_headers(&mut response.headers, &extra_checksum_headers);
|
||||
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_RESPONSE_HEADER_INJECT, header_inject_start);
|
||||
let result = Ok(response);
|
||||
let event_complete_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
|
||||
let _ = helper.complete(&result);
|
||||
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_RESPONSE_EVENT_COMPLETE, event_complete_start);
|
||||
result
|
||||
}
|
||||
|
||||
@@ -3738,9 +3773,8 @@ impl DefaultObjectUsecase {
|
||||
let metadata_filter_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
|
||||
let metadata = filter_object_metadata(&info.user_defined);
|
||||
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_METADATA_FILTER, metadata_filter_start);
|
||||
|
||||
let output_struct_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
|
||||
let parts_count = Self::get_object_parts_count(part_number, &info);
|
||||
|
||||
let output = GetObjectOutput {
|
||||
body: Some(body),
|
||||
content_length: Some(response_content_length),
|
||||
@@ -3774,6 +3808,7 @@ impl DefaultObjectUsecase {
|
||||
storage_class,
|
||||
..Default::default()
|
||||
};
|
||||
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_OUTPUT_STRUCT_BUILD, output_struct_start);
|
||||
|
||||
Ok(GetObjectOutputContext {
|
||||
output,
|
||||
@@ -4076,6 +4111,7 @@ impl DefaultObjectUsecase {
|
||||
|
||||
let manager = get_concurrency_manager();
|
||||
|
||||
let prepare_read_start = stage_metrics_enabled.then(std::time::Instant::now);
|
||||
let mut prepared_read = self
|
||||
.prepare_get_object_read_execution(
|
||||
&req,
|
||||
@@ -4091,6 +4127,7 @@ impl DefaultObjectUsecase {
|
||||
object_traffic_health.clone(),
|
||||
)
|
||||
.await;
|
||||
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_PREPARE_READ_EXECUTION, prepare_read_start);
|
||||
// An object missing locally (and only missing — other errors keep
|
||||
// their semantics) may still be served by a remote copy.
|
||||
if let Err(err) = &prepared_read
|
||||
@@ -4134,6 +4171,7 @@ impl DefaultObjectUsecase {
|
||||
return Self::complete_get_object_error(helper.version_id(version_id_for_event), err);
|
||||
}
|
||||
Some(OdmGetOutcome::RetryLocal) => {
|
||||
let prepare_read_retry_start = stage_metrics_enabled.then(std::time::Instant::now);
|
||||
prepared_read = self
|
||||
.prepare_get_object_read_execution(
|
||||
&req,
|
||||
@@ -4149,6 +4187,10 @@ impl DefaultObjectUsecase {
|
||||
object_traffic_health,
|
||||
)
|
||||
.await;
|
||||
record_get_object_s3_handler_stage_duration(
|
||||
GET_OBJECT_STAGE_PREPARE_READ_EXECUTION,
|
||||
prepare_read_retry_start,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1751,6 +1751,7 @@ impl DefaultObjectUsecase {
|
||||
)?;
|
||||
}
|
||||
|
||||
let checksum_input_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
let mut md5hex = match content_md5 {
|
||||
Some(PutObjectContentMd5::Base64(base64_md5)) => {
|
||||
let md5 = base64_simd::STANDARD
|
||||
@@ -1763,12 +1764,15 @@ impl DefaultObjectUsecase {
|
||||
};
|
||||
|
||||
let mut sha256hex = get_content_sha256_with_query(headers, query);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from("app_checksum_input_prepare", checksum_input_stage_start);
|
||||
|
||||
let mut write_plan = WritePlan::new();
|
||||
// Additional-checksum (XXHash3/64/128, SHA-512) values to echo on the PutObject
|
||||
// response (#1256); captured at want_checksum set points before opts is moved.
|
||||
let mut put_extra_checksum_headers: Vec<(&'static str, String)> = Vec::new();
|
||||
let reader_prepare_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
let mut reader = if should_compress {
|
||||
let compressed_reader_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
let body = tokio::io::BufReader::with_capacity(
|
||||
buffer_size,
|
||||
StreamReader::new(body.map(|f| f.map_err(s3s_body_error_to_io))),
|
||||
@@ -1791,51 +1795,98 @@ impl DefaultObjectUsecase {
|
||||
|
||||
size = HashReader::SIZE_PRESERVE_LAYER;
|
||||
write_plan = write_plan.with_compression(algorithm);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
"app_reader_prepare_compressed",
|
||||
compressed_reader_stage_start,
|
||||
);
|
||||
hrd
|
||||
} else {
|
||||
if use_zero_copy_eager_put_path {
|
||||
let zero_copy_start = std::time::Instant::now();
|
||||
let eager_read_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
let eager_body = read_zero_copy_put_body_exact(body, body_size as usize).await?;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from("app_body_eager_read_zero_copy", eager_read_stage_start);
|
||||
rustfs_io_metrics::record_zero_copy_write(body_size as usize, zero_copy_start.elapsed().as_secs_f64() * 1000.0);
|
||||
HashReader::from_stream(eager_body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?
|
||||
let hash_reader_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
let reader =
|
||||
HashReader::from_stream(eager_body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
"app_hash_reader_setup_zero_copy",
|
||||
hash_reader_stage_start,
|
||||
);
|
||||
reader
|
||||
} else if use_empty_or_small_eager_put_path {
|
||||
if (body_size as usize) <= POOL_BYPASS_MAX_SIZE {
|
||||
// Bypass BytesPool for very small objects to avoid Small-tier
|
||||
// Mutex contention under high concurrency. Direct allocation
|
||||
// for ≤4KiB is negligible cost.
|
||||
let eager_read_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
let eager_body = read_small_put_body_exact_direct(
|
||||
StreamReader::new(body.map(|f| f.map_err(s3s_body_error_to_io))),
|
||||
body_size as usize,
|
||||
)
|
||||
.await?;
|
||||
HashReader::from_stream(eager_body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
"app_body_eager_read_small_direct",
|
||||
eager_read_stage_start,
|
||||
);
|
||||
let hash_reader_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
let reader = HashReader::from_stream(eager_body, size, actual_size, md5hex, sha256hex, false)
|
||||
.map_err(ApiError::from)?;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
"app_hash_reader_setup_small_direct",
|
||||
hash_reader_stage_start,
|
||||
);
|
||||
reader
|
||||
} else {
|
||||
let pool = get_concurrency_manager().bytes_pool();
|
||||
let eager_read_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
let eager_body = read_small_put_body_exact_pooled(
|
||||
StreamReader::new(body.map(|f| f.map_err(s3s_body_error_to_io))),
|
||||
body_size as usize,
|
||||
pool.as_ref(),
|
||||
)
|
||||
.await?;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
"app_body_eager_read_small_pooled",
|
||||
eager_read_stage_start,
|
||||
);
|
||||
let hash_reader_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
let eager_reader = PooledBufferReader::new(eager_body, body_size as usize);
|
||||
HashReader::from_stream(eager_reader, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?
|
||||
let reader = HashReader::from_stream(eager_reader, size, actual_size, md5hex, sha256hex, false)
|
||||
.map_err(ApiError::from)?;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
"app_hash_reader_setup_small_pooled",
|
||||
hash_reader_stage_start,
|
||||
);
|
||||
reader
|
||||
}
|
||||
} else {
|
||||
let streaming_reader_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
let body = tokio::io::BufReader::with_capacity(
|
||||
buffer_size,
|
||||
StreamReader::new(body.map(|f| f.map_err(s3s_body_error_to_io))),
|
||||
);
|
||||
HashReader::from_stream(body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?
|
||||
let reader =
|
||||
HashReader::from_stream(body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
"app_reader_prepare_streaming",
|
||||
streaming_reader_stage_start,
|
||||
);
|
||||
reader
|
||||
}
|
||||
};
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from("app_reader_prepare", reader_prepare_stage_start);
|
||||
|
||||
if size >= 0 {
|
||||
let checksum_setup_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
if let Err(err) = reader.add_checksum(headers, trailer_source(trailing_headers.clone()), false) {
|
||||
return Err(ApiError::from(err).into());
|
||||
}
|
||||
|
||||
opts.want_checksum = reader.checksum();
|
||||
put_extra_checksum_headers = additional_checksum_echo_pairs(&opts.want_checksum);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from("app_reader_checksum_setup", checksum_setup_stage_start);
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_path(put_path);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from("ingress_prepare", ingress_stage_start);
|
||||
@@ -1987,7 +2038,7 @@ impl DefaultObjectUsecase {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let put_commit = spawn_traced_join({
|
||||
let put_commit_future = {
|
||||
let store = Arc::clone(&store);
|
||||
let bucket = bucket.clone();
|
||||
let key = key.clone();
|
||||
@@ -2069,17 +2120,36 @@ impl DefaultObjectUsecase {
|
||||
wait_for_put_post_store_test_hook(&bucket).await;
|
||||
|
||||
let post_store_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
let transition_enqueue_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
maybe_enqueue_transition_immediate(&obj_info, LcEventSrc::S3PutObject).await;
|
||||
let _ = invalidate_object_data_cache_after_put_success(&cache_adapter, &bucket, &key).await;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
"app_post_store_transition_enqueue",
|
||||
transition_enqueue_stage_start,
|
||||
);
|
||||
|
||||
let cache_invalidate_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
let _ = invalidate_object_data_cache_after_put_success(&cache_adapter, &bucket, &key).await;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
"app_post_store_data_cache_invalidate",
|
||||
cache_invalidate_stage_start,
|
||||
);
|
||||
|
||||
let versioning_lookup_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
let put_versioned = BucketVersioningSys::prefix_enabled(&bucket, &key).await;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
"app_post_store_versioning_lookup",
|
||||
versioning_lookup_stage_start,
|
||||
);
|
||||
// Fast in-memory update for immediate quota and admin usage consistency.
|
||||
// The previous current size comes from the prelookup when it ran,
|
||||
// otherwise from the rename_data backfill (rustfs/backlog#1009); the
|
||||
// backfill reproduces the lookup's observation bit for bit (latest
|
||||
// version's ObjectInfo.size — 0 for a delete-marker latest — or
|
||||
// not-found → None).
|
||||
let quota_size_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
let committed_size = quota_accounting_object_size(&obj_info, quota_enabled)?;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from("app_post_store_quota_size", quota_size_stage_start);
|
||||
let usage_memory_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
match prelookup_previous_current_size.or_else(|| previous_current_size_from_backfill(backfilled_old_current_size))
|
||||
{
|
||||
Some(previous_current_size) => {
|
||||
@@ -2105,16 +2175,27 @@ impl DefaultObjectUsecase {
|
||||
record_bucket_object_write_unknown_previous_memory(&bucket, committed_size, put_versioned).await;
|
||||
}
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from("app_post_store_usage_memory", usage_memory_stage_start);
|
||||
|
||||
if dsc.replicate_any() {
|
||||
let replication_schedule_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
schedule_object_replication(obj_info.clone(), store, dsc).await;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
"app_post_store_replication_schedule",
|
||||
replication_schedule_stage_start,
|
||||
);
|
||||
}
|
||||
|
||||
let dirty_usage_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
rustfs_scanner::record_dirty_usage_object_from_producer(
|
||||
&bucket,
|
||||
&key,
|
||||
rustfs_scanner::SegmentInvalidationProducerIdentity::PutObject,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
"app_post_store_dirty_usage_record",
|
||||
dirty_usage_stage_start,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from("app_post_store_bookkeeping", post_store_stage_start);
|
||||
|
||||
let capacity_update_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
@@ -2124,7 +2205,14 @@ impl DefaultObjectUsecase {
|
||||
|
||||
Ok::<_, S3Error>(PutObjectCommitResult { obj_info, put_versioned })
|
||||
}
|
||||
});
|
||||
};
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let put_commit_future = rustfs_ecstore::api::diagnostic::with_read_version_spawn_context(
|
||||
rustfs_ecstore::api::diagnostic::ReadVersionEntrySource::AppPutOwner,
|
||||
rustfs_ecstore::api::diagnostic::ReadVersionContextBoundary::AppPutOwner,
|
||||
put_commit_future,
|
||||
);
|
||||
let put_commit = spawn_traced_join(put_commit_future);
|
||||
let commit_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
let put_commit_result = if let Some(cancellation) = eager_put_commit_cancellation {
|
||||
EagerPutCommitOwner::new(put_commit, cancellation, EAGER_PUT_COMMIT_CANCELLATION_GRACE)
|
||||
@@ -4230,6 +4318,83 @@ mod tests {
|
||||
assert!(is_err_object_not_found(&lookup_err), "{lookup_err}");
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn diagnostic_s3_put_preserves_api_through_app_owner_and_tier_lookup() {
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
|
||||
use rustfs_io_metrics::s3_http_metrics::{S3HttpRequestGuard, current_s3_http_operation_name};
|
||||
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("test runtime");
|
||||
let (store, context) = runtime.block_on(Box::pin(real_store_test_context()));
|
||||
let bucket = format!("put-context-{}", Uuid::new_v4());
|
||||
runtime
|
||||
.block_on(Box::pin(store.make_bucket(&bucket, &MakeBucketOptions::default())))
|
||||
.expect("test bucket");
|
||||
let payload = Bytes::from_static(b"diagnostic PUT context");
|
||||
let input = PutObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key("object.bin".to_string())
|
||||
.body(Some(StreamingBlob::from(s3s::Body::from(payload.clone()))))
|
||||
.content_length(Some(i64::try_from(payload.len()).expect("payload length")))
|
||||
.build()
|
||||
.expect("PUT input");
|
||||
let usecase = DefaultObjectUsecase::with_context(Some(context));
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
rustfs_io_metrics::set_get_stage_metrics_enabled(true);
|
||||
let mut request = S3HttpRequestGuard::new("PUT");
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
request.in_scope(|| {
|
||||
runtime.block_on(async {
|
||||
Box::pin(usecase.execute_put_object(&FS::new(), build_request(input, Method::PUT)))
|
||||
.await
|
||||
.expect("S3 PUT must commit");
|
||||
assert_eq!(current_s3_http_operation_name(), Some("s3:PutObject"));
|
||||
assert_eq!(rustfs_ecstore::api::diagnostic::current_read_version_entry_source(), None);
|
||||
})
|
||||
});
|
||||
});
|
||||
let context_rows = snapshotter
|
||||
.snapshot()
|
||||
.into_vec()
|
||||
.into_iter()
|
||||
.filter(|(key, _, _, _)| key.key().name() == "rustfs_storage_diagnostic_context_transitions_total")
|
||||
.map(|(key, _, _, value)| {
|
||||
(
|
||||
key.key()
|
||||
.labels()
|
||||
.map(|l| (l.key().to_string(), l.value().to_string()))
|
||||
.collect::<HashMap<_, _>>(),
|
||||
value,
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
for boundary in ["app_put_owner", "put_commit_owner", "tier_free_version_source_lookup"] {
|
||||
assert!(
|
||||
context_rows.iter().any(|(labels, value)| {
|
||||
labels.get("entry_source").map(String::as_str) == Some("app_put_owner")
|
||||
&& labels.get("boundary").map(String::as_str) == Some(boundary)
|
||||
&& labels.get("parent_api_operation").map(String::as_str) == Some("s3:PutObject")
|
||||
&& labels.get("child_api_operation").map(String::as_str) == Some("s3:PutObject")
|
||||
&& labels.get("api_context").map(String::as_str) == Some("preserved")
|
||||
&& matches!(value, DebugValue::Counter(count) if *count > 0)
|
||||
}),
|
||||
"actual S3 PUT boundary must preserve the parent API: {boundary}; {context_rows:?}"
|
||||
);
|
||||
}
|
||||
let info = runtime
|
||||
.block_on(Box::pin(store.get_object_info(&bucket, "object.bin", &ObjectOptions::default())))
|
||||
.expect("committed object");
|
||||
assert_eq!(info.size, i64::try_from(payload.len()).expect("payload length"));
|
||||
assert_eq!(current_s3_http_operation_name(), None);
|
||||
assert_eq!(rustfs_ecstore::api::diagnostic::current_read_version_entry_source(), None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn execute_put_object_still_writes_plaintext_without_bucket_encryption() {
|
||||
|
||||
+83
-3
@@ -12,9 +12,23 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#[cfg(all(feature = "hotpath", feature = "hotpath-alloc", not(target_os = "windows")))]
|
||||
#[cfg(any(
|
||||
all(feature = "hotpath", feature = "hotpath-alloc", not(target_os = "windows")),
|
||||
all(
|
||||
feature = "diagnostic-allocator-live",
|
||||
not(target_os = "windows"),
|
||||
not(all(feature = "hotpath", feature = "hotpath-alloc"))
|
||||
)
|
||||
))]
|
||||
use std::alloc::{GlobalAlloc, Layout};
|
||||
#[cfg(all(feature = "hotpath", feature = "hotpath-alloc", not(target_os = "windows")))]
|
||||
#[cfg(any(
|
||||
all(feature = "hotpath", feature = "hotpath-alloc", not(target_os = "windows")),
|
||||
all(
|
||||
feature = "diagnostic-allocator-live",
|
||||
not(target_os = "windows"),
|
||||
not(all(feature = "hotpath", feature = "hotpath-alloc"))
|
||||
)
|
||||
))]
|
||||
use std::ptr::NonNull;
|
||||
|
||||
#[cfg(all(feature = "hotpath", feature = "hotpath-alloc", not(target_os = "windows")))]
|
||||
@@ -49,6 +63,60 @@ unsafe impl GlobalAlloc for MiMallocAllocator {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(
|
||||
feature = "diagnostic-allocator-live",
|
||||
not(target_os = "windows"),
|
||||
not(all(feature = "hotpath", feature = "hotpath-alloc"))
|
||||
))]
|
||||
#[derive(Default)]
|
||||
struct DiagnosticMiMalloc;
|
||||
|
||||
#[cfg(all(
|
||||
feature = "diagnostic-allocator-live",
|
||||
not(target_os = "windows"),
|
||||
not(all(feature = "hotpath", feature = "hotpath-alloc"))
|
||||
))]
|
||||
// SAFETY: allocation operations are forwarded to MiMalloc with the matching
|
||||
// GlobalAlloc layout contract. The diagnostic counter is updated only after
|
||||
// successful allocation changes and does not allocate memory itself.
|
||||
#[allow(unsafe_code)]
|
||||
unsafe impl GlobalAlloc for DiagnosticMiMalloc {
|
||||
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
|
||||
// SAFETY: the caller upholds GlobalAlloc's contract for layout.
|
||||
let ptr = unsafe { rustfs_mimalloc::MiMalloc.alloc(layout) };
|
||||
if !ptr.is_null() {
|
||||
rustfs::memory_observability::record_diagnostic_global_allocator_resize(0, layout.size());
|
||||
}
|
||||
ptr
|
||||
}
|
||||
|
||||
unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
|
||||
// SAFETY: the caller upholds GlobalAlloc's contract for layout.
|
||||
let ptr = unsafe { rustfs_mimalloc::MiMalloc.alloc_zeroed(layout) };
|
||||
if !ptr.is_null() {
|
||||
rustfs::memory_observability::record_diagnostic_global_allocator_resize(0, layout.size());
|
||||
}
|
||||
ptr
|
||||
}
|
||||
|
||||
unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
|
||||
// SAFETY: ptr came from this allocator, is non-null by GlobalAlloc's
|
||||
// dealloc contract, and layout.size() is the original allocation size.
|
||||
let ptr = unsafe { NonNull::new_unchecked(ptr) };
|
||||
unsafe { rustfs_mimalloc::MiMalloc::free_csize_nonnull(ptr, layout.size()) };
|
||||
rustfs::memory_observability::record_diagnostic_global_allocator_resize(layout.size(), 0);
|
||||
}
|
||||
|
||||
unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
|
||||
// SAFETY: ptr and layout came from this allocator and are forwarded unchanged.
|
||||
let new_ptr = unsafe { rustfs_mimalloc::MiMalloc.realloc(ptr, layout, new_size) };
|
||||
if !new_ptr.is_null() {
|
||||
rustfs::memory_observability::record_diagnostic_global_allocator_resize(layout.size(), new_size);
|
||||
}
|
||||
new_ptr
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "hotpath", feature = "hotpath-alloc", not(target_os = "windows")))]
|
||||
#[global_allocator]
|
||||
static GLOBAL: hotpath::CountingAllocator<MiMallocAllocator> = hotpath::CountingAllocator::with(MiMallocAllocator);
|
||||
@@ -57,10 +125,22 @@ static GLOBAL: hotpath::CountingAllocator<MiMallocAllocator> = hotpath::Counting
|
||||
#[global_allocator]
|
||||
static GLOBAL: hotpath::CountingAllocator<std::alloc::System> = hotpath::CountingAllocator::with(std::alloc::System);
|
||||
|
||||
#[cfg(all(not(all(feature = "hotpath", feature = "hotpath-alloc")), not(target_os = "windows")))]
|
||||
#[cfg(all(
|
||||
not(feature = "diagnostic-allocator-live"),
|
||||
not(all(feature = "hotpath", feature = "hotpath-alloc")),
|
||||
not(target_os = "windows")
|
||||
))]
|
||||
#[global_allocator]
|
||||
static GLOBAL: rustfs_mimalloc::MiMalloc = rustfs_mimalloc::MiMalloc;
|
||||
|
||||
#[cfg(all(
|
||||
feature = "diagnostic-allocator-live",
|
||||
not(target_os = "windows"),
|
||||
not(all(feature = "hotpath", feature = "hotpath-alloc"))
|
||||
))]
|
||||
#[global_allocator]
|
||||
static GLOBAL: DiagnosticMiMalloc = DiagnosticMiMalloc;
|
||||
|
||||
fn main() {
|
||||
let _hotpath_guard = hotpath::HotpathGuardBuilder::new("main").build();
|
||||
|
||||
|
||||
@@ -20,6 +20,12 @@ use serde::Serialize;
|
||||
#[cfg(any(not(target_os = "windows"), test))]
|
||||
use serde_json::Value;
|
||||
use std::path::Path;
|
||||
#[cfg(all(
|
||||
feature = "diagnostic-allocator-live",
|
||||
not(target_os = "windows"),
|
||||
not(all(feature = "hotpath", feature = "hotpath-alloc"))
|
||||
))]
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
@@ -35,6 +41,38 @@ const CGROUP_V1_MEMORY_STAT_PATH: &str = "/sys/fs/cgroup/memory/memory.stat";
|
||||
const CGROUP_V1_MEMORY_USAGE_PATH: &str = "/sys/fs/cgroup/memory/memory.usage_in_bytes";
|
||||
const CGROUP_V1_MEMORY_LIMIT_PATH: &str = "/sys/fs/cgroup/memory/memory.limit_in_bytes";
|
||||
|
||||
#[cfg(all(
|
||||
feature = "diagnostic-allocator-live",
|
||||
not(target_os = "windows"),
|
||||
not(all(feature = "hotpath", feature = "hotpath-alloc"))
|
||||
))]
|
||||
static DIAGNOSTIC_GLOBAL_ALLOCATOR_LIVE_REQUESTED_BYTES: AtomicUsize = AtomicUsize::new(0);
|
||||
|
||||
#[cfg(all(
|
||||
feature = "diagnostic-allocator-live",
|
||||
not(target_os = "windows"),
|
||||
not(all(feature = "hotpath", feature = "hotpath-alloc"))
|
||||
))]
|
||||
fn adjust_diagnostic_global_allocator_live_bytes(counter: &AtomicUsize, old_size: usize, new_size: usize) {
|
||||
if new_size > old_size {
|
||||
counter.fetch_add(new_size - old_size, Ordering::Relaxed);
|
||||
} else if old_size > new_size {
|
||||
counter.fetch_sub(old_size - new_size, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
/// Update the diagnostic-only live requested-byte counter for Rust allocations
|
||||
/// routed through the binary's `GlobalAlloc` wrapper.
|
||||
#[cfg(all(
|
||||
feature = "diagnostic-allocator-live",
|
||||
not(target_os = "windows"),
|
||||
not(all(feature = "hotpath", feature = "hotpath-alloc"))
|
||||
))]
|
||||
#[doc(hidden)]
|
||||
pub fn record_diagnostic_global_allocator_resize(old_size: usize, new_size: usize) {
|
||||
adjust_diagnostic_global_allocator_live_bytes(&DIAGNOSTIC_GLOBAL_ALLOCATOR_LIVE_REQUESTED_BYTES, old_size, new_size);
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum MemoryObservabilityServiceState {
|
||||
@@ -397,7 +435,48 @@ fn allocator_snapshot_or_unavailable(snapshot: Option<AllocatorMemorySnapshot>)
|
||||
})
|
||||
}
|
||||
|
||||
/// Record the serving Tokio runtime's worker and blocking-pool thread counts.
|
||||
/// Blocking-pool counts require a diagnostic build with `tokio_unstable`.
|
||||
fn record_tokio_runtime_metrics() {
|
||||
let runtime = tokio::runtime::Handle::current().metrics();
|
||||
metrics::gauge!("rustfs_runtime_worker_threads").set(runtime.num_workers() as f64);
|
||||
|
||||
#[cfg(tokio_unstable)]
|
||||
{
|
||||
let total = runtime.num_blocking_threads();
|
||||
let idle = runtime.num_idle_blocking_threads().min(total);
|
||||
metrics::gauge!("rustfs_runtime_blocking_pool_metrics_available").set(1.0);
|
||||
metrics::gauge!("rustfs_runtime_blocking_threads_total").set(total as f64);
|
||||
metrics::gauge!("rustfs_runtime_blocking_threads_idle").set(idle as f64);
|
||||
metrics::gauge!("rustfs_runtime_blocking_threads_active").set(total.saturating_sub(idle) as f64);
|
||||
}
|
||||
|
||||
#[cfg(not(tokio_unstable))]
|
||||
metrics::gauge!("rustfs_runtime_blocking_pool_metrics_available").set(0.0);
|
||||
|
||||
#[cfg(all(
|
||||
feature = "diagnostic-allocator-live",
|
||||
not(target_os = "windows"),
|
||||
not(all(feature = "hotpath", feature = "hotpath-alloc"))
|
||||
))]
|
||||
{
|
||||
metrics::gauge!("rustfs_memory_allocator_rust_global_requested_live_bytes_available").set(1.0);
|
||||
metrics::gauge!("rustfs_memory_allocator_rust_global_requested_live_bytes")
|
||||
.set(DIAGNOSTIC_GLOBAL_ALLOCATOR_LIVE_REQUESTED_BYTES.load(Ordering::Relaxed) as f64);
|
||||
}
|
||||
|
||||
#[cfg(not(all(
|
||||
feature = "diagnostic-allocator-live",
|
||||
not(target_os = "windows"),
|
||||
not(all(feature = "hotpath", feature = "hotpath-alloc"))
|
||||
)))]
|
||||
metrics::gauge!("rustfs_memory_allocator_rust_global_requested_live_bytes_available").set(0.0);
|
||||
}
|
||||
|
||||
async fn record_memory_snapshot(process_sampler: Arc<Mutex<ProcessSampler>>) {
|
||||
// Sample before this tick submits its own spawn_blocking snapshot task.
|
||||
record_tokio_runtime_metrics();
|
||||
|
||||
match tokio::task::spawn_blocking(move || {
|
||||
let mut sampler = process_sampler.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
let (resource, process) = sampler.snapshot_resource_and_system();
|
||||
@@ -464,6 +543,12 @@ pub fn init_memory_observability(ctx: CancellationToken) {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[cfg(all(
|
||||
feature = "diagnostic-allocator-live",
|
||||
not(target_os = "windows"),
|
||||
not(all(feature = "hotpath", feature = "hotpath-alloc"))
|
||||
))]
|
||||
use super::adjust_diagnostic_global_allocator_live_bytes;
|
||||
use super::{
|
||||
CgroupMemorySnapshot, MEMORY_OBSERVABILITY_SERVICE_NAME, MemoryObservabilityCancellationSource,
|
||||
MemoryObservabilityController, MemoryObservabilityDesiredState, MemoryObservabilityServiceState,
|
||||
@@ -472,6 +557,34 @@ mod tests {
|
||||
};
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
#[cfg(all(
|
||||
feature = "diagnostic-allocator-live",
|
||||
not(target_os = "windows"),
|
||||
not(all(feature = "hotpath", feature = "hotpath-alloc"))
|
||||
))]
|
||||
use std::sync::atomic::AtomicUsize;
|
||||
|
||||
#[cfg(all(
|
||||
feature = "diagnostic-allocator-live",
|
||||
not(target_os = "windows"),
|
||||
not(all(feature = "hotpath", feature = "hotpath-alloc"))
|
||||
))]
|
||||
#[test]
|
||||
fn diagnostic_global_allocator_counter_tracks_alloc_realloc_and_free() {
|
||||
let counter = AtomicUsize::new(0);
|
||||
|
||||
adjust_diagnostic_global_allocator_live_bytes(&counter, 0, 32);
|
||||
assert_eq!(counter.load(std::sync::atomic::Ordering::Relaxed), 32);
|
||||
|
||||
adjust_diagnostic_global_allocator_live_bytes(&counter, 32, 64);
|
||||
assert_eq!(counter.load(std::sync::atomic::Ordering::Relaxed), 64);
|
||||
|
||||
adjust_diagnostic_global_allocator_live_bytes(&counter, 64, 8);
|
||||
assert_eq!(counter.load(std::sync::atomic::Ordering::Relaxed), 8);
|
||||
|
||||
adjust_diagnostic_global_allocator_live_bytes(&counter, 8, 0);
|
||||
assert_eq!(counter.load(std::sync::atomic::Ordering::Relaxed), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_cgroup_memory_stat_extracts_tracked_numeric_fields() {
|
||||
|
||||
@@ -19,6 +19,7 @@ use hyper::{Request, Response};
|
||||
use pin_project_lite::pin_project;
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
use std::time::Instant;
|
||||
use tower::Service;
|
||||
|
||||
type BoxError = Box<dyn std::error::Error + Send + Sync + 'static>;
|
||||
@@ -35,6 +36,13 @@ pub(crate) fn is_grpc_request<B>(req: &Request<B>) -> bool {
|
||||
)
|
||||
}
|
||||
|
||||
fn rename_data_grpc_stage(path: &str) -> bool {
|
||||
matches!(
|
||||
path,
|
||||
"/node_service.NodeService/RenameData" | "/node_service.NodeService/RenameDataAtIncarnation"
|
||||
)
|
||||
}
|
||||
|
||||
/// The service that can serve both gRPC and REST HTTP Requests
|
||||
#[derive(Clone)]
|
||||
pub struct HybridService<Rest, Grpc> {
|
||||
@@ -71,8 +79,11 @@ where
|
||||
/// as a REST request
|
||||
fn call(&mut self, req: Request<Incoming>) -> Self::Future {
|
||||
if is_grpc_request(&req) {
|
||||
let observe_rename_data = rustfs_io_metrics::put_stage_metrics_enabled() && rename_data_grpc_stage(req.uri().path());
|
||||
HybridFuture::Grpc {
|
||||
grpc_future: self.grpc.call(req),
|
||||
observe_rename_data,
|
||||
service_future_started: observe_rename_data.then(Instant::now),
|
||||
}
|
||||
} else {
|
||||
HybridFuture::Rest {
|
||||
@@ -93,7 +104,10 @@ pin_project! {
|
||||
},
|
||||
Grpc {
|
||||
#[pin]
|
||||
grpc_body: GrpcBody
|
||||
grpc_body: GrpcBody,
|
||||
first_frame_started: Option<Instant>,
|
||||
body_started: Option<Instant>,
|
||||
first_frame_recorded: bool,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -123,21 +137,44 @@ where
|
||||
fn is_end_stream(&self) -> bool {
|
||||
match self {
|
||||
Self::Rest { rest_body } => rest_body.is_end_stream(),
|
||||
Self::Grpc { grpc_body } => grpc_body.is_end_stream(),
|
||||
Self::Grpc { grpc_body, .. } => grpc_body.is_end_stream(),
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_frame(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Result<Frame<Self::Data>, Self::Error>>> {
|
||||
match self.project() {
|
||||
HybridBodyProj::Rest { rest_body } => rest_body.poll_frame(cx).map_err(Into::into),
|
||||
HybridBodyProj::Grpc { grpc_body } => grpc_body.poll_frame(cx).map_err(Into::into),
|
||||
HybridBodyProj::Grpc {
|
||||
mut grpc_body,
|
||||
first_frame_started,
|
||||
body_started,
|
||||
first_frame_recorded,
|
||||
} => {
|
||||
let poll = grpc_body.as_mut().poll_frame(cx).map_err(Into::into);
|
||||
if !*first_frame_recorded && let Poll::Ready(_) = &poll {
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_FIRST_FRAME,
|
||||
first_frame_started.take(),
|
||||
);
|
||||
*first_frame_recorded = true;
|
||||
}
|
||||
let body_complete = matches!(&poll, Poll::Ready(None))
|
||||
|| (matches!(&poll, Poll::Ready(Some(Ok(_)))) && grpc_body.as_ref().get_ref().is_end_stream());
|
||||
if body_complete {
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_BODY_COMPLETE,
|
||||
body_started.take(),
|
||||
);
|
||||
}
|
||||
poll
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn size_hint(&self) -> http_body::SizeHint {
|
||||
match self {
|
||||
Self::Rest { rest_body } => rest_body.size_hint(),
|
||||
Self::Grpc { grpc_body } => grpc_body.size_hint(),
|
||||
Self::Grpc { grpc_body, .. } => grpc_body.size_hint(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -154,6 +191,8 @@ pin_project! {
|
||||
Grpc {
|
||||
#[pin]
|
||||
grpc_future: GrpcFuture,
|
||||
observe_rename_data: bool,
|
||||
service_future_started: Option<Instant>,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -174,8 +213,27 @@ where
|
||||
Poll::Ready(Err(err)) => Poll::Ready(Err(err.into())),
|
||||
Poll::Pending => Poll::Pending,
|
||||
},
|
||||
HybridFutureProj::Grpc { grpc_future } => match grpc_future.poll(cx) {
|
||||
Poll::Ready(Ok(res)) => Poll::Ready(Ok(res.map(|grpc_body| HybridBody::Grpc { grpc_body }))),
|
||||
HybridFutureProj::Grpc {
|
||||
grpc_future,
|
||||
observe_rename_data,
|
||||
service_future_started,
|
||||
} => match grpc_future.poll(cx) {
|
||||
Poll::Ready(Ok(res)) => {
|
||||
if *observe_rename_data {
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_SERVICE_FUTURE,
|
||||
service_future_started.take(),
|
||||
);
|
||||
}
|
||||
let first_frame_started = (*observe_rename_data).then(Instant::now);
|
||||
let body_started = (*observe_rename_data).then(Instant::now);
|
||||
Poll::Ready(Ok(res.map(|grpc_body| HybridBody::Grpc {
|
||||
grpc_body,
|
||||
first_frame_started,
|
||||
body_started,
|
||||
first_frame_recorded: !*observe_rename_data,
|
||||
})))
|
||||
}
|
||||
Poll::Ready(Err(err)) => Poll::Ready(Err(err.into())),
|
||||
Poll::Pending => Poll::Pending,
|
||||
},
|
||||
|
||||
@@ -863,7 +863,20 @@ where
|
||||
}
|
||||
Response::from_parts(parts, HybridBody::Rest { rest_body })
|
||||
}
|
||||
HybridBody::Grpc { grpc_body } => Response::from_parts(parts, HybridBody::Grpc { grpc_body }),
|
||||
HybridBody::Grpc {
|
||||
grpc_body,
|
||||
first_frame_started,
|
||||
body_started,
|
||||
first_frame_recorded,
|
||||
} => Response::from_parts(
|
||||
parts,
|
||||
HybridBody::Grpc {
|
||||
grpc_body,
|
||||
first_frame_started,
|
||||
body_started,
|
||||
first_frame_recorded,
|
||||
},
|
||||
),
|
||||
};
|
||||
|
||||
Ok(response)
|
||||
@@ -1012,7 +1025,20 @@ where
|
||||
}
|
||||
Response::from_parts(parts, HybridBody::Rest { rest_body })
|
||||
}
|
||||
HybridBody::Grpc { grpc_body } => Response::from_parts(parts, HybridBody::Grpc { grpc_body }),
|
||||
HybridBody::Grpc {
|
||||
grpc_body,
|
||||
first_frame_started,
|
||||
body_started,
|
||||
first_frame_recorded,
|
||||
} => Response::from_parts(
|
||||
parts,
|
||||
HybridBody::Grpc {
|
||||
grpc_body,
|
||||
first_frame_started,
|
||||
body_started,
|
||||
first_frame_recorded,
|
||||
},
|
||||
),
|
||||
};
|
||||
|
||||
Ok(response)
|
||||
@@ -1163,7 +1189,20 @@ where
|
||||
|
||||
Response::from_parts(parts, HybridBody::Rest { rest_body })
|
||||
}
|
||||
HybridBody::Grpc { grpc_body } => Response::from_parts(parts, HybridBody::Grpc { grpc_body }),
|
||||
HybridBody::Grpc {
|
||||
grpc_body,
|
||||
first_frame_started,
|
||||
body_started,
|
||||
first_frame_recorded,
|
||||
} => Response::from_parts(
|
||||
parts,
|
||||
HybridBody::Grpc {
|
||||
grpc_body,
|
||||
first_frame_started,
|
||||
body_started,
|
||||
first_frame_recorded,
|
||||
},
|
||||
),
|
||||
};
|
||||
|
||||
Ok(response)
|
||||
@@ -1240,7 +1279,20 @@ where
|
||||
},
|
||||
)
|
||||
}
|
||||
HybridBody::Grpc { grpc_body } => Response::from_parts(parts, HybridBody::Grpc { grpc_body }),
|
||||
HybridBody::Grpc {
|
||||
grpc_body,
|
||||
first_frame_started,
|
||||
body_started,
|
||||
first_frame_recorded,
|
||||
} => Response::from_parts(
|
||||
parts,
|
||||
HybridBody::Grpc {
|
||||
grpc_body,
|
||||
first_frame_started,
|
||||
body_started,
|
||||
first_frame_recorded,
|
||||
},
|
||||
),
|
||||
};
|
||||
|
||||
Ok(response)
|
||||
@@ -1308,7 +1360,20 @@ where
|
||||
rest_body: RestBody::from(Bytes::new()),
|
||||
},
|
||||
),
|
||||
HybridBody::Grpc { grpc_body } => Response::from_parts(parts, HybridBody::Grpc { grpc_body }),
|
||||
HybridBody::Grpc {
|
||||
grpc_body,
|
||||
first_frame_started,
|
||||
body_started,
|
||||
first_frame_recorded,
|
||||
} => Response::from_parts(
|
||||
parts,
|
||||
HybridBody::Grpc {
|
||||
grpc_body,
|
||||
first_frame_started,
|
||||
body_started,
|
||||
first_frame_recorded,
|
||||
},
|
||||
),
|
||||
};
|
||||
|
||||
Ok(response)
|
||||
|
||||
@@ -13,12 +13,16 @@
|
||||
// limitations under the License.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::init::{init_auto_tuner, init_update_check, print_server_info};
|
||||
use crate::startup_runtime_sources;
|
||||
use crate::storage_api::startup::storage::{ECStore, init_compression_total_memory_from_backend};
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
const GET_STAGE_LOCAL_SUMMARY_FILE: &str = "/var/log/rustfs/get-stage-summary.log";
|
||||
|
||||
pub(crate) async fn init_observability_runtime(store: Arc<ECStore>, ctx: CancellationToken) {
|
||||
print_server_info();
|
||||
init_update_check();
|
||||
@@ -27,6 +31,10 @@ pub(crate) async fn init_observability_runtime(store: Arc<ECStore>, ctx: Cancell
|
||||
let metrics_enabled = startup_runtime_sources::observability_metric_enabled();
|
||||
configure_metric_gates(metrics_enabled);
|
||||
|
||||
if rustfs_io_metrics::get_stage_local_summary_enabled() {
|
||||
init_get_stage_local_summary_runtime(ctx.clone());
|
||||
}
|
||||
|
||||
if metrics_enabled {
|
||||
// Load persisted compression stats into memory early, before any PUTs can occur.
|
||||
init_compression_total_memory_from_backend(store).await;
|
||||
@@ -37,21 +45,115 @@ pub(crate) async fn init_observability_runtime(store: Arc<ECStore>, ctx: Cancell
|
||||
}
|
||||
|
||||
fn configure_metric_gates(metrics_enabled: bool) {
|
||||
let local_summary_enabled = !metrics_enabled
|
||||
&& rustfs_utils::get_env_bool(rustfs_config::observability::ENV_OBS_GET_STAGE_LOCAL_SUMMARY_ENABLED, false);
|
||||
let sample_rate =
|
||||
rustfs_utils::get_env_u64(rustfs_config::observability::ENV_OBS_GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE, 64).max(1);
|
||||
startup_runtime_sources::set_get_stage_local_summary_enabled(local_summary_enabled);
|
||||
startup_runtime_sources::set_get_stage_local_summary_sample_rate(sample_rate);
|
||||
let put_stage_metrics_enabled = metrics_enabled
|
||||
&& rustfs_utils::get_env_bool(
|
||||
rustfs_config::observability::ENV_OBS_PUT_STAGE_METRICS_ENABLED,
|
||||
rustfs_config::DEFAULT_OBS_PUT_STAGE_METRICS_ENABLED,
|
||||
);
|
||||
startup_runtime_sources::set_put_stage_metrics_enabled(put_stage_metrics_enabled);
|
||||
startup_runtime_sources::set_get_stage_metrics_enabled(metrics_enabled);
|
||||
startup_runtime_sources::set_get_stage_metrics_enabled(metrics_enabled || local_summary_enabled);
|
||||
startup_runtime_sources::set_metrics_enabled(metrics_enabled);
|
||||
}
|
||||
|
||||
fn init_get_stage_local_summary_runtime(ctx: CancellationToken) {
|
||||
let interval_secs =
|
||||
rustfs_utils::get_env_u64(rustfs_config::observability::ENV_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS, 15).max(1);
|
||||
tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(Duration::from_secs(interval_secs));
|
||||
interval.tick().await;
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = ctx.cancelled() => break,
|
||||
_ = interval.tick() => {
|
||||
let rows = rustfs_io_metrics::take_get_stage_local_summary();
|
||||
if rows.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let summary = rows.iter().map(|row| {
|
||||
format!("{}|{}|{}|{}|{}|{}", row.path, row.stage, row.object_class, row.size_bucket, row.sampled_count, row.duration_nanoseconds)
|
||||
}).collect::<Vec<_>>().join(";");
|
||||
let timestamp_ms = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_millis();
|
||||
let line = format!(
|
||||
"timestamp_ms={timestamp_ms}\tsample_rate={}\tstage_count={}\tstages={summary}\n",
|
||||
rustfs_io_metrics::get_stage_local_summary_sample_rate(),
|
||||
rows.len(),
|
||||
);
|
||||
let Ok(mut file) = tokio::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(GET_STAGE_LOCAL_SUMMARY_FILE)
|
||||
.await
|
||||
else {
|
||||
break;
|
||||
};
|
||||
if file.write_all(line.as_bytes()).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const PUT_STAGE_ENV: &str = rustfs_config::observability::ENV_OBS_PUT_STAGE_METRICS_ENABLED;
|
||||
const LOCAL_SUMMARY_ENV: &str = rustfs_config::observability::ENV_OBS_GET_STAGE_LOCAL_SUMMARY_ENABLED;
|
||||
const LOCAL_SUMMARY_RATE_ENV: &str = rustfs_config::observability::ENV_OBS_GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE;
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn get_stage_local_summary_can_run_without_metrics_export() {
|
||||
let previous_metrics = rustfs_io_metrics::metrics_enabled();
|
||||
let previous_get_stages = rustfs_io_metrics::get_stage_metrics_enabled();
|
||||
let previous_local_summary = rustfs_io_metrics::get_stage_local_summary_enabled();
|
||||
let previous_sample_rate = rustfs_io_metrics::get_stage_local_summary_sample_rate();
|
||||
|
||||
temp_env::with_var(LOCAL_SUMMARY_ENV, Some("true"), || {
|
||||
temp_env::with_var(LOCAL_SUMMARY_RATE_ENV, Some("1"), || {
|
||||
configure_metric_gates(false);
|
||||
assert!(!rustfs_io_metrics::metrics_enabled());
|
||||
assert!(rustfs_io_metrics::get_stage_metrics_enabled());
|
||||
assert!(rustfs_io_metrics::get_stage_local_summary_enabled());
|
||||
rustfs_io_metrics::take_get_stage_local_summary();
|
||||
|
||||
rustfs_io_metrics::record_get_object_stage_duration("legacy_duplex", "metadata", 0.001);
|
||||
rustfs_io_metrics::record_get_object_stage_duration("legacy_duplex", "metadata", 0.003);
|
||||
rustfs_io_metrics::record_get_object_stage_duration_by_size(
|
||||
"legacy_duplex",
|
||||
"inline_direct.decode",
|
||||
"plain_single_part",
|
||||
"le_4kib",
|
||||
0.002,
|
||||
);
|
||||
|
||||
let rows = rustfs_io_metrics::take_get_stage_local_summary();
|
||||
let metadata = rows
|
||||
.iter()
|
||||
.find(|row| row.path == "legacy_duplex" && row.stage == "metadata")
|
||||
.expect("local summary should retain sampled metadata observations");
|
||||
assert_eq!(metadata.sampled_count, 2);
|
||||
assert_eq!(metadata.duration_nanoseconds, 4_000_000);
|
||||
assert!(rows.iter().any(|row| row.size_bucket == "le_4kib"));
|
||||
});
|
||||
});
|
||||
|
||||
startup_runtime_sources::set_metrics_enabled(previous_metrics);
|
||||
startup_runtime_sources::set_get_stage_metrics_enabled(previous_get_stages);
|
||||
startup_runtime_sources::set_get_stage_local_summary_enabled(previous_local_summary);
|
||||
startup_runtime_sources::set_get_stage_local_summary_sample_rate(previous_sample_rate);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
|
||||
@@ -98,6 +98,14 @@ pub(crate) fn set_get_stage_metrics_enabled(enabled: bool) {
|
||||
rustfs_io_metrics::set_get_stage_metrics_enabled(enabled);
|
||||
}
|
||||
|
||||
pub(crate) fn set_get_stage_local_summary_enabled(enabled: bool) {
|
||||
rustfs_io_metrics::set_get_stage_local_summary_enabled(enabled);
|
||||
}
|
||||
|
||||
pub(crate) fn set_get_stage_local_summary_sample_rate(sample_rate: u64) {
|
||||
rustfs_io_metrics::set_get_stage_local_summary_sample_rate(sample_rate);
|
||||
}
|
||||
|
||||
pub(crate) fn set_metrics_enabled(enabled: bool) {
|
||||
rustfs_io_metrics::set_metrics_enabled(enabled);
|
||||
}
|
||||
|
||||
@@ -1337,6 +1337,8 @@ impl NodeService {
|
||||
&self,
|
||||
request: Request<RenameDataRequest>,
|
||||
) -> Result<Response<RenameDataResponse>, Status> {
|
||||
let service_return_started = rustfs_io_metrics::put_stage_timer();
|
||||
let service_entry_started = rustfs_io_metrics::put_stage_timer();
|
||||
if !request.get_ref().scanner_publication_lease_token.is_empty() || !request.get_ref().bucket_incarnation_id.is_empty() {
|
||||
let has_body_digest = request
|
||||
.metadata()
|
||||
@@ -1344,14 +1346,37 @@ impl NodeService {
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.is_some_and(|value| value != "UNSIGNED-PAYLOAD");
|
||||
if !has_body_digest {
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_ENTRY,
|
||||
service_entry_started,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN,
|
||||
service_return_started,
|
||||
);
|
||||
return Err(Status::permission_denied("scanner publication lease rename requires a body-bound digest"));
|
||||
}
|
||||
}
|
||||
verify_disk_mutation_digest(
|
||||
if let Err(err) = verify_disk_mutation_digest(
|
||||
&request,
|
||||
rustfs_protos::canonical_rename_data_request_body(request.get_ref()),
|
||||
"rename_data",
|
||||
)?;
|
||||
) {
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_ENTRY,
|
||||
service_entry_started,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN,
|
||||
service_return_started,
|
||||
);
|
||||
return Err(err);
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_ENTRY,
|
||||
service_entry_started,
|
||||
);
|
||||
let before_handler_started = rustfs_io_metrics::put_stage_timer();
|
||||
let request = request.into_inner();
|
||||
let target = self.local_mutation_target();
|
||||
let bucket_incarnation = if request.bucket_incarnation_id.is_empty() {
|
||||
@@ -1362,6 +1387,14 @@ impl NodeService {
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or_else(|| Status::invalid_argument("bucket incarnation must be a non-nil UUID"))?;
|
||||
if !request.scanner_publication_lease_token.is_empty() {
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_BEFORE_HANDLER,
|
||||
before_handler_started,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN,
|
||||
service_return_started,
|
||||
);
|
||||
return Err(Status::invalid_argument(
|
||||
"heal incarnation and scanner publication lease cannot be combined",
|
||||
));
|
||||
@@ -1373,12 +1406,26 @@ impl NodeService {
|
||||
let decoded_file_info = match decode_rename_data_request_file_info(&request.file_info_bin, &request.file_info) {
|
||||
Ok(file_info) => file_info,
|
||||
Err(err) => {
|
||||
return Ok(Response::new(RenameDataResponse {
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_BEFORE_HANDLER,
|
||||
before_handler_started,
|
||||
);
|
||||
let response_build_started = rustfs_io_metrics::put_stage_timer();
|
||||
let response = Response::new(RenameDataResponse {
|
||||
success: false,
|
||||
rename_data_resp: String::new(),
|
||||
rename_data_resp_bin: Vec::new().into(),
|
||||
error: Some(DiskError::other(format!("decode FileInfo failed: {err}")).into()),
|
||||
}));
|
||||
});
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_BUILD,
|
||||
response_build_started,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN,
|
||||
service_return_started,
|
||||
);
|
||||
return Ok(response);
|
||||
}
|
||||
};
|
||||
let scanner_publication_lease_token = if request.scanner_publication_lease_token.is_empty() {
|
||||
@@ -1387,6 +1434,14 @@ impl NodeService {
|
||||
let token = Uuid::from_slice(&request.scanner_publication_lease_token)
|
||||
.map_err(|_| Status::invalid_argument("scanner publication lease token must be a UUID"))?;
|
||||
if token.is_nil() {
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_BEFORE_HANDLER,
|
||||
before_handler_started,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN,
|
||||
service_return_started,
|
||||
);
|
||||
return Err(Status::invalid_argument("scanner publication lease token must not be nil"));
|
||||
}
|
||||
Some(token)
|
||||
@@ -1394,6 +1449,11 @@ impl NodeService {
|
||||
let request_decoded_from_msgpack = decoded_file_info.from_msgpack;
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let observation = startup_cas_rename_observation(&target, &request, &decoded_file_info.value);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_BEFORE_HANDLER,
|
||||
before_handler_started,
|
||||
);
|
||||
let handler_await_started = rustfs_io_metrics::put_stage_timer();
|
||||
let result = target
|
||||
.rename_local_data(
|
||||
&request.disk,
|
||||
@@ -1404,6 +1464,11 @@ impl NodeService {
|
||||
bucket_incarnation,
|
||||
)
|
||||
.await;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_HANDLER_AWAIT,
|
||||
handler_await_started,
|
||||
);
|
||||
let after_handler_started = rustfs_io_metrics::put_stage_timer();
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
if let Some(mut observation) = observation {
|
||||
observation["ok"] = serde_json::json!(result.is_ok());
|
||||
@@ -1411,28 +1476,47 @@ impl NodeService {
|
||||
let line = format!("RUSTFS_E2E_STARTUP_CAS {observation}\n");
|
||||
let _ = std::io::Write::write_all(&mut std::io::stderr().lock(), line.as_bytes());
|
||||
}
|
||||
match result {
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_AFTER_HANDLER,
|
||||
after_handler_started,
|
||||
);
|
||||
let response_build_started = rustfs_io_metrics::put_stage_timer();
|
||||
let response_encode_started = rustfs_io_metrics::put_stage_timer();
|
||||
let response = match result {
|
||||
Ok(rename_data_resp) => match encode_rename_data_response_payloads(&rename_data_resp, request_decoded_from_msgpack) {
|
||||
Ok((rename_data_resp, rename_data_resp_bin)) => Ok(Response::new(RenameDataResponse {
|
||||
Ok((rename_data_resp, rename_data_resp_bin)) => Response::new(RenameDataResponse {
|
||||
success: true,
|
||||
rename_data_resp,
|
||||
rename_data_resp_bin: rename_data_resp_bin.into(),
|
||||
error: None,
|
||||
})),
|
||||
Err(err) => Ok(Response::new(RenameDataResponse {
|
||||
}),
|
||||
Err(err) => Response::new(RenameDataResponse {
|
||||
success: false,
|
||||
rename_data_resp: String::new(),
|
||||
rename_data_resp_bin: Vec::new().into(),
|
||||
error: Some(err.into()),
|
||||
})),
|
||||
}),
|
||||
},
|
||||
Err(err) => Ok(Response::new(RenameDataResponse {
|
||||
Err(err) => Response::new(RenameDataResponse {
|
||||
success: false,
|
||||
rename_data_resp: String::new(),
|
||||
rename_data_resp_bin: Vec::new().into(),
|
||||
error: Some(err.into()),
|
||||
})),
|
||||
}
|
||||
}),
|
||||
};
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_ENCODE,
|
||||
response_encode_started,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_BUILD,
|
||||
response_build_started,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN,
|
||||
service_return_started,
|
||||
);
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
pub(super) async fn handle_list_dir(&self, request: Request<ListDirRequest>) -> Result<Response<ListDirResponse>, Status> {
|
||||
|
||||
@@ -412,21 +412,21 @@ impl RequestTimeoutWrapper {
|
||||
|
||||
debug!(timeout_secs = timeout_duration.as_secs(), "Starting timed operation");
|
||||
|
||||
rustfs_io_metrics::record_get_object_request_started();
|
||||
rustfs_io_metrics::record_get_object_timeout_operation_start();
|
||||
|
||||
let cancel_token_for_op = self.cancel_token.clone();
|
||||
|
||||
match tokio::time::timeout(timeout_duration, operation(cancel_token_for_op)).await {
|
||||
Ok(Ok(result)) => {
|
||||
let elapsed = self.elapsed();
|
||||
rustfs_io_metrics::record_get_object_request_result("success", elapsed.as_secs_f64());
|
||||
rustfs_io_metrics::record_get_object_timeout_operation_result("success", elapsed.as_secs_f64());
|
||||
debug!(elapsed_ms = elapsed.as_millis(), "Operation completed successfully");
|
||||
|
||||
TimedGetObjectResult::Success(result)
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
let elapsed = self.elapsed();
|
||||
rustfs_io_metrics::record_get_object_request_result("error", elapsed.as_secs_f64());
|
||||
rustfs_io_metrics::record_get_object_timeout_operation_result("error", elapsed.as_secs_f64());
|
||||
debug!(elapsed_ms = elapsed.as_millis(), "Operation failed with error");
|
||||
|
||||
TimedGetObjectResult::Error(e)
|
||||
@@ -436,7 +436,7 @@ impl RequestTimeoutWrapper {
|
||||
self.cancel_token.cancel();
|
||||
|
||||
rustfs_io_metrics::record_get_object_timeout(None, Some(elapsed.as_secs_f64()));
|
||||
rustfs_io_metrics::record_get_object_request_result("timeout", elapsed.as_secs_f64());
|
||||
rustfs_io_metrics::record_get_object_timeout_operation_result("timeout", elapsed.as_secs_f64());
|
||||
|
||||
warn!(
|
||||
timeout_secs = timeout_duration.as_secs(),
|
||||
|
||||
Reference in New Issue
Block a user