Compare commits

...

10 Commits

Author SHA1 Message Date
Hauser 138bc69f6a perf: bound fdatasync group follower wait 2026-09-26 13:05:01 +08:00
Hauser 1fa7d72778 fix cargo deny check warn 2026-09-26 10:28:27 +08:00
Hauser 5b37dbbac6 chore(metrics): add PUT post-store stage probes
Add lightweight PUT post-store stage probes for issue 2615 validation.
2026-09-25 20:15:59 +08:00
Hauser 6cab5cbb0b Merge remote-tracking branch 'origin/main' into improve/base 2026-09-25 20:11:37 +08:00
Hauser 996016df2d upgrade version 2026-09-25 19:15:44 +08:00
Hauser 25382f4ff7 Merge remote-tracking branch 'origin/main' into improve/base
Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-09-22 19:32:41 +08:00
Hauser 2f9e3efcf8 perf(ecstore): split cleanup worker wait metrics
Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-09-22 19:31:09 +08:00
Hauser 47ca87f82e Merge remote-tracking branch 'origin/main' into improve/base
Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-09-22 16:46:46 +08:00
Hauser 30d416fbb4 perf(ecstore): instrument put object hot path
Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-09-22 16:44:47 +08:00
Hauser aaba1b40a4 chore: pin s3s to upstream git revision
Use the upstream s3s git repository at 50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6 for both s3s and s3s-sigv4.

Allow the official s3s git source in cargo-deny so source checks continue to pass.

Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-09-18 22:04:40 +08:00
23 changed files with 3278 additions and 345 deletions
Generated
+190 -157
View File
@@ -1193,9 +1193,9 @@ dependencies = [
[[package]]
name = "aws-smithy-http"
version = "0.64.0"
version = "0.64.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37843d9add67c3aff5856f409c6dc315d3cdff60f9c0cb5b670dab1e9920306d"
checksum = "639b4d8f8555f24a9be649811c3eb0b4d4616f4d61daf0c32e28873bc1ea9af1"
dependencies = [
"aws-smithy-eventstream",
"aws-smithy-runtime-api",
@@ -1321,9 +1321,9 @@ dependencies = [
[[package]]
name = "aws-smithy-runtime-api"
version = "1.17.0"
version = "1.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fbf162725183ec0df77a9dc82ce22d9ba0b6ea7f376e8ec4f6b695727c70698"
checksum = "9bb0deb40a69de59f7809251cfe1ab16de135f42c49d222d1202b4d0ce19720f"
dependencies = [
"aws-smithy-async",
"aws-smithy-runtime-api-macros",
@@ -1360,9 +1360,9 @@ dependencies = [
[[package]]
name = "aws-smithy-types"
version = "1.8.0"
version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16ecf7989e381d0cbb7ca06104140d940bf38d8eb9004bd3a12d23df03e3d363"
checksum = "69bb407740a197147da48238ecc94498493c9e85445732360cec180296ca45f1"
dependencies = [
"base64-simd",
"bytes",
@@ -1650,7 +1650,7 @@ version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
dependencies = [
"generic-array 0.14.9",
"generic-array 0.14.7",
]
[[package]]
@@ -1669,7 +1669,7 @@ version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
dependencies = [
"generic-array 0.14.9",
"generic-array 0.14.7",
]
[[package]]
@@ -1746,7 +1746,7 @@ checksum = "f8b851b75c23ca7873623d612fe49bd1989aeb03d08fb9432187eb253d3d4c6b"
dependencies = [
"alloc-no-stdlib 3.0.0",
"alloc-stdlib 0.3.0",
"brotli-decompressor 6.0.0",
"brotli-decompressor 6.0.1",
]
[[package]]
@@ -1761,9 +1761,9 @@ dependencies = [
[[package]]
name = "brotli-decompressor"
version = "6.0.0"
version = "6.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46087801e40b785410c352bee60e9b6226312709cf52d5fff92776cdf5cf6490"
checksum = "941cd9bd4ddab83cb46fa5a2d428f1c857b24ac78cb876cf7beb710840934bd7"
dependencies = [
"alloc-no-stdlib 3.0.0",
"alloc-stdlib 0.3.0",
@@ -1971,9 +1971,9 @@ dependencies = [
[[package]]
name = "cc"
version = "1.4.7"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54413ede23c2daf518f35156dfde027feb2374004d63bd497f983c8db9c0e313"
checksum = "207aedd8d29a0eb804986bc99eed0d43719fec84a9b7f9c8b70cf83b97cffcc1"
dependencies = [
"find-msvc-tools",
"jobserver",
@@ -2081,7 +2081,7 @@ version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
dependencies = [
"crypto-common 0.1.6",
"crypto-common 0.1.7",
"inout 0.1.4",
]
@@ -2557,7 +2557,7 @@ version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
dependencies = [
"generic-array 0.14.9",
"generic-array 0.14.7",
"rand_core 0.6.4",
"subtle",
"zeroize",
@@ -2582,11 +2582,11 @@ dependencies = [
[[package]]
name = "crypto-common"
version = "0.1.6"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
dependencies = [
"generic-array 0.14.9",
"generic-array 0.14.7",
"typenum",
]
@@ -3931,7 +3931,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
dependencies = [
"block-buffer 0.10.4",
"const-oid 0.9.6",
"crypto-common 0.1.6",
"crypto-common 0.1.7",
"subtle",
]
@@ -4176,7 +4176,7 @@ dependencies = [
"crypto-bigint 0.5.5",
"digest 0.10.7",
"ff 0.13.1",
"generic-array 0.14.9",
"generic-array 0.14.7",
"group 0.13.0",
"hkdf 0.12.4",
"pem-rfc7468 0.7.0",
@@ -4217,13 +4217,12 @@ checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
[[package]]
name = "encoding_rs"
version = "0.8.41"
version = "0.8.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b5ef0006ac9ab233c38522f5ae99cae3625151de8f706cacee1cba4b8e2832a"
checksum = "8e985e0451871ad22fb8d2b6b076e2028a502a0d3950998c2c5c0a4f9b5d9679"
dependencies = [
"cfg-if",
"core_detect",
"multiversion",
"multiversion_no_op",
"rustversion",
"scopeguard",
@@ -4406,9 +4405,9 @@ dependencies = [
[[package]]
name = "find-msvc-tools"
version = "0.1.13"
version = "0.1.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ef25905e51abafe4dcea6c15fec58c57b601cdbd0ee53d22ea1d3016c587d39b"
checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484"
[[package]]
name = "findshlibs"
@@ -4638,9 +4637,9 @@ dependencies = [
[[package]]
name = "generic-array"
version = "0.14.9"
version = "0.14.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2"
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
dependencies = [
"typenum",
"version_check",
@@ -4653,7 +4652,7 @@ version = "1.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "337d46834ee672ab3e48caca2cb0c78cc174fb12b3a68d0d88f99a0519a5e36e"
dependencies = [
"generic-array 0.14.9",
"generic-array 0.14.7",
"rustversion",
"typenum",
]
@@ -4756,9 +4755,9 @@ dependencies = [
[[package]]
name = "google-cloud-auth"
version = "1.16.0"
version = "1.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff461519b1a948200f163574be072753bcfb462a323f0eb426629d89872dd685"
checksum = "ac9870b8ea484519aa8601e14886f2efc148ea490034e43d04bef1d246170cc9"
dependencies = [
"async-trait",
"aws-lc-rs",
@@ -4785,9 +4784,9 @@ dependencies = [
[[package]]
name = "google-cloud-gax"
version = "1.14.0"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c5615cff28ee59cfe52fbb4c11b8b1e77f650296e2ea4f4c2b7757ac6b19e752"
checksum = "fc64ee071c0d22cb68662b454f1bea5424506cafeaddc51260fbc585bac1d29d"
dependencies = [
"bytes",
"futures",
@@ -4805,9 +4804,9 @@ dependencies = [
[[package]]
name = "google-cloud-gax-internal"
version = "0.7.18"
version = "0.7.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2766757d877a7a8ac23da9884cb0e3f10ed9b75a0ce59801ce6b19bf9d5819e"
checksum = "c47c73e95dc5e1de74bebe21b406394b53ff5ccfdfe0e81a0443cdef468e6744"
dependencies = [
"bytes",
"futures",
@@ -4821,9 +4820,9 @@ dependencies = [
"http-body-util",
"hyper",
"lazy_static",
"opentelemetry",
"opentelemetry-semantic-conventions",
"opentelemetry_sdk",
"opentelemetry 0.32.0",
"opentelemetry-semantic-conventions 0.32.1",
"opentelemetry_sdk 0.32.1",
"percent-encoding",
"pin-project",
"prost 0.14.4",
@@ -4839,14 +4838,14 @@ dependencies = [
"tonic-prost",
"tower",
"tracing",
"tracing-opentelemetry",
"tracing-opentelemetry 0.33.0",
]
[[package]]
name = "google-cloud-iam-v1"
version = "1.12.0"
version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f962b40234b1531e6ef73f7558871c96e117231e962086c98804323fb8d2c82"
checksum = "48f0895fa7b7bd70629ae58daaa962677e93a72c251b2f297bc058a035940821"
dependencies = [
"async-trait",
"bytes",
@@ -4862,9 +4861,9 @@ dependencies = [
[[package]]
name = "google-cloud-longrunning"
version = "1.13.0"
version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c0363c5389ffda2b55cd8a86eef4b19a3481a48467c91dc5d77f626a9572766"
checksum = "e8ea252eee22a4e2187025ea6789c682a0d791a9748ea99de2abec3026a32cb4"
dependencies = [
"async-trait",
"bytes",
@@ -4880,9 +4879,9 @@ dependencies = [
[[package]]
name = "google-cloud-lro"
version = "1.10.0"
version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47af3deef75c14a2983c430898d960c765bddbcc9f9188ca0563108e9227cfe7"
checksum = "dfb938b40ede9ef70c24f49c2687d84f289c9e9104ddfece739985e4b4844b8f"
dependencies = [
"google-cloud-gax",
"google-cloud-gax-internal",
@@ -4909,9 +4908,9 @@ dependencies = [
[[package]]
name = "google-cloud-storage"
version = "1.18.0"
version = "1.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "973399251b245c63f1d02d0768772833dcf1372ee358f593fb9159de8fe9c7d4"
checksum = "5f7fa1e66d6b47220dbfc5f243b634136e5b8d14d64cb13419bd59f976681da3"
dependencies = [
"async-trait",
"base64 0.23.1",
@@ -4962,11 +4961,11 @@ dependencies = [
[[package]]
name = "google-cloud-wkt"
version = "1.7.0"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fccf98cfd5481a5f5a285181ab0c62123d7d47cd2bb7299448440649349e4e7"
checksum = "ad4b36fefda4485f0922de7a58a3e562e984d513442a1aae2281dce25124be2e"
dependencies = [
"base64 0.22.1",
"base64 0.23.1",
"bytes",
"serde",
"serde_json",
@@ -5096,9 +5095,9 @@ dependencies = [
[[package]]
name = "headers"
version = "0.4.1"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3314d5adb5d94bcdf56771f2e50dbbc80bb4bdf88967526706205ac9eff24eb"
checksum = "fc2700e3fe555c3310aa7286cac6167449f2c87e0eb58769c9208a1c58a1d106"
dependencies = [
"base64 0.22.1",
"bytes",
@@ -5283,9 +5282,9 @@ dependencies = [
[[package]]
name = "hotpath"
version = "0.26.0"
version = "0.26.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f37234f85a8d1bda6b84d486245dd0847c5b33cca10d5f3691375e66704de0f8"
checksum = "036ff36fa1c4ae8dfb686606537eaa15b5d5da0c51bf94abeaf9dfc6ee54cac7"
dependencies = [
"arc-swap",
"async-channel",
@@ -5297,6 +5296,7 @@ dependencies = [
"futures-core",
"futures-sink",
"hdrhistogram",
"hotpath-drain",
"hotpath-macros",
"http 1.5.0",
"libc",
@@ -5315,10 +5315,16 @@ dependencies = [
]
[[package]]
name = "hotpath-macros"
version = "0.26.0"
name = "hotpath-drain"
version = "0.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ac39b5517f540cab511bdbd5a4dc8cfc62858d6cd26a095463c1d2acfb3f290"
checksum = "e149b660b611650bdf3307941ef3e9b81be3c30ed1dfdbe7f670319674f023a6"
[[package]]
name = "hotpath-macros"
version = "0.26.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ff68b15fdbcddfbad685f445597580d6fcbf28b49f01ff975e934351079e13b"
dependencies = [
"proc-macro2",
"quote",
@@ -5459,16 +5465,17 @@ dependencies = [
[[package]]
name = "hyper-util"
version = "0.1.20"
version = "0.1.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0"
checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff"
dependencies = [
"base64 0.22.1",
"base64 0.23.1",
"bytes",
"futures-channel",
"futures-util",
"http 1.5.0",
"http-body",
"httparse",
"hyper",
"ipnet",
"libc",
@@ -5646,7 +5653,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
dependencies = [
"block-padding 0.3.3",
"generic-array 0.14.9",
"generic-array 0.14.7",
]
[[package]]
@@ -5905,9 +5912,9 @@ dependencies = [
[[package]]
name = "js-sys"
version = "0.3.105"
version = "0.3.106"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e"
checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5"
dependencies = [
"cfg-if",
"futures-util",
@@ -6699,27 +6706,6 @@ version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084"
[[package]]
name = "multiversion"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4ca4bea16ffc3f443cf7d866912118196bfef4c6a1556ca00f9f9b00bb43f7c"
dependencies = [
"multiversion-macros",
]
[[package]]
name = "multiversion-macros"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d416831a7317ef4b08bee00b69cbbb9c8763da7959a7026244d6266869f9c83"
dependencies = [
"proc-macro2",
"quote",
"rustversion",
"syn 3.0.6",
]
[[package]]
name = "multiversion_no_op"
version = "1.0.0"
@@ -7335,12 +7321,26 @@ dependencies = [
]
[[package]]
name = "opentelemetry-appender-tracing"
version = "0.32.0"
name = "opentelemetry"
version = "0.33.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c0080f0dc1d7c786f467cd85a4e395fcab11ee852004f39a29a18ab7c25d837"
checksum = "6cdb0b1b267eb9db3331b434ed9ddab10d50e280a9adf9d13e5233e2002b61b5"
dependencies = [
"opentelemetry",
"futures-core",
"futures-sink",
"js-sys",
"pin-project-lite",
"thiserror 2.0.21",
"tracing",
]
[[package]]
name = "opentelemetry-appender-tracing"
version = "0.33.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2b304e8d8f11326ba366a99ca5c7aaf7cd397b67268a332437175aeb42a40db"
dependencies = [
"opentelemetry 0.33.0",
"tracing",
"tracing-core",
"tracing-log",
@@ -7349,29 +7349,30 @@ dependencies = [
[[package]]
name = "opentelemetry-http"
version = "0.32.0"
version = "0.33.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5683015d09e2df236ef005b17f6f196f0d5f6313c4fa43a7b6a53b52776e4331"
checksum = "ee2c3625b8aa04209f7e01e513bc8044da9687fa38a9eacf59628ca5f3b87300"
dependencies = [
"async-trait",
"bytes",
"http 1.5.0",
"opentelemetry",
"opentelemetry 0.33.0",
"reqwest",
]
[[package]]
name = "opentelemetry-otlp"
version = "0.32.0"
version = "0.33.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9966929966d17620d7c316c643ba62631826e10021409357772d5eea84f62c35"
checksum = "699a67345e21962a231955b9059a157e428b219fa5df8efe11f08346fb23a344"
dependencies = [
"flate2",
"http 1.5.0",
"opentelemetry",
"httpdate",
"opentelemetry 0.33.0",
"opentelemetry-http",
"opentelemetry-proto",
"opentelemetry_sdk",
"opentelemetry_sdk 0.33.0",
"prost 0.14.4",
"reqwest",
"thiserror 2.0.21",
@@ -7379,12 +7380,12 @@ dependencies = [
[[package]]
name = "opentelemetry-proto"
version = "0.32.0"
version = "0.33.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638"
checksum = "25da1ac11a0aeccf38d7f77ee0348715adaf8340f65ad46c94a02c6b20e2f65d"
dependencies = [
"opentelemetry",
"opentelemetry_sdk",
"opentelemetry 0.33.0",
"opentelemetry_sdk 0.33.0",
"prost 0.14.4",
]
@@ -7395,14 +7396,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c913ac17a6c451661ee255f4625d143e51647ae78ebd969b75e41c4442f4fe47"
[[package]]
name = "opentelemetry-stdout"
version = "0.32.0"
name = "opentelemetry-semantic-conventions"
version = "0.33.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1b1c6a247d79091f0062a5f4bd058589525cf987a8d4c169440d9c1be72f0ad"
checksum = "d288b1e1bc3590052fbb8f0f635ef248dd9022cf1fbaeb6582b7ad6081a17b87"
[[package]]
name = "opentelemetry-stdout"
version = "0.33.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f71803f481d3f5ccbe477edb42a78f473613712b2327835911a9ea0ab069824"
dependencies = [
"chrono",
"opentelemetry",
"opentelemetry_sdk",
"opentelemetry 0.33.0",
"opentelemetry_sdk 0.33.0",
]
[[package]]
@@ -7414,7 +7421,23 @@ dependencies = [
"futures-channel",
"futures-executor",
"futures-util",
"opentelemetry",
"opentelemetry 0.32.0",
"percent-encoding",
"portable-atomic",
"rand 0.9.5",
"thiserror 2.0.21",
]
[[package]]
name = "opentelemetry_sdk"
version = "0.33.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb39533d9d1c912123efd7d41d7e0c29d16917b60ce15b4c8d87cb1af7f67520"
dependencies = [
"futures-channel",
"futures-executor",
"futures-util",
"opentelemetry 0.33.0",
"percent-encoding",
"portable-atomic",
"rand 0.9.5",
@@ -8943,9 +8966,9 @@ dependencies = [
[[package]]
name = "redis"
version = "1.7.0"
version = "1.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2acbc41a996f7652b2ddd9dfd98cc4ff602cfd742ae35382f07f608405ab50ed"
checksum = "ed5b98ae99461e02895c3a96a25dbe4d852a3d5e5b815d31d5d9e4b5344d99e9"
dependencies = [
"arc-swap",
"arcstr",
@@ -9518,9 +9541,9 @@ dependencies = [
"metrics-util",
"mime_guess",
"moka",
"opentelemetry",
"opentelemetry 0.33.0",
"opentelemetry-proto",
"opentelemetry_sdk",
"opentelemetry_sdk 0.33.0",
"p256 0.14.0",
"parking_lot",
"percent-encoding",
@@ -9610,7 +9633,7 @@ dependencies = [
"tower",
"tower-http 0.7.1",
"tracing",
"tracing-opentelemetry",
"tracing-opentelemetry 0.34.0",
"tracing-subscriber",
"url",
"urlencoding",
@@ -9788,8 +9811,8 @@ dependencies = [
"metrics-util",
"moka",
"num_cpus",
"opentelemetry",
"opentelemetry_sdk",
"opentelemetry 0.33.0",
"opentelemetry_sdk 0.33.0",
"parking_lot",
"path-absolutize",
"pin-project-lite",
@@ -9850,7 +9873,7 @@ dependencies = [
"tower",
"tracing",
"tracing-core",
"tracing-opentelemetry",
"tracing-opentelemetry 0.34.0",
"tracing-subscriber",
"url",
"urlencoding",
@@ -10366,12 +10389,12 @@ dependencies = [
"metrics-util",
"num_cpus",
"nvml-wrapper",
"opentelemetry",
"opentelemetry 0.33.0",
"opentelemetry-appender-tracing",
"opentelemetry-otlp",
"opentelemetry-semantic-conventions",
"opentelemetry-semantic-conventions 0.33.0",
"opentelemetry-stdout",
"opentelemetry_sdk",
"opentelemetry_sdk 0.33.0",
"percent-encoding",
"pyroscope",
"rcgen",
@@ -10403,7 +10426,7 @@ dependencies = [
"tracing",
"tracing-appender",
"tracing-error",
"tracing-opentelemetry",
"tracing-opentelemetry 0.34.0",
"tracing-subscriber",
"url",
"zstd 0.14.0",
@@ -11157,9 +11180,9 @@ dependencies = [
[[package]]
name = "rustls-platform-verifier"
version = "0.7.0"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0"
checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98"
dependencies = [
"core-foundation 0.10.1",
"core-foundation-sys",
@@ -11178,9 +11201,9 @@ dependencies = [
[[package]]
name = "rustls-platform-verifier-android"
version = "0.1.1"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"
checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f"
[[package]]
name = "rustls-webpki"
@@ -11229,9 +11252,8 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
[[package]]
name = "s3s"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "030a43692a1fd97bbd2c594eeaad8f5c1e452a9d29ad6a3eebce1650937c4a77"
version = "0.16.0"
source = "git+https://github.com/s3s-project/s3s.git?rev=f3da7ca7684d6e93b8e8250f037aadfe3e599134#f3da7ca7684d6e93b8e8250f037aadfe3e599134"
dependencies = [
"arc-swap",
"arrayvec",
@@ -11288,9 +11310,8 @@ dependencies = [
[[package]]
name = "s3s-multipart"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a543079d42c20c81781b7e31704ecacd64b0e3819b2384db5bf4869abf6c6f01"
version = "0.17.0-alpha.1"
source = "git+https://github.com/s3s-project/s3s.git?rev=f3da7ca7684d6e93b8e8250f037aadfe3e599134#f3da7ca7684d6e93b8e8250f037aadfe3e599134"
dependencies = [
"arrayvec",
"bytes",
@@ -11302,9 +11323,8 @@ dependencies = [
[[package]]
name = "s3s-rfc2047"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "918b207c39474baef4774355100e712dfea46020e3289ddd5b4bc6e21927b826"
version = "0.16.0"
source = "git+https://github.com/s3s-project/s3s.git?rev=f3da7ca7684d6e93b8e8250f037aadfe3e599134#f3da7ca7684d6e93b8e8250f037aadfe3e599134"
dependencies = [
"base64-simd",
"thiserror 2.0.21",
@@ -11312,9 +11332,8 @@ dependencies = [
[[package]]
name = "s3s-sigv2"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0979baf8cdf30e38bdfba8e58ee2776d78e55bb3bd3d1f3502f917829da79933"
version = "0.16.0"
source = "git+https://github.com/s3s-project/s3s.git?rev=f3da7ca7684d6e93b8e8250f037aadfe3e599134#f3da7ca7684d6e93b8e8250f037aadfe3e599134"
dependencies = [
"base64-simd",
"hmac 0.13.0",
@@ -11326,9 +11345,8 @@ dependencies = [
[[package]]
name = "s3s-sigv4"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "330996b9a24804b3df4ad37e62e7817968857d99cf6029c21512b6b64b97f645"
version = "0.16.0"
source = "git+https://github.com/s3s-project/s3s.git?rev=f3da7ca7684d6e93b8e8250f037aadfe3e599134#f3da7ca7684d6e93b8e8250f037aadfe3e599134"
dependencies = [
"arrayvec",
"base64-simd",
@@ -11448,7 +11466,7 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc"
dependencies = [
"base16ct 0.2.0",
"der 0.7.10",
"generic-array 0.14.9",
"generic-array 0.14.7",
"pkcs8 0.10.2",
"subtle",
"zeroize",
@@ -11899,9 +11917,9 @@ dependencies = [
[[package]]
name = "siphasher"
version = "1.0.3"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649"
checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256"
[[package]]
name = "sketches-ddsketch"
@@ -11951,9 +11969,9 @@ dependencies = [
[[package]]
name = "smallvec"
version = "1.16.1"
version = "1.16.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891"
checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e"
dependencies = [
"serde",
]
@@ -12233,9 +12251,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
[[package]]
name = "suppaftp"
version = "12.0.1"
version = "12.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba93b140f4d07f1b3f3fb4d7264b2d422d5c8b4e925eb6a2646e1342d5d2070a"
checksum = "d79f066c76927a775d61b5642e0b712a17582ebe7f62b7e5abba246e619878f7"
dependencies = [
"async-trait",
"chrono",
@@ -12748,9 +12766,9 @@ dependencies = [
[[package]]
name = "tokio-test"
version = "0.4.5"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f6d24790a10a7af737693a3e8f1d03faef7e6ca0cc99aae5066f533766de545"
checksum = "12bc5db7778a8e08ee255090861e0e7aac55e784300077e7ed293c7be36168b2"
dependencies = [
"futures-core",
"tokio",
@@ -13043,7 +13061,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26"
dependencies = [
"js-sys",
"opentelemetry",
"opentelemetry 0.32.0",
"tracing",
"tracing-core",
"tracing-subscriber",
"web-time",
]
[[package]]
name = "tracing-opentelemetry"
version = "0.34.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a904802a1b902f43638b677ff2a650847e3b4404101b6c586d648e8c1e3e8fe"
dependencies = [
"js-sys",
"opentelemetry 0.33.0",
"smallvec",
"tracing",
"tracing-core",
@@ -13401,9 +13433,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen"
version = "0.2.128"
version = "0.2.129"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf"
checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409"
dependencies = [
"cfg-if",
"once_cell",
@@ -13414,19 +13446,20 @@ dependencies = [
[[package]]
name = "wasm-bindgen-futures"
version = "0.4.78"
version = "0.4.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ef4c5d3d2cdf5c54f4231181768f5510842e350db025faf1f7163b1030ed928"
checksum = "3cbab34de2d982e9b48e18d216d04c4a6f641066ff19ffb699980f591ee3610e"
dependencies = [
"js-sys",
"tokio",
"wasm-bindgen",
]
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.128"
version = "0.2.129"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed"
checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@@ -13434,9 +13467,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.128"
version = "0.2.129"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a"
checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7"
dependencies = [
"bumpalo",
"proc-macro2",
@@ -13447,9 +13480,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.128"
version = "0.2.129"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e"
checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6"
dependencies = [
"unicode-ident",
]
@@ -13469,9 +13502,9 @@ dependencies = [
[[package]]
name = "web-sys"
version = "0.3.105"
version = "0.3.106"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9fbddc4a036f00ec4f18c83445bd3115cb306a91da554919a099d9222fe4a7f8"
checksum = "88261b9deccee56594c11a3460c462c41f58d148598fe70ad77070126a68aba4"
dependencies = [
"js-sys",
"wasm-bindgen",
+21 -21
View File
@@ -159,7 +159,7 @@ pulsar = { default-features = false, version = "6.9.0" }
lapin = { default-features = false, version = "4.12.0" }
hyper = { version = "1.11.1" }
hyper-rustls = { default-features = false, version = "0.27.10" }
hyper-util = { version = "0.1.20" }
hyper-util = { version = "0.1.21" }
http = "1.5.0"
http-body = "1.1.0"
http-body-util = "0.1.5"
@@ -171,7 +171,7 @@ socket2 = { version = "0.6.5" }
tokio = { version = "1.53.1" }
tokio-rustls = { default-features = false, version = "0.26.5" }
tokio-stream = { version = "0.1.19" }
tokio-test = "0.4.5"
tokio-test = "0.4.6"
tokio-util = { version = "0.7.19" }
tonic = { version = "0.14.6" }
tonic-prost = { version = "0.14.6" }
@@ -249,8 +249,8 @@ aws-sdk-s3 = { default-features = false, version = "1.149.0" }
aws-sdk-sts = { default-features = false, version = "1.117.0" }
aws-smithy-async = { version = "1.3.0" }
aws-smithy-http-client = { default-features = false, version = "1.4.2" }
aws-smithy-runtime-api = { version = "1.17.0" }
aws-smithy-types = { version = "1.8.0" }
aws-smithy-runtime-api = { version = "1.18.0" }
aws-smithy-types = { version = "1.8.1" }
base64-simd = "0.8.0"
brotli = "9.0.0"
clap = { version = "4.6.7" }
@@ -268,8 +268,8 @@ enumset = "1.1.14"
faster-hex = "1.0.0"
flate2 = "1.1.10"
glob = "0.3.4"
google-cloud-storage = "1.18.0"
google-cloud-auth = "1.16.0"
google-cloud-storage = "1.19.0"
google-cloud-auth = "1.17.0"
hashbrown = { version = "0.17.1" }
# Base32 for RFC 6238 TOTP shared secrets (RFC 4648 unpadded, the alphabet
# every authenticator app expects). Already in the graph transitively.
@@ -308,17 +308,17 @@ rustfs-erasure-codec = { version = "9.0.0" }
reed-solomon-simd = "3.1.0"
regex = { version = "1.13.1" }
rumqttc = { package = "rumqttc-next", version = "0.34.0" }
redis = { version = "1.7.0" }
redis = { version = "1.7.1" }
rustify = { version = "0.7", default-features = false }
rustix = { version = "1.1.5" }
rust-embed = { version = "8.12.0" }
rustc-hash = { version = "2.1.3" }
s3s = { version = "0.17.0", features = ["minio"] }
s3s-sigv4 = "0.17.0"
s3s = { git = "https://github.com/s3s-project/s3s.git", rev = "f3da7ca7684d6e93b8e8250f037aadfe3e599134", features = ["minio"] }
s3s-sigv4 = { git = "https://github.com/s3s-project/s3s.git", rev = "f3da7ca7684d6e93b8e8250f037aadfe3e599134" }
serial_test = "4.0.1"
shadow-rs = { default-features = false, version = "2.0.0" }
siphasher = "1.0.3"
smallvec = { version = "1.16.1" }
siphasher = "1.0.4"
smallvec = { version = "1.16.2" }
compact_str = "0.10.0"
snap = "1.1.2"
starshard = { version = "2.3.0" }
@@ -332,7 +332,7 @@ tracing = { version = "0.1.44" }
tracing-appender = "0.2.5"
tracing-core = "0.1.36"
tracing-error = "0.2.1"
tracing-opentelemetry = { version = "0.33" }
tracing-opentelemetry = { version = "0.34.0" }
tracing-subscriber = { version = "0.3.23" }
transform-stream = "0.3.1"
url = "2.5.8"
@@ -352,19 +352,19 @@ zstd = "0.14.0"
metrics = "0.24.6"
metrics-util = "0.20"
dial9 = { version = "0.5.2", default-features = false }
opentelemetry = { version = "0.32.0" }
opentelemetry-appender-tracing = { version = "0.32.0" }
opentelemetry-otlp = { version = "0.32.0" }
opentelemetry-proto = { version = "0.32.0", default-features = false, features = ["metrics", "gen-tonic-messages"] }
opentelemetry_sdk = { version = "0.32.1" }
opentelemetry-semantic-conventions = { version = "0.32.1" }
opentelemetry-stdout = { version = "0.32.0" }
opentelemetry = { version = "0.33.0" }
opentelemetry-appender-tracing = { version = "0.33.0" }
opentelemetry-otlp = { version = "0.33.0" }
opentelemetry-proto = { version = "0.33.0", default-features = false, features = ["metrics", "gen-tonic-messages"] }
opentelemetry_sdk = { version = "0.33.0" }
opentelemetry-semantic-conventions = { version = "0.33.0" }
opentelemetry-stdout = { version = "0.33.0" }
pyroscope = { version = "2.1.1" }
# FTP and SFTP
libunftp = { version = "0.23.0" }
unftp-core = "0.1.0"
suppaftp = { version = "12.0.1" }
suppaftp = { version = "12.1.0" }
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
russh = { version = "0.63.3" }
russh-sftp = "3.0.0"
@@ -375,7 +375,7 @@ dav-server = "0.11.0"
# Performance Analysis and Memory Profiling
rustfs-mimalloc = { version = "0.5.6" }
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
hotpath = { version = "0.26.0", default-features = false }
hotpath = { version = "0.26.1", default-features = false }
# Snapshot testing for output format regression detection
insta = { version = "1.48" }
+3 -1
View File
@@ -584,7 +584,9 @@ pub mod set_disk {
#[cfg(feature = "test-util")]
pub mod test_util {
pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test;
pub use crate::bucket::quota::reservation::{
SafeNoQuotaCacheFastPathGuard, enable_safe_no_quota_cache_fast_path_for_test, fail_next_quota_ledger_save_for_test,
};
pub use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause};
/// Keep a namespace commit pending until the returned owner is dropped.
+40
View File
@@ -178,6 +178,13 @@ pub(crate) fn object_lock_config_state_from_authoritative_metadata(bm: &BucketMe
Ok(ObjectLockConfigState::ConfirmedAbsent)
}
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct CachedQuotaMetadataSnapshot {
pub(crate) quota: Option<BucketQuota>,
pub(crate) bucket_incarnation: Uuid,
pub(crate) quota_revision: OffsetDateTime,
}
/// Convert the persisted serving-layer configuration into the storage-level
/// [`DefaultRetention`](crate::bucket::object_lock::types::DefaultRetention)
/// the WORM evaluation code consumes (rustfs/backlog#1842). A rule without a
@@ -1330,6 +1337,9 @@ pub(crate) async fn get_quota_config_and_incarnation_from_disk_in(
BucketMetadataAuthority::Authoritative(metadata)
if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
{
if !metadata.quota_config_json.is_empty() && metadata.quota_config.is_none() {
return Err(Error::other("persisted bucket quota configuration is invalid"));
}
Ok((
metadata.quota_config.clone(),
metadata.bucket_incarnation_id,
@@ -1344,6 +1354,15 @@ pub(crate) async fn get_quota_config_and_incarnation_from_disk_in(
}
}
pub(crate) async fn get_cached_quota_config_and_incarnation_in(
ctx: &crate::runtime::instance::InstanceContext,
bucket: &str,
) -> Result<Option<CachedQuotaMetadataSnapshot>> {
let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone();
bucket_meta_sys.cached_quota_config_and_incarnation(bucket).await
}
pub async fn get_replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
@@ -2749,6 +2768,27 @@ impl BucketMetadataSys {
}
}
async fn cached_quota_config_and_incarnation(&self, bucket: &str) -> Result<Option<CachedQuotaMetadataSnapshot>> {
let metadata = {
let metadata_map = self.metadata_map.read().await;
metadata_map.get(bucket).cloned()
};
let Some(metadata) = metadata else {
return Ok(None);
};
if !metadata.bucket_incarnation_sidecar || metadata.bucket_incarnation_id.is_nil() {
return Ok(None);
}
if !metadata.quota_config_json.is_empty() && metadata.quota_config.is_none() {
return Err(Error::other("persisted bucket quota configuration is invalid"));
}
Ok(Some(CachedQuotaMetadataSnapshot {
quota: metadata.quota_config.clone(),
bucket_incarnation: metadata.bucket_incarnation_id,
quota_revision: metadata.quota_config_updated_at,
}))
}
pub async fn get_replication_config(&self, bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
let (bm, _) = self.get_config(bucket).await?;
+188 -1
View File
@@ -27,7 +27,7 @@ use futures::{StreamExt, stream};
use rustfs_lock::NamespaceLockGuard;
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use std::sync::Arc;
use std::sync::{Arc, OnceLock};
use std::time::Duration;
use time::OffsetDateTime;
use tracing::warn;
@@ -41,9 +41,13 @@ const EVENT_QUOTA_LEDGER_SETTLEMENT: &str = "quota_ledger_settlement";
const EVENT_QUOTA_ADMISSION: &str = "quota_admission";
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_QUOTA: &str = "quota";
const QUOTA_CONFIG_READ_COUNTERFACTUAL_ENV: &str = "RUSTFS_QUOTA_BEGIN_ASSUME_NO_QUOTA_COUNTERFACTUAL";
const SAFE_NO_QUOTA_CACHE_FAST_PATH_ENV: &str = "RUSTFS_QUOTA_BEGIN_SAFE_NO_QUOTA_CACHE_FAST_PATH";
#[cfg(any(test, feature = "test-util"))]
static FAIL_NEXT_LEDGER_SAVE: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
#[cfg(any(test, feature = "test-util"))]
static SAFE_NO_QUOTA_CACHE_FAST_PATH_FOR_TEST: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
// Lock order: caller-held destination object/upload, bucket metadata
// transaction (read), operation reservation, then quota ledger.
@@ -448,6 +452,11 @@ pub(crate) async fn begin(
let snapshot_admission = opts.quota_admission;
let data_movement = opts.data_movement;
if crate::bucket::utils::is_meta_bucketname(bucket) {
let meta_bucket_fast_path_started = rustfs_io_metrics::put_stage_timer();
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_META_BUCKET_FAST_PATH,
meta_bucket_fast_path_started,
);
return Ok(QuotaContext {
store: None,
bucket: bucket.to_string(),
@@ -501,9 +510,105 @@ pub(crate) async fn begin(
});
}
let metadata_lock_started = rustfs_io_metrics::put_stage_timer();
let metadata_guard = metadata_sys::acquire_bucket_metadata_transaction_read_lock_for_options_in(ctx, bucket, opts).await?;
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_METADATA_LOCK,
metadata_lock_started,
);
let cache_lookup_started = rustfs_io_metrics::put_stage_timer();
let cached_quota_snapshot = match metadata_sys::get_cached_quota_config_and_incarnation_in(ctx, bucket).await {
Ok(Some(snapshot)) => {
rustfs_io_metrics::record_put_object_quota_cache_decision("candidate", quota_cache_reason(snapshot.quota.as_ref()));
Some(snapshot)
}
Ok(None) => {
rustfs_io_metrics::record_put_object_quota_cache_decision("miss", "no_authoritative_cache");
None
}
Err(_) => {
rustfs_io_metrics::record_put_object_quota_cache_decision("miss", "invalid_cached_quota");
None
}
};
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CACHE_LOOKUP,
cache_lookup_started,
);
if safe_no_quota_cache_fast_path_enabled()
&& let Some(cached) = cached_quota_snapshot.as_ref()
&& cached.quota.is_none()
&& !cached.bucket_incarnation.is_nil()
{
let safe_hit_started = rustfs_io_metrics::put_stage_timer();
rustfs_io_metrics::record_put_object_quota_cache_decision("safe_fast_path_hit", "no_quota");
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_SAFE_NO_QUOTA_CACHE_HIT,
safe_hit_started,
);
return Ok(QuotaContext {
store: None,
bucket: bucket.to_string(),
object: object.to_string(),
ledger_object: ledger_object(bucket),
bucket_incarnation: None,
quota_revision: None,
quota_limit: None,
capability_proof: None,
snapshot_admission: None,
legacy_data_movement: false,
metadata_guard: Some(metadata_guard),
pool_index: Some(pool_index),
set_index: Some(set_index),
});
}
if quota_config_read_counterfactual_enabled() {
let counterfactual_skip_started = rustfs_io_metrics::put_stage_timer();
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONFIG_READ_COUNTERFACTUAL_SKIP,
counterfactual_skip_started,
);
return Ok(QuotaContext {
store: None,
bucket: bucket.to_string(),
object: object.to_string(),
ledger_object: ledger_object(bucket),
bucket_incarnation: None,
quota_revision: None,
quota_limit: None,
capability_proof: None,
snapshot_admission: None,
legacy_data_movement: false,
metadata_guard: Some(metadata_guard),
pool_index: Some(pool_index),
set_index: Some(set_index),
});
}
let config_read_started = rustfs_io_metrics::put_stage_timer();
let (quota, bucket_incarnation, quota_revision) =
metadata_sys::get_quota_config_and_incarnation_from_disk_in(ctx, bucket).await?;
if let Some(cached) = cached_quota_snapshot.as_ref() {
let reason = quota_cache_reason(quota.as_ref());
if cached.quota == quota && cached.bucket_incarnation == bucket_incarnation && cached.quota_revision == quota_revision {
rustfs_io_metrics::record_put_object_quota_cache_decision("authoritative_match", reason);
} else {
rustfs_io_metrics::record_put_object_quota_cache_decision("authoritative_mismatch", reason);
if cached.quota != quota {
rustfs_io_metrics::record_put_object_quota_cache_decision("authoritative_mismatch_field", "quota");
}
if cached.bucket_incarnation != bucket_incarnation {
rustfs_io_metrics::record_put_object_quota_cache_decision("authoritative_mismatch_field", "bucket_incarnation");
}
if cached.quota_revision != quota_revision {
rustfs_io_metrics::record_put_object_quota_cache_decision("authoritative_mismatch_field", "quota_revision");
}
}
}
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONFIG_READ,
config_read_started,
);
let lock_lost_check_started = rustfs_io_metrics::put_stage_timer();
if metadata_guard.is_lock_lost() {
return Err(StorageError::NamespaceLockQuorumUnavailable {
mode: "quota_config",
@@ -513,6 +618,11 @@ pub(crate) async fn begin(
achieved: 0,
});
}
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_LOCK_LOST_CHECK,
lock_lost_check_started,
);
let policy_evaluate_started = rustfs_io_metrics::put_stage_timer();
if quota
.as_ref()
.is_some_and(|quota| quota.has_unsupported_reservation_protocol())
@@ -521,6 +631,11 @@ pub(crate) async fn begin(
return Err(StorageError::PartMissingOrCorrupt);
}
let durable_quota = quota.as_ref().filter(|quota| quota.uses_durable_reservations());
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_POLICY_EVALUATE,
policy_evaluate_started,
);
let capability_proof_started = rustfs_io_metrics::put_stage_timer();
let capability_proof = if durable_quota.is_some() {
Some(
crate::services::notification_sys::acquire_cross_pool_fence_fleet_proof()
@@ -529,7 +644,12 @@ pub(crate) async fn begin(
} else {
None
};
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CAPABILITY_PROOF,
capability_proof_started,
);
let durable_quota_limit = durable_quota.and_then(|quota| quota.quota);
let snapshot_admission_started = rustfs_io_metrics::put_stage_timer();
let snapshot_admission = match quota.as_ref().filter(|quota| !quota.uses_durable_reservations()) {
Some(quota) => match (quota.quota, snapshot_admission) {
(Some(limit), Some(admission)) if admission.quota_limit() == limit => Some(admission),
@@ -548,6 +668,10 @@ pub(crate) async fn begin(
},
None => None,
};
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_SNAPSHOT_ADMISSION,
snapshot_admission_started,
);
let legacy_data_movement = durable_quota_limit.is_none()
&& quota.as_ref().and_then(|quota| quota.quota).is_some()
&& snapshot_admission.is_none()
@@ -559,11 +683,21 @@ pub(crate) async fn begin(
.then(|| quota.as_ref().and_then(|quota| quota.quota))
.flatten()
});
let object_store_lookup_started = rustfs_io_metrics::put_stage_timer();
let store = if durable_quota_limit.is_some() {
Some(metadata_sys::object_store_in(ctx).await?)
} else {
None
};
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_OBJECT_STORE_LOOKUP,
object_store_lookup_started,
);
let context_build_started = rustfs_io_metrics::put_stage_timer();
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONTEXT_BUILD,
context_build_started,
);
Ok(QuotaContext {
store,
bucket: bucket.to_string(),
@@ -581,6 +715,40 @@ pub(crate) async fn begin(
})
}
fn quota_config_read_counterfactual_enabled() -> bool {
static ENABLED: OnceLock<bool> = OnceLock::new();
*ENABLED.get_or_init(|| {
matches!(
std::env::var(QUOTA_CONFIG_READ_COUNTERFACTUAL_ENV).ok().as_deref(),
Some("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON")
)
})
}
fn safe_no_quota_cache_fast_path_enabled() -> bool {
#[cfg(any(test, feature = "test-util"))]
if SAFE_NO_QUOTA_CACHE_FAST_PATH_FOR_TEST.load(std::sync::atomic::Ordering::Relaxed) {
return true;
}
static ENABLED: OnceLock<bool> = OnceLock::new();
*ENABLED.get_or_init(|| {
matches!(
std::env::var(SAFE_NO_QUOTA_CACHE_FAST_PATH_ENV).ok().as_deref(),
Some("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON")
)
})
}
fn quota_cache_reason(quota: Option<&crate::bucket::quota::BucketQuota>) -> &'static str {
match quota {
None => "no_quota",
Some(quota) if quota.uses_durable_reservations() => "durable_quota",
Some(quota) if quota.quota.is_some() => "snapshot_quota",
Some(_) => "quota_without_limit",
}
}
fn quota_capability_error(bucket: &str, object: &str) -> StorageError {
StorageError::NamespaceLockQuorumUnavailable {
mode: "quota_capability",
@@ -918,6 +1086,25 @@ pub fn fail_next_quota_ledger_save_for_test() {
FAIL_NEXT_LEDGER_SAVE.store(true, std::sync::atomic::Ordering::SeqCst);
}
#[cfg(any(test, feature = "test-util"))]
pub struct SafeNoQuotaCacheFastPathGuard {
previous: bool,
}
#[cfg(any(test, feature = "test-util"))]
impl Drop for SafeNoQuotaCacheFastPathGuard {
fn drop(&mut self) {
SAFE_NO_QUOTA_CACHE_FAST_PATH_FOR_TEST.store(self.previous, std::sync::atomic::Ordering::Relaxed);
}
}
#[cfg(any(test, feature = "test-util"))]
#[allow(dead_code, reason = "exercised by integration tests behind `--features test-util`")]
pub fn enable_safe_no_quota_cache_fast_path_for_test() -> SafeNoQuotaCacheFastPathGuard {
let previous = SAFE_NO_QUOTA_CACHE_FAST_PATH_FOR_TEST.swap(true, std::sync::atomic::Ordering::Relaxed);
SafeNoQuotaCacheFastPathGuard { previous }
}
fn log_admission_rejected(bucket: &str, object: &str, state: &'static str) {
warn!(
event = EVENT_QUOTA_ADMISSION,
+66 -1
View File
@@ -42,6 +42,7 @@ use std::{
pin::Pin,
sync::{LazyLock, Mutex},
task::{Context, Poll},
time::Instant,
};
use tonic::{service::interceptor::InterceptedService, transport::Channel};
use tower::Service;
@@ -355,6 +356,41 @@ fn apply_peer_replay_response(
}
}
fn rename_data_grpc_stage(path: &str) -> bool {
matches!(
path,
"/node_service.NodeService/RenameData" | "/node_service.NodeService/RenameDataAtIncarnation"
)
}
async fn observe_put_stage_future<F, T>(
future: F,
duration_stage: &'static str,
pending_count_stage: &'static str,
first_pending_to_ready_stage: &'static str,
) -> T
where
F: Future<Output = T>,
{
let duration_started = rustfs_io_metrics::put_stage_timer();
let mut first_pending_at = None;
let mut pending_count = 0usize;
let mut future = std::pin::pin!(future);
let output = std::future::poll_fn(|cx| match future.as_mut().poll(cx) {
Poll::Ready(output) => Poll::Ready(output),
Poll::Pending => {
pending_count = pending_count.saturating_add(1);
first_pending_at.get_or_insert_with(Instant::now);
Poll::Pending
}
})
.await;
rustfs_io_metrics::record_put_object_stage_duration_from(duration_stage, duration_started);
rustfs_io_metrics::record_put_object_stage_duration(pending_count_stage, pending_count as f64);
rustfs_io_metrics::record_put_object_stage_duration_from(first_pending_to_ready_stage, first_pending_at);
output
}
impl<S, ReqBody, ResBody> Service<HttpRequest<ReqBody>> for ReplayScopeChannel<S>
where
S: Service<HttpRequest<ReqBody>, Response = HttpResponse<ResBody>>,
@@ -372,6 +408,12 @@ where
}
fn call(&mut self, mut request: HttpRequest<ReqBody>) -> Self::Future {
let observe_rename_data = rustfs_io_metrics::put_stage_metrics_enabled() && rename_data_grpc_stage(request.uri().path());
let request_scope_started = if observe_rename_data {
rustfs_io_metrics::put_stage_timer()
} else {
None
};
let authenticated = self.audience.as_ref().is_some_and(|_| {
request
.headers()
@@ -406,11 +448,30 @@ where
}
}
}
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_REQUEST_SCOPE,
request_scope_started,
);
let audience = self.audience.clone();
let future = self.inner.call(request);
Box::pin(async move {
let response = future.await?;
let response = if observe_rename_data {
observe_put_stage_future(
future,
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL,
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL_POLL_PENDING_COUNT,
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL_FIRST_PENDING_TO_READY,
)
.await?
} else {
future.await?
};
let replay_response_started = if observe_rename_data {
rustfs_io_metrics::put_stage_timer()
} else {
None
};
if let (Some(audience), Some(challenge)) = (audience, challenge) {
let response_state = verify_tonic_peer_replay_capabilities_response(&audience, challenge, response.headers());
if let Err(error) = &response_state
@@ -428,6 +489,10 @@ where
}
apply_peer_replay_response(audience, sent_state, response_state);
}
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_REPLAY_RESPONSE,
replay_response_started,
);
Ok(response)
})
}
+91 -7
View File
@@ -176,6 +176,42 @@ fn attach_mutation_body_digest<T>(
crate::cluster::rpc::set_tonic_rolling_canonical_body_digest(request, &canonical_body).map_err(Error::other)
}
async fn observe_put_stage_future<F, T>(
future: F,
duration_stage: &'static str,
pending_count_stage: &'static str,
first_pending_to_ready_stage: &'static str,
) -> T
where
F: Future<Output = T>,
{
// Keep the normal PUT path identical to the pre-instrumentation future
// when detailed stage metrics are disabled. In particular, do not wrap the
// RPC in poll_fn: that adds an extra poll layer and pending counter work to
// every remote rename even though no samples will be emitted.
if !rustfs_io_metrics::put_stage_metrics_enabled() {
return future.await;
}
let duration_started = rustfs_io_metrics::put_stage_timer();
let mut first_pending_at = None;
let mut pending_count = 0usize;
let mut future = std::pin::pin!(future);
let output = std::future::poll_fn(|cx| match future.as_mut().poll(cx) {
Poll::Ready(output) => Poll::Ready(output),
Poll::Pending => {
pending_count = pending_count.saturating_add(1);
first_pending_at.get_or_insert_with(Instant::now);
Poll::Pending
}
})
.await;
rustfs_io_metrics::record_put_object_stage_duration_from(duration_stage, duration_started);
rustfs_io_metrics::record_put_object_stage_duration(pending_count_stage, pending_count as f64);
rustfs_io_metrics::record_put_object_stage_duration_from(first_pending_to_ready_stage, first_pending_at);
output
}
fn decode_volume_infos(volume_infos: Vec<String>) -> Result<Vec<VolumeInfo>> {
volume_infos
.into_iter()
@@ -2137,6 +2173,7 @@ impl RemoteDisk {
self.execute_with_timeout_for_op(
"rename_data",
|| async {
let prepare_started = rustfs_io_metrics::put_stage_timer();
let file_info = compat_json(fi)?;
let file_info_bin = encode_file_info_msgpack(fi)?;
let mut client = self.get_client().await?;
@@ -2164,24 +2201,67 @@ impl RemoteDisk {
} else {
attach_mutation_body_digest(&mut request, canonical_body, "rename_data")?;
}
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_PREPARE,
prepare_started,
);
let response = if incarnation_bound {
// Older peers return Unimplemented before mutation; never downgrade.
client.rename_data_at_incarnation(request).await?
} else {
client.rename_data(request).await?
}
.into_inner();
let rpc_started = rustfs_io_metrics::put_stage_timer();
let response = observe_put_stage_future(
async {
if incarnation_bound {
// Older peers return Unimplemented before mutation; never downgrade.
client.rename_data_at_incarnation(request).await
} else {
client.rename_data(request).await
}
},
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT,
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_POLL_PENDING_COUNT,
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_FIRST_PENDING_TO_READY,
)
.await?;
let response_process_started = rustfs_io_metrics::put_stage_timer();
let response_headers_started = rustfs_io_metrics::put_stage_timer();
let _ = response.metadata().len();
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RESPONSE_HEADERS,
response_headers_started,
);
let into_inner_started = rustfs_io_metrics::put_stage_timer();
let response = response.into_inner();
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_INTO_INNER,
into_inner_started,
);
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC,
rpc_started,
);
let success_check_started = rustfs_io_metrics::put_stage_timer();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_SUCCESS_CHECK,
success_check_started,
);
let decode_started = rustfs_io_metrics::put_stage_timer();
let rename_data_resp = decode_msgpack_or_json::<RenameDataResp>(
&response.rename_data_resp_bin,
&response.rename_data_resp,
"RenameDataResp",
)?;
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_DECODE,
decode_started,
);
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RESPONSE_PROCESS,
response_process_started,
);
Ok(rename_data_resp)
},
@@ -4759,6 +4839,8 @@ mod tests {
cleanup_data_dir: Some(Uuid::new_v4()),
sign: Some(vec![0x14, 0x35]),
old_current_size: Some(crate::disk::OldCurrentSize::Present(64 * 1024)),
old_current_source_checked: false,
old_current_source: None,
};
let json = serde_json::to_string(&response).expect("legacy rename_data JSON response should encode");
let decode_before = rustfs_io_metrics::internode_metrics::global_internode_metrics().msgpack_json_decode_total_for_test();
@@ -5161,6 +5243,8 @@ mod tests {
cleanup_data_dir: Some(Uuid::new_v4()),
sign: Some(vec![1_u8; 32]),
old_current_size: Some(crate::disk::OldCurrentSize::Present(4096)),
old_current_source_checked: false,
old_current_source: None,
};
let json = serde_json::to_vec(&response).expect("rename data response json should encode");
let named_msgpack = rmp_serde::encode::to_vec_named(&response).expect("rename data response named msgpack should encode");
+53
View File
@@ -47,6 +47,33 @@ use tokio::fs;
use tracing::{info, warn};
use uuid::Uuid;
fn transitioned_old_current_source(
dst_meta_existed: bool,
dst_meta_unparsable: bool,
xlmeta: &FileMeta,
volume: &str,
path: &str,
version_id: Uuid,
) -> (bool, Option<FileInfo>) {
if dst_meta_unparsable {
return (false, None);
}
if !dst_meta_existed {
return (true, None);
}
let Ok(old_current) = xlmeta.into_fileinfo(volume, path, "", false, false, true) else {
return (false, None);
};
if old_current.version_id.unwrap_or_default() != version_id {
return (true, None);
}
if old_current.transition_status == rustfs_filemeta::TRANSITION_COMPLETE {
(true, Some(old_current))
} else {
(true, None)
}
}
/// Hold later repair publications after admitting one baseline object. The
/// fixture arms this on one replacement disk before rejoining the cluster.
#[cfg(feature = "e2e-test-hooks")]
@@ -491,6 +518,14 @@ impl LocalDisk {
}
let version_id = fi.version_id.unwrap_or_default();
let (old_current_source_checked, old_current_source) = transitioned_old_current_source(
has_dst_buf.is_some(),
dst_meta_unparsable,
&xlmeta,
dst_volume,
dst_path,
version_id,
);
let has_old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id));
let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok();
let rollback_data_dir = has_old_data_dir.or_else(|| {
@@ -924,6 +959,8 @@ impl LocalDisk {
cleanup_data_dir: has_old_data_dir,
sign: version_signature,
old_current_size,
old_current_source_checked,
old_current_source,
})
} else {
// Inline metadata preparation is blocking. The transaction lease is
@@ -937,6 +974,8 @@ impl LocalDisk {
None
};
let dst_path_for_failpoint = dst_path.to_string();
let dst_volume_for_source = dst_volume.to_string();
let dst_path_for_source = dst_path.to_string();
#[cfg(windows)]
let source_parent = src_file_parent.to_path_buf();
let rename_commit_guard_for_preparation = rename_commit_guard.clone();
@@ -981,6 +1020,14 @@ impl LocalDisk {
};
let version_id = fi.version_id.unwrap_or_default();
let (old_current_source_checked, old_current_source) = transitioned_old_current_source(
has_dst_buf.is_some(),
dst_meta_unparsable,
&xlmeta,
&dst_volume_for_source,
&dst_path_for_source,
version_id,
);
let old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id));
let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok();
let rollback_data_dir = old_data_dir.or_else(|| {
@@ -1030,6 +1077,8 @@ impl LocalDisk {
old_data_dir,
version_signature,
old_current_size,
old_current_source_checked,
old_current_source,
staged_rollback_path,
has_dst_buf.is_none(),
prepared_metadata_source,
@@ -1048,6 +1097,8 @@ impl LocalDisk {
cleanup_data_dir,
version_signature,
old_current_size,
old_current_source_checked,
old_current_source,
mut local_rollback_path,
destination_was_absent,
prepared_metadata_source,
@@ -1285,6 +1336,8 @@ impl LocalDisk {
cleanup_data_dir,
sign: version_signature,
old_current_size,
old_current_source_checked,
old_current_source,
})
}
}
+14
View File
@@ -1554,6 +1554,14 @@ pub struct RenameDataResp {
/// "cannot vote", never as `Absent`.
#[serde(default)]
pub old_current_size: Option<OldCurrentSize>,
/// Whether this disk decoded enough of the overwritten current version to
/// decide if a transitioned source has to be returned.
#[serde(default)]
pub old_current_source_checked: bool,
/// Transitioned old-current source used by PUT free-version cleanup receipt.
/// Empty when the old current is not transitioned or no source was known.
#[serde(default)]
pub old_current_source: Option<FileInfo>,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
@@ -2001,6 +2009,8 @@ mod tests {
cleanup_data_dir: Some(uuid),
sign: Some(signature.clone()),
old_current_size: Some(OldCurrentSize::Present(42)),
old_current_source_checked: false,
old_current_source: None,
};
assert_eq!(resp.old_data_dir, Some(uuid));
@@ -2021,6 +2031,8 @@ mod tests {
cleanup_data_dir: Some(Uuid::new_v4()),
sign: Some(vec![0x01, 0x02, 0x03]),
old_current_size,
old_current_source_checked: false,
old_current_source: None,
};
let encoded = rmp_serde::encode::to_vec_named(&resp).expect("named msgpack should encode");
@@ -2058,6 +2070,8 @@ mod tests {
assert_eq!(decoded.cleanup_data_dir, None);
assert_eq!(decoded.sign, legacy.sign);
assert_eq!(decoded.old_current_size, None);
assert!(!decoded.old_current_source_checked);
assert!(decoded.old_current_source.is_none());
}
/// Test constants
+137 -7
View File
@@ -522,6 +522,10 @@ const DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_ENABLE: bool = false;
const ENV_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS: &str = "RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS";
const DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS: u64 = 0;
const MAX_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS: u64 = 1_000;
const ENV_FILE_FDATASYNC_GROUP_COMMIT_FOLLOWER_TIMEOUT_MICROS: &str =
"RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_FOLLOWER_TIMEOUT_MICROS";
const DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_FOLLOWER_TIMEOUT_MICROS: u64 = 0;
const MAX_FILE_FDATASYNC_GROUP_COMMIT_FOLLOWER_TIMEOUT_MICROS: u64 = 1_000;
#[cfg(not(test))]
const MAX_DST_DIR_FSYNC_GROUPS: usize = 1024;
#[cfg(test)]
@@ -552,12 +556,22 @@ fn file_fdatasync_group_commit_wait_duration(wait_micros: u64) -> Duration {
Duration::from_micros(wait_micros.min(MAX_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS))
}
fn file_fdatasync_group_commit_follower_timeout_duration(timeout_micros: u64) -> Duration {
Duration::from_micros(timeout_micros.min(MAX_FILE_FDATASYNC_GROUP_COMMIT_FOLLOWER_TIMEOUT_MICROS))
}
static FILE_FDATASYNC_GROUP_COMMIT_WAIT: LazyLock<Duration> = LazyLock::new(|| {
file_fdatasync_group_commit_wait_duration(rustfs_utils::get_env_u64(
ENV_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS,
DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS,
))
});
static FILE_FDATASYNC_GROUP_COMMIT_FOLLOWER_TIMEOUT: LazyLock<Duration> = LazyLock::new(|| {
file_fdatasync_group_commit_follower_timeout_duration(rustfs_utils::get_env_u64(
ENV_FILE_FDATASYNC_GROUP_COMMIT_FOLLOWER_TIMEOUT_MICROS,
DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_FOLLOWER_TIMEOUT_MICROS,
))
});
#[cfg(test)]
mod dst_dir_fsync_group_commit_override {
@@ -607,6 +621,7 @@ mod file_fdatasync_group_commit_override {
static OVERRIDE: RwLock<Option<bool>> = RwLock::new(None);
static WAIT_OVERRIDE_MICROS: RwLock<Option<u64>> = RwLock::new(None);
static FOLLOWER_TIMEOUT_OVERRIDE_MICROS: RwLock<Option<u64>> = RwLock::new(None);
static SERIAL: Mutex<()> = Mutex::new(());
pub(crate) fn get() -> Option<bool> {
@@ -621,6 +636,9 @@ mod file_fdatasync_group_commit_override {
fn drop(&mut self) {
*OVERRIDE.write().unwrap_or_else(PoisonError::into_inner) = None;
*WAIT_OVERRIDE_MICROS.write().unwrap_or_else(PoisonError::into_inner) = None;
*FOLLOWER_TIMEOUT_OVERRIDE_MICROS
.write()
.unwrap_or_else(PoisonError::into_inner) = None;
}
}
@@ -637,6 +655,18 @@ mod file_fdatasync_group_commit_override {
pub(crate) fn wait_micros() -> Option<u64> {
*WAIT_OVERRIDE_MICROS.read().unwrap_or_else(PoisonError::into_inner)
}
pub(crate) fn set_follower_timeout_micros(timeout_micros: u64) {
*FOLLOWER_TIMEOUT_OVERRIDE_MICROS
.write()
.unwrap_or_else(PoisonError::into_inner) = Some(timeout_micros);
}
pub(crate) fn follower_timeout_micros() -> Option<u64> {
*FOLLOWER_TIMEOUT_OVERRIDE_MICROS
.read()
.unwrap_or_else(PoisonError::into_inner)
}
}
#[cfg(test)]
@@ -649,6 +679,11 @@ fn set_file_fdatasync_group_commit_wait_for_test(wait_micros: u64) {
file_fdatasync_group_commit_override::set_wait_micros(wait_micros);
}
#[cfg(test)]
fn set_file_fdatasync_group_commit_follower_timeout_for_test(timeout_micros: u64) {
file_fdatasync_group_commit_override::set_follower_timeout_micros(timeout_micros);
}
fn file_fdatasync_group_commit_enabled() -> bool {
#[cfg(test)]
if let Some(enabled) = file_fdatasync_group_commit_override::get() {
@@ -667,6 +702,15 @@ fn file_fdatasync_group_commit_wait() -> Duration {
*FILE_FDATASYNC_GROUP_COMMIT_WAIT
}
fn file_fdatasync_group_commit_follower_timeout() -> Duration {
#[cfg(test)]
if let Some(timeout_micros) = file_fdatasync_group_commit_override::follower_timeout_micros() {
return file_fdatasync_group_commit_follower_timeout_duration(timeout_micros);
}
*FILE_FDATASYNC_GROUP_COMMIT_FOLLOWER_TIMEOUT
}
#[derive(Clone, Eq, Hash, PartialEq)]
struct DstDirFsyncGroupKey {
canonical_path: PathBuf,
@@ -1125,7 +1169,11 @@ impl FileFdatasyncGroupCommit {
&self,
disk_permits: Arc<Semaphore>,
files: Vec<PathBuf>,
) -> io::Result<(oneshot::Receiver<SharedFileFdatasyncResult>, Option<Arc<FileFdatasyncGroup>>)> {
) -> io::Result<(
oneshot::Receiver<SharedFileFdatasyncResult>,
Option<Arc<FileFdatasyncGroup>>,
&'static str,
)> {
if files.is_empty() {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
@@ -1195,7 +1243,7 @@ impl FileFdatasyncGroupCommit {
registry.total_files += file_count;
drop(group_state);
drop(registry);
Ok((result_rx, start_worker.then_some(group)))
Ok((result_rx, start_worker.then_some(group), wait_role))
}
fn complete_batch(&self, waiters: usize, files: usize) {
@@ -1306,12 +1354,29 @@ async fn run_file_fdatasync_group_worker(group: Arc<FileFdatasyncGroup>) {
}
async fn sync_files_group_commit(files: Vec<PathBuf>, disk_permits: Arc<Semaphore>) -> io::Result<()> {
let (result_rx, worker) = FILE_FDATASYNC_GROUP_COMMIT.enqueue(disk_permits, files)?;
let fallback_files = files.clone();
let (result_rx, worker, wait_role) = FILE_FDATASYNC_GROUP_COMMIT.enqueue(disk_permits.clone(), files)?;
if let Some(group) = worker {
tokio::spawn(run_file_fdatasync_group_worker(group));
}
match result_rx.await {
let follower_timeout = file_fdatasync_group_commit_follower_timeout();
let result = if wait_role == rustfs_io_metrics::PUT_RENAME_FDATASYNC_GROUP_WAIT_ROLE_FOLLOWER && !follower_timeout.is_zero() {
match tokio::time::timeout(follower_timeout, result_rx).await {
Ok(result) => result,
Err(_) => {
rustfs_io_metrics::record_put_rename_fdatasync_group_wait(
rustfs_io_metrics::PUT_RENAME_FDATASYNC_GROUP_WAIT_ROLE_FOLLOWER_TIMEOUT,
follower_timeout.as_secs_f64() * 1000.0,
);
return run_file_sync_blocking(disk_permits, move || sync_files(&fallback_files)).await;
}
}
} else {
result_rx.await
};
match result {
Ok(Ok(())) => Ok(()),
Ok(Err(err)) => Err(err.into_error()),
Err(_) => Err(io::Error::other("file fdatasync group worker dropped the waiter")),
@@ -7396,6 +7461,71 @@ mod tests {
assert_eq!(file_fdatasync_group_commit_counts_for_test(), (0, 0, 0));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial_test::serial(file_sync_probe)]
async fn file_fdatasync_group_commit_follower_timeout_uses_independent_sync() {
use std::sync::mpsc;
let _group_commit = set_file_fdatasync_group_commit_for_test(true);
set_file_fdatasync_group_commit_wait_for_test(0);
set_file_fdatasync_group_commit_follower_timeout_for_test(1_000);
clear_file_fdatasync_group_commit_for_test();
let temp_dir = tempdir().expect("create temp dir");
let first_dir = temp_dir.path().join("first");
let second_dir = temp_dir.path().join("second");
std::fs::create_dir(&first_dir).expect("create first dir");
std::fs::create_dir(&second_dir).expect("create second dir");
std::fs::write(first_dir.join("part.1"), b"first").expect("write first part");
std::fs::write(second_dir.join("part.1"), b"second").expect("write second part");
let _probe = file_sync_probe::set_blocking(temp_dir.path());
let (entered_tx, entered_rx) = mpsc::channel();
let (release_batch_tx, release_batch_rx) = mpsc::channel();
file_sync_probe::set_before_group_batch(move || {
entered_tx.send(()).expect("signal first file fdatasync group worker");
release_batch_rx.recv().expect("wait until follower timeout path is observed");
});
let limiter = file_sync_limiter();
let first_limiter = limiter.clone();
let first_path = first_dir.clone();
let first = tokio::spawn(async move { sync_dir_files_with_limiter(first_path, first_limiter).await });
tokio::task::spawn_blocking(move || entered_rx.recv_timeout(Duration::from_secs(30)))
.await
.expect("group worker hook waiter should run")
.expect("first file fdatasync group worker should start");
let second_limiter = limiter.clone();
let second_path = second_dir.clone();
let second = tokio::spawn(async move { sync_dir_files_with_limiter(second_path, second_limiter).await });
file_sync_probe::wait_for_active(1).await;
file_sync_probe::release();
second
.await
.expect("join follower timeout file sync")
.expect("follower timeout independent sync must succeed");
assert!(
fsync_dir_recorder::was_fsynced(&second_dir),
"follower timeout path must still fsync the source directory"
);
release_batch_tx.send(()).expect("release original group batch hook");
first
.await
.expect("join leader grouped file sync")
.expect("leader grouped file sync must succeed");
assert!(
fsync_dir_recorder::was_fsynced(&first_dir),
"leader source directory must still be fsynced"
);
assert_eq!(
file_sync_probe::group_batches(),
vec![2],
"timed-out follower remains in the original best-effort group batch, but request completion is independent"
);
assert_eq!(file_fdatasync_group_commit_counts_for_test(), (0, 0, 0));
file_sync_probe::wait_for_idle().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial_test::serial(file_sync_probe)]
async fn file_fdatasync_group_commit_failure_fails_all_waiters_before_dir_fsync() {
@@ -7473,7 +7603,7 @@ mod tests {
let mut limiters = Vec::new();
for index in 0..MAX_FILE_FDATASYNC_GROUPS {
let limiter = Arc::new(Semaphore::new(1));
let (result_rx, _worker) = FILE_FDATASYNC_GROUP_COMMIT
let (result_rx, _worker, _wait_role) = FILE_FDATASYNC_GROUP_COMMIT
.enqueue(limiter.clone(), vec![PathBuf::from(format!("part-{index}"))])
.expect("group below cap should enqueue");
limiters.push(limiter);
@@ -7501,7 +7631,7 @@ mod tests {
let limiter = Arc::new(Semaphore::new(1));
let mut receivers = Vec::new();
for index in 0..MAX_FILE_FDATASYNC_WAITERS {
let (result_rx, _worker) = FILE_FDATASYNC_GROUP_COMMIT
let (result_rx, _worker, _wait_role) = FILE_FDATASYNC_GROUP_COMMIT
.enqueue(limiter.clone(), vec![PathBuf::from(format!("part-{index}"))])
.expect("waiter below cap should enqueue");
receivers.push(result_rx);
@@ -7518,7 +7648,7 @@ mod tests {
let mut receivers = Vec::new();
let file_limit_limiter = Arc::new(Semaphore::new(1));
let (result_rx, _worker) = FILE_FDATASYNC_GROUP_COMMIT
let (result_rx, _worker, _wait_role) = FILE_FDATASYNC_GROUP_COMMIT
.enqueue(
file_limit_limiter.clone(),
(0..MAX_FILE_FDATASYNC_BATCH_FILES)
+689 -24
View File
@@ -382,7 +382,7 @@ async fn flush_read_version_coalescer_pending(
fn map_batch_read_version_responses(
expected_items: &[ExpectedBatchReadVersionItem],
responses: Vec<BatchReadVersionResp>,
) -> Vec<crate::disk::error::Result<FileInfo>> {
) -> Vec<disk::error::Result<FileInfo>> {
let mut results = (0..expected_items.len())
.map(|_| Err(DiskError::other("coalesced read_version response missing")))
.collect::<Vec<_>>();
@@ -3618,6 +3618,9 @@ pub(in crate::set_disk) struct RenameDataCommit {
pub(in crate::set_disk) data_dir: Option<Uuid>,
pub(in crate::set_disk) cleanup_disks: Vec<Option<DiskStore>>,
pub(in crate::set_disk) old_current_size: Option<OldCurrentSize>,
pub(in crate::set_disk) old_current_source_checked: bool,
pub(in crate::set_disk) old_current_source_all_successful_checked: bool,
pub(in crate::set_disk) old_current_source: Option<FileInfo>,
pub(in crate::set_disk) committed_file_info: FileInfo,
pub(in crate::set_disk) tail_drain: Option<tokio::task::JoinHandle<Option<RenameTailOutcome>>>,
}
@@ -3633,6 +3636,500 @@ pub(in crate::set_disk) struct RenameTailOutcome {
pub(in crate::set_disk) cleanup: Vec<RenameTailCleanup>,
}
const RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP";
const RENAME_TAIL_CLEANUP_DEFER_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ENABLE";
const RENAME_TAIL_CLEANUP_DEFER_QUEUE_CAPACITY_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_QUEUE_CAPACITY";
const RENAME_TAIL_CLEANUP_DEFER_WORKERS_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_WORKERS";
const RENAME_TAIL_CLEANUP_DEFER_WORKER_DELAY_MS_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_WORKER_DELAY_MS";
const RENAME_TAIL_CLEANUP_DEFER_BATCH_WINDOW_MS_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_BATCH_WINDOW_MS";
const RENAME_TAIL_CLEANUP_DEFER_BATCH_MAX_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_BATCH_MAX";
const RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_ENABLE_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_ENABLE";
const RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_MIN_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_MIN";
const RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_LOW_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_LOW";
const RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_HIGH_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_HIGH";
const RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_LOW_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_LOW";
const RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_HIGH_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_HIGH";
const RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_LOW_MS_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_LOW_MS";
const RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_HIGH_MS_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_HIGH_MS";
const RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER";
const RENAME_FOREGROUND_ACTIVE_LANES_ENV: &str = "RUSTFS_PUT_RENAME_FOREGROUND_ACTIVE_LANES";
const DEFAULT_RENAME_TAIL_CLEANUP_DEFER_QUEUE_CAPACITY: usize = 1024;
const DEFAULT_RENAME_TAIL_CLEANUP_DEFER_WORKERS: usize = 2;
const DEFAULT_RENAME_TAIL_CLEANUP_DEFER_BATCH_MAX: usize = 1;
const DEFAULT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_LOW: usize = 128;
const DEFAULT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_HIGH: usize = 512;
const DEFAULT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_LOW_MS: u64 = 500;
const DEFAULT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_HIGH_MS: u64 = 2_000;
const DEFAULT_RENAME_FOREGROUND_ACTIVE_LANES: usize = 1024;
fn rename_tail_cleanup_counterfactual_skip_enabled() -> bool {
static ENABLED: OnceLock<bool> = OnceLock::new();
*ENABLED.get_or_init(|| {
matches!(
std::env::var(RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP_ENV).ok().as_deref(),
Some("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON")
)
})
}
fn rename_tail_cleanup_defer_enabled() -> bool {
static ENABLED: OnceLock<bool> = OnceLock::new();
*ENABLED.get_or_init(|| {
matches!(
std::env::var(RENAME_TAIL_CLEANUP_DEFER_ENV).ok().as_deref(),
Some("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON")
)
})
}
fn rename_tail_cleanup_defer_queue_capacity() -> usize {
static CAPACITY: OnceLock<usize> = OnceLock::new();
*CAPACITY.get_or_init(|| {
rustfs_utils::get_env_usize(
RENAME_TAIL_CLEANUP_DEFER_QUEUE_CAPACITY_ENV,
DEFAULT_RENAME_TAIL_CLEANUP_DEFER_QUEUE_CAPACITY,
)
.max(1)
})
}
fn rename_tail_cleanup_defer_workers() -> usize {
static WORKERS: OnceLock<usize> = OnceLock::new();
*WORKERS.get_or_init(|| {
rustfs_utils::get_env_usize(RENAME_TAIL_CLEANUP_DEFER_WORKERS_ENV, DEFAULT_RENAME_TAIL_CLEANUP_DEFER_WORKERS).max(1)
})
}
fn rename_tail_cleanup_defer_worker_delay() -> Duration {
static DELAY: OnceLock<Duration> = OnceLock::new();
*DELAY.get_or_init(|| Duration::from_millis(rustfs_utils::get_env_u64(RENAME_TAIL_CLEANUP_DEFER_WORKER_DELAY_MS_ENV, 0)))
}
fn rename_tail_cleanup_defer_batch_window() -> Duration {
static WINDOW: OnceLock<Duration> = OnceLock::new();
*WINDOW.get_or_init(|| Duration::from_millis(rustfs_utils::get_env_u64(RENAME_TAIL_CLEANUP_DEFER_BATCH_WINDOW_MS_ENV, 0)))
}
fn rename_tail_cleanup_defer_batch_max() -> usize {
static BATCH_MAX: OnceLock<usize> = OnceLock::new();
*BATCH_MAX.get_or_init(|| {
rustfs_utils::get_env_usize(RENAME_TAIL_CLEANUP_DEFER_BATCH_MAX_ENV, DEFAULT_RENAME_TAIL_CLEANUP_DEFER_BATCH_MAX).max(1)
})
}
fn rename_tail_cleanup_defer_adaptive_batch_enabled() -> bool {
static ENABLED: OnceLock<bool> = OnceLock::new();
*ENABLED.get_or_init(|| {
matches!(
std::env::var(RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_ENABLE_ENV)
.ok()
.as_deref(),
Some("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON")
)
})
}
fn rename_tail_cleanup_defer_adaptive_batch_min() -> usize {
static BATCH_MIN: OnceLock<usize> = OnceLock::new();
*BATCH_MIN.get_or_init(|| rustfs_utils::get_env_usize(RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_MIN_ENV, 1).max(1))
}
fn rename_tail_cleanup_defer_adaptive_batch_low() -> usize {
static BATCH_LOW: OnceLock<usize> = OnceLock::new();
*BATCH_LOW.get_or_init(|| {
rustfs_utils::get_env_usize(RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_LOW_ENV, rename_tail_cleanup_defer_batch_max())
.max(1)
})
}
fn rename_tail_cleanup_defer_adaptive_batch_high() -> usize {
static BATCH_HIGH: OnceLock<usize> = OnceLock::new();
*BATCH_HIGH.get_or_init(|| {
rustfs_utils::get_env_usize(
RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_HIGH_ENV,
rename_tail_cleanup_defer_adaptive_batch_low().max(rename_tail_cleanup_defer_batch_max()),
)
.max(1)
})
}
fn rename_tail_cleanup_defer_adaptive_queue_low() -> usize {
static QUEUE_LOW: OnceLock<usize> = OnceLock::new();
*QUEUE_LOW.get_or_init(|| {
rustfs_utils::get_env_usize(
RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_LOW_ENV,
DEFAULT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_LOW,
)
})
}
fn rename_tail_cleanup_defer_adaptive_queue_high() -> usize {
static QUEUE_HIGH: OnceLock<usize> = OnceLock::new();
*QUEUE_HIGH.get_or_init(|| {
rustfs_utils::get_env_usize(
RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_HIGH_ENV,
DEFAULT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_HIGH,
)
})
}
fn rename_tail_cleanup_defer_adaptive_age_low() -> Duration {
static AGE_LOW: OnceLock<Duration> = OnceLock::new();
*AGE_LOW.get_or_init(|| {
Duration::from_millis(rustfs_utils::get_env_u64(
RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_LOW_MS_ENV,
DEFAULT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_LOW_MS,
))
})
}
fn rename_tail_cleanup_defer_adaptive_age_high() -> Duration {
static AGE_HIGH: OnceLock<Duration> = OnceLock::new();
*AGE_HIGH.get_or_init(|| {
Duration::from_millis(rustfs_utils::get_env_u64(
RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_HIGH_MS_ENV,
DEFAULT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_HIGH_MS,
))
})
}
fn rename_tail_cleanup_defer_batch_target(queued_depth: usize, oldest_age: Option<Duration>) -> usize {
let fixed = rename_tail_cleanup_defer_batch_max();
if !rename_tail_cleanup_defer_adaptive_batch_enabled() {
return fixed;
}
let min = rename_tail_cleanup_defer_adaptive_batch_min();
let low = rename_tail_cleanup_defer_adaptive_batch_low().max(min);
let high = rename_tail_cleanup_defer_adaptive_batch_high().max(low);
let queue_low = rename_tail_cleanup_defer_adaptive_queue_low();
let queue_high = rename_tail_cleanup_defer_adaptive_queue_high().max(queue_low);
let age_low = rename_tail_cleanup_defer_adaptive_age_low();
let age_high = rename_tail_cleanup_defer_adaptive_age_high().max(age_low);
let age = oldest_age.unwrap_or_default();
if queued_depth >= queue_high || age >= age_high {
high
} else if queued_depth >= queue_low || age >= age_low {
low
} else {
min
}
}
fn rename_tail_cleanup_defer_hold_worker_enabled() -> bool {
static ENABLED: OnceLock<bool> = OnceLock::new();
*ENABLED.get_or_init(|| {
matches!(
std::env::var(RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER_ENV).ok().as_deref(),
Some("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON")
)
})
}
fn rename_foreground_active_lanes() -> &'static Vec<AtomicUsize> {
static LANES: OnceLock<Vec<AtomicUsize>> = OnceLock::new();
LANES.get_or_init(|| {
let lanes =
rustfs_utils::get_env_usize(RENAME_FOREGROUND_ACTIVE_LANES_ENV, DEFAULT_RENAME_FOREGROUND_ACTIVE_LANES).max(1);
(0..lanes).map(|_| AtomicUsize::new(0)).collect()
})
}
pub(in crate::set_disk) fn rename_foreground_active_count(disk_index: usize) -> usize {
let lanes = rename_foreground_active_lanes();
lanes[disk_index % lanes.len()].load(Ordering::Relaxed)
}
struct RenameForegroundActiveGuard {
disk_index: usize,
}
impl Drop for RenameForegroundActiveGuard {
fn drop(&mut self) {
let lanes = rename_foreground_active_lanes();
lanes[self.disk_index % lanes.len()].fetch_sub(1, Ordering::Relaxed);
}
}
fn rename_foreground_active_guard(disk_index: usize) -> RenameForegroundActiveGuard {
let lanes = rename_foreground_active_lanes();
let active = lanes[disk_index % lanes.len()].fetch_add(1, Ordering::Relaxed) + 1;
rustfs_io_metrics::record_put_object_stage_duration(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_FOREGROUND_ACTIVE,
active as f64,
);
RenameForegroundActiveGuard { disk_index }
}
type RenameTailCleanupFuture = Pin<Box<dyn Future<Output = ()> + Send + 'static>>;
struct RenameTailCleanupJob {
enqueued_at: Instant,
targets: usize,
future: RenameTailCleanupFuture,
}
struct RenameTailCleanupQueue {
sender: tokio::sync::mpsc::Sender<RenameTailCleanupJob>,
queued: Arc<AtomicUsize>,
oldest_enqueued_at: Arc<tokio::sync::Mutex<Option<Instant>>>,
}
async fn run_rename_tail_cleanup_job(job: RenameTailCleanupJob) {
rustfs_io_metrics::record_put_object_stage_duration(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_START_DELAY,
job.enqueued_at.elapsed().as_secs_f64() * 1000.0,
);
rustfs_io_metrics::record_put_object_stage_duration(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_TARGETS,
job.targets as f64,
);
let worker_delay = rename_tail_cleanup_defer_worker_delay();
if !worker_delay.is_zero() {
let delay_started = rustfs_io_metrics::put_stage_timer();
tokio::time::sleep(worker_delay).await;
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_DELAY,
delay_started,
);
}
let cleanup_started = rustfs_io_metrics::put_stage_timer();
job.future.await;
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_DURATION,
cleanup_started,
);
rustfs_io_metrics::record_put_object_stage_duration(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_COMPLETED,
1.0,
);
rustfs_io_metrics::record_put_object_stage_duration(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_COMPLETED_LAG,
job.enqueued_at.elapsed().as_secs_f64() * 1000.0,
);
}
impl RenameTailCleanupQueue {
fn new() -> Self {
let (sender, receiver) = tokio::sync::mpsc::channel::<RenameTailCleanupJob>(rename_tail_cleanup_defer_queue_capacity());
let queued = Arc::new(AtomicUsize::new(0));
let active_workers = Arc::new(AtomicUsize::new(0));
let oldest_enqueued_at = Arc::new(tokio::sync::Mutex::new(None::<Instant>));
let receiver = Arc::new(tokio::sync::Mutex::new(receiver));
if !rename_tail_cleanup_defer_hold_worker_enabled() {
for _ in 0..rename_tail_cleanup_defer_workers() {
let receiver = Arc::clone(&receiver);
let queued = Arc::clone(&queued);
let active_workers = Arc::clone(&active_workers);
let oldest_enqueued_at = Arc::clone(&oldest_enqueued_at);
tokio::spawn(async move {
loop {
let queued_before_poll = queued.load(Ordering::Relaxed);
let worker_poll_started = rustfs_io_metrics::put_stage_timer();
let Some(job) = receiver.lock().await.recv().await else {
return;
};
if queued_before_poll == 0 {
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_IDLE_WAIT,
worker_poll_started,
);
} else {
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_POLL_WAIT,
worker_poll_started,
);
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_JOB_WAIT,
worker_poll_started,
);
}
let mut batch = vec![job];
let batch_window = rename_tail_cleanup_defer_batch_window();
let queued_snapshot = queued.load(Ordering::Relaxed);
let oldest_snapshot = *oldest_enqueued_at.lock().await;
let batch_max = rename_tail_cleanup_defer_batch_target(
queued_snapshot,
oldest_snapshot.map(|oldest| oldest.elapsed()),
);
if batch_max > 1 && !batch_window.is_zero() {
let window_started = rustfs_io_metrics::put_stage_timer();
tokio::time::sleep(batch_window).await;
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BATCH_WINDOW,
window_started,
);
}
while batch.len() < batch_max {
let next = {
let mut locked = receiver.lock().await;
locked.try_recv().ok()
};
let Some(next) = next else {
break;
};
batch.push(next);
}
let queued_after_dequeue = queued.fetch_sub(batch.len(), Ordering::Relaxed).saturating_sub(batch.len());
if queued_after_dequeue == 0 {
*oldest_enqueued_at.lock().await = None;
}
rustfs_io_metrics::record_put_object_stage_duration(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH,
queued_after_dequeue as f64,
);
rustfs_io_metrics::record_put_object_stage_duration(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH_MAX_SAMPLE,
queued_after_dequeue as f64,
);
if let Some(oldest) = *oldest_enqueued_at.lock().await {
let oldest_age_ms = oldest.elapsed().as_secs_f64() * 1000.0;
rustfs_io_metrics::record_put_object_stage_duration(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE,
oldest_age_ms,
);
rustfs_io_metrics::record_put_object_stage_duration(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE_MAX_SAMPLE,
oldest_age_ms,
);
}
rustfs_io_metrics::record_put_object_stage_duration(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BATCH_SIZE,
batch.len() as f64,
);
if batch.len() > 1 {
rustfs_io_metrics::record_put_object_stage_duration(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BURST_BATCH_SIZE,
batch.len() as f64,
);
}
let active = active_workers.fetch_add(1, Ordering::Relaxed) + 1;
rustfs_io_metrics::record_put_object_stage_duration(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_ACTIVE,
active as f64,
);
for job in batch {
run_rename_tail_cleanup_job(job).await;
}
active_workers.fetch_sub(1, Ordering::Relaxed);
}
});
}
}
Self {
sender,
queued,
oldest_enqueued_at,
}
}
async fn try_enqueue(&self, job: RenameTailCleanupJob) -> std::result::Result<(), RenameTailCleanupJob> {
let queue_push_started = rustfs_io_metrics::put_stage_timer();
{
let mut oldest = self.oldest_enqueued_at.lock().await;
if oldest.is_none() {
*oldest = Some(job.enqueued_at);
}
}
let queued_after_enqueue = self.queued.fetch_add(1, Ordering::Relaxed) + 1;
if rename_tail_cleanup_defer_hold_worker_enabled() {
rustfs_io_metrics::record_put_object_stage_duration(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_ENQUEUED,
1.0,
);
rustfs_io_metrics::record_put_object_stage_duration(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER,
1.0,
);
rustfs_io_metrics::record_put_object_stage_duration(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH,
queued_after_enqueue as f64,
);
rustfs_io_metrics::record_put_object_stage_duration(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH_MAX_SAMPLE,
queued_after_enqueue as f64,
);
if let Some(oldest) = *self.oldest_enqueued_at.lock().await {
let oldest_age_ms = oldest.elapsed().as_secs_f64() * 1000.0;
rustfs_io_metrics::record_put_object_stage_duration(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE,
oldest_age_ms,
);
rustfs_io_metrics::record_put_object_stage_duration(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE_MAX_SAMPLE,
oldest_age_ms,
);
}
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_PUSH,
queue_push_started,
);
return Ok(());
}
match self.sender.try_send(job) {
Ok(()) => {
rustfs_io_metrics::record_put_object_stage_duration(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_ENQUEUED,
1.0,
);
rustfs_io_metrics::record_put_object_stage_duration(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH,
queued_after_enqueue as f64,
);
rustfs_io_metrics::record_put_object_stage_duration(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH_MAX_SAMPLE,
queued_after_enqueue as f64,
);
if let Some(oldest) = *self.oldest_enqueued_at.lock().await {
let oldest_age_ms = oldest.elapsed().as_secs_f64() * 1000.0;
rustfs_io_metrics::record_put_object_stage_duration(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE,
oldest_age_ms,
);
rustfs_io_metrics::record_put_object_stage_duration(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE_MAX_SAMPLE,
oldest_age_ms,
);
}
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_PUSH,
queue_push_started,
);
Ok(())
}
Err(err) => {
self.queued.fetch_sub(1, Ordering::Relaxed);
if self.queued.load(Ordering::Relaxed) == 0 {
*self.oldest_enqueued_at.lock().await = None;
}
rustfs_io_metrics::record_put_object_stage_duration(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_FALLBACK_SYNC,
1.0,
);
rustfs_io_metrics::record_put_object_stage_duration(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH,
self.queued.load(Ordering::Relaxed) as f64,
);
rustfs_io_metrics::record_put_object_stage_duration(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH_MAX_SAMPLE,
self.queued.load(Ordering::Relaxed) as f64,
);
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_PUSH,
queue_push_started,
);
Err(err.into_inner())
}
}
}
}
fn rename_tail_cleanup_queue() -> &'static RenameTailCleanupQueue {
static QUEUE: OnceLock<RenameTailCleanupQueue> = OnceLock::new();
QUEUE.get_or_init(RenameTailCleanupQueue::new)
}
const EVENT_SET_DISK_RENAME_ROLLBACK: &str = "set_disk_rename_rollback";
#[derive(Clone, Copy)]
@@ -3727,10 +4224,9 @@ async fn inspect_incomplete_rename_rollback(
request.recreate_missing = Some(false);
request.update_parity = Some(false);
request.recursive = Some(false);
match tokio::time::timeout(Duration::from_secs(1), submitter(request)).await {
Ok(result) => result,
Err(_) => ReadRepairAdmissionOutcome::Failed("rollback inspection admission timed out".to_string()),
}
tokio::time::timeout(Duration::from_secs(1), submitter(request))
.await
.unwrap_or_else(|_| ReadRepairAdmissionOutcome::Failed("rollback inspection admission timed out".to_string()))
}
fn rename_rollback_task_outcome(
@@ -4017,21 +4513,24 @@ pub(in crate::set_disk) async fn finish_rename_tail_heal<
SubmitFuture,
>(
tail_drain: tokio::task::JoinHandle<Option<RenameTailOutcome>>,
guard_release: tokio::sync::oneshot::Receiver<bool>,
guard_release: oneshot::Receiver<bool>,
guards: Guards,
request: rustfs_heal_contracts::heal_channel::HealChannelRequest,
finalize: Finalize,
cleanup: Cleanup,
submit: Submit,
) where
Guards: Send,
Guards: Send + 'static,
Finalize: FnOnce() -> FinalizeFuture + Send,
FinalizeFuture: Future<Output = ()> + Send,
Cleanup: FnOnce(Guards, Vec<RenameTailCleanup>) -> CleanupFuture + Send,
CleanupFuture: Future<Output = ()> + Send,
Cleanup: 'static,
CleanupFuture: 'static,
Submit: FnOnce(rustfs_heal_contracts::heal_channel::HealChannelRequest) -> SubmitFuture + Send,
SubmitFuture: Future<Output = ()> + Send,
{
let tail_async_started = rustfs_io_metrics::put_stage_timer();
let (needs_heal, tail_cleanup, tail_complete) = match tail_drain.await {
Ok(Some(outcome)) => (outcome.convergence.needs_heal(), outcome.cleanup, true),
Ok(None) => {
@@ -4060,16 +4559,80 @@ pub(in crate::set_disk) async fn finish_rename_tail_heal<
(true, Vec::new(), false)
}
};
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_ASYNC_WAIT,
tail_async_started,
);
// A dropped sender means the request continuation was cancelled. Cleanup
// must still run; only the explicit `false` used by the hard-crash harness
// suppresses all in-process continuation work.
let guard_release_wait_started = rustfs_io_metrics::put_stage_timer();
if matches!(guard_release.await, Ok(false)) {
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_GUARD_RELEASE_WAIT,
guard_release_wait_started,
);
drop(guards);
return;
}
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_GUARD_RELEASE_WAIT,
guard_release_wait_started,
);
let finalize_started = rustfs_io_metrics::put_stage_timer();
finalize().await;
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_FINALIZE,
finalize_started,
);
if tail_complete {
cleanup(guards, tail_cleanup).await;
let cleanup_submit_started = rustfs_io_metrics::put_stage_timer();
if rename_tail_cleanup_counterfactual_skip_enabled() {
rustfs_io_metrics::record_put_object_stage_duration(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP,
1.0,
);
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_SUBMIT,
cleanup_submit_started,
);
drop(guards);
} else if rename_tail_cleanup_defer_enabled() {
let targets = tail_cleanup.len();
let job = RenameTailCleanupJob {
enqueued_at: Instant::now(),
targets,
future: Box::pin(cleanup(guards, tail_cleanup)),
};
if let Err(job) = rename_tail_cleanup_queue().try_enqueue(job).await {
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_SUBMIT,
cleanup_submit_started,
);
let cleanup_started = rustfs_io_metrics::put_stage_timer();
job.future.await;
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP,
cleanup_started,
);
} else {
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_SUBMIT,
cleanup_submit_started,
);
}
} else {
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_SUBMIT,
cleanup_submit_started,
);
let cleanup_started = rustfs_io_metrics::put_stage_timer();
cleanup(guards, tail_cleanup).await;
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP,
cleanup_started,
);
}
} else {
// The outer coordinator may fail while a disk rename still owns the
// staging source. Preserve it for later healing/GC instead of racing
@@ -4077,7 +4640,12 @@ pub(in crate::set_disk) async fn finish_rename_tail_heal<
drop(guards);
}
if needs_heal {
let heal_submit_started = rustfs_io_metrics::put_stage_timer();
submit(request).await;
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_HEAL_SUBMIT,
heal_submit_started,
);
}
}
@@ -4131,8 +4699,8 @@ impl SetDisks {
bucket: &str,
object: &str,
write_quorum: usize,
) -> crate::error::Result<(Vec<Option<DiskStore>>, Vec<Option<SnapshotLeaseToken>>)> {
let fence_path = crate::disk::quota_mutation_fence_path(bucket, object);
) -> Result<(Vec<Option<DiskStore>>, Vec<Option<SnapshotLeaseToken>>)> {
let fence_path = disk::quota_mutation_fence_path(bucket, object);
let results = join_all(disks.iter().map(|disk| {
let disk = disk.clone();
let fence_path = fence_path.clone();
@@ -4174,8 +4742,8 @@ impl SetDisks {
bucket: &str,
object: &str,
write_quorum: usize,
) -> crate::error::Result<()> {
let fence_path = crate::disk::quota_mutation_fence_path(bucket, object);
) -> Result<()> {
let fence_path = disk::quota_mutation_fence_path(bucket, object);
let results = join_all(disks.iter().zip(tokens).filter_map(|(disk, token)| {
let disk = disk.as_ref()?.clone();
let token = (*token)?;
@@ -4204,11 +4772,25 @@ impl SetDisks {
errs: &[Option<DiskError>],
cleanup_data_dirs: &[Option<Uuid>],
old_current_sizes: &[Option<OldCurrentSize>],
old_current_source_checked: &[bool],
old_current_sources: &[Option<FileInfo>],
write_quorum: usize,
) -> disk::error::Result<RenameDataCommit> {
let data_dir = Self::reduce_common_data_dir(cleanup_data_dirs, write_quorum);
let convergence = Self::classify_rename_convergence(disk_versions, errs);
let old_current_size = Self::reduce_common_old_current_size(old_current_sizes, write_quorum);
let old_current_source_quorum_checked =
old_current_source_checked.iter().filter(|checked| **checked).count() >= write_quorum;
let old_current_source_all_successful_checked = errs
.iter()
.enumerate()
.filter(|(_, err)| err.is_none())
.all(|(idx, _)| old_current_source_checked.get(idx).copied().unwrap_or(false));
let old_current_source = if old_current_source_quorum_checked {
Self::reduce_common_old_current_source(old_current_sources, write_quorum)
} else {
None
};
let online_disks = Self::eval_disks(disks, errs);
let committed_slot = online_disks.iter().position(Option::is_some).ok_or(DiskError::Unexpected)?;
let committed_file_info = file_infos.get(committed_slot).cloned().ok_or(DiskError::Unexpected)?;
@@ -4236,6 +4818,9 @@ impl SetDisks {
data_dir,
cleanup_disks,
old_current_size,
old_current_source_checked: old_current_source_quorum_checked,
old_current_source_all_successful_checked,
old_current_source,
committed_file_info,
tail_drain: None,
})
@@ -4353,9 +4938,10 @@ impl SetDisks {
let src_object = Arc::new(src_object.to_string());
let dst_bucket = Arc::new(dst_bucket.to_string());
let dst_object = Arc::new(dst_object.to_string());
let (commit_tx, commit_rx) = tokio::sync::oneshot::channel();
let (commit_tx, commit_rx) = oneshot::channel();
let coordinator_failure_receipt = rollback_receipt.clone();
let early_ack_spawn_to_quorum_wait_started = rustfs_io_metrics::put_stage_timer();
let tail_drain = tokio::spawn({
let fanout_src_bucket = src_bucket.clone();
let fanout_src_object = src_object.clone();
@@ -4388,6 +4974,11 @@ impl SetDisks {
let Some(disk) = disk else {
return Err(DiskError::DiskNotFound);
};
let disk_wait_location_stage = if disk.is_local() {
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_LOCAL
} else {
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_REMOTE
};
let is_delete_marker = file_info.is_canonical_delete_marker();
// Clone FileInfo and set erasure.index for this disk
@@ -4407,6 +4998,7 @@ impl SetDisks {
let disk_wait_started = rustfs_io_metrics::put_stage_timer();
dispatch_state = RenameDispatchState::MayHavePublished;
let _foreground_active_guard = rename_foreground_active_guard(i);
let observed = disk
.rename_data_borrowed_with_fence_observed(
&src_bucket,
@@ -4414,7 +5006,7 @@ impl SetDisks {
&file_info,
&dst_bucket,
&dst_object,
crate::disk::RenameDataGuards {
disk::RenameDataGuards {
scanner_publication_lease_token,
namespace_owner: namespace_commit_guard
.clone()
@@ -4435,6 +5027,7 @@ impl SetDisks {
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT,
duration_ms,
);
rustfs_io_metrics::record_put_object_stage_duration(disk_wait_location_stage, duration_ms);
let position = if result.is_ok() {
let rank = successful_rename_completion_rank
.as_ref()
@@ -4473,9 +5066,12 @@ impl SetDisks {
let mut data_dirs = vec![None; disk_count];
let mut cleanup_data_dirs = vec![None; disk_count];
let mut old_current_sizes = vec![None; disk_count];
let mut old_current_source_checked = vec![false; disk_count];
let mut old_current_sources = vec![None; disk_count];
let mut capacity_scope_generation = None;
let mut cleanup_at_snapshot = vec![false; disk_count];
let mut tail_after_quorum_started = None;
while let Some(joined) = tasks.join_next().await {
results_seen += 1;
match joined {
@@ -4485,6 +5081,8 @@ impl SetDisks {
cleanup_data_dirs[idx] = res.cleanup_data_dir;
disk_versions[idx] = res.sign;
old_current_sizes[idx] = res.old_current_size;
old_current_source_checked[idx] = res.old_current_source_checked;
old_current_sources[idx] = res.old_current_source;
errs[idx] = None;
success_count += 1;
}
@@ -4510,6 +5108,8 @@ impl SetDisks {
&errs,
&cleanup_data_dirs,
&old_current_sizes,
&old_current_source_checked,
&old_current_sources,
write_quorum,
);
if let Ok(commit) = snapshot_commit.as_ref() {
@@ -4520,8 +5120,13 @@ impl SetDisks {
let _ = commit_tx.send(snapshot_commit);
}
sent_commit = true;
tail_after_quorum_started = rustfs_io_metrics::put_stage_timer();
}
}
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_AFTER_QUORUM_WAIT,
tail_after_quorum_started,
);
#[cfg(test)]
rollback_fault_injection::after_fanout(&fanout_dst_object);
@@ -4641,6 +5246,10 @@ impl SetDisks {
}
});
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_SPAWN_TO_QUORUM_WAIT,
early_ack_spawn_to_quorum_wait_started,
);
let quorum_wait_started = rustfs_io_metrics::put_stage_timer();
let commit = match commit_rx.await {
Ok(commit) => commit,
@@ -4653,8 +5262,13 @@ impl SetDisks {
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_QUORUM_WAIT,
quorum_wait_started,
);
let attach_tail_owner_started = rustfs_io_metrics::put_stage_timer();
commit.map(|mut commit| {
commit.tail_drain = Some(tail_drain);
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_ATTACH_TAIL_OWNER,
attach_tail_owner_started,
);
commit
})
}
@@ -4777,6 +5391,11 @@ impl SetDisks {
let Some(disk) = disk else {
return Err(DiskError::DiskNotFound);
};
let disk_wait_location_stage = if disk.is_local() {
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_LOCAL
} else {
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_REMOTE
};
let is_delete_marker = file_info.is_canonical_delete_marker();
let mut local_file_info;
@@ -4810,6 +5429,7 @@ impl SetDisks {
let disk_wait_started = rustfs_io_metrics::put_stage_timer();
dispatch_state = RenameDispatchState::MayHavePublished;
let _foreground_active_guard = rename_foreground_active_guard(i);
let observed = disk
.rename_data_borrowed_with_fence_observed(
&src_bucket,
@@ -4817,7 +5437,7 @@ impl SetDisks {
file_info,
&dst_bucket,
&dst_object,
crate::disk::RenameDataGuards {
disk::RenameDataGuards {
scanner_publication_lease_token,
namespace_owner: namespace_commit_guard
.clone()
@@ -4838,6 +5458,7 @@ impl SetDisks {
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT,
duration_ms,
);
rustfs_io_metrics::record_put_object_stage_duration(disk_wait_location_stage, duration_ms);
let position = if result.is_ok() {
let rank = successful_rename_completion_rank
.as_ref()
@@ -4873,9 +5494,16 @@ impl SetDisks {
let mut data_dirs = vec![None; disk_count];
let mut cleanup_data_dirs = vec![None; disk_count];
let mut old_current_sizes = vec![None; disk_count];
let mut old_current_source_checked = vec![false; disk_count];
let mut old_current_sources = vec![None; disk_count];
let quorum_wait_started = rustfs_io_metrics::put_stage_timer();
let sync_full_fanout_started = rustfs_io_metrics::put_stage_timer();
let fanout_result = fanout.await;
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_SYNC_FULL_FANOUT_WAIT,
sync_full_fanout_started,
);
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_QUORUM_WAIT,
quorum_wait_started,
@@ -4916,6 +5544,8 @@ impl SetDisks {
cleanup_data_dirs[idx] = res.cleanup_data_dir;
disk_versions[idx].clone_from(&res.sign);
old_current_sizes[idx] = res.old_current_size;
old_current_source_checked[idx] = res.old_current_source_checked;
old_current_sources[idx] = res.old_current_source.clone();
errs.push(None);
}
Ok(Err(e)) => {
@@ -5052,6 +5682,18 @@ impl SetDisks {
let data_dir = Self::reduce_common_data_dir(&cleanup_data_dirs, write_quorum);
let convergence = Self::classify_rename_convergence(&disk_versions, &errs);
let old_current_size = Self::reduce_common_old_current_size(&old_current_sizes, write_quorum);
let old_current_source_quorum_checked =
old_current_source_checked.iter().filter(|checked| **checked).count() >= write_quorum;
let old_current_source_all_successful_checked = errs
.iter()
.enumerate()
.filter(|(_, err)| err.is_none())
.all(|(idx, _)| old_current_source_checked.get(idx).copied().unwrap_or(false));
let old_current_source = if old_current_source_quorum_checked {
Self::reduce_common_old_current_source(&old_current_sources, write_quorum)
} else {
None
};
let online_disks = Self::eval_disks(disks, &errs);
let committed_slot = online_disks.iter().position(Option::is_some).ok_or(DiskError::Unexpected)?;
let committed_file_info = std::mem::take(&mut file_infos[committed_slot]);
@@ -5079,11 +5721,34 @@ impl SetDisks {
data_dir,
cleanup_disks,
old_current_size,
old_current_source_checked: old_current_source_quorum_checked,
old_current_source_all_successful_checked,
old_current_source,
committed_file_info,
tail_drain: None,
})
}
pub(in crate::set_disk) fn reduce_common_old_current_source(
old_current_sources: &[Option<FileInfo>],
write_quorum: usize,
) -> Option<FileInfo> {
let mut counts: HashMap<[u8; 32], (usize, FileInfo)> = HashMap::new();
for source in old_current_sources.iter().flatten() {
let entry = counts
.entry(Self::file_info_quorum_hash(source))
.or_insert_with(|| (0, source.clone()));
entry.0 += 1;
}
counts
.into_values()
.filter(|(count, _)| *count >= write_quorum)
.max_by_key(|(count, _)| *count)
.map(|(_, source)| source)
}
/// rustfs/backlog#1009: reduce the per-disk observations of the
/// destination's previous current version to one set-level value, mirroring
/// `reduce_common_data_dir`: the observation reported by at least
@@ -7555,9 +8220,9 @@ mod tests {
);
scope.try_begin().expect("delete scope should enter flight");
let scope_guard = crate::object_api::ScannerPublicationCommitScopeGuard::new(scope.clone());
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
let (finished_tx, finished_rx) = tokio::sync::oneshot::channel();
let (started_tx, started_rx) = oneshot::channel();
let (release_tx, release_rx) = oneshot::channel();
let (finished_tx, finished_rx) = oneshot::channel();
let waiter = tokio::spawn(run_scanner_publication_delete_owner(Some(scope.clone()), move || async move {
started_tx.send(()).expect("delete owner should start");
@@ -9987,7 +10652,7 @@ mod tests {
#[allow(unreachable_code)]
None
});
let (release, released) = tokio::sync::oneshot::channel::<bool>();
let (release, released) = oneshot::channel::<bool>();
drop(release);
finish_rename_tail_heal(
@@ -11194,7 +11859,7 @@ mod tests {
info.size = 11;
info.parts.clear();
info.add_object_part(1, "new-etag".to_string(), 11, None, 11, None, None);
let crate::disk::Disk::Local(local) = disk.as_ref() else {
let disk::Disk::Local(local) = disk.as_ref() else {
panic!("physical publication fixture requires local disks");
};
// Linux IO paths use a mount FD, which is also the namespace lock key.
@@ -11202,7 +11867,7 @@ mod tests {
.get_disk()
.get_object_path_for_io(bucket, object)
.expect("the publication path must resolve through the disk's mount lease");
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
let (entered_tx, entered_rx) = oneshot::channel();
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
hooks.push(os::prepared_publication_test_hooks::install(
&destination.join(STORAGE_FORMAT_FILE),
@@ -11388,11 +12053,11 @@ mod tests {
info.add_object_part(1, "new-etag".to_string(), 11, None, 11, None, None);
}
let disk = disks[3].as_ref().expect("tail disk");
let crate::disk::Disk::Local(local) = disk.as_ref() else {
let disk::Disk::Local(local) = disk.as_ref() else {
panic!("local fixture");
};
let destination = local.get_disk().get_object_path_for_io(bucket, object).expect("tail IO path");
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
let (entered_tx, entered_rx) = oneshot::channel();
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
let _hook = os::prepared_publication_test_hooks::install(&destination.join(STORAGE_FORMAT_FILE), move || {
let _ = entered_tx.send(());
@@ -11404,7 +12069,7 @@ mod tests {
let receipt = RenameRollbackReceipt::default();
let caller_receipt = receipt.clone();
let caller_disks = disks.clone();
let (ack_tx, ack_rx) = tokio::sync::oneshot::channel();
let (ack_tx, ack_rx) = oneshot::channel();
let caller = tokio::spawn(async move {
let commit = SetDisks::rename_data_owned_with_fence(
&caller_disks,
+20 -1
View File
@@ -166,7 +166,7 @@ use sha2::Sha256;
use std::hash::{BuildHasher, Hash, Hasher};
use std::mem::{self};
use std::pin::Pin;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, OnceLock};
use std::task::{Context, Poll};
use std::time::{Instant, SystemTime, UNIX_EPOCH};
@@ -410,6 +410,12 @@ impl Drop for ObjectLockDiagGuard {
}
let hold = self.acquired_at.elapsed();
if self.op == "put_object_commit" && self.mode == "write" && rustfs_io_metrics::put_stage_metrics_enabled() {
rustfs_io_metrics::record_put_object_stage_duration(
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD,
hold.as_secs_f64() * 1000.0,
);
}
record_object_lock_diag_hold_duration(self.op, self.mode, hold);
let threshold = get_object_lock_diag_slow_hold_threshold();
if hold >= threshold {
@@ -3904,6 +3910,9 @@ pub struct SetDisks {
/// writes skip the global registry mutex (backlog#1315). `Arc` so clones of
/// a set share one generation marker.
capacity_dirty_generation: Arc<AtomicU64>,
/// Per-set proof that successful rename_data peers return the old-current
/// source capability needed before PUT may skip its pre-rename lookup.
tier_free_version_rename_data_source_capable: Arc<AtomicBool>,
/// Orphan prefixes whose last purge scan met data that can never be
/// purged by listing (residue without a committed marker, an in-flight
/// write), keyed by `bucket/prefix` with the time of that scan. Empty
@@ -4767,6 +4776,7 @@ impl SetDisks {
ctx,
capacity_scope_cache: Arc::new(std::sync::RwLock::new(CapacityScopeCache::default())),
capacity_dirty_generation: Arc::new(AtomicU64::new(u64::MAX)),
tier_free_version_rename_data_source_capable: Arc::new(AtomicBool::new(false)),
orphan_purge_backoff: Arc::new(std::sync::Mutex::new(HashMap::new())),
#[cfg(test)]
storage_class_config_override: Arc::new(std::sync::RwLock::new(None)),
@@ -4781,6 +4791,15 @@ impl SetDisks {
&self.ctx
}
pub(in crate::set_disk) fn tier_free_version_rename_data_source_capable(&self) -> bool {
self.tier_free_version_rename_data_source_capable.load(Ordering::Acquire)
}
pub(in crate::set_disk) fn observe_tier_free_version_rename_data_source_capability(&self, capable: bool) {
self.tier_free_version_rename_data_source_capable
.store(capable, Ordering::Release);
}
#[cfg(test)]
pub(crate) fn set_instance_ctx_for_test(&mut self, ctx: Arc<InstanceContext>) {
self.local_lock_manager = ctx.lock_manager();
File diff suppressed because it is too large Load Diff
+28
View File
@@ -18,6 +18,11 @@ use crate::disk::error::{DiskError, Result as DiskResult};
use crate::disk::{DeleteOptions, Disk, RenameDataGuards, RenameDataResp};
use crate::runtime::instance::{InstanceContext, NamespaceCommitGuard};
use crate::runtime::sources as runtime_sources;
use rustfs_io_metrics::{
PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_ADMIT, PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOCAL_RENAME,
PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOOKUP, PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_OVERHEAD,
PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_TOTAL,
};
use tracing::{debug, error};
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
@@ -331,15 +336,38 @@ async fn rename_local_data_with_ctx(
destination: (&str, &str),
mut guards: RenameDataGuards,
) -> DiskResult<RenameDataResp> {
let handler_started = rustfs_io_metrics::put_stage_timer();
let lookup_started = rustfs_io_metrics::put_stage_timer();
let (disk, disk_id) = local_disk_candidate(ctx, disk_ref).await?;
let lookup_ms = lookup_started.map(|started| started.elapsed().as_secs_f64() * 1000.0);
if let Some(duration_ms) = lookup_ms {
rustfs_io_metrics::record_put_object_stage_duration(PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOOKUP, duration_ms);
}
let mutates_namespace = !is_meta_bucketname(source.0) || !is_meta_bucketname(destination.0);
let admit_started = rustfs_io_metrics::put_stage_timer();
let owner = admit_local_disk(ctx, &disk, disk_id, mutates_namespace).await?;
let admit_ms = admit_started.map(|started| started.elapsed().as_secs_f64() * 1000.0);
if let Some(duration_ms) = admit_ms {
rustfs_io_metrics::record_put_object_stage_duration(PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_ADMIT, duration_ms);
}
guards.namespace_owner = owner.as_ref().map(|owner| owner.clone() as Arc<dyn Send + Sync>);
let local_rename_started = rustfs_io_metrics::put_stage_timer();
let result = disk
.rename_data_borrowed_with_fence_observed(source.0, source.1, fi, destination.0, destination.1, guards)
.await
.result;
let local_rename_ms = local_rename_started.map(|started| started.elapsed().as_secs_f64() * 1000.0);
if let Some(duration_ms) = local_rename_ms {
rustfs_io_metrics::record_put_object_stage_duration(PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOCAL_RENAME, duration_ms);
}
drop(owner);
if let Some(handler_started) = handler_started {
let handler_ms = handler_started.elapsed().as_secs_f64() * 1000.0;
rustfs_io_metrics::record_put_object_stage_duration(PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_TOTAL, handler_ms);
let child_ms = lookup_ms.unwrap_or(0.0) + admit_ms.unwrap_or(0.0) + local_rename_ms.unwrap_or(0.0);
let overhead_ms = (handler_ms - child_ms).max(0.0);
rustfs_io_metrics::record_put_object_stage_duration(PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_OVERHEAD, overhead_ms);
}
result
}
@@ -20,8 +20,8 @@ use std::sync::Arc;
use std::time::Duration;
use storage_api::metadata_lock::{
BucketOperations, CompletePart, Error, MakeBucketOptions, MultipartOperations, NamespaceLocking, ObjectIO, ObjectOperations,
ObjectOptions, PutObjReader, PutObjectCommitBarrier, PutObjectCommitPause, init_bucket_metadata_sys,
isolated_store_over_temp_disks,
ObjectOptions, PutObjReader, PutObjectCommitBarrier, PutObjectCommitPause, enable_safe_no_quota_cache_fast_path_for_test,
init_bucket_metadata_sys, isolated_store_over_temp_disks,
};
use tokio::io::AsyncReadExt;
use tokio::time::timeout;
@@ -170,6 +170,103 @@ async fn reused_metadata_lock_still_enforces_quota() {
.expect("acquire writer");
}
#[tokio::test]
async fn safe_no_quota_cache_fast_path_does_not_ignore_later_quota() {
let _guard = enable_safe_no_quota_cache_fast_path_for_test();
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
let bucket = "safe-no-quota-cache-quota-set";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
let mut first = PutObjReader::from_vec(b"cache no quota".to_vec());
store
.put_object(bucket, "warm", &mut first, &ObjectOptions::default())
.await
.expect("warm no-quota metadata cache");
store
.update_bucket_metadata_config(bucket, "quota.json", br#"{"quota":1}"#.to_vec())
.await
.expect("set quota after no-quota cache was observed");
let mut opts = quota_snapshot_options(&store, bucket).await;
assert!(opts.set_quota_admission(0, 1));
let mut data = PutObjReader::from_vec(b"too large".to_vec());
let error = store
.put_object(bucket, "object", &mut data, &opts)
.await
.expect_err("quota set after a no-quota cache hit must still reject growth");
assert!(matches!(error, Error::QuotaExceeded { limit: 1, .. }), "{error:?}");
}
#[tokio::test]
async fn safe_no_quota_cache_fast_path_allows_write_after_quota_is_cleared() {
let _guard = enable_safe_no_quota_cache_fast_path_for_test();
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
let bucket = "safe-no-quota-cache-quota-cleared";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
store
.update_bucket_metadata_config(bucket, "quota.json", br#"{"quota":1}"#.to_vec())
.await
.expect("set quota");
store
.update_bucket_metadata_config(bucket, "quota.json", Vec::new())
.await
.expect("clear quota");
let mut data = PutObjReader::from_vec(b"allowed after clear".to_vec());
store
.put_object(bucket, "object", &mut data, &ObjectOptions::default())
.await
.expect("cleared quota should allow writes through the no-quota fast path");
let mut reader = store
.get_object_reader(bucket, "object", None, Default::default(), &ObjectOptions::default())
.await
.expect("read object written after quota clear");
let mut bytes = Vec::new();
reader.stream.read_to_end(&mut bytes).await.expect("read body");
assert_eq!(bytes, b"allowed after clear");
}
#[tokio::test]
async fn safe_no_quota_cache_fast_path_fails_closed_on_invalid_quota_json() {
let _guard = enable_safe_no_quota_cache_fast_path_for_test();
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
let bucket = "safe-no-quota-cache-invalid-quota";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
store
.update_bucket_metadata_config(bucket, "quota.json", b"{not-json".to_vec())
.await
.expect("persist invalid quota bytes for fail-closed read");
let mut data = PutObjReader::from_vec(b"must fail".to_vec());
store
.put_object(bucket, "object", &mut data, &ObjectOptions::default())
.await
.expect_err("invalid quota JSON must not degrade to no-quota");
}
async fn quota_snapshot_options(store: &Arc<storage_api::ECStore>, bucket: &str) -> ObjectOptions {
ObjectOptions {
versioned: true,
version_id: Some(Uuid::new_v4().to_string()),
expected_bucket_incarnation_id: Some(store.bucket_incarnation_id(bucket).await.expect("load incarnation")),
object_lock_config_snapshot: Some(store.object_lock_config_snapshot(bucket).await.expect("capture snapshot")),
..Default::default()
}
}
#[tokio::test]
async fn cancelled_put_releases_shared_metadata_guard() {
let (_dirs, store) = isolated_store_over_temp_disks().await;
+3 -1
View File
@@ -15,7 +15,9 @@ pub(crate) mod metadata_lock {
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{
init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks,
};
pub(crate) use rustfs_ecstore::api::set_disk::test_util::{PutObjectCommitBarrier, PutObjectCommitPause};
pub(crate) use rustfs_ecstore::api::set_disk::test_util::{
PutObjectCommitBarrier, PutObjectCommitPause, enable_safe_no_quota_cache_fast_path_for_test,
};
}
pub(crate) mod contract_compat {
+395 -2
View File
@@ -110,8 +110,213 @@ pub fn put_stage_timer() -> Option<std::time::Instant> {
}
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT: &str = "put_object_commit_namespace_lock_wait";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD: &str = "put_object_commit_namespace_lock_held";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_RENAME: &str = "put_object_commit_namespace_lock_held_rename";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_PRE_RENAME_PREPARE: &str =
"put_object_commit_namespace_lock_held_pre_rename_prepare";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_RENAME_RESULT_TO_TAIL_HANDOFF: &str =
"put_object_commit_namespace_lock_held_rename_result_to_tail_handoff";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_TAIL_HANDOFF: &str =
"put_object_commit_namespace_lock_held_tail_handoff";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_POST_RENAME: &str = "put_object_commit_namespace_lock_held_post_rename";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_GUARD_RELEASE: &str =
"put_object_commit_namespace_lock_held_guard_release";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE: &str =
"put_object_commit_namespace_lock_held_quota_fence_release";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_COMMIT: &str =
"put_object_commit_namespace_lock_held_quota_commit";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT: &str = "put_object_commit_namespace_lock_held_heal_submit";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT: &str =
"put_object_commit_namespace_lock_held_cleanup_receipt";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_METADATA_INVALIDATE: &str =
"put_object_commit_namespace_lock_held_metadata_invalidate";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_OBJECT_INFO_BUILD: &str =
"put_object_commit_namespace_lock_held_object_info_build";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_POST_OBJECT_INFO_TO_RETURN: &str =
"put_object_commit_namespace_lock_held_post_object_info_to_return";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_TASK_WAIT: &str = "put_object_commit_task_wait";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_FUTURE_TOTAL: &str = "put_object_commit_future_total";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_OPERATION_SELECT_WAIT: &str = "put_object_commit_operation_select_wait";
pub const PUT_STAGE_PUT_OBJECT_OPERATION_AWAIT_TOTAL: &str = "put_object_operation_await_total";
pub const PUT_STAGE_PUT_OBJECT_OPERATION_BODY_TO_COMMIT_SUBMIT: &str = "put_object_operation_body_to_commit_submit";
pub const PUT_STAGE_PUT_OBJECT_OPERATION_RESULT_RETURN: &str = "put_object_operation_result_return";
pub const PUT_STAGE_PUT_OBJECT_OPERATION_FIRST_POLL_DELAY: &str = "put_object_operation_first_poll_delay";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_ACQUIRED_TO_COMMIT_READY: &str =
"put_object_commit_namespace_lock_acquired_to_commit_ready";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_PRECONDITION_AND_TIMESTAMP: &str =
"put_object_commit_ready_precondition_and_timestamp";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_EXPECTED_VERSION_LOOKUP: &str = "put_object_commit_ready_expected_version_lookup";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_EXPLICIT_VERSION_LOOKUP: &str = "put_object_commit_ready_explicit_version_lookup";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_RESTORE_VERIFY: &str = "put_object_commit_ready_restore_verify";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_FENCE_VERIFY: &str = "put_object_commit_ready_fence_verify";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_DECOMMISSION_CAPACITY_FENCE: &str =
"put_object_commit_ready_decommission_capacity_fence";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_TRANSACTION_EPOCH_FENCE_READ: &str =
"put_object_commit_ready_transaction_epoch_fence_read";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_BEGIN: &str = "put_object_commit_ready_quota_begin";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_METADATA_ADJUST: &str = "put_object_commit_ready_metadata_adjust";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_RESERVE: &str = "put_object_commit_ready_quota_reserve";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_MUTATION_FENCE_PREPARE: &str =
"put_object_commit_ready_quota_mutation_fence_prepare";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_CONTEXT_PREPARE: &str = "put_object_commit_ready_context_prepare";
pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_META_BUCKET_FAST_PATH: &str = "put_object_quota_begin_meta_bucket_fast_path";
pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_METADATA_LOCK: &str = "put_object_quota_begin_metadata_lock";
pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CACHE_LOOKUP: &str = "put_object_quota_begin_cache_lookup";
pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONFIG_READ: &str = "put_object_quota_begin_config_read";
pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONFIG_READ_COUNTERFACTUAL_SKIP: &str =
"put_object_quota_begin_config_read_counterfactual_skip";
pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_SAFE_NO_QUOTA_CACHE_HIT: &str = "put_object_quota_begin_safe_no_quota_cache_hit";
pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_LOCK_LOST_CHECK: &str = "put_object_quota_begin_lock_lost_check";
pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_POLICY_EVALUATE: &str = "put_object_quota_begin_policy_evaluate";
pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CAPABILITY_PROOF: &str = "put_object_quota_begin_capability_proof";
pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_SNAPSHOT_ADMISSION: &str = "put_object_quota_begin_snapshot_admission";
pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_OBJECT_STORE_LOOKUP: &str = "put_object_quota_begin_object_store_lookup";
pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONTEXT_BUILD: &str = "put_object_quota_begin_context_build";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_SUBMIT_TO_CLOSURE_ENTER: &str = "put_object_commit_submit_to_closure_enter";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_CLOSURE_ENTER_TO_FIRST_AWAIT: &str = "put_object_commit_closure_enter_to_first_await";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_CLOSURE_TOTAL: &str = "put_object_commit_closure_total";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_BEFORE_RENAME: &str = "put_object_commit_before_rename";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_AWAIT_TOTAL: &str = "put_object_commit_pre_rename_await_total";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_QUOTA_MARK_COMMIT_STARTED: &str =
"put_object_commit_pre_rename_quota_mark_commit_started";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_FIRST_FENCE_CHECK: &str = "put_object_commit_pre_rename_first_fence_check";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_RESTORE_VERIFY: &str = "put_object_commit_pre_rename_restore_verify";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_TRANSACTION_FENCE_VERIFY: &str =
"put_object_commit_pre_rename_transaction_fence_verify";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_SECOND_FENCE_CHECK: &str = "put_object_commit_pre_rename_second_fence_check";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_SCANNER_SCOPE_CHECK: &str = "put_object_commit_pre_rename_scanner_scope_check";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_TIER_FREE_VERSION_SOURCE_LOOKUP: &str =
"put_object_commit_pre_rename_tier_free_version_source_lookup";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_ASSIGN_RENAME_INDEXES: &str =
"put_object_commit_pre_rename_assign_rename_indexes";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_POST_AWAIT_TO_RENAME: &str = "put_object_commit_pre_rename_post_await_to_rename";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_AFTER_RENAME_TO_RETURN: &str = "put_object_commit_after_rename_to_return";
pub const PUT_STAGE_SET_DISK_RENAME_QUORUM_WAIT: &str = "set_disk_rename_quorum_wait";
pub const PUT_STAGE_SET_DISK_RENAME_SYNC_FULL_FANOUT_WAIT: &str = "set_disk_rename_sync_full_fanout_wait";
pub const PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_SPAWN_TO_QUORUM_WAIT: &str = "set_disk_rename_early_ack_spawn_to_quorum_wait";
pub const PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_ATTACH_TAIL_OWNER: &str = "set_disk_rename_early_ack_attach_tail_owner";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_AFTER_QUORUM_WAIT: &str = "set_disk_rename_tail_after_quorum_wait";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_ASYNC_WAIT: &str = "set_disk_rename_tail_async_wait";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_GUARD_RELEASE_WAIT: &str = "set_disk_rename_tail_guard_release_wait";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_FINALIZE: &str = "set_disk_rename_tail_finalize";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP: &str = "set_disk_rename_tail_cleanup";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_SUBMIT: &str = "set_disk_rename_tail_cleanup_submit";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP: &str = "set_disk_rename_tail_cleanup_counterfactual_skip";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_ENQUEUED: &str = "set_disk_rename_tail_cleanup_defer_enqueued";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_FALLBACK_SYNC: &str = "set_disk_rename_tail_cleanup_defer_fallback_sync";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_PUSH: &str = "set_disk_rename_tail_cleanup_defer_queue_push";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH: &str = "set_disk_rename_tail_cleanup_defer_queue_depth";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH_MAX_SAMPLE: &str =
"set_disk_rename_tail_cleanup_defer_queue_depth_max_sample";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE: &str = "set_disk_rename_tail_cleanup_defer_oldest_age";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE_MAX_SAMPLE: &str =
"set_disk_rename_tail_cleanup_defer_oldest_age_max_sample";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_TARGETS: &str = "set_disk_rename_tail_cleanup_defer_targets";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_START_DELAY: &str = "set_disk_rename_tail_cleanup_defer_start_delay";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_COMPLETED_LAG: &str = "set_disk_rename_tail_cleanup_defer_completed_lag";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER: &str = "set_disk_rename_tail_cleanup_defer_hold_worker";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_DELAY: &str = "set_disk_rename_tail_cleanup_defer_worker_delay";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_PER_DISK_PERMIT_WAIT: &str = "set_disk_rename_tail_cleanup_per_disk_permit_wait";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_PER_DISK_INFLIGHT: &str = "set_disk_rename_tail_cleanup_per_disk_inflight";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_FOREGROUND_YIELD: &str = "set_disk_rename_tail_cleanup_foreground_yield";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFERRED_FOREGROUND_ACTIVE: &str =
"set_disk_rename_tail_cleanup_deferred_foreground_active";
pub const PUT_STAGE_SET_DISK_RENAME_FOREGROUND_ACTIVE: &str = "set_disk_rename_foreground_active";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BATCH_WINDOW: &str = "set_disk_rename_tail_cleanup_defer_batch_window";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BATCH_SIZE: &str = "set_disk_rename_tail_cleanup_defer_batch_size";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BURST_BATCH_SIZE: &str =
"set_disk_rename_tail_cleanup_defer_burst_batch_size";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_ACTIVE: &str = "set_disk_rename_tail_cleanup_defer_worker_active";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_POLL_WAIT: &str =
"set_disk_rename_tail_cleanup_defer_worker_poll_wait";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_IDLE_WAIT: &str =
"set_disk_rename_tail_cleanup_defer_worker_idle_wait";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_JOB_WAIT: &str =
"set_disk_rename_tail_cleanup_defer_worker_job_wait";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_COMPLETED: &str = "set_disk_rename_tail_cleanup_defer_completed";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_DURATION: &str =
"set_disk_rename_tail_cleanup_defer_worker_duration";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_RECEIPT: &str = "set_disk_rename_tail_cleanup_target_receipt";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_COMMIT_CAPACITY: &str =
"set_disk_rename_tail_cleanup_target_commit_capacity";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_REPORT: &str = "set_disk_rename_tail_cleanup_target_report";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_COUNT: &str = "set_disk_rename_tail_cleanup_target_count";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_ZERO_TARGET: &str = "set_disk_rename_tail_cleanup_zero_target";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TMP_DELETE: &str = "set_disk_rename_tail_cleanup_tmp_delete";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TMP_DELETE_COUNTERFACTUAL_SKIP: &str =
"set_disk_rename_tail_cleanup_tmp_delete_counterfactual_skip";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DECOMMISSION_GUARD_DROP: &str =
"set_disk_rename_tail_cleanup_decommission_guard_drop";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_DEPTH: &str = "set_disk_rename_tail_cleanup_lane_queue_depth";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_PUSH: &str = "set_disk_rename_tail_cleanup_lane_queue_push";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_DEPTH_MAX_SAMPLE: &str =
"set_disk_rename_tail_cleanup_lane_queue_depth_max_sample";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_OLDEST_AGE: &str = "set_disk_rename_tail_cleanup_lane_oldest_age";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_OLDEST_AGE_MAX_SAMPLE: &str =
"set_disk_rename_tail_cleanup_lane_oldest_age_max_sample";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_BATCH_SIZE: &str = "set_disk_rename_tail_cleanup_lane_batch_size";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_BURST_BATCH_SIZE: &str =
"set_disk_rename_tail_cleanup_lane_burst_batch_size";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_ACTIVE: &str = "set_disk_rename_tail_cleanup_lane_worker_active";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_POLL_WAIT: &str =
"set_disk_rename_tail_cleanup_lane_worker_poll_wait";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_IDLE_WAIT: &str =
"set_disk_rename_tail_cleanup_lane_worker_idle_wait";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_JOB_WAIT: &str = "set_disk_rename_tail_cleanup_lane_worker_job_wait";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_SERVICE_TIME: &str = "set_disk_rename_tail_cleanup_lane_service_time";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_REQUEUE: &str = "set_disk_rename_tail_cleanup_lane_requeue";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_FOREGROUND_ACTIVE_HIT: &str =
"set_disk_rename_tail_cleanup_lane_foreground_active_hit";
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_HEAL_SUBMIT: &str = "set_disk_rename_tail_heal_submit";
pub const PUT_STAGE_SET_DISK_RENAME_DISK_WAIT: &str = "set_disk_rename_disk_wait";
pub const PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_LOCAL: &str = "set_disk_rename_disk_wait_local";
pub const PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_REMOTE: &str = "set_disk_rename_disk_wait_remote";
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_PREPARE: &str = "set_disk_rename_remote_client_prepare";
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC: &str = "set_disk_rename_remote_client_rpc";
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT: &str = "set_disk_rename_remote_client_rpc_await";
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_POLL_PENDING_COUNT: &str =
"set_disk_rename_remote_client_rpc_await_poll_pending_count";
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_FIRST_PENDING_TO_READY: &str =
"set_disk_rename_remote_client_rpc_await_first_pending_to_ready";
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_REQUEST_SCOPE: &str = "set_disk_rename_remote_client_request_scope";
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL: &str = "set_disk_rename_remote_client_transport_call";
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL_POLL_PENDING_COUNT: &str =
"set_disk_rename_remote_client_transport_call_poll_pending_count";
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL_FIRST_PENDING_TO_READY: &str =
"set_disk_rename_remote_client_transport_call_first_pending_to_ready";
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_REPLAY_RESPONSE: &str = "set_disk_rename_remote_client_replay_response";
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_INTO_INNER: &str = "set_disk_rename_remote_client_into_inner";
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_SUCCESS_CHECK: &str = "set_disk_rename_remote_client_success_check";
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_DECODE: &str = "set_disk_rename_remote_client_decode";
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RESPONSE_PROCESS: &str = "set_disk_rename_remote_client_response_process";
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RESPONSE_HEADERS: &str = "set_disk_rename_remote_client_response_headers";
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_ENCODE: &str = "set_disk_rename_remote_service_response_encode";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE_TRIGGERED: &str =
"put_object_commit_namespace_lock_held_quota_fence_release_triggered";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE_SKIPPED: &str =
"put_object_commit_namespace_lock_held_quota_fence_release_skipped";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT_TRIGGERED: &str =
"put_object_commit_namespace_lock_held_heal_submit_triggered";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT_SKIPPED: &str =
"put_object_commit_namespace_lock_held_heal_submit_skipped";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT_TRIGGERED: &str =
"put_object_commit_namespace_lock_held_cleanup_receipt_triggered";
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT_SKIPPED: &str =
"put_object_commit_namespace_lock_held_cleanup_receipt_skipped";
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_TOTAL: &str = "set_disk_rename_remote_handler_total";
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_OVERHEAD: &str = "set_disk_rename_remote_handler_overhead";
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOOKUP: &str = "set_disk_rename_remote_handler_lookup";
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_ADMIT: &str = "set_disk_rename_remote_handler_admit";
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOCAL_RENAME: &str = "set_disk_rename_remote_handler_local_rename";
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_ENTRY: &str = "set_disk_rename_remote_service_entry";
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_BEFORE_HANDLER: &str = "set_disk_rename_remote_service_before_handler";
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_HANDLER_AWAIT: &str = "set_disk_rename_remote_service_handler_await";
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_AFTER_HANDLER: &str = "set_disk_rename_remote_service_after_handler";
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_BUILD: &str = "set_disk_rename_remote_service_response_build";
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN: &str = "set_disk_rename_remote_service_response_return";
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_SERVICE_FUTURE: &str = "set_disk_rename_remote_transport_service_future";
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_FIRST_FRAME: &str = "set_disk_rename_remote_transport_first_frame";
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_BODY_COMPLETE: &str = "set_disk_rename_remote_transport_body_complete";
pub const PUT_STAGE_SET_DISK_RENAME_FILE_SYNC_PERMIT_WAIT: &str = "set_disk_rename_file_sync_permit_wait";
pub const PUT_STAGE_SET_DISK_RENAME_GLOBAL_FILE_SYNC_PERMIT_WAIT: &str = "set_disk_rename_global_file_sync_permit_wait";
pub const PUT_STAGE_SET_DISK_RENAME_FILE_FDATASYNC: &str = "set_disk_rename_file_fdatasync";
@@ -135,6 +340,7 @@ pub const PUT_RENAME_FDATASYNC_BATCH_MODE_SERIAL: &str = "serial";
pub const PUT_RENAME_FDATASYNC_BATCH_MODE_PARALLEL: &str = "parallel";
pub const PUT_RENAME_FDATASYNC_GROUP_WAIT_ROLE_LEADER: &str = "leader";
pub const PUT_RENAME_FDATASYNC_GROUP_WAIT_ROLE_FOLLOWER: &str = "follower";
pub const PUT_RENAME_FDATASYNC_GROUP_WAIT_ROLE_FOLLOWER_TIMEOUT: &str = "follower_timeout";
pub const PUT_RENAME_FDATASYNC_GROUP_OUTSTANDING_STATE_ENQUEUE_WAITERS: &str = "enqueue_waiters";
pub const PUT_RENAME_FDATASYNC_GROUP_OUTSTANDING_STATE_ENQUEUE_FILES: &str = "enqueue_files";
pub const PUT_RENAME_FDATASYNC_GROUP_OUTSTANDING_STATE_BATCH_WAITERS: &str = "batch_waiters";
@@ -2072,6 +2278,14 @@ pub fn record_put_object_stage_duration_from(stage: &'static str, started_at: Op
}
}
#[inline(always)]
pub fn record_put_object_quota_cache_decision(decision: &'static str, reason: &'static str) {
if !put_stage_metrics_enabled() {
return;
}
counter!("rustfs_s3_put_object_quota_cache_total", "decision" => decision, "reason" => reason).increment(1);
}
#[inline(always)]
pub fn record_put_object_commit_lock_admission(budget: &'static str, outcome: &'static str) {
if !put_stage_metrics_enabled() {
@@ -3171,8 +3385,166 @@ mod tests {
assert_eq!(PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT, "put_object_commit_namespace_lock_wait");
let stages = [
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT,
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD,
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_RENAME,
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_PRE_RENAME_PREPARE,
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_RENAME_RESULT_TO_TAIL_HANDOFF,
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_TAIL_HANDOFF,
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_POST_RENAME,
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_GUARD_RELEASE,
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE,
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_COMMIT,
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT,
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT,
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_METADATA_INVALIDATE,
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_OBJECT_INFO_BUILD,
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_POST_OBJECT_INFO_TO_RETURN,
PUT_STAGE_PUT_OBJECT_COMMIT_TASK_WAIT,
PUT_STAGE_PUT_OBJECT_COMMIT_FUTURE_TOTAL,
PUT_STAGE_PUT_OBJECT_COMMIT_OPERATION_SELECT_WAIT,
PUT_STAGE_PUT_OBJECT_OPERATION_AWAIT_TOTAL,
PUT_STAGE_PUT_OBJECT_OPERATION_BODY_TO_COMMIT_SUBMIT,
PUT_STAGE_PUT_OBJECT_OPERATION_RESULT_RETURN,
PUT_STAGE_PUT_OBJECT_OPERATION_FIRST_POLL_DELAY,
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_ACQUIRED_TO_COMMIT_READY,
PUT_STAGE_PUT_OBJECT_COMMIT_READY_PRECONDITION_AND_TIMESTAMP,
PUT_STAGE_PUT_OBJECT_COMMIT_READY_EXPECTED_VERSION_LOOKUP,
PUT_STAGE_PUT_OBJECT_COMMIT_READY_EXPLICIT_VERSION_LOOKUP,
PUT_STAGE_PUT_OBJECT_COMMIT_READY_RESTORE_VERIFY,
PUT_STAGE_PUT_OBJECT_COMMIT_READY_FENCE_VERIFY,
PUT_STAGE_PUT_OBJECT_COMMIT_READY_DECOMMISSION_CAPACITY_FENCE,
PUT_STAGE_PUT_OBJECT_COMMIT_READY_TRANSACTION_EPOCH_FENCE_READ,
PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_BEGIN,
PUT_STAGE_PUT_OBJECT_COMMIT_READY_METADATA_ADJUST,
PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_RESERVE,
PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_MUTATION_FENCE_PREPARE,
PUT_STAGE_PUT_OBJECT_COMMIT_READY_CONTEXT_PREPARE,
PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_META_BUCKET_FAST_PATH,
PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_METADATA_LOCK,
PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CACHE_LOOKUP,
PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONFIG_READ,
PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONFIG_READ_COUNTERFACTUAL_SKIP,
PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_SAFE_NO_QUOTA_CACHE_HIT,
PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_LOCK_LOST_CHECK,
PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_POLICY_EVALUATE,
PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CAPABILITY_PROOF,
PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_SNAPSHOT_ADMISSION,
PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_OBJECT_STORE_LOOKUP,
PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONTEXT_BUILD,
PUT_STAGE_PUT_OBJECT_COMMIT_SUBMIT_TO_CLOSURE_ENTER,
PUT_STAGE_PUT_OBJECT_COMMIT_CLOSURE_ENTER_TO_FIRST_AWAIT,
PUT_STAGE_PUT_OBJECT_COMMIT_CLOSURE_TOTAL,
PUT_STAGE_PUT_OBJECT_COMMIT_BEFORE_RENAME,
PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_AWAIT_TOTAL,
PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_QUOTA_MARK_COMMIT_STARTED,
PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_FIRST_FENCE_CHECK,
PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_RESTORE_VERIFY,
PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_TRANSACTION_FENCE_VERIFY,
PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_SECOND_FENCE_CHECK,
PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_SCANNER_SCOPE_CHECK,
PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_TIER_FREE_VERSION_SOURCE_LOOKUP,
PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_ASSIGN_RENAME_INDEXES,
PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_POST_AWAIT_TO_RENAME,
PUT_STAGE_PUT_OBJECT_COMMIT_AFTER_RENAME_TO_RETURN,
PUT_STAGE_SET_DISK_RENAME_QUORUM_WAIT,
PUT_STAGE_SET_DISK_RENAME_SYNC_FULL_FANOUT_WAIT,
PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_SPAWN_TO_QUORUM_WAIT,
PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_ATTACH_TAIL_OWNER,
PUT_STAGE_SET_DISK_RENAME_TAIL_AFTER_QUORUM_WAIT,
PUT_STAGE_SET_DISK_RENAME_TAIL_ASYNC_WAIT,
PUT_STAGE_SET_DISK_RENAME_TAIL_GUARD_RELEASE_WAIT,
PUT_STAGE_SET_DISK_RENAME_TAIL_FINALIZE,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_SUBMIT,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_ENQUEUED,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_FALLBACK_SYNC,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_PUSH,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH_MAX_SAMPLE,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE_MAX_SAMPLE,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_TARGETS,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_START_DELAY,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_COMPLETED_LAG,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_DELAY,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_PER_DISK_PERMIT_WAIT,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_PER_DISK_INFLIGHT,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_FOREGROUND_YIELD,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFERRED_FOREGROUND_ACTIVE,
PUT_STAGE_SET_DISK_RENAME_FOREGROUND_ACTIVE,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BATCH_WINDOW,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BATCH_SIZE,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BURST_BATCH_SIZE,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_ACTIVE,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_POLL_WAIT,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_IDLE_WAIT,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_JOB_WAIT,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_COMPLETED,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_DURATION,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_RECEIPT,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_COMMIT_CAPACITY,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_REPORT,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_COUNT,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_ZERO_TARGET,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TMP_DELETE,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TMP_DELETE_COUNTERFACTUAL_SKIP,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DECOMMISSION_GUARD_DROP,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_DEPTH,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_PUSH,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_DEPTH_MAX_SAMPLE,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_OLDEST_AGE,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_OLDEST_AGE_MAX_SAMPLE,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_BATCH_SIZE,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_BURST_BATCH_SIZE,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_ACTIVE,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_POLL_WAIT,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_IDLE_WAIT,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_JOB_WAIT,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_SERVICE_TIME,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_REQUEUE,
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_FOREGROUND_ACTIVE_HIT,
PUT_STAGE_SET_DISK_RENAME_TAIL_HEAL_SUBMIT,
PUT_STAGE_SET_DISK_RENAME_DISK_WAIT,
PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_LOCAL,
PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_REMOTE,
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_PREPARE,
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC,
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT,
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_POLL_PENDING_COUNT,
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_FIRST_PENDING_TO_READY,
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_REQUEST_SCOPE,
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL,
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL_POLL_PENDING_COUNT,
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL_FIRST_PENDING_TO_READY,
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_REPLAY_RESPONSE,
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_INTO_INNER,
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_SUCCESS_CHECK,
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_DECODE,
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RESPONSE_PROCESS,
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RESPONSE_HEADERS,
PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_ENCODE,
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE_TRIGGERED,
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE_SKIPPED,
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT_TRIGGERED,
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT_SKIPPED,
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT_TRIGGERED,
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT_SKIPPED,
PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_TOTAL,
PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_OVERHEAD,
PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOOKUP,
PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_ADMIT,
PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOCAL_RENAME,
PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_ENTRY,
PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_BEFORE_HANDLER,
PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_HANDLER_AWAIT,
PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_AFTER_HANDLER,
PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_BUILD,
PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN,
PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_SERVICE_FUTURE,
PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_FIRST_FRAME,
PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_BODY_COMPLETE,
PUT_STAGE_SET_DISK_RENAME_FILE_SYNC_PERMIT_WAIT,
PUT_STAGE_SET_DISK_RENAME_GLOBAL_FILE_SYNC_PERMIT_WAIT,
PUT_STAGE_SET_DISK_RENAME_FILE_FDATASYNC,
@@ -3185,7 +3557,26 @@ mod tests {
let unique = stages.iter().copied().collect::<HashSet<_>>();
assert_eq!(unique.len(), stages.len());
assert!(stages.iter().all(|stage| {
(stage.starts_with("set_disk_rename_") || *stage == PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT)
(stage.starts_with("set_disk_rename_")
|| *stage == PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT
|| *stage == PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD
|| *stage == PUT_STAGE_PUT_OBJECT_COMMIT_TASK_WAIT
|| *stage == PUT_STAGE_PUT_OBJECT_COMMIT_FUTURE_TOTAL
|| *stage == PUT_STAGE_PUT_OBJECT_COMMIT_OPERATION_SELECT_WAIT
|| *stage == PUT_STAGE_PUT_OBJECT_OPERATION_AWAIT_TOTAL
|| *stage == PUT_STAGE_PUT_OBJECT_OPERATION_BODY_TO_COMMIT_SUBMIT
|| *stage == PUT_STAGE_PUT_OBJECT_OPERATION_RESULT_RETURN
|| *stage == PUT_STAGE_PUT_OBJECT_OPERATION_FIRST_POLL_DELAY
|| *stage == PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_ACQUIRED_TO_COMMIT_READY
|| stage.starts_with("put_object_commit_ready_")
|| stage.starts_with("put_object_quota_begin_")
|| *stage == PUT_STAGE_PUT_OBJECT_COMMIT_SUBMIT_TO_CLOSURE_ENTER
|| *stage == PUT_STAGE_PUT_OBJECT_COMMIT_CLOSURE_ENTER_TO_FIRST_AWAIT
|| *stage == PUT_STAGE_PUT_OBJECT_COMMIT_CLOSURE_TOTAL
|| *stage == PUT_STAGE_PUT_OBJECT_COMMIT_BEFORE_RENAME
|| stage.starts_with("put_object_commit_pre_rename_")
|| *stage == PUT_STAGE_PUT_OBJECT_COMMIT_AFTER_RENAME_TO_RETURN
|| stage.starts_with("put_object_commit_namespace_lock_held_"))
&& !stage.contains('/')
&& !stage.contains('{')
}));
@@ -3318,6 +3709,7 @@ mod tests {
record_put_rename_fdatasync_batch(PUT_RENAME_FDATASYNC_BATCH_MODE_PARALLEL, 9);
record_put_rename_fdatasync_group_wait(PUT_RENAME_FDATASYNC_GROUP_WAIT_ROLE_LEADER, 1.0);
record_put_rename_fdatasync_group_wait(PUT_RENAME_FDATASYNC_GROUP_WAIT_ROLE_FOLLOWER, 2.0);
record_put_rename_fdatasync_group_wait(PUT_RENAME_FDATASYNC_GROUP_WAIT_ROLE_FOLLOWER_TIMEOUT, 3.0);
record_put_rename_fdatasync_group_outstanding(PUT_RENAME_FDATASYNC_GROUP_OUTSTANDING_STATE_ENQUEUE_WAITERS, 2);
record_put_rename_fdatasync_group_outstanding(PUT_RENAME_FDATASYNC_GROUP_OUTSTANDING_STATE_ENQUEUE_FILES, 4);
record_put_rename_fdatasync_group_outstanding(PUT_RENAME_FDATASYNC_GROUP_OUTSTANDING_STATE_BATCH_WAITERS, 3);
@@ -3352,7 +3744,7 @@ mod tests {
assert_eq!(quorum_samples, vec![0.0, 1.0, 3.0, 3.0, 4.0]);
assert_eq!(
histogram_samples(&rows, "rustfs_s3_put_object_rename_fdatasync_group_wait_ms"),
vec![1.0, 2.0]
vec![1.0, 2.0, 3.0]
);
assert_eq!(
histogram_samples(&rows, "rustfs_s3_put_object_rename_fdatasync_group_outstanding"),
@@ -3382,6 +3774,7 @@ mod tests {
HashSet::from([
PUT_RENAME_FDATASYNC_GROUP_WAIT_ROLE_LEADER.to_string(),
PUT_RENAME_FDATASYNC_GROUP_WAIT_ROLE_FOLLOWER.to_string(),
PUT_RENAME_FDATASYNC_GROUP_WAIT_ROLE_FOLLOWER_TIMEOUT.to_string(),
])
);
let group_outstanding_states = rows
-3
View File
@@ -19,9 +19,6 @@ no-default-features = false
version = 2
yanked = "deny"
ignore = [
# `paste 1.0.15` — unmaintained. No direct dependency.
# owner: rustfs-maintainers review: 2026-07
{ id = "RUSTSEC-2024-0436", reason = "paste unmaintained; transitive only; tracked for upgrade" },
# `rsa` Marvin timing sidechannel (RUSTSEC-2023-0071). Pulled in via
# `openidconnect` (transitive) and historically used directly. No upstream
# fix is available yet. Tracked separately for follow-up; remove this
+55 -11
View File
@@ -221,6 +221,16 @@ const GET_OBJECT_STAGE_LIFECYCLE_EXPIRATION: &str = "lifecycle_expiration";
const GET_OBJECT_STAGE_METADATA_FILTER: &str = "metadata_filter";
const GET_OBJECT_STAGE_BODY_CACHE_HOOK_HIT: &str = "body_cache_hook_hit";
const GET_OBJECT_STAGE_BODY_CACHE_LOOKUP: &str = "body_cache_lookup";
const GET_OBJECT_STAGE_BODY_CACHE_FILL_HANDOFF: &str = "body_cache_fill_handoff";
const GET_OBJECT_STAGE_BODY_BUFFERED_HANDOFF: &str = "body_buffered_handoff";
const GET_OBJECT_STAGE_OUTPUT_STRUCT_BUILD: &str = "output_struct_build";
const GET_OBJECT_STAGE_PREPARE_READ_EXECUTION: &str = "prepare_read_execution";
const GET_OBJECT_STAGE_RESPONSE_CORS_WRAP: &str = "response_cors_wrap";
const GET_OBJECT_STAGE_RESPONSE_HEADER_INJECT: &str = "response_header_inject";
const GET_OBJECT_STAGE_RESPONSE_EVENT_COMPLETE: &str = "response_event_complete";
const GET_OBJECT_STREAM_WARN_THRESHOLD: Duration = Duration::from_secs(5);
static GET_OBJECT_BUFFER_THRESHOLD_WARNED: AtomicBool = AtomicBool::new(false);
@@ -499,10 +509,10 @@ fn get_object_stream_failure_reason(error_class: &'static str) -> &'static str {
}
}
/// A mid-stream reopen can recover only when another representation of the
/// committed object may exist. A single-disk, no-parity object has no alternate
/// shard after a bitrot/read-quorum failure, so retrying the same read only
/// repeats the failure and adds latency.
/// A mid-stream reopen can only recover when another representation of the
/// committed object may exist. Single-disk, no-parity objects have no alternate
/// shard to discover after a bitrot/read-quorum failure, so arming the normal
/// three-attempt resume loop only repeats the same failing read and adds delay.
fn should_attach_get_object_resume(info: &ObjectInfo) -> bool {
info.parity_blocks > 0 || info.data_blocks > 1 || !info.transitioned_object.tier.is_empty()
}
@@ -3322,12 +3332,15 @@ impl DefaultObjectUsecase {
// already supplied this body, the request-level plan was built before
// the authoritative lookup. Serve it without planning a second time.
if cache_hook_served && let Some(bytes) = buffered_body.take() {
return Ok(Self::build_memory_bytes_blob(
let hook_hit_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
let body = Self::build_memory_bytes_blob(
bytes,
response_content_length,
GET_MEMORY_BODY_SOURCE_OBJECT_DATA_CACHE,
lifecycle,
));
);
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_CACHE_HOOK_HIT, hook_hit_start);
return Ok(body);
}
if !cache_fill_allowed {
@@ -3368,20 +3381,27 @@ impl DefaultObjectUsecase {
// authoritative because it ran after fresh metadata resolution, so the
// app layer skips its own lookup and only uses the plan to fill.
if !cache_hook_probed {
match lookup_get_object_body_cache_hit(cache_adapter, &cache_plan).await {
let cache_lookup_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
let cache_lookup = lookup_get_object_body_cache_hit(cache_adapter, &cache_plan).await;
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_CACHE_LOOKUP, cache_lookup_start);
match cache_lookup {
GetObjectBodyCacheLookup::Hit(bytes) => {
return Ok(Self::build_memory_bytes_blob(
let cache_hit_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
let body = Self::build_memory_bytes_blob(
bytes,
response_content_length,
GET_MEMORY_BODY_SOURCE_OBJECT_DATA_CACHE,
lifecycle,
));
);
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_CACHE_HOOK_HIT, cache_hit_start);
return Ok(body);
}
GetObjectBodyCacheLookup::Disabled | GetObjectBodyCacheLookup::Skip | GetObjectBodyCacheLookup::Miss => {}
}
}
if let Some(buffered_body) = buffered_body {
let cache_fill_handoff_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
// ODC-15: the body is already fully in hand, so keep the fill off the
// response's critical path. For a cacheable plan, run the fill in a
// detached task (Bytes is a cheap clone) and return immediately. For
@@ -3397,13 +3417,17 @@ impl DefaultObjectUsecase {
} else if cache_fill_allowed {
let _ = fill_get_object_body_cache_from_buffered_body(cache_adapter, &cache_plan, &buffered_body).await;
}
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_CACHE_FILL_HANDOFF, cache_fill_handoff_start);
return Ok(Self::build_memory_bytes_blob(
let buffered_handoff_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
let body = Self::build_memory_bytes_blob(
buffered_body,
response_content_length,
GET_MEMORY_BODY_SOURCE_BUFFERED_BODY,
lifecycle,
));
);
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_BUFFERED_HANDOFF, buffered_handoff_start);
return Ok(body);
}
let should_materialize_for_cache = cache_adapter.materialize_fill_enabled()
@@ -3457,6 +3481,7 @@ impl DefaultObjectUsecase {
{
Ok(buf) => {
let bytes = Bytes::from(buf);
let cache_fill_handoff_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
// ODC-15: fill off the response's critical path (see the
// buffered-body branch above).
let cache_adapter = cache_adapter.clone();
@@ -3465,6 +3490,10 @@ impl DefaultObjectUsecase {
tokio::spawn(async move {
let _ = fill_get_object_body_cache_from_materialized_body(&cache_adapter, &cache_plan, &fill_bytes).await;
});
record_get_object_s3_handler_stage_duration(
GET_OBJECT_STAGE_BODY_CACHE_FILL_HANDOFF,
cache_fill_handoff_start,
);
return Ok(Self::build_memory_bytes_blob(
bytes,
@@ -3603,13 +3632,19 @@ impl DefaultObjectUsecase {
None => helper,
};
let helper = helper.version_id(version_id_for_event);
let cors_wrap_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
let mut response = wrap_response_with_cors(bucket, method, headers, output).await;
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_RESPONSE_CORS_WRAP, cors_wrap_start);
let header_inject_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
inject_accept_ranges_header(&mut response.headers);
// Emit XXHash3/64/128 and SHA-512 checksums that s3s GetObjectOutput cannot
// carry (#1257). This is the download-side integrity path AWS SDKs verify.
inject_additional_checksum_headers(&mut response.headers, &extra_checksum_headers);
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_RESPONSE_HEADER_INJECT, header_inject_start);
let result = Ok(response);
let event_complete_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
let _ = helper.complete(&result);
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_RESPONSE_EVENT_COMPLETE, event_complete_start);
result
}
@@ -3720,6 +3755,7 @@ impl DefaultObjectUsecase {
let metadata = filter_object_metadata(&info.user_defined);
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_METADATA_FILTER, metadata_filter_start);
let output_struct_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
let output = GetObjectOutput {
body: Some(body),
content_length: Some(response_content_length),
@@ -3748,6 +3784,7 @@ impl DefaultObjectUsecase {
storage_class,
..Default::default()
};
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_OUTPUT_STRUCT_BUILD, output_struct_start);
Ok(GetObjectOutputContext {
output,
@@ -4048,6 +4085,7 @@ impl DefaultObjectUsecase {
let manager = get_concurrency_manager();
let prepare_read_start = stage_metrics_enabled.then(std::time::Instant::now);
let mut prepared_read = self
.prepare_get_object_read_execution(
&req,
@@ -4063,6 +4101,7 @@ impl DefaultObjectUsecase {
object_traffic_health.clone(),
)
.await;
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_PREPARE_READ_EXECUTION, prepare_read_start);
// An object missing locally (and only missing — other errors keep
// their semantics) may still be served by a remote copy.
if let Err(err) = &prepared_read
@@ -4100,6 +4139,7 @@ impl DefaultObjectUsecase {
return Self::complete_get_object_error(helper.version_id(version_id_for_event), err);
}
Some(OdmGetOutcome::RetryLocal) => {
let prepare_read_retry_start = stage_metrics_enabled.then(std::time::Instant::now);
prepared_read = self
.prepare_get_object_read_execution(
&req,
@@ -4115,6 +4155,10 @@ impl DefaultObjectUsecase {
object_traffic_health,
)
.await;
record_get_object_s3_handler_stage_duration(
GET_OBJECT_STAGE_PREPARE_READ_EXECUTION,
prepare_read_retry_start,
);
}
}
}
+86 -5
View File
@@ -1721,6 +1721,7 @@ impl DefaultObjectUsecase {
)?;
}
let checksum_input_stage_start = put_stage_metrics_enabled.then(Instant::now);
let mut md5hex = match content_md5 {
Some(PutObjectContentMd5::Base64(base64_md5)) => {
let md5 = base64_simd::STANDARD
@@ -1733,12 +1734,15 @@ impl DefaultObjectUsecase {
};
let mut sha256hex = get_content_sha256_with_query(headers, query);
rustfs_io_metrics::record_put_object_stage_duration_from("app_checksum_input_prepare", checksum_input_stage_start);
let mut write_plan = WritePlan::new();
// Additional-checksum (XXHash3/64/128, SHA-512) values to echo on the PutObject
// response (#1256); captured at want_checksum set points before opts is moved.
let mut put_extra_checksum_headers: Vec<(&'static str, String)> = Vec::new();
let reader_prepare_stage_start = put_stage_metrics_enabled.then(Instant::now);
let mut reader = if should_compress {
let compressed_reader_stage_start = put_stage_metrics_enabled.then(Instant::now);
let body = tokio::io::BufReader::with_capacity(
buffer_size,
StreamReader::new(body.map(|f| f.map_err(s3s_body_error_to_io))),
@@ -1761,51 +1765,98 @@ impl DefaultObjectUsecase {
size = HashReader::SIZE_PRESERVE_LAYER;
write_plan = write_plan.with_compression(algorithm);
rustfs_io_metrics::record_put_object_stage_duration_from(
"app_reader_prepare_compressed",
compressed_reader_stage_start,
);
hrd
} else {
if use_zero_copy_eager_put_path {
let zero_copy_start = std::time::Instant::now();
let eager_read_stage_start = put_stage_metrics_enabled.then(Instant::now);
let eager_body = read_zero_copy_put_body_exact(body, body_size as usize).await?;
rustfs_io_metrics::record_put_object_stage_duration_from("app_body_eager_read_zero_copy", eager_read_stage_start);
rustfs_io_metrics::record_zero_copy_write(body_size as usize, zero_copy_start.elapsed().as_secs_f64() * 1000.0);
HashReader::from_stream(eager_body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?
let hash_reader_stage_start = put_stage_metrics_enabled.then(Instant::now);
let reader =
HashReader::from_stream(eager_body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?;
rustfs_io_metrics::record_put_object_stage_duration_from(
"app_hash_reader_setup_zero_copy",
hash_reader_stage_start,
);
reader
} else if use_empty_or_small_eager_put_path {
if (body_size as usize) <= POOL_BYPASS_MAX_SIZE {
// Bypass BytesPool for very small objects to avoid Small-tier
// Mutex contention under high concurrency. Direct allocation
// for ≤4KiB is negligible cost.
let eager_read_stage_start = put_stage_metrics_enabled.then(Instant::now);
let eager_body = read_small_put_body_exact_direct(
StreamReader::new(body.map(|f| f.map_err(s3s_body_error_to_io))),
body_size as usize,
)
.await?;
HashReader::from_stream(eager_body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?
rustfs_io_metrics::record_put_object_stage_duration_from(
"app_body_eager_read_small_direct",
eager_read_stage_start,
);
let hash_reader_stage_start = put_stage_metrics_enabled.then(Instant::now);
let reader = HashReader::from_stream(eager_body, size, actual_size, md5hex, sha256hex, false)
.map_err(ApiError::from)?;
rustfs_io_metrics::record_put_object_stage_duration_from(
"app_hash_reader_setup_small_direct",
hash_reader_stage_start,
);
reader
} else {
let pool = get_concurrency_manager().bytes_pool();
let eager_read_stage_start = put_stage_metrics_enabled.then(Instant::now);
let eager_body = read_small_put_body_exact_pooled(
StreamReader::new(body.map(|f| f.map_err(s3s_body_error_to_io))),
body_size as usize,
pool.as_ref(),
)
.await?;
rustfs_io_metrics::record_put_object_stage_duration_from(
"app_body_eager_read_small_pooled",
eager_read_stage_start,
);
let hash_reader_stage_start = put_stage_metrics_enabled.then(Instant::now);
let eager_reader = PooledBufferReader::new(eager_body, body_size as usize);
HashReader::from_stream(eager_reader, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?
let reader = HashReader::from_stream(eager_reader, size, actual_size, md5hex, sha256hex, false)
.map_err(ApiError::from)?;
rustfs_io_metrics::record_put_object_stage_duration_from(
"app_hash_reader_setup_small_pooled",
hash_reader_stage_start,
);
reader
}
} else {
let streaming_reader_stage_start = put_stage_metrics_enabled.then(Instant::now);
let body = tokio::io::BufReader::with_capacity(
buffer_size,
StreamReader::new(body.map(|f| f.map_err(s3s_body_error_to_io))),
);
HashReader::from_stream(body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?
let reader =
HashReader::from_stream(body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?;
rustfs_io_metrics::record_put_object_stage_duration_from(
"app_reader_prepare_streaming",
streaming_reader_stage_start,
);
reader
}
};
rustfs_io_metrics::record_put_object_stage_duration_from("app_reader_prepare", reader_prepare_stage_start);
if size >= 0 {
let checksum_setup_stage_start = put_stage_metrics_enabled.then(Instant::now);
if let Err(err) = reader.add_checksum(headers, trailer_source(trailing_headers.clone()), false) {
return Err(ApiError::from(err).into());
}
opts.want_checksum = reader.checksum();
put_extra_checksum_headers = additional_checksum_echo_pairs(&opts.want_checksum);
rustfs_io_metrics::record_put_object_stage_duration_from("app_reader_checksum_setup", checksum_setup_stage_start);
}
rustfs_io_metrics::record_put_object_path(put_path);
rustfs_io_metrics::record_put_object_stage_duration_from("ingress_prepare", ingress_stage_start);
@@ -2035,17 +2086,36 @@ impl DefaultObjectUsecase {
wait_for_put_post_store_test_hook(&bucket).await;
let post_store_stage_start = put_stage_metrics_enabled.then(Instant::now);
let transition_enqueue_stage_start = put_stage_metrics_enabled.then(Instant::now);
maybe_enqueue_transition_immediate(&obj_info, LcEventSrc::S3PutObject).await;
let _ = invalidate_object_data_cache_after_put_success(&cache_adapter, &bucket, &key).await;
rustfs_io_metrics::record_put_object_stage_duration_from(
"app_post_store_transition_enqueue",
transition_enqueue_stage_start,
);
let cache_invalidate_stage_start = put_stage_metrics_enabled.then(Instant::now);
let _ = invalidate_object_data_cache_after_put_success(&cache_adapter, &bucket, &key).await;
rustfs_io_metrics::record_put_object_stage_duration_from(
"app_post_store_data_cache_invalidate",
cache_invalidate_stage_start,
);
let versioning_lookup_stage_start = put_stage_metrics_enabled.then(Instant::now);
let put_versioned = BucketVersioningSys::prefix_enabled(&bucket, &key).await;
rustfs_io_metrics::record_put_object_stage_duration_from(
"app_post_store_versioning_lookup",
versioning_lookup_stage_start,
);
// Fast in-memory update for immediate quota and admin usage consistency.
// The previous current size comes from the prelookup when it ran,
// otherwise from the rename_data backfill (rustfs/backlog#1009); the
// backfill reproduces the lookup's observation bit for bit (latest
// version's ObjectInfo.size — 0 for a delete-marker latest — or
// not-found → None).
let quota_size_stage_start = put_stage_metrics_enabled.then(Instant::now);
let committed_size = quota_accounting_object_size(&obj_info, quota_enabled)?;
rustfs_io_metrics::record_put_object_stage_duration_from("app_post_store_quota_size", quota_size_stage_start);
let usage_memory_stage_start = put_stage_metrics_enabled.then(Instant::now);
match prelookup_previous_current_size.or_else(|| previous_current_size_from_backfill(backfilled_old_current_size))
{
Some(previous_current_size) => {
@@ -2071,16 +2141,27 @@ impl DefaultObjectUsecase {
record_bucket_object_write_unknown_previous_memory(&bucket, committed_size, put_versioned).await;
}
}
rustfs_io_metrics::record_put_object_stage_duration_from("app_post_store_usage_memory", usage_memory_stage_start);
if dsc.replicate_any() {
let replication_schedule_stage_start = put_stage_metrics_enabled.then(Instant::now);
schedule_object_replication(obj_info.clone(), store, dsc).await;
rustfs_io_metrics::record_put_object_stage_duration_from(
"app_post_store_replication_schedule",
replication_schedule_stage_start,
);
}
let dirty_usage_stage_start = put_stage_metrics_enabled.then(Instant::now);
rustfs_scanner::record_dirty_usage_object_from_producer(
&bucket,
&key,
rustfs_scanner::SegmentInvalidationProducerIdentity::PutObject,
);
rustfs_io_metrics::record_put_object_stage_duration_from(
"app_post_store_dirty_usage_record",
dirty_usage_stage_start,
);
rustfs_io_metrics::record_put_object_stage_duration_from("app_post_store_bookkeeping", post_store_stage_start);
let capacity_update_stage_start = put_stage_metrics_enabled.then(Instant::now);
+64 -6
View File
@@ -19,6 +19,7 @@ use hyper::{Request, Response};
use pin_project_lite::pin_project;
use std::pin::Pin;
use std::task::{Context, Poll};
use std::time::Instant;
use tower::Service;
type BoxError = Box<dyn std::error::Error + Send + Sync + 'static>;
@@ -35,6 +36,13 @@ pub(crate) fn is_grpc_request<B>(req: &Request<B>) -> bool {
)
}
fn rename_data_grpc_stage(path: &str) -> bool {
matches!(
path,
"/node_service.NodeService/RenameData" | "/node_service.NodeService/RenameDataAtIncarnation"
)
}
/// The service that can serve both gRPC and REST HTTP Requests
#[derive(Clone)]
pub struct HybridService<Rest, Grpc> {
@@ -71,8 +79,11 @@ where
/// as a REST request
fn call(&mut self, req: Request<Incoming>) -> Self::Future {
if is_grpc_request(&req) {
let observe_rename_data = rename_data_grpc_stage(req.uri().path());
HybridFuture::Grpc {
grpc_future: self.grpc.call(req),
observe_rename_data,
service_future_started: rustfs_io_metrics::put_stage_timer(),
}
} else {
HybridFuture::Rest {
@@ -93,7 +104,10 @@ pin_project! {
},
Grpc {
#[pin]
grpc_body: GrpcBody
grpc_body: GrpcBody,
first_frame_started: Option<Instant>,
body_started: Option<Instant>,
first_frame_recorded: bool,
},
}
}
@@ -123,21 +137,44 @@ where
fn is_end_stream(&self) -> bool {
match self {
Self::Rest { rest_body } => rest_body.is_end_stream(),
Self::Grpc { grpc_body } => grpc_body.is_end_stream(),
Self::Grpc { grpc_body, .. } => grpc_body.is_end_stream(),
}
}
fn poll_frame(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Result<Frame<Self::Data>, Self::Error>>> {
match self.project() {
HybridBodyProj::Rest { rest_body } => rest_body.poll_frame(cx).map_err(Into::into),
HybridBodyProj::Grpc { grpc_body } => grpc_body.poll_frame(cx).map_err(Into::into),
HybridBodyProj::Grpc {
mut grpc_body,
first_frame_started,
body_started,
first_frame_recorded,
} => {
let poll = grpc_body.as_mut().poll_frame(cx).map_err(Into::into);
if !*first_frame_recorded && let Poll::Ready(_) = &poll {
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_FIRST_FRAME,
first_frame_started.take(),
);
*first_frame_recorded = true;
}
let body_complete = matches!(&poll, Poll::Ready(None))
|| (matches!(&poll, Poll::Ready(Some(Ok(_)))) && grpc_body.as_ref().get_ref().is_end_stream());
if body_complete {
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_BODY_COMPLETE,
body_started.take(),
);
}
poll
}
}
}
fn size_hint(&self) -> http_body::SizeHint {
match self {
Self::Rest { rest_body } => rest_body.size_hint(),
Self::Grpc { grpc_body } => grpc_body.size_hint(),
Self::Grpc { grpc_body, .. } => grpc_body.size_hint(),
}
}
}
@@ -154,6 +191,8 @@ pin_project! {
Grpc {
#[pin]
grpc_future: GrpcFuture,
observe_rename_data: bool,
service_future_started: Option<Instant>,
},
}
}
@@ -174,8 +213,27 @@ where
Poll::Ready(Err(err)) => Poll::Ready(Err(err.into())),
Poll::Pending => Poll::Pending,
},
HybridFutureProj::Grpc { grpc_future } => match grpc_future.poll(cx) {
Poll::Ready(Ok(res)) => Poll::Ready(Ok(res.map(|grpc_body| HybridBody::Grpc { grpc_body }))),
HybridFutureProj::Grpc {
grpc_future,
observe_rename_data,
service_future_started,
} => match grpc_future.poll(cx) {
Poll::Ready(Ok(res)) => {
if *observe_rename_data {
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_SERVICE_FUTURE,
service_future_started.take(),
);
}
let first_frame_started = (*observe_rename_data).then(Instant::now);
let body_started = (*observe_rename_data).then(Instant::now);
Poll::Ready(Ok(res.map(|grpc_body| HybridBody::Grpc {
grpc_body,
first_frame_started,
body_started,
first_frame_recorded: !*observe_rename_data,
})))
}
Poll::Ready(Err(err)) => Poll::Ready(Err(err.into())),
Poll::Pending => Poll::Pending,
},
+70 -5
View File
@@ -861,7 +861,20 @@ where
}
Response::from_parts(parts, HybridBody::Rest { rest_body })
}
HybridBody::Grpc { grpc_body } => Response::from_parts(parts, HybridBody::Grpc { grpc_body }),
HybridBody::Grpc {
grpc_body,
first_frame_started,
body_started,
first_frame_recorded,
} => Response::from_parts(
parts,
HybridBody::Grpc {
grpc_body,
first_frame_started,
body_started,
first_frame_recorded,
},
),
};
Ok(response)
@@ -1010,7 +1023,20 @@ where
}
Response::from_parts(parts, HybridBody::Rest { rest_body })
}
HybridBody::Grpc { grpc_body } => Response::from_parts(parts, HybridBody::Grpc { grpc_body }),
HybridBody::Grpc {
grpc_body,
first_frame_started,
body_started,
first_frame_recorded,
} => Response::from_parts(
parts,
HybridBody::Grpc {
grpc_body,
first_frame_started,
body_started,
first_frame_recorded,
},
),
};
Ok(response)
@@ -1161,7 +1187,20 @@ where
Response::from_parts(parts, HybridBody::Rest { rest_body })
}
HybridBody::Grpc { grpc_body } => Response::from_parts(parts, HybridBody::Grpc { grpc_body }),
HybridBody::Grpc {
grpc_body,
first_frame_started,
body_started,
first_frame_recorded,
} => Response::from_parts(
parts,
HybridBody::Grpc {
grpc_body,
first_frame_started,
body_started,
first_frame_recorded,
},
),
};
Ok(response)
@@ -1238,7 +1277,20 @@ where
},
)
}
HybridBody::Grpc { grpc_body } => Response::from_parts(parts, HybridBody::Grpc { grpc_body }),
HybridBody::Grpc {
grpc_body,
first_frame_started,
body_started,
first_frame_recorded,
} => Response::from_parts(
parts,
HybridBody::Grpc {
grpc_body,
first_frame_started,
body_started,
first_frame_recorded,
},
),
};
Ok(response)
@@ -1306,7 +1358,20 @@ where
rest_body: RestBody::from(Bytes::new()),
},
),
HybridBody::Grpc { grpc_body } => Response::from_parts(parts, HybridBody::Grpc { grpc_body }),
HybridBody::Grpc {
grpc_body,
first_frame_started,
body_started,
first_frame_recorded,
} => Response::from_parts(
parts,
HybridBody::Grpc {
grpc_body,
first_frame_started,
body_started,
first_frame_recorded,
},
),
};
Ok(response)
+96 -12
View File
@@ -1337,6 +1337,8 @@ impl NodeService {
&self,
request: Request<RenameDataRequest>,
) -> Result<Response<RenameDataResponse>, Status> {
let service_return_started = rustfs_io_metrics::put_stage_timer();
let service_entry_started = rustfs_io_metrics::put_stage_timer();
if !request.get_ref().scanner_publication_lease_token.is_empty() || !request.get_ref().bucket_incarnation_id.is_empty() {
let has_body_digest = request
.metadata()
@@ -1344,14 +1346,37 @@ impl NodeService {
.and_then(|value| value.to_str().ok())
.is_some_and(|value| value != "UNSIGNED-PAYLOAD");
if !has_body_digest {
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_ENTRY,
service_entry_started,
);
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN,
service_return_started,
);
return Err(Status::permission_denied("scanner publication lease rename requires a body-bound digest"));
}
}
verify_disk_mutation_digest(
if let Err(err) = verify_disk_mutation_digest(
&request,
rustfs_protos::canonical_rename_data_request_body(request.get_ref()),
"rename_data",
)?;
) {
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_ENTRY,
service_entry_started,
);
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN,
service_return_started,
);
return Err(err);
}
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_ENTRY,
service_entry_started,
);
let before_handler_started = rustfs_io_metrics::put_stage_timer();
let request = request.into_inner();
let target = self.local_mutation_target();
let bucket_incarnation = if request.bucket_incarnation_id.is_empty() {
@@ -1362,6 +1387,14 @@ impl NodeService {
.filter(|id| !id.is_nil())
.ok_or_else(|| Status::invalid_argument("bucket incarnation must be a non-nil UUID"))?;
if !request.scanner_publication_lease_token.is_empty() {
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_BEFORE_HANDLER,
before_handler_started,
);
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN,
service_return_started,
);
return Err(Status::invalid_argument(
"heal incarnation and scanner publication lease cannot be combined",
));
@@ -1373,12 +1406,26 @@ impl NodeService {
let decoded_file_info = match decode_rename_data_request_file_info(&request.file_info_bin, &request.file_info) {
Ok(file_info) => file_info,
Err(err) => {
return Ok(Response::new(RenameDataResponse {
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_BEFORE_HANDLER,
before_handler_started,
);
let response_build_started = rustfs_io_metrics::put_stage_timer();
let response = Response::new(RenameDataResponse {
success: false,
rename_data_resp: String::new(),
rename_data_resp_bin: Vec::new().into(),
error: Some(DiskError::other(format!("decode FileInfo failed: {err}")).into()),
}));
});
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_BUILD,
response_build_started,
);
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN,
service_return_started,
);
return Ok(response);
}
};
let scanner_publication_lease_token = if request.scanner_publication_lease_token.is_empty() {
@@ -1387,6 +1434,14 @@ impl NodeService {
let token = Uuid::from_slice(&request.scanner_publication_lease_token)
.map_err(|_| Status::invalid_argument("scanner publication lease token must be a UUID"))?;
if token.is_nil() {
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_BEFORE_HANDLER,
before_handler_started,
);
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN,
service_return_started,
);
return Err(Status::invalid_argument("scanner publication lease token must not be nil"));
}
Some(token)
@@ -1394,6 +1449,11 @@ impl NodeService {
let request_decoded_from_msgpack = decoded_file_info.from_msgpack;
#[cfg(feature = "e2e-test-hooks")]
let observation = startup_cas_rename_observation(&target, &request, &decoded_file_info.value);
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_BEFORE_HANDLER,
before_handler_started,
);
let handler_await_started = rustfs_io_metrics::put_stage_timer();
let result = target
.rename_local_data(
&request.disk,
@@ -1404,6 +1464,11 @@ impl NodeService {
bucket_incarnation,
)
.await;
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_HANDLER_AWAIT,
handler_await_started,
);
let after_handler_started = rustfs_io_metrics::put_stage_timer();
#[cfg(feature = "e2e-test-hooks")]
if let Some(mut observation) = observation {
observation["ok"] = serde_json::json!(result.is_ok());
@@ -1411,28 +1476,47 @@ impl NodeService {
let line = format!("RUSTFS_E2E_STARTUP_CAS {observation}\n");
let _ = std::io::Write::write_all(&mut std::io::stderr().lock(), line.as_bytes());
}
match result {
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_AFTER_HANDLER,
after_handler_started,
);
let response_build_started = rustfs_io_metrics::put_stage_timer();
let response_encode_started = rustfs_io_metrics::put_stage_timer();
let response = match result {
Ok(rename_data_resp) => match encode_rename_data_response_payloads(&rename_data_resp, request_decoded_from_msgpack) {
Ok((rename_data_resp, rename_data_resp_bin)) => Ok(Response::new(RenameDataResponse {
Ok((rename_data_resp, rename_data_resp_bin)) => Response::new(RenameDataResponse {
success: true,
rename_data_resp,
rename_data_resp_bin: rename_data_resp_bin.into(),
error: None,
})),
Err(err) => Ok(Response::new(RenameDataResponse {
}),
Err(err) => Response::new(RenameDataResponse {
success: false,
rename_data_resp: String::new(),
rename_data_resp_bin: Vec::new().into(),
error: Some(err.into()),
})),
}),
},
Err(err) => Ok(Response::new(RenameDataResponse {
Err(err) => Response::new(RenameDataResponse {
success: false,
rename_data_resp: String::new(),
rename_data_resp_bin: Vec::new().into(),
error: Some(err.into()),
})),
}
}),
};
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_ENCODE,
response_encode_started,
);
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_BUILD,
response_build_started,
);
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN,
service_return_started,
);
Ok(response)
}
pub(super) async fn handle_list_dir(&self, request: Request<ListDirRequest>) -> Result<Response<ListDirResponse>, Status> {