mirror of
https://github.com/rustfs/rustfs.git
synced 2026-10-27 14:27:10 +00:00
Compare commits
241 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 1f8d013b8d | |||
| ceddcdd878 | |||
| b8858d0c70 | |||
| 7728eeab1a | |||
| 4fd444852c | |||
| 13dc635d51 | |||
| 05f84793a3 | |||
| 2fa9a433ad | |||
| 08811ec243 | |||
| 0f89e9b8b8 | |||
| de8b465c41 | |||
| ead77d799f | |||
| 67db0f241f | |||
| a94f625bc8 | |||
| b7835f0f9a | |||
| 299609bb83 | |||
| 1800b90d06 | |||
| 2ccc8d8647 | |||
| 124cbfc00a | |||
| 3aa7964c58 | |||
| 42d376c6b7 | |||
| 017b85a5bf | |||
| 55023ecd86 | |||
| 03f77844b7 | |||
| 1cd62c4249 | |||
| b44c76db52 | |||
| d0096af742 | |||
| 35e07b44f4 | |||
| 7d91176cea | |||
| 001d935569 | |||
| cc157adca8 | |||
| e0917fb617 | |||
| 6a30694765 | |||
| 991d647c78 | |||
| aafa85c6e5 | |||
| 01f5aa9c3a | |||
| e27bfb2006 | |||
| f095c6a476 | |||
| a223e74c83 | |||
| 1d1964215d | |||
| 5b23462652 | |||
| 02f6d87201 | |||
| 37a54d3ff2 | |||
| d1aaaee9bf | |||
| 0f24511535 | |||
| 9517db6cf7 | |||
| 3bcba78ccb | |||
| bebd5d0631 | |||
| 0de18a3577 | |||
| af4ce32528 | |||
| 04756b0a93 | |||
| 6fc62edcf7 | |||
| 32c4593587 | |||
| 2f90fb19f5 | |||
| 68d1790b74 | |||
| 6fde372f5b | |||
| 503a852214 | |||
| ab7a8c400f | |||
| 85663e9647 | |||
| 17aa75a175 | |||
| 61e4cfbc2a | |||
| 675722e5f8 | |||
| f6dfe86c58 | |||
| 3c9b9589f2 | |||
| a5ad283c8c | |||
| dcdd53ccf3 | |||
| bdd7efc96d | |||
| 9fbf5d0553 | |||
| e7bebb0fc5 | |||
| cacdaa65a0 | |||
| efe2255ccf | |||
| f4b694ed19 | |||
| aa95a32ea2 | |||
| 3f3e892d86 | |||
| 874d2babcc | |||
| 7c85783a08 | |||
| 541fbbe74a | |||
| 08ba7a7976 | |||
| 81f214c99a | |||
| dff05785de | |||
| cf2e3af4d9 | |||
| e0f0628bc1 | |||
| 6de3034e36 | |||
| ae259e127d | |||
| 8233e3fe4c | |||
| f88b7b6ba5 | |||
| 0307c9a7e2 | |||
| 6a8d0aba58 | |||
| ee6c2afe43 | |||
| 9ca8bc3ca6 | |||
| 3678703a98 | |||
| fe798c13e7 | |||
| 24b77349aa | |||
| 8c55396bfe | |||
| 5be8badb20 | |||
| 46daaadbb3 | |||
| f3b6c4df6b | |||
| 7183849227 | |||
| 942ce79c4e | |||
| fec1262f1a | |||
| 5eb46cb29d | |||
| 2cd904d0e7 | |||
| 61eec6da18 | |||
| 5b8cd2a5dc | |||
| d7824049b2 | |||
| d46e4c54a6 | |||
| 3ca0b5f35e | |||
| 00b5ddc0ad | |||
| ad4c2bc9b1 | |||
| ef5fe69d52 | |||
| ba6f37e58d | |||
| 31eb2e5a4a | |||
| 2f3654793d | |||
| ce09e7817c | |||
| 31dc8a0fea | |||
| d243a4a29e | |||
| 1048374cbd | |||
| de32dece41 | |||
| f79f318ec4 | |||
| 60fe68a04a | |||
| 498be1fa53 | |||
| b798d291d7 | |||
| 57c4d4c719 | |||
| 560b083278 | |||
| 36afe8bb57 | |||
| 57963574b7 | |||
| c56a597374 | |||
| 5354e6dccf | |||
| 7608c5af11 | |||
| c926e10de0 | |||
| 972dafccde | |||
| 5116bb2bbb | |||
| 08d6e4343d | |||
| 7d3a6acc57 | |||
| aabb26b519 | |||
| 163e6064da | |||
| c7a9b76543 | |||
| 7cc4ce5447 | |||
| 44fe49e5fa | |||
| dde2c50309 | |||
| d5a8f022bd | |||
| 150d349ade | |||
| 38056f3783 | |||
| 0b40e35e76 | |||
| 458b144ac7 | |||
| d58b80d54e | |||
| f8d0174fa7 | |||
| a57cc0d44c | |||
| 8006cb7eae | |||
| 8d40a995a3 | |||
| a451b53bcc | |||
| a6127c8fde | |||
| f346c13e5b | |||
| 2634e829ca | |||
| dc15444256 | |||
| ebb19907a9 | |||
| d29babf251 | |||
| d8a9522919 | |||
| 60dc84bf0e | |||
| 061e89ed2b | |||
| 3065142735 | |||
| c6a8e30f5a | |||
| f51b52d7e9 | |||
| c54160c22c | |||
| 1f67b65ab8 | |||
| a7806e65f0 | |||
| 6a3678997a | |||
| 64ea8b8e09 | |||
| 7773aac0fd | |||
| 8763f87ca7 | |||
| 9bbac59977 | |||
| 3bd57517ff | |||
| 8ee78c6891 | |||
| 21c594e625 | |||
| 748095280a | |||
| 960195458f | |||
| de58fdfc7d | |||
| 869d5ec24b | |||
| 19902438a6 | |||
| bd244d13f5 | |||
| 38395d35ec | |||
| 68064b5ad9 | |||
| c1da0c361b | |||
| cb7626c1f1 | |||
| 8186a7175c | |||
| c9e1446126 | |||
| 11d21ec6f4 | |||
| 974ab6f785 | |||
| c3f4cdf27c | |||
| e82d7c107a | |||
| 9d2d6a8e46 | |||
| ff2cbc3c47 | |||
| d65ef3e29a | |||
| 0a2f58c7a9 | |||
| 4d97501327 | |||
| 3b2af12c64 | |||
| c73ddde848 | |||
| ef72a37fd7 | |||
| e46cf750dd | |||
| fe955fa3ee | |||
| 64578c067e | |||
| 78715f0921 | |||
| 3284aafd73 | |||
| 9286442f87 | |||
| b8aad1c168 | |||
| 5fcb229760 | |||
| 94baef7462 | |||
| 60d90f085a | |||
| dfa5d50565 | |||
| a902886737 | |||
| 8e53b87421 | |||
| f7631e91b1 | |||
| f9bc4e921f | |||
| 781584307a | |||
| 59d641a5fe | |||
| c618b57870 | |||
| bbe1d54098 | |||
| 9e8ea4dcdb | |||
| 63cf8e2f2f | |||
| b7fe40f4b3 | |||
| 5b872b8e56 | |||
| ca5be1ef95 | |||
| 9373e8fa3e | |||
| cb32965522 | |||
| fb6cf49a67 | |||
| 8c97049e68 | |||
| 17d4fe27cc | |||
| fb0c7972b2 | |||
| eba7967755 | |||
| 451c1c31ad | |||
| 8780dd670e | |||
| f29bf2c902 | |||
| fd03436090 | |||
| 1f1e7b1c97 | |||
| 45a952609e | |||
| 323190a95a | |||
| 38cdda9292 | |||
| 85c84fa73d | |||
| 51402087c5 | |||
| a994984fe2 | |||
| 48eb0e402c |
@@ -1,2 +1,2 @@
|
||||
sha256-linux=cf31ed27e4e6bf42da5bf81f248d9d2619ac8e8615186cdacb30bd63996d7607
|
||||
sha256-darwin=cf31ed27e4e6bf42da5bf81f248d9d2619ac8e8615186cdacb30bd63996d7607
|
||||
sha256-linux=74a1d6975e2d6dfe01363585cb9bfa3232d79556a154d4f599653fe2ed1ce696
|
||||
sha256-darwin=74a1d6975e2d6dfe01363585cb9bfa3232d79556a154d4f599653fe2ed1ce696
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=f21d495ca0c780f1f6e6bc626065599e3f16ea152bdc3ec08403af2fdf8b1203
|
||||
sha256-linux=609fca1f70ab7c58e2a6c3839a6dcb5abe8177d478cd243ed679a12d20eaedfa
|
||||
sha256-darwin=b53fe6d46cdffedfad6a83da0dfffa9aec10cd7c8ace53469f129587bc9f4d50
|
||||
sha256-linux=6be90f5c3886e9852ac27d51f2d47e6d2fa8bb424e0d03ee931268dcd0020db8
|
||||
|
||||
@@ -25,6 +25,8 @@ on:
|
||||
- "crates/ecstore/src/object_api/write_commit_context.rs"
|
||||
- "crates/ecstore/src/bucket/utils.rs"
|
||||
- "crates/ecstore/Cargo.toml"
|
||||
- "crates/storage-api/src/bucket.rs"
|
||||
- "crates/storage-api/Cargo.toml"
|
||||
- "crates/utils/src/dunce.rs"
|
||||
- "crates/utils/src/os/**"
|
||||
- "crates/utils/Cargo.toml"
|
||||
@@ -48,6 +50,8 @@ on:
|
||||
- "crates/ecstore/src/object_api/write_commit_context.rs"
|
||||
- "crates/ecstore/src/bucket/utils.rs"
|
||||
- "crates/ecstore/Cargo.toml"
|
||||
- "crates/storage-api/src/bucket.rs"
|
||||
- "crates/storage-api/Cargo.toml"
|
||||
- "crates/utils/src/dunce.rs"
|
||||
- "crates/utils/src/os/**"
|
||||
- "crates/utils/Cargo.toml"
|
||||
@@ -140,6 +144,17 @@ jobs:
|
||||
shell: pwsh
|
||||
run: "cargo test -p rustfs-ecstore --lib bucket::utils:: -- --nocapture"
|
||||
|
||||
- name: Test artifact prefix Windows file-name rules
|
||||
shell: pwsh
|
||||
run: |
|
||||
$testName = 'bucket::tests::artifact_prefix_preserves_host_file_name_rules'
|
||||
cargo test --locked -p rustfs-storage-api --lib $testName -- --exact --nocapture --color never 2>&1 |
|
||||
Tee-Object -Variable testOutput
|
||||
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
|
||||
if (($testOutput -join "`n") -notmatch 'test result: ok\. 1 passed; 0 failed; 0 ignored;') {
|
||||
throw 'Expected exactly one passing artifact prefix test on Windows'
|
||||
}
|
||||
|
||||
- name: Test Windows file-name rules
|
||||
shell: pwsh
|
||||
run: "cargo test -p rustfs-utils --no-default-features --features path --lib dunce:: -- --nocapture"
|
||||
|
||||
Generated
+55
-58
@@ -255,7 +255,7 @@ version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
|
||||
dependencies = [
|
||||
"windows-sys 0.60.2",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -266,7 +266,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"once_cell_polyfill",
|
||||
"windows-sys 0.60.2",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1461,12 +1461,6 @@ version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fd307490d624467aa6f74b0eabb77633d1f758a7b25f12bceb0b22e08d9726f6"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.21.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.22.1"
|
||||
@@ -1618,7 +1612,7 @@ version = "0.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
||||
dependencies = [
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1637,7 +1631,7 @@ version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
|
||||
dependencies = [
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1869,7 +1863,7 @@ dependencies = [
|
||||
"maybe-owned",
|
||||
"rustix",
|
||||
"rustix-linux-procfs",
|
||||
"windows-sys 0.60.2",
|
||||
"windows-sys 0.61.2",
|
||||
"winx",
|
||||
]
|
||||
|
||||
@@ -2073,7 +2067,7 @@ version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
|
||||
dependencies = [
|
||||
"crypto-common 0.1.6",
|
||||
"crypto-common 0.1.7",
|
||||
"inout 0.1.4",
|
||||
]
|
||||
|
||||
@@ -2296,9 +2290,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "const_format"
|
||||
version = "0.2.36"
|
||||
version = "0.2.37"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4481a617ad9a412be3b97c5d403fef8ed023103368908b9c50af598ff467cc1e"
|
||||
checksum = "a3decb4d6965448659923544a634a0571ba44cede4e11ef15a5a317126840454"
|
||||
dependencies = [
|
||||
"const_format_proc_macros",
|
||||
"konst",
|
||||
@@ -2306,9 +2300,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "const_format_proc_macros"
|
||||
version = "0.2.34"
|
||||
version = "0.2.37"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d57c2eccfb16dbac1f4e61e206105db5820c9d26c3c472bc17c774259ef7744"
|
||||
checksum = "941b570269a642506f08cb5b688c04bfa6cfd41a26f8bcdc49db8fd00c0ab659"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -2548,9 +2542,9 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
|
||||
|
||||
[[package]]
|
||||
name = "crypto-bigint"
|
||||
version = "0.7.6"
|
||||
version = "0.7.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6bb08299a6046833a879ace115697f12fd2bd6c8d3e93c909ffc9e0738e7cae0"
|
||||
checksum = "cc907c45bb7e9ef8fc3778defd0f3cd8c81288f1b4e748eba064948f20c47122"
|
||||
dependencies = [
|
||||
"cpubits",
|
||||
"ctutils",
|
||||
@@ -2565,11 +2559,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.6"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
|
||||
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
||||
dependencies = [
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
"typenum",
|
||||
]
|
||||
|
||||
@@ -3910,7 +3904,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||
dependencies = [
|
||||
"block-buffer 0.10.4",
|
||||
"crypto-common 0.1.6",
|
||||
"crypto-common 0.1.7",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
@@ -3945,7 +3939,7 @@ dependencies = [
|
||||
"libc",
|
||||
"option-ext",
|
||||
"redox_users",
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4137,9 +4131,9 @@ checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be"
|
||||
|
||||
[[package]]
|
||||
name = "elliptic-curve"
|
||||
version = "0.14.1"
|
||||
version = "0.14.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9d65aa39b3a5c1c9c1b745c9a019234bb7a21b77abcb4f4d266d706e2d577d65"
|
||||
checksum = "0ad4496221fcfc21015fd8c0e26fcfab00504832dc0604f52acdd7bad2a8dc81"
|
||||
dependencies = [
|
||||
"base16ct",
|
||||
"crypto-bigint",
|
||||
@@ -4258,7 +4252,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4575,9 +4569,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.9"
|
||||
version = "0.14.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2"
|
||||
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
"version_check",
|
||||
@@ -4589,7 +4583,7 @@ version = "1.4.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "337d46834ee672ab3e48caca2cb0c78cc174fb12b3a68d0d88f99a0519a5e36e"
|
||||
dependencies = [
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
"rustversion",
|
||||
"typenum",
|
||||
]
|
||||
@@ -5241,9 +5235,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hotpath"
|
||||
version = "0.29.0"
|
||||
version = "0.29.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "73f137bde4c97275aef220f3a1566f39721fbd53086a444c8d841b5f345719dc"
|
||||
checksum = "62f2351902fa259b758c7303505273393f6b08172ab5bdd5739a2d2bc6d554e8"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-channel",
|
||||
@@ -5276,15 +5270,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hotpath-drain"
|
||||
version = "0.29.0"
|
||||
version = "0.29.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "87df2604170a3a0b512a837eba4d5d18671e54df8d450cc894af36d41291dba5"
|
||||
checksum = "83dab2340134d9976d62f9f5172f9c6584c7f521b092f258e0e58e412e160b53"
|
||||
|
||||
[[package]]
|
||||
name = "hotpath-macros"
|
||||
version = "0.29.0"
|
||||
version = "0.29.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "25dcd4a80153dd66b34b8ae5d02572d71cc3ff55aff397578cd471d645a8a740"
|
||||
checksum = "a04bbe5701b097f224a08187a78b75bb9d9b4bbaff6a279c2a50ba73edbb7aac"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -5613,7 +5607,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
|
||||
dependencies = [
|
||||
"block-padding 0.3.3",
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -6264,9 +6258,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lzma-rust2"
|
||||
version = "0.22.0"
|
||||
version = "0.23.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7277c9742229dc25243236d578b6471a798c554512c6f6a04365ec2e87c2a3bb"
|
||||
checksum = "f1c46f3fcd2e888afdb01a07efc071cb21300e07c68d32c3e8b01b5c8cb1fcaa"
|
||||
dependencies = [
|
||||
"sha2 0.11.1",
|
||||
]
|
||||
@@ -6871,7 +6865,7 @@ version = "0.50.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
|
||||
dependencies = [
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -7004,7 +6998,7 @@ name = "oauth2"
|
||||
version = "5.1.0"
|
||||
source = "git+https://github.com/ramosbugs/oauth2-rs?rev=89585f4a280afd7b285eaf1c9be0068874e03d22#89585f4a280afd7b285eaf1c9be0068874e03d22"
|
||||
dependencies = [
|
||||
"base64 0.21.7",
|
||||
"base64 0.22.1",
|
||||
"chrono",
|
||||
"getrandom 0.2.17",
|
||||
"http 1.5.0",
|
||||
@@ -8580,7 +8574,7 @@ dependencies = [
|
||||
"once_cell",
|
||||
"socket2",
|
||||
"tracing",
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10125,18 +10119,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-mimalloc"
|
||||
version = "0.7.1"
|
||||
version = "0.7.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "91c0ac4b4c6f8246bb843c985b47f7f8ea3e654e02d965a5d6c19fc006813584"
|
||||
checksum = "8301a578772fc9823d68e083e42c3b414613d588a0eac594c1ca6bc0bbe0653f"
|
||||
dependencies = [
|
||||
"rustfs-mimalloc-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-mimalloc-sys"
|
||||
version = "0.7.1"
|
||||
version = "0.7.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "265332babe807f64b800b1a47d6d39e6897f249c153b962a6b1050f073816342"
|
||||
checksum = "6eb4082853043cb29556b4b776b05f28c3146706f43c53869dbf1ef262e00e91"
|
||||
dependencies = [
|
||||
"cc",
|
||||
]
|
||||
@@ -10304,6 +10298,7 @@ dependencies = [
|
||||
"tokio",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10693,6 +10688,7 @@ dependencies = [
|
||||
"hotpath",
|
||||
"insta",
|
||||
"rustfs-filemeta",
|
||||
"rustfs-utils",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"time",
|
||||
@@ -10960,7 +10956,7 @@ dependencies = [
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11043,7 +11039,7 @@ dependencies = [
|
||||
"security-framework",
|
||||
"security-framework-sys",
|
||||
"webpki-root-certs",
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11804,9 +11800,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.16.3"
|
||||
version = "1.16.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b"
|
||||
checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
@@ -11845,7 +11841,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.60.2",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11996,7 +11992,7 @@ dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"psm",
|
||||
"windows-sys 0.60.2",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -12302,10 +12298,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.3.4",
|
||||
"getrandom 0.4.3",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -12978,9 +12974,9 @@ checksum = "86a801b3cea342a06d468c8710662aa29e5e05e4f5c0d62f00bbb7f2ad7941c2"
|
||||
|
||||
[[package]]
|
||||
name = "typed-path"
|
||||
version = "0.12.4"
|
||||
version = "0.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "54ec2755506471f54baf5e4bb5499d8e510cc1caf271270e778a736bd2d7336c"
|
||||
checksum = "9ce898823d8cd0f028662d2f349694b1af0a5cc9be83f1d213a00d3147445e1e"
|
||||
|
||||
[[package]]
|
||||
name = "typeid"
|
||||
@@ -13399,7 +13395,7 @@ version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||
dependencies = [
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -13960,9 +13956,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zip"
|
||||
version = "9.0.5"
|
||||
version = "9.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e04976da629d3b04204ba8105b396a193af8ad2583b2c79bfb3ab8af580a11e"
|
||||
checksum = "2886b00003a72c4ccee3f42cfc34908d33b2508a74d827dc2a4244706475c704"
|
||||
dependencies = [
|
||||
"aes 0.9.3",
|
||||
"bzip2",
|
||||
@@ -13981,6 +13977,7 @@ dependencies = [
|
||||
"sha1 0.11.0",
|
||||
"time",
|
||||
"typed-path",
|
||||
"windows-sys 0.61.2",
|
||||
"zeroize",
|
||||
"zopfli",
|
||||
"zstd 0.14.3",
|
||||
|
||||
+4
-4
@@ -327,7 +327,7 @@ s3s-sigv4 = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fc
|
||||
serial_test = "4.0.1"
|
||||
shadow-rs = { default-features = false, version = "2.0.0" }
|
||||
siphasher = "1.0.4"
|
||||
smallvec = { version = "1.16.3" }
|
||||
smallvec = { version = "1.16.4" }
|
||||
compact_str = "0.10.0"
|
||||
snap = "1.1.2"
|
||||
starshard = { version = "2.4.0" }
|
||||
@@ -354,7 +354,7 @@ winapi-util = "0.1.11"
|
||||
windows = { version = "0.62.2" }
|
||||
windows-sys = "0.61.2"
|
||||
xxhash-rust = { version = "0.8.19" }
|
||||
zip = "9.0.5"
|
||||
zip = "9.1.0"
|
||||
zstd = "0.14.3"
|
||||
|
||||
# Observability and Metrics
|
||||
@@ -382,9 +382,9 @@ russh-sftp = "3.0.1"
|
||||
dav-server = "0.11.0"
|
||||
|
||||
# Performance Analysis and Memory Profiling
|
||||
rustfs-mimalloc = { version = "0.7.1" }
|
||||
rustfs-mimalloc = { version = "0.7.2" }
|
||||
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
|
||||
hotpath = { version = "0.29.0", default-features = false }
|
||||
hotpath = { version = "0.29.1", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.49" }
|
||||
|
||||
|
||||
@@ -49,12 +49,28 @@ impl Workers {
|
||||
|
||||
/// Acquire a worker slot, waiting until one becomes available.
|
||||
pub async fn take(&self) {
|
||||
struct AcquireCancelGuard<'a> {
|
||||
workers: &'a Workers,
|
||||
completed: bool,
|
||||
}
|
||||
impl Drop for AcquireCancelGuard<'_> {
|
||||
fn drop(&mut self) {
|
||||
if !self.completed {
|
||||
self.workers.in_use.send_modify(|_| {});
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut guard = AcquireCancelGuard {
|
||||
workers: self,
|
||||
completed: false,
|
||||
};
|
||||
let permit = match self.semaphore.acquire().await {
|
||||
Ok(permit) => permit,
|
||||
// The semaphore is owned by `self` and never closed.
|
||||
Err(_) => unreachable!("worker semaphore is never closed"),
|
||||
};
|
||||
permit.forget(); // returned manually via give()
|
||||
guard.completed = true;
|
||||
self.in_use.send_modify(|held| *held += 1);
|
||||
trace!(
|
||||
event = "worker_slot.acquire",
|
||||
@@ -82,6 +98,7 @@ impl Workers {
|
||||
});
|
||||
if released {
|
||||
self.semaphore.add_permits(1);
|
||||
self.in_use.send_modify(|_| {});
|
||||
}
|
||||
trace!(
|
||||
event = "worker_slot.release",
|
||||
@@ -99,7 +116,9 @@ impl Workers {
|
||||
pub async fn wait(&self) {
|
||||
let mut rx = self.in_use.subscribe();
|
||||
// The sender is owned by `self`, so it outlives this borrow.
|
||||
let _ = rx.wait_for(|&held| held == 0).await;
|
||||
let _ = rx
|
||||
.wait_for(|&held| held == 0 && self.semaphore.available_permits() == self.limit)
|
||||
.await;
|
||||
debug!(
|
||||
event = "worker_slot.wait",
|
||||
component = "concurrency",
|
||||
@@ -114,7 +133,9 @@ impl Workers {
|
||||
|
||||
/// Return the current number of available worker slots.
|
||||
pub async fn available(&self) -> usize {
|
||||
self.limit.saturating_sub(*self.in_use.borrow())
|
||||
self.limit
|
||||
.saturating_sub(*self.in_use.borrow())
|
||||
.min(self.semaphore.available_permits())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -217,4 +238,36 @@ mod tests {
|
||||
}
|
||||
assert_eq!(workers.available().await, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_wait_does_not_complete_while_permit_is_in_transit_to_queued_taker() {
|
||||
use std::future::Future;
|
||||
use std::pin::pin;
|
||||
use std::task::{Context, Poll, Waker};
|
||||
|
||||
let workers = Workers::new(1).unwrap();
|
||||
workers.take().await;
|
||||
|
||||
let mut taker = pin!(workers.take());
|
||||
let mut cx = Context::from_waker(Waker::noop());
|
||||
assert!(matches!(taker.as_mut().poll(&mut cx), Poll::Pending));
|
||||
|
||||
let mut waiter = pin!(workers.wait());
|
||||
assert!(matches!(waiter.as_mut().poll(&mut cx), Poll::Pending));
|
||||
|
||||
// Hand off the slot to the queued taker before `taker` is polled again.
|
||||
workers.give().await;
|
||||
assert_eq!(workers.available().await, 0);
|
||||
assert!(
|
||||
matches!(waiter.as_mut().poll(&mut cx), Poll::Pending),
|
||||
"wait() must not complete while a permit is in transit to a woken taker"
|
||||
);
|
||||
|
||||
assert!(matches!(taker.as_mut().poll(&mut cx), Poll::Ready(())));
|
||||
assert!(matches!(waiter.as_mut().poll(&mut cx), Poll::Pending));
|
||||
|
||||
workers.give().await;
|
||||
assert!(matches!(waiter.as_mut().poll(&mut cx), Poll::Ready(())));
|
||||
assert_eq!(workers.available().await, 1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -200,8 +200,8 @@ python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run
|
||||
python3 scripts/e2e_binary.py run --binary "$RUSTFS_E2E_STARTUP_CAS_BINARY" --features e2e-test-hooks -- cargo nextest run --profile e2e-full -p e2e_test
|
||||
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-nightly -p e2e_test
|
||||
|
||||
# Distributed 4-node 4-disk lane uses the fault-test hooks server; the
|
||||
# scanner_heal_topology_ fault cases fail closed without it.
|
||||
# Distributed 4-node 4-disk lane includes physical-worker fault injection and
|
||||
# scanner_heal_topology_ cases; both require the fault-test hooks server.
|
||||
# Upgrade cases require RUSTFS_UPGRADE_SOURCE_BINARY and fail closed without it.
|
||||
python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks
|
||||
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-distributed -p e2e_test
|
||||
|
||||
@@ -0,0 +1,378 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#![cfg(unix)]
|
||||
|
||||
use super::artifact_seal_test::{assert_exact_version, confirm_seal, create_artifact_attempt};
|
||||
use super::harness::{
|
||||
DistCluster, DistLayout, TestResult, enable_versioning, payload_for, s3_probe_client, unique_bucket, wait_for_ready,
|
||||
};
|
||||
use crate::chaos::{VersionShardCensus, census_object_version_on_disk};
|
||||
use crate::common::init_logging;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::types::{Delete, ObjectIdentifier};
|
||||
use serde_json::json;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
use tokio::time::timeout;
|
||||
|
||||
const PUBLIC_INTERNAL_ERROR: &str = "We encountered an internal error, please try again.";
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub(super) struct PhysicalEntry {
|
||||
identity: (u64, u64),
|
||||
changed: (i64, i64),
|
||||
modified: (i64, i64),
|
||||
bytes: Option<Vec<u8>>,
|
||||
}
|
||||
|
||||
// Include directory identity/ctime as well as file bytes. Creating and removing
|
||||
// a temporary marker/rollback file cannot hide behind a restored final xl.meta.
|
||||
// Reads change atime, which is deliberately excluded.
|
||||
pub(super) fn physical_snapshot(dist: &DistCluster, bucket: &str) -> TestResult<BTreeMap<(usize, PathBuf), PhysicalEntry>> {
|
||||
let mut snapshot = BTreeMap::new();
|
||||
for (node, instance) in dist.cluster.nodes.iter().enumerate() {
|
||||
let root = Path::new(&instance.data_dirs[0]).join(bucket);
|
||||
for entry in walkdir::WalkDir::new(&root) {
|
||||
let entry = entry?;
|
||||
let metadata = std::fs::symlink_metadata(entry.path())?;
|
||||
assert!(!metadata.file_type().is_symlink(), "unexpected fixture symlink");
|
||||
snapshot.insert(
|
||||
(node, entry.path().strip_prefix(&root)?.to_path_buf()),
|
||||
PhysicalEntry {
|
||||
identity: (metadata.dev(), metadata.ino()),
|
||||
changed: (metadata.ctime(), metadata.ctime_nsec()),
|
||||
modified: (metadata.mtime(), metadata.mtime_nsec()),
|
||||
bytes: if metadata.is_dir() {
|
||||
None
|
||||
} else {
|
||||
Some(std::fs::read(entry.path())?)
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(snapshot)
|
||||
}
|
||||
|
||||
struct Version {
|
||||
key: String,
|
||||
id: String,
|
||||
body: Vec<u8>,
|
||||
manifests: Vec<VersionShardCensus>,
|
||||
}
|
||||
|
||||
impl Version {
|
||||
async fn put(dist: &DistCluster, bucket: &str, key: &str, seed: &str) -> TestResult<Self> {
|
||||
let body = payload_for(seed, 2 * 1024 * 1024);
|
||||
let output = timeout(
|
||||
Duration::from_secs(30),
|
||||
dist.client(1)?
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(body.clone().into())
|
||||
.send(),
|
||||
)
|
||||
.await??;
|
||||
let id = output.version_id().ok_or("fixture PUT omitted version ID")?.to_owned();
|
||||
let manifests = dist
|
||||
.cluster
|
||||
.nodes
|
||||
.iter()
|
||||
.map(|node| census_object_version_on_disk(Path::new(&node.data_dirs[0]), bucket, key, Some(&id)))
|
||||
.collect::<TestResult<Vec<_>>>()?;
|
||||
assert!(
|
||||
manifests
|
||||
.iter()
|
||||
.all(|manifest| manifest.is_complete() && !manifest.expected_part_numbers.is_empty())
|
||||
);
|
||||
Ok(Self {
|
||||
key: key.to_owned(),
|
||||
id,
|
||||
body,
|
||||
manifests,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn target(key: &str, version: Option<&str>) -> TestResult<ObjectIdentifier> {
|
||||
Ok(ObjectIdentifier::builder()
|
||||
.key(key)
|
||||
.set_version_id(version.map(str::to_owned))
|
||||
.build()?)
|
||||
}
|
||||
|
||||
async fn assert_fixture(
|
||||
dist: &DistCluster,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
versions: &[Version],
|
||||
marker: (&str, &str),
|
||||
) -> TestResult {
|
||||
let expected: BTreeSet<_> = versions
|
||||
.iter()
|
||||
.map(|version| (version.key.clone(), version.id.clone()))
|
||||
.collect();
|
||||
for node in 0..dist.cluster.nodes.len() {
|
||||
let client = dist.client(node)?;
|
||||
let listed = timeout(
|
||||
Duration::from_secs(30),
|
||||
client.list_object_versions().bucket(bucket).prefix(prefix).send(),
|
||||
)
|
||||
.await??;
|
||||
let actual: BTreeSet<_> = listed
|
||||
.versions()
|
||||
.iter()
|
||||
.map(|version| {
|
||||
(
|
||||
version.key().unwrap_or_default().to_owned(),
|
||||
version.version_id().unwrap_or_default().to_owned(),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(listed.versions().len(), versions.len());
|
||||
assert_eq!(actual, expected, "delete changed exact data versions on node {node}");
|
||||
assert_eq!(listed.delete_markers().len(), 1);
|
||||
assert_eq!(listed.delete_markers()[0].key(), Some(marker.0));
|
||||
assert_eq!(listed.delete_markers()[0].version_id(), Some(marker.1));
|
||||
assert_eq!(listed.delete_markers()[0].is_latest(), Some(true));
|
||||
for version in versions {
|
||||
assert_exact_version(&client, bucket, &version.key, &version.id, &version.body).await?;
|
||||
let disk = Path::new(&dist.cluster.nodes[node].data_dirs[0]);
|
||||
assert!(
|
||||
census_object_version_on_disk(disk, bucket, &version.key, Some(&version.id))?
|
||||
.matches_manifest(&version.manifests[node])
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_single_denied(dist: &DistCluster, node: usize, bucket: &str, key: &str, version: Option<&str>) -> TestResult {
|
||||
let before = physical_snapshot(dist, bucket)?;
|
||||
// One attempt: a later retry must not conceal a first local mutation.
|
||||
let client = s3_probe_client(&dist.client(node)?);
|
||||
let error = timeout(
|
||||
Duration::from_secs(30),
|
||||
client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.set_version_id(version.map(str::to_owned))
|
||||
.send(),
|
||||
)
|
||||
.await?
|
||||
.expect_err("ordinary deletion succeeded in a managed bucket");
|
||||
let service = error
|
||||
.as_service_error()
|
||||
.ok_or("delete failed without an S3 service rejection")?;
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(500));
|
||||
assert_eq!(service.code(), Some("InternalError"));
|
||||
assert_eq!(service.message(), Some(PUBLIC_INTERNAL_ERROR));
|
||||
assert!(
|
||||
physical_snapshot(dist, bucket)? == before,
|
||||
"first delete through node {node} wrote or rolled back managed data"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_batch_denied(dist: &DistCluster, node: usize, bucket: &str, objects: &[ObjectIdentifier]) -> TestResult {
|
||||
let before = physical_snapshot(dist, bucket)?;
|
||||
let client = s3_probe_client(&dist.client(node)?);
|
||||
let response = timeout(
|
||||
Duration::from_secs(30),
|
||||
client
|
||||
.delete_objects()
|
||||
.bucket(bucket)
|
||||
.delete(Delete::builder().set_objects(Some(objects.to_vec())).quiet(false).build()?)
|
||||
.send(),
|
||||
)
|
||||
.await??;
|
||||
assert!(response.deleted().is_empty(), "managed batch reported a successful delete");
|
||||
assert_eq!(response.errors().len(), objects.len(), "batch must reject every requested occurrence");
|
||||
for (error, requested) in response.errors().iter().zip(objects) {
|
||||
assert_eq!(error.key(), Some(requested.key()));
|
||||
assert_eq!(error.version_id(), requested.version_id());
|
||||
assert_eq!(error.code(), Some("InternalError"));
|
||||
assert_eq!(error.message(), Some(PUBLIC_INTERNAL_ERROR));
|
||||
}
|
||||
assert!(
|
||||
physical_snapshot(dist, bucket)? == before,
|
||||
"first batch through node {node} wrote or rolled back managed data"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn copy_existing_object(source: &Path, destination: &Path) -> TestResult {
|
||||
assert!(!destination.exists());
|
||||
for entry in walkdir::WalkDir::new(source) {
|
||||
let entry = entry?;
|
||||
let target = destination.join(entry.path().strip_prefix(source)?);
|
||||
if entry.file_type().is_dir() {
|
||||
std::fs::create_dir_all(&target)?;
|
||||
} else {
|
||||
assert!(entry.file_type().is_file());
|
||||
std::fs::copy(entry.path(), target)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn ordinary_delete_control(dist: &DistCluster, bucket: &str) -> TestResult {
|
||||
let client = s3_probe_client(&dist.client(0)?);
|
||||
let key = "ordinary-observer-control.bin";
|
||||
let version = Version::put(dist, bucket, key, key).await?;
|
||||
let before = physical_snapshot(dist, bucket)?;
|
||||
let deleted = client.delete_object().bucket(bucket).key(key).send().await?;
|
||||
assert_eq!(deleted.delete_marker(), Some(true));
|
||||
let marker = deleted.version_id().ok_or("ordinary delete omitted marker ID")?;
|
||||
assert!(
|
||||
physical_snapshot(dist, bucket)? != before,
|
||||
"physical observer missed real marker creation"
|
||||
);
|
||||
client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.version_id(marker)
|
||||
.send()
|
||||
.await?;
|
||||
// The logical version inventory is restored, but ctime/identity still
|
||||
// expose the two writes. This checks the observer used by managed negatives.
|
||||
let listed = client.list_object_versions().bucket(bucket).prefix(key).send().await?;
|
||||
assert_eq!(listed.versions().len(), 1);
|
||||
assert_eq!(listed.versions()[0].version_id(), Some(version.id.as_str()));
|
||||
assert!(listed.delete_markers().is_empty());
|
||||
assert!(
|
||||
physical_snapshot(dist, bucket)? != before,
|
||||
"marker round-trip hid transient physical writes"
|
||||
);
|
||||
assert_exact_version(&client, bucket, key, &version.id, &version.body).await?;
|
||||
client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.version_id(&version.id)
|
||||
.send()
|
||||
.await?;
|
||||
assert!(
|
||||
client
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.prefix(key)
|
||||
.send()
|
||||
.await?
|
||||
.versions()
|
||||
.is_empty()
|
||||
);
|
||||
|
||||
let current = Version::put(dist, bucket, "ordinary-batch-current.bin", "current").await?;
|
||||
let exact = Version::put(dist, bucket, "ordinary-batch-exact.bin", "exact").await?;
|
||||
let response = client
|
||||
.delete_objects()
|
||||
.bucket(bucket)
|
||||
.delete(
|
||||
Delete::builder()
|
||||
.objects(target(¤t.key, None)?)
|
||||
.objects(target(&exact.key, Some(&exact.id))?)
|
||||
.objects(target(&exact.key, Some(&exact.id))?)
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
assert!(response.errors().is_empty());
|
||||
assert_eq!(response.deleted().len(), 3);
|
||||
let listed = client
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.prefix(¤t.key)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(listed.versions().len(), 1);
|
||||
assert_eq!(listed.delete_markers().len(), 1);
|
||||
assert!(
|
||||
client
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.prefix(&exact.key)
|
||||
.send()
|
||||
.await?
|
||||
.versions()
|
||||
.is_empty()
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_artifact_delete_entrypoints_preserve_versions_and_physical_state() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
|
||||
let (identity, admin) = create_artifact_attempt(&dist).await?;
|
||||
let bucket = identity["bucket"]
|
||||
.as_str()
|
||||
.ok_or("managed receipt omitted bucket")?
|
||||
.to_owned();
|
||||
let prefix = identity["attempt_prefix"]
|
||||
.as_str()
|
||||
.ok_or("managed receipt omitted prefix")?
|
||||
.to_owned();
|
||||
let key = format!("{prefix}native.bin");
|
||||
let mut versions = vec![
|
||||
Version::put(&dist, &bucket, &key, "native-old").await?,
|
||||
Version::put(&dist, &bucket, &key, "native-new").await?,
|
||||
];
|
||||
let ordinary = unique_bucket("delete-control");
|
||||
dist.create_bucket(&ordinary).await?;
|
||||
enable_versioning(&dist.client(0)?, &ordinary).await?;
|
||||
|
||||
// A new managed bucket cannot legitimately create a marker. Model existing
|
||||
// persisted state from before adoption/upgrade using a real S3-generated
|
||||
// ordinary object+marker, copied disk-for-disk in the same erasure set.
|
||||
// Only this object's files move; bucket authority/configuration is untouched.
|
||||
let marker_key = format!("{prefix}existing-marker.bin");
|
||||
versions.push(Version::put(&dist, &ordinary, &marker_key, "existing-data").await?);
|
||||
let output = dist
|
||||
.client(2)?
|
||||
.delete_object()
|
||||
.bucket(&ordinary)
|
||||
.key(&marker_key)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(output.delete_marker(), Some(true));
|
||||
let marker = output.version_id().ok_or("ordinary fixture omitted marker ID")?.to_owned();
|
||||
dist.cluster.stop();
|
||||
for node in &dist.cluster.nodes {
|
||||
let disk = Path::new(&node.data_dirs[0]);
|
||||
copy_existing_object(&disk.join(&ordinary).join(&marker_key), &disk.join(&bucket).join(&marker_key))?;
|
||||
}
|
||||
dist.cluster.start().await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
assert_fixture(&dist, &bucket, &prefix, &versions, (&marker_key, &marker)).await?;
|
||||
ordinary_delete_control(&dist, &ordinary).await?;
|
||||
let batch = vec![
|
||||
target(&key, None)?,
|
||||
target(&key, Some(&versions[0].id))?,
|
||||
target(&key, Some(&versions[1].id))?,
|
||||
target(&marker_key, Some(&marker))?,
|
||||
target(&key, Some(&versions[0].id))?,
|
||||
target(&marker_key, Some(&versions[2].id))?,
|
||||
];
|
||||
for phase in ["open", "sealed", "restarted"] {
|
||||
if phase == "sealed" {
|
||||
confirm_seal(&dist, &admin, &json!({"version": 1, "target": identity}).to_string(), &identity).await?;
|
||||
} else if phase == "restarted" {
|
||||
dist.restart_with_current_binary().await?;
|
||||
}
|
||||
for node in 0..dist.cluster.nodes.len() {
|
||||
assert_single_denied(&dist, node, &bucket, &key, None).await?;
|
||||
assert_single_denied(&dist, node, &bucket, &key, Some(&versions[0].id)).await?;
|
||||
assert_single_denied(&dist, node, &bucket, &marker_key, Some(&marker)).await?;
|
||||
assert_batch_denied(&dist, node, &bucket, &batch).await?;
|
||||
}
|
||||
assert_fixture(&dist, &bucket, &prefix, &versions, (&marker_key, &marker)).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,434 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#![cfg(unix)]
|
||||
|
||||
use super::artifact_delete_test::physical_snapshot;
|
||||
use super::artifact_seal_test::{ArtifactAdmin, create_artifact_attempt};
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, wait_for_ready};
|
||||
use crate::common::{
|
||||
AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, init_logging,
|
||||
signed_s3_request,
|
||||
};
|
||||
use futures::{StreamExt as _, TryStreamExt as _};
|
||||
use http::{Method, StatusCode};
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Value, json};
|
||||
use std::{collections::BTreeSet, time::Duration};
|
||||
use tokio::time::timeout;
|
||||
use uuid::Uuid;
|
||||
|
||||
const ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/inventory";
|
||||
const BUDGET: Duration = Duration::from_secs(45);
|
||||
|
||||
struct Reader {
|
||||
access: String,
|
||||
secret: String,
|
||||
}
|
||||
|
||||
impl Reader {
|
||||
async fn create(dist: &DistCluster, target: &Value, action: &str) -> TestResult<Self> {
|
||||
let reader = Self {
|
||||
access: format!("inventory{}", &Uuid::new_v4().simple().to_string()[..12]),
|
||||
secret: Uuid::new_v4().simple().to_string(),
|
||||
};
|
||||
let policy_name = format!("inventory{}", &Uuid::new_v4().simple().to_string()[..12]);
|
||||
let bucket = target["bucket"].as_str().ok_or("target bucket")?;
|
||||
let prefix = target["attempt_prefix"]
|
||||
.as_str()
|
||||
.ok_or("target prefix")?
|
||||
.trim_end_matches('/');
|
||||
let resource = if action == "s3:ListBucket" {
|
||||
format!("arn:aws:s3:::{bucket}")
|
||||
} else {
|
||||
format!("arn:aws:s3:::{bucket}/{prefix}")
|
||||
};
|
||||
let policy =
|
||||
json!({"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":action,"Resource":resource}]}).to_string();
|
||||
let url = &dist.cluster.nodes[0].url;
|
||||
let access = &dist.cluster.access_key;
|
||||
let secret = &dist.cluster.secret_key;
|
||||
admin_create_user_via(AdminTransport::Signed, url, access, secret, &reader.access, &reader.secret).await?;
|
||||
admin_add_canned_policy_via(AdminTransport::Signed, url, access, secret, &policy_name, &policy).await?;
|
||||
admin_attach_user_policy_via(AdminTransport::Signed, url, access, secret, &policy_name, &reader.access).await?;
|
||||
Ok(reader)
|
||||
}
|
||||
|
||||
async fn request(&self, dist: &DistCluster, target: &Value, invocation: Uuid) -> TestResult<reqwest::Response> {
|
||||
timeout(
|
||||
BUDGET,
|
||||
signed_s3_request(
|
||||
Method::POST,
|
||||
&format!("{}{ROUTE}", dist.cluster.nodes[0].url),
|
||||
Some(json!({"version":1,"invocation_id":invocation,"target":target}).to_string()),
|
||||
Some("application/json"),
|
||||
&self.access,
|
||||
&self.secret,
|
||||
),
|
||||
)
|
||||
.await?
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct Frame {
|
||||
version: u32,
|
||||
invocation_id: Uuid,
|
||||
target: Value,
|
||||
sequence: u64,
|
||||
payload: Payload,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
|
||||
enum Payload {
|
||||
Observation {
|
||||
disk_id: Uuid,
|
||||
key: String,
|
||||
version_id: Uuid,
|
||||
delete_marker: bool,
|
||||
},
|
||||
Complete {
|
||||
members: usize,
|
||||
observations: u64,
|
||||
},
|
||||
}
|
||||
|
||||
async fn read_frames(mut response: reqwest::Response) -> TestResult<(Vec<Frame>, bool)> {
|
||||
let mut pending = Vec::new();
|
||||
let mut frames = Vec::new();
|
||||
let mut clean = true;
|
||||
loop {
|
||||
match timeout(BUDGET, response.chunk()).await? {
|
||||
Ok(Some(chunk)) => {
|
||||
pending.extend_from_slice(&chunk);
|
||||
while let Some(end) = pending.iter().position(|byte| *byte == b'\n') {
|
||||
assert!(end < 1024 * 1024, "HTTP inventory line exceeds encoded bound");
|
||||
let line: Vec<_> = pending.drain(..=end).collect();
|
||||
frames.push(serde_json::from_slice(&line[..end])?);
|
||||
}
|
||||
assert!(pending.len() < 1024 * 1024, "unbounded incomplete HTTP line");
|
||||
}
|
||||
Ok(None) => break,
|
||||
Err(_) => {
|
||||
clean = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok((frames, clean && pending.is_empty()))
|
||||
}
|
||||
|
||||
async fn seal(dist: &DistCluster, user: &ArtifactAdmin, target: &Value) -> TestResult {
|
||||
let (status, body) = user
|
||||
.request(
|
||||
dist,
|
||||
0,
|
||||
"/rustfs/admin/v3/plugin-artifacts/seal-writes",
|
||||
json!({"version":1,"target":target}).to_string(),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(status, StatusCode::OK, "actual all-member closure: {body}");
|
||||
assert_eq!(serde_json::from_str::<Value>(&body)?["all_writes_sealed"], true);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn exact_inventory(frames: &[Frame], target: &Value, invocation: Uuid, versions: &BTreeSet<(String, Uuid, bool)>) {
|
||||
let mut observed = BTreeSet::new();
|
||||
let mut disks = BTreeSet::new();
|
||||
let mut count = 0u64;
|
||||
for (sequence, frame) in frames.iter().enumerate() {
|
||||
assert_eq!(frame.version, 1);
|
||||
assert_eq!(frame.invocation_id, invocation);
|
||||
assert_eq!(&frame.target, target);
|
||||
assert_eq!(frame.sequence, u64::try_from(sequence).expect("bounded fixture sequence"));
|
||||
match &frame.payload {
|
||||
Payload::Observation {
|
||||
disk_id,
|
||||
key,
|
||||
version_id,
|
||||
delete_marker,
|
||||
} => {
|
||||
assert!(!disk_id.is_nil() && !version_id.is_nil());
|
||||
assert!(
|
||||
observed.insert((*disk_id, key.clone(), *version_id, *delete_marker)),
|
||||
"duplicate physical observation"
|
||||
);
|
||||
disks.insert(*disk_id);
|
||||
count += 1;
|
||||
}
|
||||
Payload::Complete { members, observations } => {
|
||||
assert_eq!(sequence + 1, frames.len(), "Complete must be the last HTTP frame");
|
||||
assert_eq!(*members, 4);
|
||||
assert_eq!(*observations, count);
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
matches!(frames.last().map(|frame| &frame.payload), Some(Payload::Complete { .. })),
|
||||
"A03I3A_HTTP_COMPLETE"
|
||||
);
|
||||
if !versions.is_empty() {
|
||||
assert_eq!(disks.len(), 4);
|
||||
}
|
||||
for disk in disks {
|
||||
let actual: BTreeSet<_> = observed
|
||||
.iter()
|
||||
.filter(|(id, ..)| *id == disk)
|
||||
.map(|(_, key, version, marker)| (key.clone(), *version, *marker))
|
||||
.collect();
|
||||
assert_eq!(&actual, versions, "A03I3A_EXACT_VERSIONS");
|
||||
}
|
||||
assert_eq!(observed.len(), versions.len() * 4);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_inventory_http_exact_versions() -> TestResult {
|
||||
crate::common::rustfs_binary_path_with_features(Some("e2e-test-hooks"));
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
|
||||
let (target, admin) = create_artifact_attempt(&dist).await?;
|
||||
let bucket = target["bucket"].as_str().ok_or("bucket")?;
|
||||
let prefix = target["attempt_prefix"].as_str().ok_or("prefix")?;
|
||||
let mut versions = BTreeSet::new();
|
||||
for (name, body) in [
|
||||
("same", b"first".to_vec()),
|
||||
("same", b"second".to_vec()),
|
||||
("zero", Vec::new()),
|
||||
] {
|
||||
let key = format!("{prefix}{name}");
|
||||
let output = timeout(BUDGET, dist.client(1)?.put_object().bucket(bucket).key(&key).body(body.into()).send()).await??;
|
||||
let version = Uuid::parse_str(output.version_id().ok_or("actual PUT version")?)?;
|
||||
versions.insert((key, version, false));
|
||||
}
|
||||
seal(&dist, &admin, &target).await?;
|
||||
// A wildcard action is still restricted to this exact attempt resource.
|
||||
// The separate authority case exercises the dedicated action alone.
|
||||
let reader = Reader::create(&dist, &target, "admin:*").await?;
|
||||
let before = physical_snapshot(&dist, bucket)?;
|
||||
let invocation = Uuid::new_v4();
|
||||
let response = reader.request(&dist, &target, invocation).await?;
|
||||
assert_eq!(response.status(), StatusCode::OK, "A03I3A_HTTP_ROUTE");
|
||||
assert_eq!(
|
||||
response.headers().get("content-type").and_then(|value| value.to_str().ok()),
|
||||
Some("application/x-ndjson")
|
||||
);
|
||||
let (frames, clean) = read_frames(response).await?;
|
||||
assert!(clean, "Complete requires clean HTTP EOF");
|
||||
assert!(frames.len() > 2, "real multi-frame HTTP body");
|
||||
exact_inventory(&frames, &target, invocation, &versions);
|
||||
assert!(
|
||||
before == physical_snapshot(&dist, bucket)?,
|
||||
"inventory mutated physical artifact namespace"
|
||||
);
|
||||
let (empty, empty_admin) = create_artifact_attempt(&dist).await?;
|
||||
seal(&dist, &empty_admin, &empty).await?;
|
||||
let empty_reader = Reader::create(&dist, &empty, "admin:*").await?;
|
||||
let invocation = Uuid::new_v4();
|
||||
let response = empty_reader.request(&dist, &empty, invocation).await?;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let (frames, clean) = read_frames(response).await?;
|
||||
assert!(clean);
|
||||
exact_inventory(&frames, &empty, invocation, &BTreeSet::new());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_inventory_http_scoped_authority() -> TestResult {
|
||||
crate::common::rustfs_binary_path_with_features(Some("e2e-test-hooks"));
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
|
||||
let (target, admin) = create_artifact_attempt(&dist).await?;
|
||||
seal(&dist, &admin, &target).await?;
|
||||
let reader = Reader::create(&dist, &target, "admin:InventoryPluginArtifactAttempt").await?;
|
||||
let invocation = Uuid::new_v4();
|
||||
let response = reader.request(&dist, &target, invocation).await?;
|
||||
assert_eq!(response.status(), StatusCode::OK, "dedicated inventory authority");
|
||||
let (frames, clean) = read_frames(response).await?;
|
||||
assert!(clean);
|
||||
exact_inventory(&frames, &target, invocation, &BTreeSet::new());
|
||||
let mut other_prefix = target.clone();
|
||||
other_prefix["attempt_prefix"] = json!(format!("outputs/6169/{}/{}/", Uuid::new_v4(), Uuid::new_v4()));
|
||||
let mut other_bucket = target.clone();
|
||||
other_bucket["bucket"] = json!("another-artifact-bucket");
|
||||
for denied in [&other_prefix, &other_bucket] {
|
||||
assert_eq!(
|
||||
reader.request(&dist, denied, Uuid::new_v4()).await?.status(),
|
||||
StatusCode::FORBIDDEN,
|
||||
"A03I3A_SCOPED_AUTHORITY"
|
||||
);
|
||||
}
|
||||
let (status, _) = admin
|
||||
.request(
|
||||
&dist,
|
||||
0,
|
||||
ROUTE,
|
||||
json!({"version":1,"invocation_id":Uuid::new_v4(),"target":target}).to_string(),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(status, StatusCode::FORBIDDEN, "A03I3A_SCOPED_AUTHORITY seal-only user");
|
||||
let listed = Reader::create(&dist, &target, "s3:ListBucket").await?;
|
||||
dist.client_with_credentials(0, &listed.access, &listed.secret)?
|
||||
.list_objects_v2()
|
||||
.bucket(target["bucket"].as_str().ok_or("bucket")?)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
listed.request(&dist, &target, Uuid::new_v4()).await?.status(),
|
||||
StatusCode::FORBIDDEN,
|
||||
"A03I3A_SCOPED_AUTHORITY S3 list-only user"
|
||||
);
|
||||
let root_response = signed_s3_request(
|
||||
Method::POST,
|
||||
&format!("{}{ROUTE}", dist.cluster.nodes[0].url),
|
||||
Some(json!({"version":1,"invocation_id":Uuid::new_v4(),"target":target}).to_string()),
|
||||
Some("application/json"),
|
||||
&dist.cluster.access_key,
|
||||
&dist.cluster.secret_key,
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(root_response.status(), StatusCode::FORBIDDEN, "original scoped user required");
|
||||
for body in [
|
||||
json!({"version":1,"invocation_id":Uuid::nil(),"target":target}),
|
||||
json!({"version":1,"invocation_id":Uuid::new_v4(),"target":target,"extra":true}),
|
||||
] {
|
||||
let response = signed_s3_request(
|
||||
Method::POST,
|
||||
&format!("{}{ROUTE}", dist.cluster.nodes[0].url),
|
||||
Some(body.to_string()),
|
||||
Some("application/json"),
|
||||
&reader.access,
|
||||
&reader.secret,
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST, "strict inventory request");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_incomplete(response: reqwest::Response) -> TestResult<Vec<Frame>> {
|
||||
assert_eq!(
|
||||
response.status(),
|
||||
StatusCode::OK,
|
||||
"valid request must exercise the streaming failure boundary"
|
||||
);
|
||||
let (frames, _) = read_frames(response).await?;
|
||||
assert!(
|
||||
!frames.iter().any(|frame| matches!(frame.payload, Payload::Complete { .. })),
|
||||
"A03I3A_NO_FALSE_COMPLETE"
|
||||
);
|
||||
Ok(frames)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_inventory_http_incomplete_has_no_terminal() -> TestResult {
|
||||
crate::common::rustfs_binary_path_with_features(Some("e2e-test-hooks"));
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
|
||||
let (target, admin) = create_artifact_attempt(&dist).await?;
|
||||
let bucket = target["bucket"].as_str().ok_or("bucket")?;
|
||||
let prefix = target["attempt_prefix"].as_str().ok_or("prefix")?;
|
||||
// Real PUTs produce more data than a stalled HTTP/2 flow-control window.
|
||||
futures::stream::iter(0..256)
|
||||
.map(|index| {
|
||||
let client = dist.client(1);
|
||||
let key = format!("{prefix}slow/{index:04}/{}/{}", "a".repeat(190), "b".repeat(190));
|
||||
async move {
|
||||
let output = client?
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(vec![1].into())
|
||||
.send()
|
||||
.await?;
|
||||
assert!(!Uuid::parse_str(output.version_id().ok_or("actual PUT version")?)?.is_nil());
|
||||
Ok::<_, Box<dyn std::error::Error + Send + Sync>>(())
|
||||
}
|
||||
})
|
||||
.buffer_unordered(8)
|
||||
.try_collect::<Vec<_>>()
|
||||
.await?;
|
||||
seal(&dist, &admin, &target).await?;
|
||||
let reader = Reader::create(&dist, &target, "admin:InventoryPluginArtifactAttempt").await?;
|
||||
dist.cluster.stop_node(3)?;
|
||||
assert_incomplete(reader.request(&dist, &target, Uuid::new_v4()).await?).await?;
|
||||
dist.cluster.start_node(3).await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
let mut wrong = target.clone();
|
||||
wrong["bucket_incarnation"] = json!(Uuid::new_v4());
|
||||
assert_incomplete(reader.request(&dist, &wrong, Uuid::new_v4()).await?).await?;
|
||||
// The configured endpoint map is ordered. Damage its final receiver, so
|
||||
// earlier real observations are emitted before the physical scan fails.
|
||||
let node = dist
|
||||
.cluster
|
||||
.nodes
|
||||
.iter()
|
||||
.max_by_key(|node| format!("{}{}", node.url, node.data_dirs[0]))
|
||||
.ok_or("configured members")?;
|
||||
let metadata = std::path::Path::new(&node.data_dirs[0])
|
||||
.join(bucket)
|
||||
.join(format!("{prefix}slow/0000/{}/{}", "a".repeat(190), "b".repeat(190)))
|
||||
.join("xl.meta");
|
||||
let original = std::fs::read(&metadata)?;
|
||||
std::fs::write(&metadata, b"corrupt inventory metadata")?;
|
||||
let before = physical_snapshot(&dist, bucket)?;
|
||||
let outcome = assert_incomplete(reader.request(&dist, &target, Uuid::new_v4()).await?).await;
|
||||
let after = physical_snapshot(&dist, bucket)?;
|
||||
std::fs::write(&metadata, original)?;
|
||||
let frames = outcome?;
|
||||
assert!(before == after, "failed inventory repaired or mutated a physical object");
|
||||
assert!(
|
||||
frames
|
||||
.iter()
|
||||
.any(|frame| matches!(frame.payload, Payload::Observation { .. })),
|
||||
"late physical failure must follow real HTTP observations"
|
||||
);
|
||||
let client = reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.http2_prior_knowledge()
|
||||
.http2_initial_stream_window_size(1)
|
||||
.http2_initial_connection_window_size(1)
|
||||
.http2_adaptive_window(false)
|
||||
.build()?;
|
||||
let url = format!("{}{ROUTE}", dist.cluster.nodes[0].url);
|
||||
let request = |invocation| {
|
||||
json!({"version":1,"invocation_id":invocation,"target":target})
|
||||
.to_string()
|
||||
.into_bytes()
|
||||
};
|
||||
let response = crate::common::signed_request_with_client(
|
||||
&client,
|
||||
Method::POST,
|
||||
&url,
|
||||
&reader.access,
|
||||
&reader.secret,
|
||||
Some(request(Uuid::new_v4())),
|
||||
Some("application/json"),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
// Do not poll the body: its one-byte HTTP/2 receive window stalls transport.
|
||||
// This waits beyond the fixed30s service budget, not for readiness.
|
||||
tokio::time::sleep(Duration::from_secs(35)).await;
|
||||
assert_incomplete(response).await?;
|
||||
let response = crate::common::signed_request_with_client(
|
||||
&client,
|
||||
Method::POST,
|
||||
&url,
|
||||
&reader.access,
|
||||
&reader.secret,
|
||||
Some(request(Uuid::new_v4())),
|
||||
Some("application/json"),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
drop(response);
|
||||
let response = timeout(Duration::from_secs(10), reader.request(&dist, &target, Uuid::new_v4())).await??;
|
||||
let (frames, clean) = timeout(Duration::from_secs(10), read_frames(response)).await??;
|
||||
assert!(
|
||||
clean && matches!(frames.last().map(|frame| &frame.payload), Some(Payload::Complete { .. })),
|
||||
"fresh scan must succeed promptly after an actual disconnected HTTP body"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,269 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use super::artifact_repair_tail_test::PhysicalBarrier;
|
||||
use super::artifact_seal_test::{assert_exact_version, confirm_seal, create_artifact_attempt};
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, payload_for, wait_until};
|
||||
use crate::chaos::census_object_version_on_disk;
|
||||
use crate::common::init_logging;
|
||||
use crate::storage_api::artifact_rpc::{
|
||||
DiskError, TONIC_RPC_PREFIX, gen_tonic_signature_headers, set_tonic_canonical_body_digest,
|
||||
};
|
||||
use http::StatusCode;
|
||||
use rustfs_filemeta::FileMeta;
|
||||
use rustfs_protos::proto_gen::node_service::{RenameDataRequest, node_service_client::NodeServiceClient};
|
||||
use serde_json::{Value, json};
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
use tokio::time::timeout;
|
||||
use tonic::Request;
|
||||
use uuid::Uuid;
|
||||
|
||||
const PUT_SEAL_ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/seal-attempt";
|
||||
|
||||
fn signed_put_request(body: RenameDataRequest, audience: &str) -> TestResult<Request<RenameDataRequest>> {
|
||||
let canonical = rustfs_protos::canonical_artifact_rename_data_request_body(&body)?;
|
||||
let mut request = Request::new(body);
|
||||
request.set_timeout(Duration::from_secs(15));
|
||||
set_tonic_canonical_body_digest(&mut request, &canonical)?;
|
||||
let digest = request
|
||||
.metadata()
|
||||
.get("x-rustfs-content-sha256")
|
||||
.ok_or("missing canonical digest")?
|
||||
.to_str()?
|
||||
.to_owned();
|
||||
request.metadata_mut().as_mut().extend(gen_tonic_signature_headers(
|
||||
audience,
|
||||
TONIC_RPC_PREFIX.trim_start_matches('/'),
|
||||
"ArtifactRenameData",
|
||||
Some(&digest),
|
||||
)?);
|
||||
Ok(request)
|
||||
}
|
||||
|
||||
fn publication_snapshot(paths: &[&Path]) -> TestResult<BTreeMap<PathBuf, Option<Vec<u8>>>> {
|
||||
let mut snapshot = BTreeMap::new();
|
||||
for root in paths {
|
||||
for entry in walkdir::WalkDir::new(root) {
|
||||
let entry = entry?;
|
||||
let contents = if entry.file_type().is_dir() {
|
||||
None
|
||||
} else {
|
||||
Some(std::fs::read(entry.path())?)
|
||||
};
|
||||
snapshot.insert(entry.path().to_path_buf(), contents);
|
||||
}
|
||||
}
|
||||
Ok(snapshot)
|
||||
}
|
||||
|
||||
async fn assert_raw_put_closed(body: &RenameDataRequest, url: &str, audience: &str, paths: &[&Path]) -> TestResult {
|
||||
let before = publication_snapshot(paths)?;
|
||||
// Re-sign the exact original request; replay protection is not the rejection.
|
||||
let request = signed_put_request(body.clone(), audience)?;
|
||||
let response = timeout(Duration::from_secs(20), async {
|
||||
let mut client = NodeServiceClient::connect(url.to_owned()).await?;
|
||||
Ok::<_, Box<dyn std::error::Error + Send + Sync>>(client.artifact_rename_data(request).await?.into_inner())
|
||||
})
|
||||
.await??;
|
||||
assert!(!response.success, "sealed attempt admitted another ordinary PUT");
|
||||
let error = response.error.ok_or("rejected raw PUT omitted error")?;
|
||||
assert_eq!(error.code, DiskError::Io(std::io::ErrorKind::PermissionDenied.into()).to_u32());
|
||||
assert_eq!(error.error_info, "io error artifact attempt mutation domain is closed");
|
||||
assert_eq!(publication_snapshot(paths)?, before, "rejected raw PUT mutated staged or published files");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_artifact_put_seal_waits_for_disconnected_physical_tail() -> TestResult {
|
||||
crate::common::rustfs_binary_path_with_features(Some("e2e-test-hooks"));
|
||||
init_logging();
|
||||
let _ = rustfs_credentials::set_global_rpc_secret("rustfs-internode-signature-e2e-secret".to_owned());
|
||||
let rpc_secret = rustfs_credentials::try_get_rpc_token()?;
|
||||
let dist = DistCluster::start_with_env(DistLayout::FourNodeFourDisk, &[("RUSTFS_RPC_SECRET", &rpc_secret)]).await?;
|
||||
let (target, user) = create_artifact_attempt(&dist).await?;
|
||||
let bucket = target["bucket"].as_str().ok_or("target omitted bucket")?;
|
||||
let prefix = target["attempt_prefix"].as_str().ok_or("target omitted prefix")?;
|
||||
let key = format!("{prefix}detached-put.bin");
|
||||
let payload = payload_for(&key, 2 * 1024 * 1024);
|
||||
let output = timeout(
|
||||
Duration::from_secs(30),
|
||||
dist.client(1)?
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(&key)
|
||||
.body(payload.clone().into())
|
||||
.send(),
|
||||
)
|
||||
.await??;
|
||||
let version = output.version_id().ok_or("managed PUT omitted version ID")?;
|
||||
let request_body = json!({"version": 1, "target": target}).to_string();
|
||||
let baselines = dist
|
||||
.cluster
|
||||
.nodes
|
||||
.iter()
|
||||
.map(|node| census_object_version_on_disk(Path::new(&node.data_dirs[0]), bucket, &key, Some(version)))
|
||||
.collect::<TestResult<Vec<_>>>()?;
|
||||
assert!(
|
||||
baselines
|
||||
.iter()
|
||||
.all(|baseline| baseline.is_complete() && !baseline.expected_part_numbers.is_empty())
|
||||
);
|
||||
|
||||
let node = &dist.cluster.nodes[3];
|
||||
let disk = PathBuf::from(&node.data_dirs[0]).canonicalize()?;
|
||||
let object = disk.join(bucket).join(&key);
|
||||
let metadata_bytes = std::fs::read(object.join("xl.meta"))?;
|
||||
let fi = FileMeta::load(&metadata_bytes)?.into_fileinfo(bucket, &key, version, true, false, true)?;
|
||||
let admitted_file_info = serde_json::to_value(&fi)?;
|
||||
assert!(!fi.is_healing(), "ordinary publication must not carry the repair marker");
|
||||
let data_dir = fi
|
||||
.data_dir
|
||||
.ok_or("physical-tail fixture requires external shards")?
|
||||
.to_string();
|
||||
let source = Uuid::new_v4().to_string();
|
||||
let staging = disk.join(".rustfs.sys/tmp").join(&source);
|
||||
std::fs::create_dir_all(&staging)?;
|
||||
let destination = object.join(&data_dir);
|
||||
// Replay the receiver publication of a real ordinary PUT while its attempt
|
||||
// remains open. Preserve its referenced version metadata. The admitted
|
||||
// owned operation may still finish metadata publication after RPC detach.
|
||||
std::fs::rename(&destination, staging.join(&data_dir))?;
|
||||
let barrier = PhysicalBarrier::arm(&disk, &destination)?;
|
||||
let format: Value = serde_json::from_slice(&std::fs::read(disk.join(".rustfs.sys/format.json"))?)?;
|
||||
let disk_id = Uuid::parse_str(format["xl"]["this"].as_str().ok_or("disk format omitted xl.this")?)?;
|
||||
let incarnation = Uuid::parse_str(target["bucket_incarnation"].as_str().ok_or("target omitted incarnation")?)?;
|
||||
let put = RenameDataRequest {
|
||||
disk: disk_id.to_string(),
|
||||
src_volume: ".rustfs.sys/tmp".to_owned(),
|
||||
src_path: source,
|
||||
dst_volume: bucket.to_owned(),
|
||||
dst_path: key.clone(),
|
||||
file_info: serde_json::to_string(&fi)?,
|
||||
file_info_bin: Default::default(),
|
||||
scanner_publication_lease_token: Default::default(),
|
||||
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
|
||||
};
|
||||
let request = signed_put_request(put.clone(), &node.address)?;
|
||||
// Direct RPC takes no coordinator attempt lock and owns its own connection.
|
||||
let url = node.url.clone();
|
||||
let mut rpc = tokio::spawn(async move {
|
||||
timeout(Duration::from_secs(30), async move {
|
||||
let mut client = NodeServiceClient::connect(url).await?;
|
||||
Ok::<_, Box<dyn std::error::Error + Send + Sync>>(client.artifact_rename_data(request).await?)
|
||||
})
|
||||
.await
|
||||
});
|
||||
tokio::select! {
|
||||
result = &mut rpc => {
|
||||
let response = result???.into_inner();
|
||||
panic!(
|
||||
"ordinary PUT receiver completed before the physical barrier: success={}, error={:?}",
|
||||
response.success, response.error
|
||||
);
|
||||
}
|
||||
arrived = barrier.observed("arrived") => arrived?,
|
||||
}
|
||||
assert!(!destination.exists(), "physical PUT escaped its barrier");
|
||||
rpc.abort();
|
||||
assert!(rpc.await.expect_err("disconnect the ordinary PUT waiter").is_cancelled());
|
||||
// The ordinary PUT wrapper retains an owned task. Observe its RPC-facing
|
||||
// waiter dropping, rather than cancellation of the continuing disk task.
|
||||
barrier.observed("rpc-waiter-dropped").await?;
|
||||
|
||||
let seal = user.request(&dist, 0, PUT_SEAL_ROUTE, request_body.clone());
|
||||
tokio::pin!(seal);
|
||||
let closed = wait_until(
|
||||
Duration::from_secs(15),
|
||||
|| async {
|
||||
Ok(walkdir::WalkDir::new(disk.join(".rustfs-artifact-put-seals"))
|
||||
.min_depth(2)
|
||||
.max_depth(2)
|
||||
.into_iter()
|
||||
.filter_map(Result::ok)
|
||||
.any(|entry| entry.file_name() == "closed"))
|
||||
},
|
||||
"PUT receiver persisted closing before physical drain",
|
||||
);
|
||||
tokio::pin!(closed);
|
||||
tokio::select! {
|
||||
result = &mut seal => {
|
||||
let (status, response) = result?;
|
||||
panic!("PUT seal returned before its blocked receiver closed admission: {status} {response}");
|
||||
}
|
||||
result = &mut closed => result?,
|
||||
}
|
||||
let (status, response) = timeout(Duration::from_secs(40), &mut seal).await??;
|
||||
assert_eq!(
|
||||
status,
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"PUT seal confirmed an unfinished physical tail: {response}"
|
||||
);
|
||||
assert!(response.contains("artifact operation unconfirmed"));
|
||||
assert!(!destination.exists());
|
||||
assert_raw_put_closed(&put, &node.url, &node.address, &[&staging, &object]).await?;
|
||||
assert!(!destination.exists(), "a second admission published while the worker was paused");
|
||||
|
||||
barrier.release()?;
|
||||
wait_until(
|
||||
Duration::from_secs(15),
|
||||
|| async { Ok(destination.join("part.1").is_file()) },
|
||||
"detached ordinary PUT worker finished",
|
||||
)
|
||||
.await?;
|
||||
confirm_seal(&dist, &user, &request_body, &target).await?;
|
||||
// Seal confirmation is the boundary: already admitted work may re-encode
|
||||
// metadata before this point, but the complete FileInfo must be unchanged.
|
||||
let published_objects: Vec<_> = dist
|
||||
.cluster
|
||||
.nodes
|
||||
.iter()
|
||||
.map(|node| PathBuf::from(&node.data_dirs[0]).join(bucket).join(&key))
|
||||
.collect();
|
||||
let published_paths: Vec<_> = published_objects.iter().map(PathBuf::as_path).collect();
|
||||
let sealed_snapshot = publication_snapshot(&published_paths)?;
|
||||
let sealed_file_info =
|
||||
FileMeta::load(&std::fs::read(object.join("xl.meta"))?)?.into_fileinfo(bucket, &key, version, true, false, true)?;
|
||||
assert_eq!(
|
||||
serde_json::to_value(sealed_file_info)?,
|
||||
admitted_file_info,
|
||||
"admitted PUT changed FileInfo semantics"
|
||||
);
|
||||
for (index, baseline) in baselines.iter().enumerate() {
|
||||
assert!(
|
||||
census_object_version_on_disk(Path::new(&dist.cluster.nodes[index].data_dirs[0]), bucket, &key, Some(version))?
|
||||
.matches_manifest(baseline)
|
||||
);
|
||||
assert_exact_version(&dist.client(index)?, bucket, &key, version, &payload).await?;
|
||||
}
|
||||
|
||||
// Stage the exact request again. A source metadata sentinel makes a missing
|
||||
// admission check observable even if the existing destination later causes
|
||||
// ENOTEMPTY: ordinary rename rewrites source metadata before that syscall.
|
||||
let staged_data = staging.join(&data_dir);
|
||||
std::fs::create_dir_all(&staged_data)?;
|
||||
for entry in std::fs::read_dir(&destination)? {
|
||||
let entry = entry?;
|
||||
assert!(entry.file_type()?.is_file(), "unexpected nested shard fixture");
|
||||
std::fs::copy(entry.path(), staged_data.join(entry.file_name()))?;
|
||||
}
|
||||
std::fs::write(staging.join("xl.meta"), b"closed-admission-must-preserve-this-staged-metadata")?;
|
||||
assert_raw_put_closed(&put, &node.url, &node.address, &[&staging, &object]).await?;
|
||||
let listed = dist
|
||||
.client(0)?
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.prefix(&key)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(listed.versions().len(), 1, "raw PUT retries created another version");
|
||||
assert_eq!(listed.versions()[0].version_id(), Some(version));
|
||||
assert!(listed.delete_markers().is_empty());
|
||||
confirm_seal(&dist, &user, &request_body, &target).await?;
|
||||
assert!(
|
||||
publication_snapshot(&published_paths)? == sealed_snapshot,
|
||||
"published files changed after the first successful physical-drain confirmation"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use super::artifact_seal_test::{assert_exact_version, confirm_seal, create_artifact_attempt};
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, payload_for, wait_until};
|
||||
use crate::chaos::census_object_version_on_disk;
|
||||
use crate::common::init_logging;
|
||||
use crate::storage_api::artifact_rpc::{TONIC_RPC_PREFIX, gen_tonic_signature_headers, set_tonic_canonical_body_digest};
|
||||
use http::StatusCode;
|
||||
use rustfs_filemeta::FileMeta;
|
||||
use rustfs_protos::proto_gen::node_service::{
|
||||
ArtifactRepairRenameDataRequest, RenameDataRequest, node_service_client::NodeServiceClient,
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
use tokio::time::timeout;
|
||||
use tonic::Request;
|
||||
use uuid::Uuid;
|
||||
|
||||
const REPAIR_SEAL_ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/seal-repairs";
|
||||
|
||||
pub(super) struct PhysicalBarrier(PathBuf);
|
||||
|
||||
impl PhysicalBarrier {
|
||||
pub(super) fn arm(disk: &Path, destination: &Path) -> TestResult<Self> {
|
||||
let path = disk.join(".rustfs.sys/e2e-physical-rename-barrier");
|
||||
std::fs::write(&path, destination.to_str().ok_or("non-UTF8 fixture destination")?)?;
|
||||
Ok(Self(path))
|
||||
}
|
||||
|
||||
pub(super) async fn observed(&self, stage: &str) -> TestResult {
|
||||
wait_until(Duration::from_secs(15), || async { Ok(self.0.with_extension(stage).is_file()) }, stage).await
|
||||
}
|
||||
|
||||
pub(super) fn release(&self) -> TestResult {
|
||||
std::fs::write(self.0.with_extension("release"), b"release physical worker")?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for PhysicalBarrier {
|
||||
fn drop(&mut self) {
|
||||
// Release even on assertion failure; the worker also has its own deadline.
|
||||
let _ = self.release();
|
||||
let _ = std::fs::remove_file(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_artifact_repair_seal_waits_for_disconnected_physical_tail() -> TestResult {
|
||||
crate::common::rustfs_binary_path_with_features(Some("e2e-test-hooks"));
|
||||
init_logging();
|
||||
// Match the other raw internode signature tests sharing the OnceLock.
|
||||
let _ = rustfs_credentials::set_global_rpc_secret("rustfs-internode-signature-e2e-secret".to_owned());
|
||||
let rpc_secret = rustfs_credentials::try_get_rpc_token()?;
|
||||
let dist = DistCluster::start_with_env(DistLayout::FourNodeFourDisk, &[("RUSTFS_RPC_SECRET", &rpc_secret)]).await?;
|
||||
let (target, user) = create_artifact_attempt(&dist).await?;
|
||||
let bucket = target["bucket"].as_str().ok_or("target omitted bucket")?;
|
||||
let prefix = target["attempt_prefix"].as_str().ok_or("target omitted prefix")?;
|
||||
let key = format!("{prefix}detached-rename.bin");
|
||||
let body = payload_for(&key, 2 * 1024 * 1024);
|
||||
let output = timeout(
|
||||
Duration::from_secs(30),
|
||||
dist.client(1)?
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(&key)
|
||||
.body(body.clone().into())
|
||||
.send(),
|
||||
)
|
||||
.await??;
|
||||
let version = output.version_id().ok_or("managed PUT omitted version ID")?;
|
||||
let request_body = json!({"version": 1, "target": target}).to_string();
|
||||
confirm_seal(&dist, &user, &request_body, &target).await?;
|
||||
|
||||
let node = &dist.cluster.nodes[3];
|
||||
let disk = PathBuf::from(&node.data_dirs[0]).canonicalize()?;
|
||||
let object = disk.join(bucket).join(&key);
|
||||
let metadata = FileMeta::load(&std::fs::read(object.join("xl.meta"))?)?;
|
||||
let mut fi = metadata.into_fileinfo(bucket, &key, version, true, false, true)?;
|
||||
fi.set_healing();
|
||||
let data_dir = fi
|
||||
.data_dir
|
||||
.ok_or("physical-tail fixture requires external shard files")?
|
||||
.to_string();
|
||||
let baseline = census_object_version_on_disk(&disk, bucket, &key, Some(version))?;
|
||||
assert!(baseline.is_complete() && !baseline.expected_part_numbers.is_empty());
|
||||
let source = Uuid::new_v4().to_string();
|
||||
let staging = disk.join(".rustfs.sys/tmp").join(&source);
|
||||
std::fs::create_dir_all(&staging)?;
|
||||
let destination = object.join(&data_dir);
|
||||
std::fs::rename(&destination, staging.join(&data_dir))?;
|
||||
let barrier = PhysicalBarrier::arm(&disk, &destination)?;
|
||||
let format: Value = serde_json::from_slice(&std::fs::read(disk.join(".rustfs.sys/format.json"))?)?;
|
||||
let disk_id = Uuid::parse_str(format["xl"]["this"].as_str().ok_or("disk format omitted xl.this")?)?;
|
||||
let incarnation = Uuid::parse_str(target["bucket_incarnation"].as_str().ok_or("target omitted incarnation")?)?;
|
||||
let repair = ArtifactRepairRenameDataRequest {
|
||||
rename: Some(RenameDataRequest {
|
||||
disk: disk_id.to_string(),
|
||||
src_volume: ".rustfs.sys/tmp".to_owned(),
|
||||
src_path: source,
|
||||
dst_volume: bucket.to_owned(),
|
||||
dst_path: key.clone(),
|
||||
file_info: serde_json::to_string(&fi)?,
|
||||
file_info_bin: Default::default(),
|
||||
scanner_publication_lease_token: Default::default(),
|
||||
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
|
||||
}),
|
||||
expected_disk_id: disk_id.as_bytes().to_vec().into(),
|
||||
};
|
||||
let canonical = rustfs_protos::canonical_artifact_repair_rename_data_request_body(&repair)?;
|
||||
let mut request = Request::new(repair);
|
||||
request.set_timeout(Duration::from_secs(15));
|
||||
set_tonic_canonical_body_digest(&mut request, &canonical)?;
|
||||
let digest = request
|
||||
.metadata()
|
||||
.get("x-rustfs-content-sha256")
|
||||
.ok_or("missing canonical digest")?
|
||||
.to_str()?
|
||||
.to_owned();
|
||||
let headers = gen_tonic_signature_headers(
|
||||
&node.address,
|
||||
TONIC_RPC_PREFIX.trim_start_matches('/'),
|
||||
"ArtifactRepairRenameData",
|
||||
Some(&digest),
|
||||
)?;
|
||||
request.metadata_mut().as_mut().extend(headers);
|
||||
// An independent channel and direct receiver RPC take no coordinator attempt
|
||||
// lock. Dropping this task closes the client stream while rename is paused.
|
||||
let url = node.url.clone();
|
||||
let mut rpc = tokio::spawn(async move {
|
||||
timeout(Duration::from_secs(30), async move {
|
||||
let mut client = NodeServiceClient::connect(url).await?;
|
||||
client
|
||||
.artifact_repair_rename_data(request)
|
||||
.await
|
||||
.map_err(|error| -> Box<dyn std::error::Error + Send + Sync> { Box::new(error) })
|
||||
})
|
||||
.await
|
||||
});
|
||||
tokio::select! {
|
||||
result = &mut rpc => {
|
||||
let response = result???.into_inner();
|
||||
panic!(
|
||||
"repair receiver completed before the physical barrier: success={}, error={:?}",
|
||||
response.success, response.error
|
||||
);
|
||||
}
|
||||
arrived = barrier.observed("arrived") => arrived?,
|
||||
}
|
||||
assert!(!destination.exists(), "physical publication ran before its barrier");
|
||||
rpc.abort();
|
||||
assert!(rpc.await.expect_err("cancel the disconnected RPC waiter").is_cancelled());
|
||||
// This acknowledgement is emitted by the receiver's dropped async waiter,
|
||||
// not by the client and not by a delayed response.
|
||||
barrier.observed("waiter-dropped").await?;
|
||||
|
||||
let seal = user.request(&dist, 0, REPAIR_SEAL_ROUTE, request_body.clone());
|
||||
tokio::pin!(seal);
|
||||
let closed = wait_until(
|
||||
Duration::from_secs(15),
|
||||
|| async {
|
||||
Ok(walkdir::WalkDir::new(disk.join(".rustfs-artifact-repair-seals"))
|
||||
.min_depth(2)
|
||||
.max_depth(2)
|
||||
.into_iter()
|
||||
.filter_map(Result::ok)
|
||||
.any(|entry| entry.file_name() == "closed"))
|
||||
},
|
||||
"receiver persisted repair closure while its physical worker remains paused",
|
||||
);
|
||||
tokio::pin!(closed);
|
||||
tokio::select! {
|
||||
result = &mut seal => {
|
||||
let (status, response) = result?;
|
||||
panic!("repair seal returned before the blocked receiver closed admission: {status} {response}");
|
||||
}
|
||||
result = &mut closed => result?,
|
||||
}
|
||||
// Cover a complete receiver drain timeout, so early physical-owner release
|
||||
// causes a successful response here and fails this runtime assertion.
|
||||
let (status, response) = timeout(Duration::from_secs(40), &mut seal).await??;
|
||||
assert_eq!(
|
||||
status,
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"seal confirmed an unfinished physical tail: {response}"
|
||||
);
|
||||
assert!(response.contains("artifact operation unconfirmed"));
|
||||
assert!(!destination.exists(), "worker escaped the physical barrier");
|
||||
barrier.release()?;
|
||||
wait_until(
|
||||
Duration::from_secs(15),
|
||||
|| async { Ok(destination.join("part.1").is_file()) },
|
||||
"detached physical rename completes after release",
|
||||
)
|
||||
.await?;
|
||||
wait_until(
|
||||
Duration::from_secs(45),
|
||||
|| async {
|
||||
let (status, response) = user.request(&dist, 0, REPAIR_SEAL_ROUTE, request_body.clone()).await?;
|
||||
if status == StatusCode::SERVICE_UNAVAILABLE && response.contains("artifact operation unconfirmed") {
|
||||
return Ok(false);
|
||||
}
|
||||
assert_eq!(status, StatusCode::OK, "repair seal did not confirm after physical drain: {response}");
|
||||
assert_eq!(
|
||||
serde_json::from_str::<Value>(&response)?,
|
||||
json!({"version": 1, "target": target, "attempt_repairs_sealed": true})
|
||||
);
|
||||
Ok(true)
|
||||
},
|
||||
"exact retry confirms all four receivers after physical drain",
|
||||
)
|
||||
.await?;
|
||||
assert!(census_object_version_on_disk(&disk, bucket, &key, Some(version))?.matches_manifest(&baseline));
|
||||
assert_exact_version(&dist.client(0)?, bucket, &key, version, &body).await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,327 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use super::artifact_seal_test::{ArtifactAdmin, assert_exact_version, confirm_seal, create_artifact_attempt};
|
||||
use super::harness::{
|
||||
DistCluster, DistLayout, TestResult, enable_versioning, payload_for, unique_bucket, wait_for_ready, wait_until,
|
||||
};
|
||||
use crate::chaos::{VersionShardCensus, census_object_version_on_disk, signed_admin_post, start_root_heal_when_control_ready};
|
||||
use crate::common::init_logging;
|
||||
use http::StatusCode;
|
||||
use serde_json::{Value, json};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
use tokio::time::timeout;
|
||||
|
||||
const REPAIR_SEAL_ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/seal-repairs";
|
||||
const HEAL_BODY: &str =
|
||||
r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
|
||||
const HEAL_BUDGET: Duration = Duration::from_secs(120);
|
||||
|
||||
pub(super) struct RepairObject {
|
||||
pub(super) key: String,
|
||||
pub(super) version: String,
|
||||
pub(super) body: Vec<u8>,
|
||||
manifests: Vec<VersionShardCensus>,
|
||||
}
|
||||
|
||||
fn roots(dist: &DistCluster) -> Vec<PathBuf> {
|
||||
dist.cluster
|
||||
.nodes
|
||||
.iter()
|
||||
.map(|node| PathBuf::from(&node.data_dirs[0]))
|
||||
.collect()
|
||||
}
|
||||
|
||||
impl RepairObject {
|
||||
pub(super) async fn put(dist: &DistCluster, bucket: &str, key: String) -> TestResult<Self> {
|
||||
let body = payload_for(&key, 2 * 1024 * 1024);
|
||||
let output = timeout(
|
||||
Duration::from_secs(30),
|
||||
dist.client(1)?
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(&key)
|
||||
.body(body.clone().into())
|
||||
.send(),
|
||||
)
|
||||
.await??;
|
||||
let version = output.version_id().ok_or("managed PUT omitted version ID")?.to_owned();
|
||||
let manifests = roots(dist)
|
||||
.iter()
|
||||
.map(|disk| census_object_version_on_disk(disk, bucket, &key, Some(&version)))
|
||||
.collect::<TestResult<Vec<_>>>()?;
|
||||
for manifest in &manifests {
|
||||
assert!(manifest.is_complete(), "repair needs a complete baseline: {manifest:?}");
|
||||
assert!(!manifest.expected_part_numbers.is_empty(), "repair must use actual shard files");
|
||||
}
|
||||
Ok(Self {
|
||||
key,
|
||||
version,
|
||||
body,
|
||||
manifests,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn part(&self, disk: &Path, bucket: &str, node: usize) -> TestResult<PathBuf> {
|
||||
let data_dir = self.manifests[node]
|
||||
.data_dir
|
||||
.as_ref()
|
||||
.ok_or("external shard omitted data directory")?;
|
||||
Ok(disk.join(bucket).join(&self.key).join(data_dir).join("part.1"))
|
||||
}
|
||||
|
||||
pub(super) async fn assert_repaired(&self, dist: &DistCluster, bucket: &str) -> TestResult {
|
||||
wait_until(
|
||||
HEAL_BUDGET,
|
||||
|| async {
|
||||
for (disk, baseline) in roots(dist).iter().zip(&self.manifests) {
|
||||
if !census_object_version_on_disk(disk, bucket, &self.key, Some(&self.version))?.matches_manifest(baseline) {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
},
|
||||
"every configured receiver restored the exact version and original shard fingerprints",
|
||||
)
|
||||
.await?;
|
||||
for node in 0..dist.cluster.nodes.len() {
|
||||
assert_exact_version(&dist.client(node)?, bucket, &self.key, &self.version, &self.body).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn start_heal(dist: &DistCluster, bucket: &str, prefix: &str, remove: bool) -> TestResult<String> {
|
||||
let url = format!("{}/rustfs/admin/v3/heal/{bucket}/{prefix}?forceStart=true", dist.cluster.nodes[0].url);
|
||||
let body = if remove {
|
||||
HEAL_BODY.replace("\"remove\":false", "\"remove\":true")
|
||||
} else {
|
||||
HEAL_BODY.to_owned()
|
||||
};
|
||||
let response = timeout(
|
||||
HEAL_BUDGET,
|
||||
start_root_heal_when_control_ready(&url, &body, &dist.cluster.access_key, &dist.cluster.secret_key),
|
||||
)
|
||||
.await??;
|
||||
let response: Value = serde_json::from_str(&response)?;
|
||||
Ok(response["clientToken"]
|
||||
.as_str()
|
||||
.ok_or("heal omitted its task token")?
|
||||
.to_owned())
|
||||
}
|
||||
|
||||
pub(super) async fn finish_heal(dist: &DistCluster, token: &str, rejected_key: Option<&str>) -> TestResult {
|
||||
let url = format!("{}/rustfs/admin/v3/heal/?clientToken={token}", dist.cluster.nodes[0].url);
|
||||
wait_until(
|
||||
HEAL_BUDGET,
|
||||
|| async {
|
||||
let response = signed_admin_post(&url, None, &dist.cluster.access_key, &dist.cluster.secret_key).await?;
|
||||
let status: Value = serde_json::from_str(&response)?;
|
||||
if status["summary"] == "running" {
|
||||
return Ok(false);
|
||||
}
|
||||
if let Some(key) = rejected_key {
|
||||
assert_eq!(status["summary"], "stopped", "closed repair must report failure: {status}");
|
||||
assert!(
|
||||
status["outcome"]["counters"]["failed"]
|
||||
.as_u64()
|
||||
.is_some_and(|count| count > 0)
|
||||
);
|
||||
assert!(
|
||||
status["outcome"]["objects"].as_array().is_some_and(|objects| objects
|
||||
.iter()
|
||||
.any(|object| { object["identity"]["object"] == key && object["disposition"]["state"] == "failed" })),
|
||||
"heal must actually attempt the rejected object: {status}"
|
||||
);
|
||||
} else {
|
||||
assert_eq!(status["summary"], "finished", "heal did not complete: {status}");
|
||||
}
|
||||
Ok(true)
|
||||
},
|
||||
"exact admin heal task reached its expected terminal outcome",
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn repair_seal(dist: &DistCluster, user: &ArtifactAdmin, target: &Value, confirmed: bool) -> TestResult {
|
||||
wait_until(
|
||||
Duration::from_secs(45),
|
||||
|| async {
|
||||
let request = json!({"version": 1, "target": target}).to_string();
|
||||
let (status, body) = user.request(dist, 0, REPAIR_SEAL_ROUTE, request).await?;
|
||||
if !confirmed {
|
||||
assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "offline receiver repair seal: {body}");
|
||||
assert!(body.contains("<Code>ServiceUnavailable</Code>"));
|
||||
assert!(body.contains("artifact operation unconfirmed"));
|
||||
return Ok(true);
|
||||
}
|
||||
if status == StatusCode::SERVICE_UNAVAILABLE
|
||||
&& body.contains("<Code>ServiceUnavailable</Code>")
|
||||
&& body.contains("<Message>artifact operation unconfirmed; retry the same identity</Message>")
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
assert_eq!(status, StatusCode::OK, "repair seal failed: {body}");
|
||||
assert_eq!(
|
||||
serde_json::from_str::<Value>(&body)?,
|
||||
json!({"version": 1, "target": target, "attempt_repairs_sealed": true})
|
||||
);
|
||||
Ok(true)
|
||||
},
|
||||
"repair seal confirms every configured receiver",
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_artifact_shard_and_minority_metadata_repair_after_put_seal() -> TestResult {
|
||||
init_logging();
|
||||
// Sub-quorum metadata recovery supports genuine legacy shards only.
|
||||
let mut dist =
|
||||
DistCluster::start_with_env(DistLayout::FourNodeFourDisk, &[("RUSTFS_SHARD_INTEGRITY_WRITE", "false")]).await?;
|
||||
let (target, user) = create_artifact_attempt(&dist).await?;
|
||||
let bucket = target["bucket"].as_str().ok_or("target omitted bucket")?;
|
||||
let prefix = target["attempt_prefix"].as_str().ok_or("target omitted prefix")?;
|
||||
let shard = RepairObject::put(&dist, bucket, format!("{prefix}shard.bin")).await?;
|
||||
let metadata = RepairObject::put(&dist, bucket, format!("{prefix}metadata.bin")).await?;
|
||||
let later = dist
|
||||
.client(2)?
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(&shard.key)
|
||||
.body(b"newer-version".to_vec().into())
|
||||
.send()
|
||||
.await?;
|
||||
let later_version = later.version_id().ok_or("second PUT omitted version ID")?.to_owned();
|
||||
assert_ne!(shard.version, later_version);
|
||||
confirm_seal(&dist, &user, &json!({"version": 1, "target": target}).to_string(), &target).await?;
|
||||
|
||||
dist.cluster.stop();
|
||||
let disks = roots(&dist);
|
||||
std::fs::remove_file(shard.part(&disks[3], bucket, 3)?)?;
|
||||
for disk in &disks[1..] {
|
||||
std::fs::remove_file(disk.join(bucket).join(&metadata.key).join("xl.meta"))?;
|
||||
}
|
||||
assert!(!census_object_version_on_disk(&disks[3], bucket, &shard.key, Some(&shard.version))?.is_complete());
|
||||
assert!(disks[0].join(bucket).join(&metadata.key).join("xl.meta").is_file());
|
||||
assert!(
|
||||
disks[1..]
|
||||
.iter()
|
||||
.all(|disk| !disk.join(bucket).join(&metadata.key).join("xl.meta").exists())
|
||||
);
|
||||
dist.cluster.start().await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
let token = start_heal(&dist, bucket, prefix, false).await?;
|
||||
shard.assert_repaired(&dist, bucket).await?;
|
||||
metadata.assert_repaired(&dist, bucket).await?;
|
||||
finish_heal(&dist, &token, None).await?;
|
||||
assert_exact_version(&dist.client(3)?, bucket, &shard.key, &later_version, b"newer-version").await?;
|
||||
let listed = dist
|
||||
.client(0)?
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.prefix(prefix)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(listed.versions().len(), 3, "repair must not publish replacement versions");
|
||||
assert!(listed.delete_markers().is_empty());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_artifact_repair_seal_denies_shard_and_metadata_repair_after_restart() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist =
|
||||
DistCluster::start_with_env(DistLayout::FourNodeFourDisk, &[("RUSTFS_SHARD_INTEGRITY_WRITE", "false")]).await?;
|
||||
let (target, user) = create_artifact_attempt(&dist).await?;
|
||||
let bucket = target["bucket"].as_str().ok_or("target omitted bucket")?;
|
||||
let prefix = target["attempt_prefix"].as_str().ok_or("target omitted prefix")?;
|
||||
let shard = RepairObject::put(&dist, bucket, format!("{prefix}shard.bin")).await?;
|
||||
let metadata = RepairObject::put(&dist, bucket, format!("{prefix}metadata.bin")).await?;
|
||||
confirm_seal(&dist, &user, &json!({"version": 1, "target": target}).to_string(), &target).await?;
|
||||
repair_seal(&dist, &user, &target, true).await?;
|
||||
let ordinary_bucket = unique_bucket("repair-control");
|
||||
dist.create_bucket(&ordinary_bucket).await?;
|
||||
enable_versioning(&dist.client(0)?, &ordinary_bucket).await?;
|
||||
let ordinary = RepairObject::put(&dist, &ordinary_bucket, "ordinary.bin".to_owned()).await?;
|
||||
let disks = roots(&dist);
|
||||
let part = shard.part(&disks[3], bucket, 3)?;
|
||||
std::fs::remove_file(&part)?;
|
||||
for disk in &disks[1..] {
|
||||
std::fs::remove_file(disk.join(bucket).join(&metadata.key).join("xl.meta"))?;
|
||||
}
|
||||
for restarted in [false, true] {
|
||||
if restarted {
|
||||
dist.restart_with_current_binary().await?;
|
||||
}
|
||||
// The same receivers must still execute an ordinary bucket repair.
|
||||
std::fs::remove_file(ordinary.part(&disks[3], &ordinary_bucket, 3)?)?;
|
||||
let control = start_heal(&dist, &ordinary_bucket, &ordinary.key, false).await?;
|
||||
ordinary.assert_repaired(&dist, &ordinary_bucket).await?;
|
||||
finish_heal(&dist, &control, None).await?;
|
||||
for object in [&shard, &metadata] {
|
||||
let token = start_heal(&dist, bucket, &object.key, false).await?;
|
||||
finish_heal(&dist, &token, Some(&object.key)).await?;
|
||||
}
|
||||
assert!(!part.exists(), "sealed repair recreated the missing shard");
|
||||
assert!(
|
||||
disks[1..]
|
||||
.iter()
|
||||
.all(|disk| !disk.join(bucket).join(&metadata.key).join("xl.meta").exists())
|
||||
);
|
||||
assert_exact_version(&dist.client(2)?, bucket, &shard.key, &shard.version, &shard.body).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_artifact_repair_seal_requires_every_receiver_on_exact_retry() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
|
||||
let (target, user) = create_artifact_attempt(&dist).await?;
|
||||
let request = json!({"version": 1, "target": target}).to_string();
|
||||
confirm_seal(&dist, &user, &request, &target).await?;
|
||||
// Repeat after global confirmation: its persisted receipt cannot replace a
|
||||
// fresh acknowledgement from an offline configured receiver.
|
||||
for _ in 0..2 {
|
||||
dist.cluster.stop_node(3)?;
|
||||
repair_seal(&dist, &user, &target, false).await?;
|
||||
dist.cluster.start_node(3).await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
repair_seal(&dist, &user, &target, true).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_artifact_dangling_repair_preserves_surviving_metadata() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist =
|
||||
DistCluster::start_with_env(DistLayout::FourNodeFourDisk, &[("RUSTFS_SHARD_INTEGRITY_WRITE", "false")]).await?;
|
||||
let (target, user) = create_artifact_attempt(&dist).await?;
|
||||
let bucket = target["bucket"].as_str().ok_or("target omitted bucket")?;
|
||||
let prefix = target["attempt_prefix"].as_str().ok_or("target omitted prefix")?;
|
||||
let object = RepairObject::put(&dist, bucket, format!("{prefix}dangling.bin")).await?;
|
||||
confirm_seal(&dist, &user, &json!({"version": 1, "target": target}).to_string(), &target).await?;
|
||||
dist.cluster.stop();
|
||||
let disks = roots(&dist);
|
||||
let survivor = disks[0].join(bucket).join(&object.key).join("xl.meta");
|
||||
let before = std::fs::read(&survivor)?;
|
||||
for (node, disk) in disks.iter().enumerate() {
|
||||
std::fs::remove_file(object.part(disk, bucket, node)?)?;
|
||||
if node != 0 {
|
||||
std::fs::remove_file(disk.join(bucket).join(&object.key).join("xl.meta"))?;
|
||||
}
|
||||
}
|
||||
dist.cluster.start().await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
let token = start_heal(&dist, bucket, &object.key, true).await?;
|
||||
finish_heal(&dist, &token, Some(&object.key)).await?;
|
||||
assert_eq!(
|
||||
std::fs::read(survivor)?,
|
||||
before,
|
||||
"distributed repair must not authorize managed dangling deletion"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,347 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use super::harness::{
|
||||
DistCluster, DistLayout, TestResult, assert_object_bytes, put_object, s3_probe_client, unique_bucket, wait_for_ready,
|
||||
wait_until,
|
||||
};
|
||||
use crate::common::{
|
||||
AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, admin_request, init_logging,
|
||||
};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use http::{Method, StatusCode};
|
||||
use serde_json::{Value, json};
|
||||
use std::time::Duration;
|
||||
use tokio::time::timeout;
|
||||
use uuid::Uuid;
|
||||
|
||||
const BUCKET_ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/buckets";
|
||||
const INSPECT_ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/buckets/inspect";
|
||||
const SEAL_ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/seal-attempt";
|
||||
const ADMIN_BUDGET: Duration = Duration::from_secs(45);
|
||||
|
||||
pub(super) struct ArtifactAdmin {
|
||||
access_key: String,
|
||||
secret_key: String,
|
||||
}
|
||||
|
||||
impl ArtifactAdmin {
|
||||
pub(super) async fn request(
|
||||
&self,
|
||||
dist: &DistCluster,
|
||||
node: usize,
|
||||
route: &str,
|
||||
body: String,
|
||||
) -> TestResult<(StatusCode, String)> {
|
||||
timeout(
|
||||
ADMIN_BUDGET,
|
||||
admin_request(
|
||||
&dist.cluster.nodes[node].url,
|
||||
Method::POST,
|
||||
route,
|
||||
Some(body),
|
||||
&self.access_key,
|
||||
&self.secret_key,
|
||||
),
|
||||
)
|
||||
.await?
|
||||
}
|
||||
}
|
||||
|
||||
async fn create_artifact_admin(dist: &DistCluster, bucket: &str, prefix: &str) -> TestResult<ArtifactAdmin> {
|
||||
let user = ArtifactAdmin {
|
||||
access_key: format!("artifact{}", &Uuid::new_v4().simple().to_string()[..12]),
|
||||
secret_key: Uuid::new_v4().simple().to_string(),
|
||||
};
|
||||
let policy_name = format!("artifact{}", &Uuid::new_v4().simple().to_string()[..12]);
|
||||
let policy = json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["admin:CreatePluginArtifactBucket", "admin:GetPluginArtifactBucket"],
|
||||
"Resource": format!("arn:aws:s3:::{bucket}"),
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["admin:SealPluginArtifactAttempt", "admin:SealPluginArtifactRepairs", "admin:SealPluginArtifactWrites"],
|
||||
"Resource": format!("arn:aws:s3:::{bucket}/{}", prefix.trim_end_matches('/')),
|
||||
},
|
||||
],
|
||||
})
|
||||
.to_string();
|
||||
let url = &dist.cluster.nodes[0].url;
|
||||
let access = &dist.cluster.access_key;
|
||||
let secret = &dist.cluster.secret_key;
|
||||
admin_create_user_via(AdminTransport::Signed, url, access, secret, &user.access_key, &user.secret_key).await?;
|
||||
admin_add_canned_policy_via(AdminTransport::Signed, url, access, secret, &policy_name, &policy).await?;
|
||||
admin_attach_user_policy_via(AdminTransport::Signed, url, access, secret, &policy_name, &user.access_key).await?;
|
||||
Ok(user)
|
||||
}
|
||||
|
||||
pub(super) async fn create_artifact_attempt(dist: &DistCluster) -> TestResult<(Value, ArtifactAdmin)> {
|
||||
let bucket = unique_bucket("artifact");
|
||||
let prefix = format!("outputs/6169/{}/{}/", Uuid::new_v4(), Uuid::new_v4());
|
||||
let user = create_artifact_admin(dist, &bucket, &prefix).await?;
|
||||
let body = json!({"version": 1, "bucket": bucket, "base_prefix": "outputs/"}).to_string();
|
||||
let (status, response) = user.request(dist, 0, BUCKET_ROUTE, body).await?;
|
||||
assert_eq!(status, StatusCode::OK, "artifact bucket creation failed: {response}");
|
||||
let receipt: Value = serde_json::from_str(&response)?;
|
||||
assert_eq!(receipt["version"], 1);
|
||||
assert_eq!(receipt["bucket"], bucket);
|
||||
assert_eq!(receipt["base_prefix"], "outputs/");
|
||||
assert_eq!(receipt["versioning_enabled"], true);
|
||||
let incarnation = receipt["bucket_incarnation"]
|
||||
.as_str()
|
||||
.ok_or("artifact receipt omitted incarnation")?;
|
||||
assert!(!Uuid::parse_str(incarnation)?.is_nil());
|
||||
let target = json!({
|
||||
"bucket": bucket,
|
||||
"bucket_incarnation": incarnation,
|
||||
"attempt_prefix": prefix,
|
||||
});
|
||||
assert_bucket_identity(dist, &user, 1, &target).await?;
|
||||
Ok((target, user))
|
||||
}
|
||||
|
||||
async fn assert_bucket_identity(dist: &DistCluster, user: &ArtifactAdmin, node: usize, target: &Value) -> TestResult {
|
||||
let body = json!({"version": 1, "bucket": target["bucket"], "base_prefix": "outputs/"}).to_string();
|
||||
wait_until(
|
||||
ADMIN_BUDGET,
|
||||
|| async {
|
||||
let (status, response) = user.request(dist, node, INSPECT_ROUTE, body.clone()).await?;
|
||||
// IAM peers may not have received the new user and policy yet.
|
||||
// Storage failures must fail immediately, including the old gate's 503.
|
||||
if matches!(status, StatusCode::FORBIDDEN | StatusCode::UNAUTHORIZED)
|
||||
&& (response.contains("<Code>AccessDenied</Code>") || response.contains("<Code>InvalidAccessKeyId</Code>"))
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
assert_eq!(status, StatusCode::OK, "cross-node artifact inspect failed: {response}");
|
||||
assert_eq!(
|
||||
serde_json::from_str::<Value>(&response)?,
|
||||
json!({
|
||||
"version": 1,
|
||||
"bucket": target["bucket"],
|
||||
"base_prefix": "outputs/",
|
||||
"bucket_incarnation": target["bucket_incarnation"],
|
||||
"versioning_enabled": true,
|
||||
}),
|
||||
"inspect must return the created managed bucket identity"
|
||||
);
|
||||
Ok(true)
|
||||
},
|
||||
"artifact IAM user and policy propagation",
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn confirm_seal(dist: &DistCluster, user: &ArtifactAdmin, request: &str, target: &Value) -> TestResult {
|
||||
wait_until(
|
||||
ADMIN_BUDGET,
|
||||
|| async {
|
||||
let (status, response) = user.request(dist, 0, SEAL_ROUTE, request.to_owned()).await?;
|
||||
// HTTP readiness does not confirm that remote disk handles have
|
||||
// recovered. Retry only the API's explicit same-identity response.
|
||||
if status == StatusCode::SERVICE_UNAVAILABLE
|
||||
&& response.contains("<Code>ServiceUnavailable</Code>")
|
||||
&& response.contains("<Message>artifact operation unconfirmed; retry the same identity</Message>")
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
assert_eq!(status, StatusCode::OK, "artifact seal failed: {response}");
|
||||
let receipt: Value = serde_json::from_str(&response)?;
|
||||
assert_eq!(receipt, json!({"version": 1, "target": target, "ordinary_put_sealed": true}));
|
||||
Ok(true)
|
||||
},
|
||||
"exact artifact seal confirmation after receiver recovery",
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn assert_seal_unconfirmed(dist: &DistCluster, user: &ArtifactAdmin, request: &str) -> TestResult {
|
||||
let (status, body) = user.request(dist, 0, SEAL_ROUTE, request.to_owned()).await?;
|
||||
assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "offline receiver seal response: {body}");
|
||||
assert!(
|
||||
body.contains("<Code>ServiceUnavailable</Code>"),
|
||||
"seal failure lost its service code: {body}"
|
||||
);
|
||||
assert!(
|
||||
body.contains("artifact operation unconfirmed"),
|
||||
"seal failure lost its retry contract: {body}"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn assert_exact_version(client: &Client, bucket: &str, key: &str, version: &str, body: &[u8]) -> TestResult {
|
||||
let output = timeout(
|
||||
Duration::from_secs(30),
|
||||
client.get_object().bucket(bucket).key(key).version_id(version).send(),
|
||||
)
|
||||
.await??;
|
||||
assert_eq!(output.version_id(), Some(version));
|
||||
let actual = timeout(Duration::from_secs(30), output.body.collect()).await??.into_bytes();
|
||||
assert_eq!(actual.as_ref(), body);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_put_denied(client: &Client, ordinary_bucket: &str, bucket: &str, key: &str) -> TestResult {
|
||||
// The same receiver must still accept a normal write before its sealed probe.
|
||||
put_object(client, ordinary_bucket, key, b"ordinary-control".to_vec()).await?;
|
||||
assert_object_bytes(client, ordinary_bucket, key, b"ordinary-control").await?;
|
||||
let client = s3_probe_client(client);
|
||||
let result = timeout(
|
||||
Duration::from_secs(30),
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(b"must-not-publish".to_vec().into())
|
||||
.send(),
|
||||
)
|
||||
.await?;
|
||||
let error = result.expect_err("sealed artifact attempt accepted a new PUT");
|
||||
let service = error
|
||||
.as_service_error()
|
||||
.ok_or("sealed PUT failed without a service rejection")?;
|
||||
assert_eq!(
|
||||
error.raw_response().map(|response| response.status().as_u16()),
|
||||
Some(500),
|
||||
"sealed PUT must return its current HTTP status"
|
||||
);
|
||||
assert_eq!(service.code(), Some("InternalError"), "sealed PUT must return its current service code");
|
||||
// StorageError::Io is redacted at the API boundary; the sealed reason is
|
||||
// not exposed. Success controls and exact inventory checks prove the effect.
|
||||
assert_eq!(
|
||||
service.message(),
|
||||
Some("We encountered an internal error, please try again."),
|
||||
"sealed PUT must retain the public redacted message"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_artifact_put_seal_survives_full_restart() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
|
||||
let (target, user) = create_artifact_attempt(&dist).await?;
|
||||
let bucket = target["bucket"].as_str().ok_or("target omitted bucket")?;
|
||||
let prefix = target["attempt_prefix"].as_str().ok_or("target omitted attempt prefix")?;
|
||||
let key = format!("{prefix}result.bin");
|
||||
let mut versions = Vec::new();
|
||||
for (node, body) in [(1, b"first-version".as_slice()), (2, b"second-version".as_slice())] {
|
||||
let client = dist.client(node)?;
|
||||
let output = timeout(
|
||||
Duration::from_secs(30),
|
||||
client.put_object().bucket(bucket).key(&key).body(body.to_vec().into()).send(),
|
||||
)
|
||||
.await??;
|
||||
let version = output.version_id().ok_or("artifact PUT omitted version ID")?.to_owned();
|
||||
assert!(!Uuid::parse_str(&version)?.is_nil());
|
||||
assert_exact_version(&dist.client(3)?, bucket, &key, &version, body).await?;
|
||||
versions.push((version, body.to_vec()));
|
||||
}
|
||||
assert_ne!(versions[0].0, versions[1].0);
|
||||
let ordinary = unique_bucket("ordinary");
|
||||
dist.create_bucket(&ordinary).await?;
|
||||
put_object(&dist.client(2)?, &ordinary, "normal.bin", b"before-seal".to_vec()).await?;
|
||||
assert_object_bytes(&dist.client(3)?, &ordinary, "normal.bin", b"before-seal").await?;
|
||||
|
||||
let request = json!({"version": 1, "target": target}).to_string();
|
||||
confirm_seal(&dist, &user, &request, &target).await?;
|
||||
let other_prefix = format!("outputs/6169/{}/{}/", Uuid::new_v4(), Uuid::new_v4());
|
||||
for phase in ["sealed", "restarted"] {
|
||||
if phase == "restarted" {
|
||||
dist.cluster.stop();
|
||||
dist.cluster.start().await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
}
|
||||
assert_bucket_identity(&dist, &user, 2, &target).await?;
|
||||
for node in 0..dist.cluster.nodes.len() {
|
||||
// A second attempt in the managed bucket must remain writable;
|
||||
// a broken bucket-wide write path cannot stand in for sealing.
|
||||
let client = dist.client(node)?;
|
||||
let other_key = format!("{other_prefix}{phase}-{node}.bin");
|
||||
let output = timeout(
|
||||
Duration::from_secs(30),
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(&other_key)
|
||||
.body(b"other-attempt".to_vec().into())
|
||||
.send(),
|
||||
)
|
||||
.await??;
|
||||
let version = output.version_id().ok_or("other attempt PUT omitted version ID")?;
|
||||
assert!(!Uuid::parse_str(version)?.is_nil());
|
||||
assert_exact_version(
|
||||
&dist.client((node + 1) % dist.cluster.nodes.len())?,
|
||||
bucket,
|
||||
&other_key,
|
||||
version,
|
||||
b"other-attempt",
|
||||
)
|
||||
.await?;
|
||||
assert_put_denied(&dist.client(node)?, &ordinary, bucket, &format!("{prefix}{phase}-{node}.bin")).await?;
|
||||
for (version, body) in &versions {
|
||||
assert_exact_version(&dist.client(node)?, bucket, &key, version, body).await?;
|
||||
}
|
||||
}
|
||||
let listed = dist
|
||||
.client(0)?
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.prefix(prefix)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(listed.versions().len(), 2, "denied PUT left an artifact version: {listed:?}");
|
||||
assert!(listed.delete_markers().is_empty(), "artifact seal created a delete marker");
|
||||
put_object(&dist.client(1)?, &ordinary, "normal.bin", phase.as_bytes().to_vec()).await?;
|
||||
assert_object_bytes(&dist.client(2)?, &ordinary, "normal.bin", phase.as_bytes()).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_artifact_seal_requires_every_configured_receiver_on_exact_retry() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
|
||||
let (target, user) = create_artifact_attempt(&dist).await?;
|
||||
let bucket = target["bucket"].as_str().ok_or("target omitted bucket")?;
|
||||
let prefix = target["attempt_prefix"].as_str().ok_or("target omitted attempt prefix")?;
|
||||
let key = format!("{prefix}existing.bin");
|
||||
let output = dist
|
||||
.client(1)?
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(&key)
|
||||
.body(b"existing".to_vec().into())
|
||||
.send()
|
||||
.await?;
|
||||
let version = output.version_id().ok_or("artifact PUT omitted version ID")?.to_owned();
|
||||
assert_exact_version(&dist.client(3)?, bucket, &key, &version, b"existing").await?;
|
||||
let ordinary = unique_bucket("ordinary");
|
||||
dist.create_bucket(&ordinary).await?;
|
||||
let request = json!({"version": 1, "target": target}).to_string();
|
||||
|
||||
// The second iteration starts from a fully acknowledged global seal. Its
|
||||
// persisted record cannot replace a fresh acknowledgement from every disk.
|
||||
for _ in 0..2 {
|
||||
dist.cluster.stop_node(3)?;
|
||||
assert_seal_unconfirmed(&dist, &user, &request).await?;
|
||||
dist.cluster.start_node(3).await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
confirm_seal(&dist, &user, &request, &target).await?;
|
||||
}
|
||||
assert_put_denied(&dist.client(3)?, &ordinary, bucket, &format!("{prefix}late.bin")).await?;
|
||||
assert_exact_version(&dist.client(2)?, bucket, &key, &version, b"existing").await?;
|
||||
let listed = timeout(
|
||||
Duration::from_secs(30),
|
||||
dist.client(0)?.list_object_versions().bucket(bucket).prefix(prefix).send(),
|
||||
)
|
||||
.await??;
|
||||
assert_eq!(listed.versions().len(), 1, "rejected PUT added a version");
|
||||
assert!(listed.delete_markers().is_empty(), "seal added a delete marker");
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,235 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#![cfg(unix)]
|
||||
|
||||
use super::artifact_delete_test::physical_snapshot;
|
||||
use super::artifact_repair_test::{RepairObject, finish_heal, start_heal};
|
||||
use super::artifact_seal_test::{ArtifactAdmin, assert_exact_version, confirm_seal, create_artifact_attempt};
|
||||
use super::harness::{
|
||||
DistCluster, DistLayout, TestResult, enable_versioning, s3_probe_client, unique_bucket, wait_for_ready, wait_until,
|
||||
};
|
||||
use crate::common::init_logging;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::types::{
|
||||
ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockRetention,
|
||||
ObjectLockRetentionMode,
|
||||
};
|
||||
use http::StatusCode;
|
||||
use serde_json::{Value, json};
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
|
||||
const WRITES_ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/seal-writes";
|
||||
const INTERNAL_MESSAGE: &str = "We encountered an internal error, please try again.";
|
||||
|
||||
async fn seal_writes(dist: &DistCluster, user: &ArtifactAdmin, target: &Value, confirmed: bool) -> TestResult {
|
||||
wait_until(Duration::from_secs(45), || async {
|
||||
let (status, response) = user.request(dist, 0, WRITES_ROUTE, json!({"version":1,"target":target}).to_string()).await?;
|
||||
if status == StatusCode::SERVICE_UNAVAILABLE && response.contains("artifact operation unconfirmed; retry the same identity") {
|
||||
return Ok(!confirmed);
|
||||
}
|
||||
assert!(confirmed, "offline configured receiver produced a complete receipt: {status} {response}");
|
||||
assert_eq!(status, StatusCode::OK, "combined writes seal: {response}");
|
||||
assert_eq!(serde_json::from_str::<Value>(&response)?, json!({"version":1,"target":target,"ordinary_put_sealed":true,"attempt_repairs_sealed":true,"all_writes_sealed":true}));
|
||||
Ok(true)
|
||||
}, "combined seal acknowledges both domains on every receiver").await
|
||||
}
|
||||
|
||||
async fn assert_put_closed(dist: &DistCluster, bucket: &str, key: &str) -> TestResult {
|
||||
let before = physical_snapshot(dist, bucket)?;
|
||||
let client = s3_probe_client(&dist.client(2)?);
|
||||
let error = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(b"must-not-publish".to_vec().into())
|
||||
.send()
|
||||
.await
|
||||
.expect_err("combined seal admitted a new ordinary PUT");
|
||||
let service = error.as_service_error().ok_or("PUT was not a service rejection")?;
|
||||
assert_eq!(service.code(), Some("InternalError"));
|
||||
assert_eq!(service.message(), Some(INTERNAL_MESSAGE));
|
||||
assert!(
|
||||
physical_snapshot(dist, bucket)? == before,
|
||||
"rejected PUT changed the managed physical namespace"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn lock_metadata_entrypoints(dist: &DistCluster, managed: &Value, object: &RepairObject, ordinary: &str) -> TestResult {
|
||||
let bucket = managed["bucket"].as_str().ok_or("bucket identity")?;
|
||||
let client = s3_probe_client(&dist.client(0)?);
|
||||
for bucket in [bucket, ordinary] {
|
||||
client
|
||||
.put_object_lock_configuration()
|
||||
.bucket(bucket)
|
||||
.object_lock_configuration(
|
||||
ObjectLockConfiguration::builder()
|
||||
.object_lock_enabled(ObjectLockEnabled::Enabled)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
wait_until(
|
||||
Duration::from_secs(30),
|
||||
|| async {
|
||||
for node in 0..4 {
|
||||
let output = match dist.client(node)?.get_object_lock_configuration().bucket(bucket).send().await {
|
||||
Ok(output) => output,
|
||||
Err(error)
|
||||
if error.as_service_error().and_then(ProvideErrorMetadata::code)
|
||||
== Some("ObjectLockConfigurationNotFoundError") =>
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
if output
|
||||
.object_lock_configuration()
|
||||
.and_then(|configuration| configuration.object_lock_enabled())
|
||||
!= Some(&ObjectLockEnabled::Enabled)
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
},
|
||||
"actual Object Lock configuration is enabled on all request nodes",
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
let control = RepairObject::put(dist, ordinary, "lock-control.bin".to_owned()).await?;
|
||||
let retention = ObjectLockRetention::builder()
|
||||
.mode(ObjectLockRetentionMode::Governance)
|
||||
.retain_until_date(aws_sdk_s3::primitives::DateTime::from_secs(chrono::Utc::now().timestamp() + 86400))
|
||||
.build();
|
||||
let hold = ObjectLockLegalHold::builder().status(ObjectLockLegalHoldStatus::On).build();
|
||||
client
|
||||
.put_object_retention()
|
||||
.bucket(ordinary)
|
||||
.key(&control.key)
|
||||
.version_id(&control.version)
|
||||
.retention(retention.clone())
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
client
|
||||
.get_object_retention()
|
||||
.bucket(ordinary)
|
||||
.key(&control.key)
|
||||
.version_id(&control.version)
|
||||
.send()
|
||||
.await?
|
||||
.retention()
|
||||
.and_then(|value| value.mode()),
|
||||
Some(&ObjectLockRetentionMode::Governance)
|
||||
);
|
||||
client
|
||||
.put_object_legal_hold()
|
||||
.bucket(ordinary)
|
||||
.key(&control.key)
|
||||
.version_id(&control.version)
|
||||
.legal_hold(hold.clone())
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
client
|
||||
.get_object_legal_hold()
|
||||
.bucket(ordinary)
|
||||
.key(&control.key)
|
||||
.version_id(&control.version)
|
||||
.send()
|
||||
.await?
|
||||
.legal_hold()
|
||||
.and_then(|value| value.status()),
|
||||
Some(&ObjectLockLegalHoldStatus::On)
|
||||
);
|
||||
let before = physical_snapshot(dist, bucket)?;
|
||||
let error = client
|
||||
.put_object_retention()
|
||||
.bucket(bucket)
|
||||
.key(&object.key)
|
||||
.version_id(&object.version)
|
||||
.retention(retention)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("managed Retention must reach and fail its unsupported metadata write");
|
||||
let service = error.as_service_error().ok_or("retention was not a service rejection")?;
|
||||
assert_eq!(service.code(), Some("InternalError"));
|
||||
assert_eq!(service.message(), Some(INTERNAL_MESSAGE));
|
||||
assert!(physical_snapshot(dist, bucket)? == before, "retention changed managed metadata");
|
||||
let error = client
|
||||
.put_object_legal_hold()
|
||||
.bucket(bucket)
|
||||
.key(&object.key)
|
||||
.version_id(&object.version)
|
||||
.legal_hold(hold)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("managed LegalHold must reach and fail its unsupported metadata write");
|
||||
let service = error.as_service_error().ok_or("legal hold was not a service rejection")?;
|
||||
assert_eq!(service.code(), Some("InternalError"));
|
||||
assert_eq!(service.message(), Some("Io error: artifact bucket does not support this write operation"));
|
||||
assert!(physical_snapshot(dist, bucket)? == before, "legal hold changed managed metadata");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_artifact_writes_seal_first_partial_retry_offline_and_restart() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
|
||||
let ordinary = unique_bucket("writes-control");
|
||||
dist.create_bucket(&ordinary).await?;
|
||||
enable_versioning(&dist.client(0)?, &ordinary).await?;
|
||||
let (fresh, fresh_user) = create_artifact_attempt(&dist).await?;
|
||||
let fresh_bucket = fresh["bucket"].as_str().ok_or("fresh bucket")?;
|
||||
let fresh_prefix = fresh["attempt_prefix"].as_str().ok_or("fresh prefix")?;
|
||||
let first = RepairObject::put(&dist, fresh_bucket, format!("{fresh_prefix}existing.bin")).await?;
|
||||
lock_metadata_entrypoints(&dist, &fresh, &first, &ordinary).await?;
|
||||
seal_writes(&dist, &fresh_user, &fresh, true).await?;
|
||||
seal_writes(&dist, &fresh_user, &fresh, true).await?;
|
||||
assert_put_closed(&dist, fresh_bucket, &format!("{fresh_prefix}closed.bin")).await?;
|
||||
|
||||
let (partial, partial_user) = create_artifact_attempt(&dist).await?;
|
||||
let bucket = partial["bucket"].as_str().ok_or("partial bucket")?;
|
||||
let prefix = partial["attempt_prefix"].as_str().ok_or("partial prefix")?;
|
||||
let repaired = RepairObject::put(&dist, bucket, format!("{prefix}repair.bin")).await?;
|
||||
// Enter the combined operation with only its ordinary domain closed.
|
||||
confirm_seal(&dist, &partial_user, &json!({"version":1,"target":partial}).to_string(), &partial).await?;
|
||||
let part = repaired.part(Path::new(&dist.cluster.nodes[3].data_dirs[0]), bucket, 3)?;
|
||||
std::fs::remove_file(&part)?;
|
||||
let token = start_heal(&dist, bucket, &repaired.key, false).await?;
|
||||
repaired.assert_repaired(&dist, bucket).await?;
|
||||
finish_heal(&dist, &token, None).await?;
|
||||
for _ in 0..2 {
|
||||
dist.cluster.stop_node(3)?;
|
||||
seal_writes(&dist, &partial_user, &partial, false).await?;
|
||||
dist.cluster.start_node(3).await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
seal_writes(&dist, &partial_user, &partial, true).await?;
|
||||
}
|
||||
let ordinary_object = RepairObject::put(&dist, &ordinary, "repair-control.bin".to_owned()).await?;
|
||||
for restarted in [false, true] {
|
||||
if restarted {
|
||||
dist.restart_with_current_binary().await?;
|
||||
}
|
||||
seal_writes(&dist, &partial_user, &partial, true).await?;
|
||||
assert_put_closed(&dist, bucket, &format!("{prefix}closed-{restarted}.bin")).await?;
|
||||
if part.exists() {
|
||||
std::fs::remove_file(&part)?;
|
||||
}
|
||||
let before = physical_snapshot(&dist, bucket)?;
|
||||
let token = start_heal(&dist, bucket, &repaired.key, false).await?;
|
||||
finish_heal(&dist, &token, Some(&repaired.key)).await?;
|
||||
assert!(!part.exists(), "combined seal admitted a remote repair publication");
|
||||
assert!(physical_snapshot(&dist, bucket)? == before, "rejected repair changed metadata or shards");
|
||||
assert_exact_version(&dist.client(2)?, bucket, &repaired.key, &repaired.version, &repaired.body).await?;
|
||||
let ordinary_part = ordinary_object.part(Path::new(&dist.cluster.nodes[3].data_dirs[0]), &ordinary, 3)?;
|
||||
std::fs::remove_file(ordinary_part)?;
|
||||
let token = start_heal(&dist, &ordinary, &ordinary_object.key, false).await?;
|
||||
ordinary_object.assert_repaired(&dist, &ordinary).await?;
|
||||
finish_heal(&dist, &token, None).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -4,12 +4,13 @@
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, unique_bucket};
|
||||
use crate::tls_gen;
|
||||
use aws_sdk_s3::{Client, config::retry::RetryConfig, primitives::ByteStream};
|
||||
use futures::future::try_join_all;
|
||||
use rand::{Rng, SeedableRng, rngs::StdRng};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::io::{Read, Seek, SeekFrom, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::task::JoinSet;
|
||||
|
||||
const MIB: u64 = 1024 * 1024;
|
||||
const READ_TIMEOUT: Duration = Duration::from_secs(180);
|
||||
@@ -85,8 +86,10 @@ async fn verified_get(
|
||||
expected: &ExpectedRead,
|
||||
concurrent: bool,
|
||||
) -> TestResult<serde_json::Value> {
|
||||
let started = Instant::now();
|
||||
let mut received = 0u64;
|
||||
let mut stage = "response headers";
|
||||
tokio::time::timeout(READ_TIMEOUT, async {
|
||||
let started = Instant::now();
|
||||
let mut request = client.get_object().bucket(bucket).key(&object.key);
|
||||
if let Some((start, end)) = expected.range {
|
||||
request = request.range(format!("bytes={start}-{end}"));
|
||||
@@ -98,24 +101,37 @@ async fn verified_get(
|
||||
} else {
|
||||
assert_eq!(output.content_range(), None);
|
||||
}
|
||||
let mut count = 0u64;
|
||||
stage = "response body";
|
||||
let mut digest = Sha256::new();
|
||||
while let Some(chunk) = output.body.next().await {
|
||||
let chunk = chunk?;
|
||||
count += u64::try_from(chunk.len())?;
|
||||
assert!(count <= expected.length, "node {node}: GET exceeded the requested length");
|
||||
received += u64::try_from(chunk.len())?;
|
||||
assert!(received <= expected.length, "node {node}: GET exceeded the requested length");
|
||||
digest.update(&chunk);
|
||||
}
|
||||
let actual_sha256 = hex_simd::encode_to_string(digest.finalize(), hex_simd::AsciiCase::Lower);
|
||||
assert_eq!(count, expected.length, "node {node}: truncated GET {}", object.key);
|
||||
assert_eq!(received, expected.length, "node {node}: truncated GET {}", object.key);
|
||||
assert_eq!(actual_sha256, expected.sha256, "node {node}: corrupt GET {}", object.key);
|
||||
eprintln!(
|
||||
"TLS acceptance: GET node={node}, key={}, range={:?}, concurrent={concurrent}, bytes={received}, elapsed={:?}",
|
||||
object.key, expected.range, started.elapsed()
|
||||
);
|
||||
Ok(serde_json::json!({
|
||||
"operation": "get", "node": node, "key": object.key,
|
||||
"range": expected.range, "bytes": count, "sha256": actual_sha256,
|
||||
"range": expected.range, "bytes": received, "sha256": actual_sha256,
|
||||
"concurrent": concurrent, "elapsed_ms": started.elapsed().as_secs_f64() * 1000.0,
|
||||
}))
|
||||
})
|
||||
.await?
|
||||
.await
|
||||
.map_err(|error| {
|
||||
std::io::Error::new(
|
||||
std::io::ErrorKind::TimedOut,
|
||||
format!(
|
||||
"TLS GET timed out: node={node}, key={}, range={:?}, concurrent={concurrent}, stage={stage}, received={received}/{}, elapsed={:?}: {error}",
|
||||
object.key, expected.range, expected.length, started.elapsed()
|
||||
),
|
||||
)
|
||||
})?
|
||||
}
|
||||
|
||||
/// This case always exercises the full acceptance sizes; do not substitute
|
||||
@@ -157,9 +173,13 @@ async fn four_node_tls_large_and_range_gets_preserve_bytes() -> TestResult {
|
||||
{
|
||||
return Err("acceptance report paths must be unused".into());
|
||||
}
|
||||
let log_directory = report_path
|
||||
.as_ref()
|
||||
.map_or_else(|| directory.join("logs"), |path| path.with_extension("logs"));
|
||||
let log_directory = report_path.as_ref().map_or_else(
|
||||
|| {
|
||||
std::env::var_os("RUSTFS_E2E_LOG_DIR")
|
||||
.map_or_else(|| directory.join("logs"), |root| PathBuf::from(root).join(unique_bucket("internode-tls")))
|
||||
},
|
||||
|path| path.with_extension("logs"),
|
||||
);
|
||||
std::fs::create_dir(&log_directory)?;
|
||||
dist.capture_logs_under(&log_directory)?;
|
||||
dist.cluster.start().await?;
|
||||
@@ -183,7 +203,8 @@ async fn four_node_tls_large_and_range_gets_preserve_bytes() -> TestResult {
|
||||
let mut results = Vec::new();
|
||||
for length in [32 * 1024, 100 * MIB, 1024 * MIB] {
|
||||
let fixture_directory = directory.clone();
|
||||
let object = tokio::task::spawn_blocking(move || fixture(&fixture_directory, length)).await??;
|
||||
let object = Arc::new(tokio::task::spawn_blocking(move || fixture(&fixture_directory, length)).await??);
|
||||
eprintln!("TLS acceptance: fixture key={}, bytes={length}", object.key);
|
||||
let started = Instant::now();
|
||||
tokio::time::timeout(
|
||||
READ_TIMEOUT,
|
||||
@@ -194,7 +215,18 @@ async fn four_node_tls_large_and_range_gets_preserve_bytes() -> TestResult {
|
||||
.body(ByteStream::from_path(&object.path).await?)
|
||||
.send(),
|
||||
)
|
||||
.await??;
|
||||
.await
|
||||
.map_err(|error| {
|
||||
std::io::Error::new(
|
||||
std::io::ErrorKind::TimedOut,
|
||||
format!(
|
||||
"TLS PUT timed out: key={}, bytes={length}, elapsed={:?}: {error}",
|
||||
object.key,
|
||||
started.elapsed()
|
||||
),
|
||||
)
|
||||
})??;
|
||||
eprintln!("TLS acceptance: PUT key={}, bytes={length}, elapsed={:?}", object.key, started.elapsed());
|
||||
results.push(
|
||||
serde_json::json!({"operation": "put", "bytes": length, "elapsed_ms": started.elapsed().as_secs_f64() * 1000.0}),
|
||||
);
|
||||
@@ -216,15 +248,19 @@ async fn four_node_tls_large_and_range_gets_preserve_bytes() -> TestResult {
|
||||
results.push(verified_get(client, node, &bucket, &object, expected, false).await?);
|
||||
}
|
||||
}
|
||||
results.extend(
|
||||
try_join_all(
|
||||
clients
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(node, client)| verified_get(client, node, &bucket, &object, &object.reads[0], true)),
|
||||
)
|
||||
.await?,
|
||||
);
|
||||
// Each reader owns a Tokio task and its cooperative budget. Polling
|
||||
// every body and synchronous digest in one parent task can let an
|
||||
// always-ready reader starve its siblings and their deadlines.
|
||||
let mut readers = JoinSet::new();
|
||||
for (node, client) in clients.iter().enumerate() {
|
||||
let client = client.clone();
|
||||
let bucket = bucket.clone();
|
||||
let object = Arc::clone(&object);
|
||||
readers.spawn(async move { verified_get(&client, node, &bucket, &object, &object.reads[0], true).await });
|
||||
}
|
||||
while let Some(result) = readers.join_next().await {
|
||||
results.push(result??);
|
||||
}
|
||||
tokio::fs::remove_file(&object.path).await?;
|
||||
}
|
||||
for node in 0..clients.len() {
|
||||
|
||||
@@ -18,6 +18,13 @@
|
||||
//! `.github/workflows/e2e-distributed.yml`. Excluded from `e2e-full` because
|
||||
//! each case starts four real `rustfs` processes.
|
||||
|
||||
mod artifact_delete_test;
|
||||
mod artifact_inventory_test;
|
||||
mod artifact_put_tail_test;
|
||||
mod artifact_repair_tail_test;
|
||||
mod artifact_repair_test;
|
||||
mod artifact_seal_test;
|
||||
mod artifact_writes_test;
|
||||
mod chaos_test;
|
||||
mod concurrency_stability_test;
|
||||
mod concurrent_data_movement_test;
|
||||
|
||||
@@ -13,7 +13,8 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{
|
||||
TestResult, assert_object_bytes, cluster_admin_ok, configure_node_logs, payload_for, retrying_put, unique_bucket, wait_until,
|
||||
TestResult, assert_object_bytes, cluster_admin_ok, configure_node_logs, payload_for, retrying_put, sha256_hex, unique_bucket,
|
||||
wait_until,
|
||||
};
|
||||
use crate::common::{ClusterTopology, RustFSTestClusterEnvironment};
|
||||
use crate::rpc_fault_proxy::{RpcFaultProxy, RpcPause};
|
||||
@@ -86,12 +87,19 @@ fn create(
|
||||
tokio::spawn(async move { request.send().await })
|
||||
}
|
||||
|
||||
async fn wait_for_faults(pauses: &[RpcPause]) -> TestResult {
|
||||
// Distributed locks use adaptive fan-out and may never contact an
|
||||
// unnecessary voter. Every selected peer remains gated; require a real
|
||||
// request hit without waiting for an unused peer to exhaust the S3 budget.
|
||||
let (result, _, _) = futures::future::select_all(pauses.iter().map(|pause| Box::pin(pause.wait_for_hits(1)))).await;
|
||||
result?;
|
||||
async fn wait_for_faults(pauses: &[RpcPause], methods: &[&str]) -> TestResult {
|
||||
if methods.iter().any(|method| matches!(*method, "Lock" | "LockBatch")) {
|
||||
// Adaptive lock fan-out may not contact an unnecessary voter. All
|
||||
// selected peers remain gated; require an actual scoped request hit.
|
||||
let (result, _, _) = futures::future::select_all(pauses.iter().map(|pause| Box::pin(pause.wait_for_hits(1)))).await;
|
||||
result?;
|
||||
} else {
|
||||
// Storage mutations fan out eagerly. Every selected peer must really
|
||||
// receive this upload's RPC, so an unrelated failure cannot fake quorum.
|
||||
for pause in pauses {
|
||||
pause.wait_for_hits(1).await?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -112,39 +120,14 @@ fn assert_unavailable<E: std::fmt::Debug + ProvideErrorMetadata>(error: &SdkErro
|
||||
eprintln!("MPU acceptance: cause={cause}, retry={retry:?}, request_id={request_id}");
|
||||
}
|
||||
|
||||
async fn abort(client: &Client, bucket: &str, key: &str, upload: &str, allow_missing: bool) -> TestResult {
|
||||
// Fault retirement resets outstanding RPC streams. Probe recovery within
|
||||
// the shared deadline while channels and late lock cleanup converge.
|
||||
wait_until(
|
||||
DEADLINE,
|
||||
|| async {
|
||||
match client
|
||||
.abort_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.upload_id(upload)
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
Ok(_) => Ok(true),
|
||||
Err(error) if allow_missing && error.as_service_error().and_then(|e| e.meta().code()) == Some("NoSuchUpload") => {
|
||||
Ok(true)
|
||||
}
|
||||
Err(error)
|
||||
if error.as_service_error().and_then(|e| e.meta().code()) == Some("ServiceUnavailable")
|
||||
&& error.raw_response().is_some_and(|response| {
|
||||
response.status().as_u16() == 503
|
||||
&& response.headers().get("x-rustfs-error-cause") == Some("namespace_lock_timeout")
|
||||
}) =>
|
||||
{
|
||||
Ok(false)
|
||||
}
|
||||
Err(error) => Err(error.into()),
|
||||
}
|
||||
},
|
||||
"multipart abort after RPC fault retirement",
|
||||
)
|
||||
.await?;
|
||||
async fn abort(client: &Client, bucket: &str, key: &str, upload: &str) -> TestResult {
|
||||
client
|
||||
.abort_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.upload_id(upload)
|
||||
.send()
|
||||
.await?;
|
||||
let missing = client
|
||||
.list_parts()
|
||||
.bucket(bucket)
|
||||
@@ -157,6 +140,88 @@ async fn abort(client: &Client, bucket: &str, key: &str, upload: &str, allow_mis
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Fault removal does not synchronously clear existing server leases or drive
|
||||
// health state. Keep the injected operation single-attempt, then require
|
||||
// bounded recovery of the exact cleanup API instead of assuming instant heal.
|
||||
async fn recover_after_fault<T, E, F, Fut>(operation: &str, mut call: F) -> TestResult<Result<T, SdkError<E>>>
|
||||
where
|
||||
E: std::fmt::Debug + ProvideErrorMetadata,
|
||||
F: FnMut() -> Fut,
|
||||
Fut: std::future::Future<Output = Result<T, SdkError<E>>>,
|
||||
{
|
||||
let started = Instant::now();
|
||||
let mut attempts = 0;
|
||||
let mut last_failure = String::new();
|
||||
let outcome = tokio::time::timeout(DEADLINE, async {
|
||||
loop {
|
||||
attempts += 1;
|
||||
let result = call().await;
|
||||
let retry = result.as_ref().err().and_then(|error| {
|
||||
let response = error.raw_response()?;
|
||||
if response.status().as_u16() != 503
|
||||
|| error.as_service_error()?.meta().code() != Some("ServiceUnavailable")
|
||||
|| !matches!(
|
||||
response.headers().get("x-rustfs-error-cause"),
|
||||
Some(
|
||||
"namespace_lock_timeout"
|
||||
| "namespace_lock_unavailable"
|
||||
| "storage_write_quorum"
|
||||
| "storage_read_quorum"
|
||||
)
|
||||
)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let delay = match response.headers().get("retry-after") {
|
||||
Some("1") => Duration::from_secs(1),
|
||||
None => Duration::from_millis(100),
|
||||
Some(_) => return None,
|
||||
};
|
||||
last_failure = format!(
|
||||
"cause={:?}, request_id={:?}",
|
||||
response.headers().get("x-rustfs-error-cause"),
|
||||
response.headers().get("x-amz-request-id")
|
||||
);
|
||||
Some(delay)
|
||||
});
|
||||
match retry {
|
||||
Some(delay) => tokio::time::sleep(delay).await,
|
||||
None => return result,
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
eprintln!(
|
||||
"MPU recovery: operation={operation}, attempts={attempts}, elapsed={:?}, last={last_failure}",
|
||||
started.elapsed()
|
||||
);
|
||||
outcome.map_err(|_| format!("{operation} did not recover within {DEADLINE:?}: {last_failure}").into())
|
||||
}
|
||||
|
||||
async fn abort_after_fault(client: &Client, bucket: &str, key: &str, upload: &str, allow_missing: bool) -> TestResult {
|
||||
match recover_after_fault(&format!("Abort {key}"), || {
|
||||
client
|
||||
.abort_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.upload_id(upload)
|
||||
.send()
|
||||
})
|
||||
.await?
|
||||
{
|
||||
Ok(_) => {}
|
||||
Err(error) if allow_missing && error.as_service_error().and_then(|e| e.meta().code()) == Some("NoSuchUpload") => {}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
let missing = recover_after_fault(&format!("ListParts {key}"), || {
|
||||
client.list_parts().bucket(bucket).key(key).upload_id(upload).send()
|
||||
})
|
||||
.await?
|
||||
.expect_err("a recovered Abort must remove the upload");
|
||||
assert_eq!(missing.as_service_error().and_then(|e| e.meta().code()), Some("NoSuchUpload"));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn round_trip(clients: &[Client], bucket: &str, key: &str, size: usize) -> TestResult {
|
||||
let upload = clients[0].create_multipart_upload().bucket(bucket).key(key).send().await?;
|
||||
let upload_id = upload.upload_id().ok_or("missing upload ID")?;
|
||||
@@ -207,6 +272,31 @@ async fn assert_no_uploads(clients: &[Client], bucket: &str) -> TestResult {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_no_uploads_after_fault(clients: &[Client], bucket: &str) -> TestResult {
|
||||
for client in clients {
|
||||
let uploads = recover_after_fault("ListMultipartUploads after fault", || {
|
||||
client.list_multipart_uploads().bucket(bucket).send()
|
||||
})
|
||||
.await??;
|
||||
assert!(uploads.uploads().is_empty(), "unexpected residual uploads: {:?}", uploads.uploads());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_object_bytes_after_fault(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
|
||||
let output = recover_after_fault(&format!("GetObject {key} after fault"), || {
|
||||
client.get_object().bucket(bucket).key(key).send()
|
||||
})
|
||||
.await??;
|
||||
// A successful response must have a complete, correct body. Streaming
|
||||
// failures are not retried or converted into a successful comparison.
|
||||
let bytes = output.body.collect().await?.into_bytes();
|
||||
if bytes.as_ref() != expected {
|
||||
return Err(format!("object {key} bytes differ: expected {} bytes, got {}", expected.len(), bytes.len()).into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// This fixture inspects RustFS RPCs, so use the server-generated upload's
|
||||
// canonical suffix and fail explicitly if its encoding changes.
|
||||
fn upload_scope(upload_id: &str) -> TestResult<String> {
|
||||
@@ -218,6 +308,14 @@ fn upload_scope(upload_id: &str) -> TestResult<String> {
|
||||
Ok(suffix.to_owned())
|
||||
}
|
||||
|
||||
fn upload_path_scope(bucket: &str, key: &str, upload_id: &str) -> TestResult<String> {
|
||||
Ok(format!(
|
||||
"{}/{}",
|
||||
sha256_hex(format!("{bucket}/{key}").as_bytes()),
|
||||
upload_scope(upload_id)?
|
||||
))
|
||||
}
|
||||
|
||||
async fn with_paused_peers<T: Send + 'static>(
|
||||
proxies: &[RpcFaultProxy],
|
||||
methods: &[&'static str],
|
||||
@@ -231,7 +329,7 @@ async fn with_paused_peers<T: Send + 'static>(
|
||||
.map(|(proxy, limit)| proxy.pause(methods, *limit, Some(scope)))
|
||||
.collect::<Result<_, _>>()?;
|
||||
let task = tokio::spawn(request);
|
||||
wait_for_faults(&pauses).await?;
|
||||
wait_for_faults(&pauses, methods).await?;
|
||||
let result = tokio::time::timeout(DEADLINE, task).await??;
|
||||
for pause in &pauses {
|
||||
pause.discard_pending();
|
||||
@@ -284,6 +382,42 @@ async fn wait_for_lock_peer_recovery(proxies: &[RpcFaultProxy], client: &Client,
|
||||
.await
|
||||
}
|
||||
|
||||
async fn wait_for_storage_peer_recovery(
|
||||
proxies: &[RpcFaultProxy],
|
||||
client: &Client,
|
||||
bucket: &str,
|
||||
upload: &str,
|
||||
drives_per_node: usize,
|
||||
) -> TestResult {
|
||||
let scope = upload_scope(upload)?;
|
||||
tokio::time::timeout(
|
||||
DEADLINE,
|
||||
wait_until(
|
||||
DEADLINE,
|
||||
|| async {
|
||||
let observers = proxies[1..]
|
||||
.iter()
|
||||
.map(|proxy| proxy.pause(&["ReadVersion"], usize::MAX, Some(&scope)))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
for observer in &observers {
|
||||
observer.release();
|
||||
}
|
||||
// A fresh absent key forces a full metadata read without mutating
|
||||
// this upload. Count distinct configured disk routes, not retries
|
||||
// from the same healthy disk or just lock-RPC reachability.
|
||||
let key = format!("storage-ready-{}", uuid::Uuid::new_v4());
|
||||
let _ = client.list_parts().bucket(bucket).key(key).upload_id(upload).send().await;
|
||||
Ok(observers
|
||||
.iter()
|
||||
.all(|observer| observer.forwarded_routes() >= drives_per_node))
|
||||
},
|
||||
"all storage disk routes recover after fault retirement",
|
||||
),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| "storage route recovery deadline exceeded".to_string())?
|
||||
}
|
||||
|
||||
async fn phase_quorum_matrix(
|
||||
proxies: &[RpcFaultProxy],
|
||||
clients: &[Client],
|
||||
@@ -324,13 +458,25 @@ async fn phase_quorum_matrix(
|
||||
let request = clients[0].create_multipart_upload().bucket(bucket).key(&key);
|
||||
let result = with_paused_peers(proxies, methods, &limits, bucket, async move { request.send().await }).await?;
|
||||
if let Some(upload) = quorum_outcome(result, unavailable == unavailable_at_quorum, locks)? {
|
||||
abort(&clients[0], bucket, &key, upload.upload_id().ok_or("missing upload ID")?, false).await?;
|
||||
let upload_id = upload.upload_id().ok_or("missing upload ID")?;
|
||||
wait_for_storage_peer_recovery(proxies, &clients[0], bucket, upload_id, drives_per_node).await?;
|
||||
abort_after_fault(&clients[0], bucket, &key, upload_id, false).await?;
|
||||
}
|
||||
} else {
|
||||
let upload = clients[0].create_multipart_upload().bucket(bucket).key(&key).send().await?;
|
||||
let upload = clients[0]
|
||||
.create_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(&key)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| format!("setup Create failed for {key}: {error:?}"))?;
|
||||
let upload_id = upload.upload_id().ok_or("missing upload ID")?;
|
||||
let scope = upload_scope(upload_id)?;
|
||||
let body = payload_for(&key, 1024 * 1024);
|
||||
let scope = if !locks && phase == "abort" {
|
||||
upload_path_scope(bucket, &key, upload_id)?
|
||||
} else {
|
||||
upload_scope(upload_id)?
|
||||
};
|
||||
let body = payload_for(&key, if drives_per_node == 4 { 9 * 1024 * 1024 } else { 1024 * 1024 });
|
||||
let initial = if phase != "part" {
|
||||
Some(
|
||||
clients[0]
|
||||
@@ -358,7 +504,7 @@ async fn phase_quorum_matrix(
|
||||
let result =
|
||||
with_paused_peers(proxies, methods, &limits, &scope, async move { request.send().await }).await?;
|
||||
if let Some(part) = quorum_outcome(result, unavailable == unavailable_at_quorum, locks)? {
|
||||
clients[0]
|
||||
let completion = clients[0]
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(&key)
|
||||
@@ -372,11 +518,11 @@ async fn phase_quorum_matrix(
|
||||
.build(),
|
||||
)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
);
|
||||
recover_after_fault(&format!("Complete {key} after part fault"), || completion.clone().send())
|
||||
.await??;
|
||||
for reader in clients {
|
||||
assert_object_bytes(reader, bucket, &key, &body).await?;
|
||||
assert_object_bytes_after_fault(reader, bucket, &key, &body).await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -401,7 +547,7 @@ async fn phase_quorum_matrix(
|
||||
with_paused_peers(proxies, methods, &limits, &scope, async move { request.send().await }).await?;
|
||||
if quorum_outcome(result, unavailable == unavailable_at_quorum, locks)?.is_some() {
|
||||
for reader in clients {
|
||||
assert_object_bytes(reader, bucket, &key, &body).await?;
|
||||
assert_object_bytes_after_fault(reader, bucket, &key, &body).await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -417,14 +563,19 @@ async fn phase_quorum_matrix(
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
// A failed completion can be ambiguous to an S3 client.
|
||||
// Cleanup also accepts an already completed/aborted session.
|
||||
abort(&clients[0], bucket, &key, upload_id, true).await?;
|
||||
clients[0].delete_object().bucket(bucket).key(&key).send().await?;
|
||||
wait_for_storage_peer_recovery(proxies, &clients[0], bucket, upload_id, drives_per_node).await?;
|
||||
// A rejected Complete must retain authority for ordinary
|
||||
// Abort. NoSuchUpload must not hide partial staging loss.
|
||||
let allow_missing = phase != "complete" || unavailable == unavailable_at_quorum;
|
||||
abort_after_fault(&clients[0], bucket, &key, upload_id, allow_missing).await?;
|
||||
recover_after_fault(&format!("DeleteObject {key} after {phase}"), || {
|
||||
clients[0].delete_object().bucket(bucket).key(&key).send()
|
||||
})
|
||||
.await??;
|
||||
}
|
||||
wait_for_lock_peer_recovery(proxies, &clients[0], bucket).await?;
|
||||
round_trip(clients, bucket, "recovered", 9 * 1024 * 1024).await?;
|
||||
assert_no_uploads(clients, bucket).await?;
|
||||
assert_no_uploads_after_fault(clients, bucket).await?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
@@ -463,7 +614,7 @@ async fn cancelled_metadata_wait_releases_fence_for_bucket_recreation() -> TestR
|
||||
.map(|proxy| proxy.pause(&["WriteMetadata"], usize::MAX, Some(&bucket)))
|
||||
.collect::<Result<_, _>>()?;
|
||||
let task = create(&clients[0], &bucket, "cancelled");
|
||||
wait_for_faults(&pauses).await?;
|
||||
wait_for_faults(&pauses, &["WriteMetadata"]).await?;
|
||||
task.abort();
|
||||
assert!(task.await.expect_err("cancelled S3 future").is_cancelled());
|
||||
drop(pauses);
|
||||
@@ -547,7 +698,7 @@ async fn eight_node_eight_drive_multipart_load_leaves_no_residual_uploads() -> T
|
||||
.body(ByteStream::from(vec![0xa5; 8 * 1024 * 1024]))
|
||||
.send()
|
||||
.await?;
|
||||
abort(&clients[(round + 2) % clients.len()], &bucket, "aborted", upload_id, false).await?;
|
||||
abort(&clients[(round + 2) % clients.len()], &bucket, "aborted", upload_id).await?;
|
||||
assert_no_uploads(&clients, &bucket).await?;
|
||||
eprintln!(
|
||||
"MPU 8x8 acceptance: round={}/{rounds}, completed={}, aborted={}, elapsed={:?}",
|
||||
@@ -572,7 +723,7 @@ async fn resume_at_rpc<T: Send + 'static>(
|
||||
.collect::<Result<_, _>>()?;
|
||||
eprintln!("MPU delay acceptance: methods={methods:?}, scope={scope}");
|
||||
let task = tokio::spawn(request);
|
||||
wait_for_faults(&pauses).await?;
|
||||
wait_for_faults(&pauses, methods).await?;
|
||||
assert!(
|
||||
!task.is_finished(),
|
||||
"the selected commit must wait while only one shard/voter is available"
|
||||
@@ -649,6 +800,11 @@ async fn each_multipart_phase_resumes_after_metadata_and_lock_delays() -> TestRe
|
||||
let upload = client.create_multipart_upload().bucket(&bucket).key(&key).send().await?;
|
||||
let upload_id = upload.upload_id().ok_or("missing upload ID")?;
|
||||
let scope = upload_scope(upload_id)?;
|
||||
let scope = if locks {
|
||||
scope
|
||||
} else {
|
||||
upload_path_scope(&bucket, &key, upload_id)?
|
||||
};
|
||||
let request = client.abort_multipart_upload().bucket(&bucket).key(&key).upload_id(upload_id);
|
||||
resume_at_rpc(&proxies, if locks { &["Lock", "LockBatch"] } else { &["Delete"] }, &scope, async move {
|
||||
Ok(request.send().await?)
|
||||
|
||||
@@ -1206,7 +1206,9 @@ mod tests {
|
||||
let interval = interval
|
||||
.filter(|interval| *interval != 0)
|
||||
.ok_or("scanner schedule has no positive interval")?;
|
||||
let next_cycle = cycle_end
|
||||
// A new leader may already be sleeping before completing its first cycle.
|
||||
let schedule_start = if cycle_end == 0 { now } else { cycle_end };
|
||||
let next_cycle = schedule_start
|
||||
.checked_add(interval)
|
||||
.ok_or("scanner schedule timestamp overflows")?;
|
||||
let budget = next_cycle
|
||||
@@ -1223,6 +1225,11 @@ mod tests {
|
||||
Ok(Duration::from_secs(259)),
|
||||
);
|
||||
assert_eq!(scanner_cycle_completion_budget(100, Some(1), 110, false), Ok(Duration::from_secs(60)));
|
||||
// A newly elected leader can advertise a sleep before its first completed cycle.
|
||||
assert_eq!(
|
||||
scanner_cycle_completion_budget(0, Some(289), 1_792_913_211, false),
|
||||
Ok(Duration::from_secs(349)),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1243,7 +1250,7 @@ mod tests {
|
||||
previous_cycle_end: u64,
|
||||
) -> Result<u64, Box<dyn Error + Send + Sync>> {
|
||||
let mut deadline = Instant::now() + Duration::from_secs(60);
|
||||
let mut schedule_accounted = false;
|
||||
let mut accounted_schedules = HashSet::new();
|
||||
loop {
|
||||
let mut latest_cycle_end = 0;
|
||||
let mut versions_observed = false;
|
||||
@@ -1290,15 +1297,27 @@ mod tests {
|
||||
return Err("scanner cycle timestamp is in the future".into());
|
||||
}
|
||||
let schedule = &status["cycle_schedule"];
|
||||
if !schedule_accounted && schedule["execution_role"].as_str() == Some("leader") {
|
||||
if schedule["execution_role"].as_str() == Some("leader") {
|
||||
let active = metrics["current_cycle_active"]
|
||||
.as_bool()
|
||||
.ok_or("scanner status is missing its active-cycle flag")?;
|
||||
if active || (cycle_end != 0 && schedule["effective_interval_available"].as_bool() == Some(true)) {
|
||||
let interval = schedule["effective_interval_seconds"].as_u64();
|
||||
let budget = scanner_cycle_completion_budget(cycle_end, interval, now, active)?;
|
||||
deadline = Instant::now().checked_add(budget).ok_or("scanner deadline overflows")?;
|
||||
schedule_accounted = true;
|
||||
if active || schedule["effective_interval_available"].as_bool() == Some(true) {
|
||||
let interval = if active {
|
||||
None
|
||||
} else {
|
||||
Some(
|
||||
schedule["effective_interval_seconds"]
|
||||
.as_u64()
|
||||
.ok_or("scanner schedule is missing its effective interval")?,
|
||||
)
|
||||
};
|
||||
// Reconsider a changed leader or retry plan, but never renew
|
||||
// the deadline on repeated observations of the same sleep.
|
||||
if accounted_schedules.insert((node_index, cycle_end, interval)) {
|
||||
let budget = scanner_cycle_completion_budget(cycle_end, interval, now, active)?;
|
||||
let scheduled_deadline = Instant::now().checked_add(budget).ok_or("scanner deadline overflows")?;
|
||||
deadline = deadline.max(scheduled_deadline);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Superseded walks complete all sets while withholding quota authority.
|
||||
|
||||
@@ -541,6 +541,50 @@ impl NodeService for MinimalLockNodeService {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn artifact_rename_data(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::RenameDataRequest>,
|
||||
) -> Result<Response<rustfs_protos::proto_gen::node_service::RenameDataResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn artifact_repair_rename_data(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::ArtifactRepairRenameDataRequest>,
|
||||
) -> Result<Response<rustfs_protos::proto_gen::node_service::RenameDataResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn artifact_repair_write_metadata(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::ArtifactRepairWriteMetadataRequest>,
|
||||
) -> Result<Response<rustfs_protos::proto_gen::node_service::WriteMetadataResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
type InventoryArtifactAttemptStream =
|
||||
Pin<Box<dyn Stream<Item = Result<rustfs_protos::proto_gen::node_service::ArtifactInventoryFrame, Status>> + Send>>;
|
||||
async fn inventory_artifact_attempt(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::ArtifactInventoryRequest>,
|
||||
) -> Result<Response<Self::InventoryArtifactAttemptStream>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn seal_artifact_attempt_repairs(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsRequest>,
|
||||
) -> Result<Response<rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn seal_artifact_attempt_puts(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsRequest>,
|
||||
) -> Result<Response<rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn rename_data(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::RenameDataRequest>,
|
||||
|
||||
@@ -22,7 +22,10 @@ use hyper_util::{
|
||||
rt::{TokioExecutor, TokioIo},
|
||||
server::conn::auto,
|
||||
};
|
||||
use prost::Message as _;
|
||||
use rustfs_protos::proto_gen::node_service::{DeleteRequest, ReadVersionRequest};
|
||||
use std::{
|
||||
collections::HashSet,
|
||||
io,
|
||||
net::SocketAddr,
|
||||
sync::{
|
||||
@@ -54,6 +57,7 @@ struct PauseState {
|
||||
matched: AtomicUsize,
|
||||
blocked: AtomicUsize,
|
||||
forwarded: AtomicUsize,
|
||||
forwarded_routes: Mutex<HashSet<String>>,
|
||||
reached: Notify,
|
||||
release: watch::Sender<Disposition>,
|
||||
}
|
||||
@@ -75,6 +79,10 @@ impl RpcPause {
|
||||
self.state.forwarded.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub(crate) fn forwarded_routes(&self) -> usize {
|
||||
self.state.forwarded_routes.lock().expect("RPC route observations").len()
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_for_hits(&self, count: usize) -> io::Result<()> {
|
||||
tokio::time::timeout(Duration::from_secs(20), async {
|
||||
loop {
|
||||
@@ -184,6 +192,7 @@ impl RpcFaultProxy {
|
||||
matched: AtomicUsize::new(0),
|
||||
blocked: AtomicUsize::new(0),
|
||||
forwarded: AtomicUsize::new(0),
|
||||
forwarded_routes: Mutex::new(HashSet::new()),
|
||||
reached: Notify::new(),
|
||||
release,
|
||||
});
|
||||
@@ -195,6 +204,29 @@ impl RpcFaultProxy {
|
||||
}
|
||||
}
|
||||
|
||||
fn unary_payload(body: &[u8]) -> io::Result<&[u8]> {
|
||||
if body.len() < 5 || body[0] != 0 {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "scoped RPC needs an uncompressed unary frame"));
|
||||
}
|
||||
let mut encoded_length = [0; 4];
|
||||
encoded_length.copy_from_slice(&body[1..5]);
|
||||
if u32::try_from(body.len() - 5).ok() != Some(u32::from_be_bytes(encoded_length)) {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "scoped RPC frame length mismatch"));
|
||||
}
|
||||
Ok(&body[5..])
|
||||
}
|
||||
|
||||
fn scope_matches(method: &str, body: &[u8], scope: &[u8]) -> io::Result<bool> {
|
||||
if method != "Delete" {
|
||||
return Ok(body.windows(scope.len()).any(|window| window == scope));
|
||||
}
|
||||
let request = DeleteRequest::decode(unary_payload(body)?).map_err(io::Error::other)?;
|
||||
let options: serde_json::Value = serde_json::from_str(&request.options).map_err(io::Error::other)?;
|
||||
Ok(request.volume == ".rustfs.sys/multipart"
|
||||
&& request.path.trim_end_matches('/').as_bytes() == scope
|
||||
&& options.get("recursive").and_then(serde_json::Value::as_bool) == Some(true))
|
||||
}
|
||||
|
||||
async fn forward_request(
|
||||
request: Request<Incoming>,
|
||||
client: ProxyClient,
|
||||
@@ -206,14 +238,23 @@ async fn forward_request(
|
||||
let mut observed = None;
|
||||
if let Some(pause) = pause {
|
||||
let method = parts.uri.path().strip_prefix("/node_service.NodeService/");
|
||||
if method.is_some_and(|name| pause.methods.contains(&name)) {
|
||||
if let Some(method) = method.filter(|name| pause.methods.contains(name)) {
|
||||
let mut route = None;
|
||||
let matches = if let Some(scope) = pause.scope.as_deref() {
|
||||
// Scope the control packet to this bucket or upload without
|
||||
// decoding or changing bytes covered by the RPC signature.
|
||||
// Inspect bounded control routing while preserving the exact
|
||||
// signed bytes and trailers forwarded to the backend.
|
||||
let collected = Limited::new(body, 1024 * 1024).collect().await.map_err(io::Error::other)?;
|
||||
let trailers = collected.trailers().cloned();
|
||||
let bytes = collected.to_bytes();
|
||||
let matches = bytes.windows(scope.len()).any(|window| window == scope);
|
||||
let matches = scope_matches(method, &bytes, scope)?;
|
||||
if matches && method == "ReadVersion" {
|
||||
let disk = ReadVersionRequest::decode(unary_payload(&bytes)?)
|
||||
.map_err(io::Error::other)?
|
||||
.disk;
|
||||
if !disk.is_empty() {
|
||||
route = Some(disk);
|
||||
}
|
||||
}
|
||||
body = Full::new(bytes)
|
||||
.map_err(|never| match never {})
|
||||
.with_trailers(async move { trailers.map(Ok::<_, io::Error>) })
|
||||
@@ -238,7 +279,7 @@ async fn forward_request(
|
||||
if disposition == Disposition::Discard {
|
||||
return Err(io::Error::new(io::ErrorKind::ConnectionAborted, "injected RPC packet loss"));
|
||||
}
|
||||
observed = Some(pause);
|
||||
observed = Some((pause, route));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -260,11 +301,14 @@ async fn forward_request(
|
||||
.request(Request::from_parts(parts, body))
|
||||
.await
|
||||
.map_err(io::Error::other)?;
|
||||
if let Some(pause) = observed
|
||||
if let Some((pause, route)) = observed
|
||||
&& response.status().is_success()
|
||||
&& response.headers().get("grpc-status").is_none_or(|status| status == "0")
|
||||
{
|
||||
pause.forwarded.fetch_add(1, Ordering::SeqCst);
|
||||
if let Some(route) = route {
|
||||
pause.forwarded_routes.lock().expect("RPC route observations").insert(route);
|
||||
}
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
@@ -461,6 +505,30 @@ mod tests {
|
||||
.expect("observer forwards without pausing");
|
||||
another.into_body().collect().await.expect("second body");
|
||||
assert_eq!(observer.forwarded(), 2);
|
||||
drop(observer);
|
||||
let routes = proxy
|
||||
.pause(&["ReadVersion"], usize::MAX, Some("recovery-route"))
|
||||
.expect("disk-route observer");
|
||||
routes.release();
|
||||
for (disk, expected) in [("drive-a", 1), ("drive-a", 1), ("drive-b", 2)] {
|
||||
let request = ReadVersionRequest {
|
||||
disk: disk.to_owned(),
|
||||
path: "recovery-route".to_owned(),
|
||||
..Default::default()
|
||||
}
|
||||
.encode_to_vec();
|
||||
let mut frame = vec![0];
|
||||
frame.extend_from_slice(&u32::try_from(request.len()).expect("small frame").to_be_bytes());
|
||||
frame.extend_from_slice(&request);
|
||||
call_body(&client, address, "ReadVersion", Bytes::from(frame))
|
||||
.await
|
||||
.expect("routed read")
|
||||
.into_body()
|
||||
.collect()
|
||||
.await
|
||||
.expect("routed response");
|
||||
assert_eq!(routes.forwarded_routes(), expected, "repeated traffic from one disk is only one voter");
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("passive observer deadline");
|
||||
@@ -504,4 +572,43 @@ mod tests {
|
||||
.await
|
||||
.expect("scoped discard deadline");
|
||||
}
|
||||
#[test]
|
||||
fn delete_scope_matches_only_the_upload_root() {
|
||||
let frame = |volume: &str, path: &str| {
|
||||
let message = DeleteRequest {
|
||||
volume: volume.to_owned(),
|
||||
path: path.to_owned(),
|
||||
options: r#"{"recursive":true}"#.to_owned(),
|
||||
..Default::default()
|
||||
};
|
||||
let payload = message.encode_to_vec();
|
||||
let length = u32::try_from(payload.len()).expect("small control packet");
|
||||
let mut bytes = vec![0];
|
||||
bytes.extend_from_slice(&length.to_be_bytes());
|
||||
bytes.extend_from_slice(&payload);
|
||||
bytes
|
||||
};
|
||||
let root = b"hash/upload";
|
||||
assert!(scope_matches("Delete", &frame(".rustfs.sys/multipart", "hash/upload"), root).expect("root frame"));
|
||||
assert!(scope_matches("Delete", &frame(".rustfs.sys/multipart", "hash/upload/"), root).expect("directory frame"));
|
||||
for path in ["hash/upload/part.1", "hash/upload/.abort-authority", "hash/upload-other"] {
|
||||
assert!(!scope_matches("Delete", &frame(".rustfs.sys/multipart", path), root).expect("other path"));
|
||||
}
|
||||
assert!(!scope_matches("Delete", &frame(".rustfs.sys/tmp", "hash/upload"), root).expect("other volume"));
|
||||
let non_recursive = DeleteRequest {
|
||||
volume: ".rustfs.sys/multipart".to_owned(),
|
||||
path: "hash/upload".to_owned(),
|
||||
options: r#"{"recursive":false}"#.to_owned(),
|
||||
..Default::default()
|
||||
}
|
||||
.encode_to_vec();
|
||||
let mut non_recursive_frame = vec![0];
|
||||
non_recursive_frame.extend_from_slice(&u32::try_from(non_recursive.len()).expect("small frame").to_be_bytes());
|
||||
non_recursive_frame.extend_from_slice(&non_recursive);
|
||||
assert!(!scope_matches("Delete", &non_recursive_frame, root).expect("non-recursive frame"));
|
||||
assert!(
|
||||
scope_matches("Delete", &[0, 0, 0, 0, 1], root).is_err(),
|
||||
"truncation must not become a match"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,12 +15,14 @@
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::disk::{DiskOption, Endpoint, RUSTFS_META_BUCKET, VolumeInfo, WalkDirOptions, new_disk};
|
||||
pub(crate) use rustfs_ecstore::api::disk::{
|
||||
DiskError, DiskOption, Endpoint, RUSTFS_META_BUCKET, VolumeInfo, WalkDirOptions, new_disk,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{
|
||||
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
||||
verify_tonic_boot_epoch_response,
|
||||
set_tonic_canonical_body_digest, verify_tonic_boot_epoch_response,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{gen_tonic_signature_interceptor, node_service_time_out_client};
|
||||
@@ -36,6 +38,11 @@ pub(crate) mod grpc_lock {
|
||||
pub(crate) use super::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod artifact_rpc {
|
||||
pub(crate) use super::{DiskError, TONIC_RPC_PREFIX, gen_tonic_signature_headers, set_tonic_canonical_body_digest};
|
||||
}
|
||||
|
||||
/// Signing/transport surface used by the cross-process internode RPC signature
|
||||
/// acceptance tests (backlog#1327). The signing helpers are what let a test mint
|
||||
/// the one legitimately signed request an on-path attacker is assumed to have
|
||||
|
||||
@@ -382,6 +382,43 @@ pub mod config {
|
||||
with_config_object_read_lock, with_config_object_write_lock, with_server_config_read_lock,
|
||||
with_server_config_write_lock,
|
||||
};
|
||||
|
||||
/// Observe persisted config bytes and their reader-associated metadata within a byte limit.
|
||||
pub async fn read_config_observation(
|
||||
api: std::sync::Arc<crate::store::ECStore>,
|
||||
file: &str,
|
||||
max_bytes: usize,
|
||||
) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
|
||||
crate::config::com::read_config_observation(api, file, max_bytes).await
|
||||
}
|
||||
|
||||
/// Observe bounded config bytes while the caller retains the object's write owner.
|
||||
pub async fn read_config_observation_no_lock(
|
||||
api: std::sync::Arc<crate::store::ECStore>,
|
||||
file: &str,
|
||||
max_bytes: usize,
|
||||
guard: &crate::object_api::WriteCommitGuard,
|
||||
) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
|
||||
crate::config::com::read_config_observation_no_lock(api, file, max_bytes, guard).await
|
||||
}
|
||||
|
||||
/// Read config bytes and metadata without treating an empty payload as absent.
|
||||
pub async fn read_config_preserve_empty_with_metadata_opts(
|
||||
api: std::sync::Arc<crate::store::ECStore>,
|
||||
file: &str,
|
||||
opts: &crate::object_api::ObjectOptions,
|
||||
) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
|
||||
crate::config::com::read_config_preserve_empty_with_metadata_opts(api, file, opts).await
|
||||
}
|
||||
|
||||
pub async fn read_config_limited_preserve_empty_with_metadata_opts(
|
||||
api: std::sync::Arc<crate::store::ECStore>,
|
||||
file: &str,
|
||||
opts: &crate::object_api::ObjectOptions,
|
||||
max_bytes: usize,
|
||||
) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
|
||||
crate::config::com::read_config_limited_preserve_empty_with_metadata_opts(api, file, opts, max_bytes).await
|
||||
}
|
||||
}
|
||||
|
||||
pub mod storageclass {
|
||||
@@ -431,16 +468,19 @@ pub mod disk {
|
||||
pub use crate::disk::local::{ReplacementExecutionLease, ScanGuard};
|
||||
#[cfg(all(feature = "test-util", not(windows)))]
|
||||
pub use crate::disk::os::{LocalPublicationPause, LocalPublicationStage};
|
||||
pub use crate::disk::{ArtifactInventoryFrame, ArtifactInventoryVersion};
|
||||
pub use crate::disk::{
|
||||
BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
|
||||
CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration, CurrentDeleteTransition,
|
||||
DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest,
|
||||
DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DeleteRecoveryState,
|
||||
Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore, FileInfoVersions, FileReader, FileWriter,
|
||||
GuardedDeleteVersionsPayload, GuardedFileInfoVersions, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize,
|
||||
PartTransactionAction, QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp,
|
||||
ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
|
||||
is_quota_mutation_fence_path, new_disk, validate_batch_read_version_item_count,
|
||||
GuardedDeleteVersionsPayload, GuardedFileInfoVersions, HEALING_MARKER_PATH, LegacyFileStream, LegacyFileWrite,
|
||||
LegacyMetadataWrite, LegacyPartMutation, LegacyPartOperation, LegacyPathDelete, LegacyPathDeleteOperation,
|
||||
LegacyRenameData, LegacyRenameFile, NsScannerOpenRequest, OldCurrentSize, PartTransactionAction,
|
||||
QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
|
||||
STORAGE_FORMAT_FILE, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, is_quota_mutation_fence_path,
|
||||
new_disk, validate_batch_read_version_item_count,
|
||||
};
|
||||
pub use bytes::Bytes;
|
||||
pub use endpoint::Endpoint;
|
||||
@@ -635,6 +675,7 @@ pub mod store_list {
|
||||
}
|
||||
|
||||
pub mod storage {
|
||||
pub use crate::bucket::artifact::inventory::ArtifactAttemptInventoryFrame;
|
||||
pub use crate::core::pools::{HealLifecycleExpiryContext, POOL_META_NAME};
|
||||
pub use crate::store::HealWalkVersion;
|
||||
pub use crate::store::{
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,120 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use super::*;
|
||||
use crate::bucket::metadata::BUCKET_INCARNATION_FILE;
|
||||
use crate::config::com::read_config_observation;
|
||||
use crate::object_api::WriteCompletion;
|
||||
|
||||
impl ECStore {
|
||||
/// Permanently retain this managed bucket's versions for historical restores.
|
||||
/// Lifecycle WRITE drains earlier exact deletes. The detached owner keeps
|
||||
/// lifecycle -> metadata transaction locks until every metadata write drains.
|
||||
pub async fn retain_artifact_backups(self: Arc<Self>, bucket: &str, incarnation: Uuid, base_prefix: &str) -> Result<()> {
|
||||
let bucket = bucket.to_owned();
|
||||
let base_prefix = base_prefix.to_owned();
|
||||
let ctx = Arc::clone(&self.ctx);
|
||||
ctx.run_detached_mutation(async move {
|
||||
require_put_seal_topology(&self).await?;
|
||||
if self.ctx.lock_manager().is_disabled() {
|
||||
return Err(Error::other("artifact backup retention requires namespace locking"));
|
||||
}
|
||||
let lifecycle = self.acquire_bucket_lifecycle_write_lock(&bucket).await?;
|
||||
let transaction = metadata_sys::acquire_bucket_metadata_transaction_lock_in(&self.ctx, &bucket).await?;
|
||||
let mut metadata = self.backup_retention_metadata(&bucket, incarnation, &base_prefix).await?;
|
||||
let check = || {
|
||||
if lifecycle.is_lock_lost() || transaction.is_lock_lost() {
|
||||
Err(Error::other("artifact backup retention fence was lost"))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
};
|
||||
check()?;
|
||||
// Even an idempotent retry rewrites with TailDrained: an earlier
|
||||
// interrupted call may have reached only some metadata replicas.
|
||||
metadata.artifact_backup_retained = true;
|
||||
let mut opts = ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
};
|
||||
opts.add_bucket_lifecycle_lock_guard(&lifecycle);
|
||||
opts.add_namespace_lock_guard(&transaction);
|
||||
metadata.save_with_store_options(self.clone(), &opts).await?;
|
||||
check()?;
|
||||
let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
|
||||
let sys = sys.read().await.clone();
|
||||
sys.set_validated(metadata).await?;
|
||||
check()
|
||||
})
|
||||
.await
|
||||
.map_err(|_| Error::other("artifact backup retention owner failed"))?
|
||||
}
|
||||
|
||||
/// Observe the immutable bucket identity and retention in one locked read.
|
||||
pub async fn inspect_artifact_backup_retention(&self, bucket: &str, incarnation: Uuid, base_prefix: &str) -> Result<bool> {
|
||||
require_put_seal_topology(self).await?;
|
||||
if self.ctx.lock_manager().is_disabled() {
|
||||
return Err(Error::other("artifact backup retention requires namespace locking"));
|
||||
}
|
||||
let lifecycle = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
|
||||
let metadata = self.backup_retention_metadata(bucket, incarnation, base_prefix).await?;
|
||||
if lifecycle.is_lock_lost() {
|
||||
return Err(Error::other("artifact backup retention observation fence was lost"));
|
||||
}
|
||||
Ok(metadata.artifact_backup_retained)
|
||||
}
|
||||
|
||||
async fn backup_retention_metadata(&self, bucket: &str, incarnation: Uuid, base_prefix: &str) -> Result<BucketMetadata> {
|
||||
if incarnation.is_nil() || !crate::storage_api_contracts::bucket::valid_artifact_base_prefix(base_prefix) {
|
||||
return Err(Error::other("artifact backup retention identity is invalid"));
|
||||
}
|
||||
let mut metadata = self.read_backup_retention_metadata(bucket).await?;
|
||||
let store = metadata_sys::object_store_in(&self.ctx).await?;
|
||||
let (sidecar, _) =
|
||||
read_config_observation(store, &format!("{BUCKET_META_PREFIX}/{bucket}/{BUCKET_INCARNATION_FILE}"), 16).await?;
|
||||
metadata.parse_all_configs()?;
|
||||
if metadata.bucket_incarnation_id != incarnation
|
||||
|| Uuid::from_slice(&sidecar).ok() != Some(incarnation)
|
||||
|| metadata.artifact_base_prefix.as_deref() != Some(base_prefix)
|
||||
|| !metadata.bucket_creation_committed
|
||||
|| !metadata.versioning_config.as_ref().is_some_and(|versioning| {
|
||||
versioning.enabled()
|
||||
&& !versioning.exclude_folders.unwrap_or(false)
|
||||
&& versioning.excluded_prefixes.as_ref().is_none_or(Vec::is_empty)
|
||||
})
|
||||
{
|
||||
return Err(Error::other("artifact backup retention identity mismatch"));
|
||||
}
|
||||
metadata.bucket_incarnation_sidecar = true;
|
||||
Ok(metadata)
|
||||
}
|
||||
|
||||
/// Caller must hold the bucket lifecycle fence. An unreadable or missing
|
||||
/// record cannot prove that historical backup protection was never enabled.
|
||||
pub(crate) async fn require_no_artifact_backup_retention(&self, bucket: &str) -> Result<()> {
|
||||
if self.read_backup_retention_metadata(bucket).await?.artifact_backup_retained {
|
||||
return Err(Error::other("artifact versions are retained for backups"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn read_backup_retention_metadata(&self, bucket: &str) -> Result<BucketMetadata> {
|
||||
if bucket.is_empty()
|
||||
|| super::super::utils::has_bad_path_component(bucket)
|
||||
|| super::super::utils::is_meta_bucketname(bucket)
|
||||
{
|
||||
return Err(Error::other("artifact backup retention bucket is invalid"));
|
||||
}
|
||||
let store = metadata_sys::object_store_in(&self.ctx).await?;
|
||||
if store.id != self.id {
|
||||
return Err(Error::other("artifact backup retention belongs to another store"));
|
||||
}
|
||||
let (bytes, _) = read_config_observation(store, &BucketMetadata::new(bucket).save_file_path(), 64 * 1024 * 1024).await?;
|
||||
let metadata = BucketMetadata::decode_file(&bytes)?;
|
||||
if metadata.name != bucket {
|
||||
return Err(Error::other("artifact backup retention bucket mismatch"));
|
||||
}
|
||||
Ok(metadata)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use super::*;
|
||||
use crate::storage_api_contracts::object::ObjectOperations as _;
|
||||
|
||||
/// In-process authority for exactly one drained managed version. No caller can
|
||||
/// construct this capability or serialize it into a disk/RPC request.
|
||||
#[doc(hidden)]
|
||||
pub struct ArtifactDeleteTarget {
|
||||
store: Uuid,
|
||||
record: ArtifactVersionRetirementRecord,
|
||||
lifecycle: Arc<rustfs_lock::NamespaceLockGuard>,
|
||||
attempt: WriteCommitGuard,
|
||||
repair: WriteCommitGuard,
|
||||
}
|
||||
|
||||
impl ArtifactDeleteTarget {
|
||||
fn check_live(&self) -> Result<()> {
|
||||
if self.lifecycle.is_lock_lost() || self.attempt.is_lock_lost() || self.repair.is_lock_lost() {
|
||||
return Err(Error::other("artifact exact delete owned fence was lost"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn lifecycle(&self) -> Arc<rustfs_lock::NamespaceLockGuard> {
|
||||
Arc::clone(&self.lifecycle)
|
||||
}
|
||||
|
||||
pub(crate) fn validate(&self, store: &ECStore, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
|
||||
if self.store != store.id
|
||||
|| bucket != self.record.bucket
|
||||
|| rustfs_utils::path::encode_dir_object(object) != self.record.object
|
||||
|| opts.expected_bucket_incarnation_id != Some(self.record.incarnation_id)
|
||||
|| opts.version_id.as_deref() != Some(self.record.version_id.to_string().as_str())
|
||||
|| !opts.versioned
|
||||
|| opts.version_suspended
|
||||
|| opts.delete_prefix
|
||||
|| opts.delete_marker
|
||||
|| opts.data_movement
|
||||
|| opts.lifecycle_delete_all.is_some()
|
||||
|| self.lifecycle.is_lock_lost()
|
||||
|| self.attempt.is_lock_lost()
|
||||
|| self.repair.is_lock_lost()
|
||||
{
|
||||
return Err(Error::other("artifact exact delete target or owned fence mismatch"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
/// Read-only capability observation; each mutation revalidates the topology.
|
||||
pub async fn supports_artifact_exact_delete(&self) -> bool {
|
||||
require_local_put_seal_topology(self).await.is_ok() && !self.ctx.lock_manager().is_disabled()
|
||||
}
|
||||
|
||||
/// Delete an exact version in a fully sealed attempt. This authenticates the
|
||||
/// storage identity and drains writers; the caller still owns GC eligibility
|
||||
/// and cleanup IAM. Distributed receiver deletion is not yet supported.
|
||||
pub async fn delete_artifact_object_version(
|
||||
self: Arc<Self>,
|
||||
bucket: &str,
|
||||
expected_incarnation: Uuid,
|
||||
attempt_prefix: &str,
|
||||
object: &str,
|
||||
version: Uuid,
|
||||
) -> Result<()> {
|
||||
require_local_put_seal_topology(&self).await?;
|
||||
if self.ctx.lock_manager().is_disabled() {
|
||||
return Err(Error::other("artifact exact deletion requires namespace locking"));
|
||||
}
|
||||
let lifecycle = Arc::new(self.acquire_bucket_lifecycle_read_lock(bucket).await?);
|
||||
self.require_no_artifact_backup_retention(bucket).await?;
|
||||
let put = self
|
||||
.inventory_artifact_record(bucket, expected_incarnation, attempt_prefix)
|
||||
.await?;
|
||||
let repair = ArtifactRepairSealRecord::new(&put);
|
||||
let record = ArtifactVersionRetirementRecord::new(&put, object, version)?;
|
||||
let object = object.to_owned();
|
||||
let store = Arc::clone(&self);
|
||||
// Lifecycle -> attempt -> repair seal -> metadata transaction -> object.
|
||||
// The spawned owner, and the disk deletion tail's typed capability, keep
|
||||
// these locks after cancellation of the HTTP caller.
|
||||
with_config_object_write_lock(self, put.path(), move |attempt| async move {
|
||||
if !put.exists(Arc::clone(&store), &attempt).await? {
|
||||
return Err(Error::other("artifact exact deletion requires sealed PUTs"));
|
||||
}
|
||||
let lock = store.new_ns_lock(RUSTFS_META_BUCKET, &repair.path()).await?;
|
||||
let repair_owner = WriteCommitGuard::acquire(&lock, get_lock_acquire_timeout()).await?;
|
||||
if !repair.exists(Arc::clone(&store), &repair_owner).await? {
|
||||
return Err(Error::other("artifact exact deletion requires sealed repairs"));
|
||||
}
|
||||
let receivers = store.capture_artifact_receivers().await?;
|
||||
store
|
||||
.close_artifact_receiver_snapshot(&put, crate::disk::local::ArtifactLeaseDomain::Put, &receivers)
|
||||
.await?;
|
||||
store
|
||||
.close_artifact_receiver_snapshot(&put, crate::disk::local::ArtifactLeaseDomain::Repair, &receivers)
|
||||
.await?;
|
||||
store.verify_artifact_receiver_snapshot(&put, &receivers).await?;
|
||||
let target = Arc::new(ArtifactDeleteTarget {
|
||||
store: store.id,
|
||||
record,
|
||||
lifecycle,
|
||||
attempt,
|
||||
repair: repair_owner,
|
||||
});
|
||||
let mut opts = ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(version.to_string()),
|
||||
expected_bucket_incarnation_id: Some(expected_incarnation),
|
||||
artifact_delete_target: Some(Arc::clone(&target)),
|
||||
delete_mutation_owner: Some(target.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
opts.add_write_commit_guard(&target.attempt);
|
||||
opts.add_write_commit_guard(&target.repair);
|
||||
// These owners fence seal records, not the object. Ordinary DELETE
|
||||
// must still acquire and retain its exact object WRITE owner.
|
||||
opts.no_lock = false;
|
||||
target.check_live()?;
|
||||
// Preserve the normal exact DELETE's Object Lock, quorum, journal,
|
||||
// cache and capacity behavior. Its success is logical deletion, not
|
||||
// a claim that every offline shard has been physically erased.
|
||||
match store.delete_object(&put.bucket, &object, opts).await {
|
||||
Ok(info) => {
|
||||
// Match the S3 request layer's committed-generation quota
|
||||
// delta. A strict absence retry must not subtract twice.
|
||||
if !info.name.is_empty()
|
||||
&& !info.delete_marker
|
||||
&& let Ok(size) = crate::data_usage::quota_object_size(&info)
|
||||
{
|
||||
crate::data_usage::record_bucket_object_delete_memory(&put.bucket, size, false).await;
|
||||
}
|
||||
}
|
||||
Err(Error::VersionNotFound(bucket, object, version))
|
||||
if bucket == target.record.bucket
|
||||
&& rustfs_utils::path::encode_dir_object(&object) == target.record.object
|
||||
&& version == target.record.version_id.to_string() => {}
|
||||
Err(Error::ObjectNotFound(bucket, object))
|
||||
if bucket == target.record.bucket
|
||||
&& rustfs_utils::path::encode_dir_object(&object) == target.record.object => {}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
store.verify_artifact_receiver_snapshot(&put, &receivers).await?;
|
||||
store
|
||||
.inventory_artifact_record(&put.bucket, expected_incarnation, &put.attempt_prefix)
|
||||
.await?;
|
||||
target.check_live()
|
||||
})
|
||||
.await?
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use super::*;
|
||||
use crate::disk::{ArtifactInventoryFrame, ArtifactInventoryVersion, Disk};
|
||||
use rustfs_protos::proto_gen::node_service::ArtifactInventoryPhase;
|
||||
use tokio::{sync::mpsc, time::Instant};
|
||||
|
||||
/// Physical observations retain their member. Replicas are not extra business
|
||||
/// versions and this stream makes no quorum-readability or GC claim.
|
||||
#[derive(Debug)]
|
||||
pub enum ArtifactAttemptInventoryFrame {
|
||||
Observations {
|
||||
endpoint: String,
|
||||
disk_id: Uuid,
|
||||
versions: Vec<ArtifactInventoryVersion>,
|
||||
},
|
||||
Complete {
|
||||
members: usize,
|
||||
observations: u64,
|
||||
},
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
/// Inventory one fully sealed attempt across every configured receiver.
|
||||
/// Earlier output is staging only; only the final frame plus Ok is complete.
|
||||
pub async fn inventory_artifact_attempt(
|
||||
self: Arc<Self>,
|
||||
bucket: &str,
|
||||
incarnation: Uuid,
|
||||
prefix: &str,
|
||||
deadline: Instant,
|
||||
output: mpsc::Sender<ArtifactAttemptInventoryFrame>,
|
||||
) -> Result<()> {
|
||||
tokio::time::timeout_at(deadline, Box::pin(async move {
|
||||
require_put_seal_topology(&self).await?;
|
||||
if self.ctx.lock_manager().is_disabled() { return Err(Error::other("artifact inventory requires namespace locking")); }
|
||||
let lifecycle = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
|
||||
let put = self.inventory_artifact_record(bucket, incarnation, prefix).await?;
|
||||
let repair = ArtifactRepairSealRecord::new(&put);
|
||||
// Lifecycle -> attempt -> repair record. Receivers do not reenter
|
||||
// distributed locks; actual two-domain disk owners guard their reads.
|
||||
let store = self.clone();
|
||||
let bucket = bucket.to_owned();
|
||||
let prefix = prefix.to_owned();
|
||||
with_config_object_write_lock(self, put.path(), move |attempt| async move {
|
||||
tokio::time::timeout_at(deadline, async {
|
||||
if !put.exists(store.clone(), &attempt).await? { return Err(Error::other("artifact inventory requires PUT seal record")); }
|
||||
let lock = store.new_ns_lock(RUSTFS_META_BUCKET, &repair.path()).await?;
|
||||
let repair_owner = WriteCommitGuard::acquire(&lock, get_lock_acquire_timeout()).await?;
|
||||
if !repair.exists(store.clone(), &repair_owner).await? { return Err(Error::other("artifact inventory requires repair seal record")); }
|
||||
let receivers = store.capture_artifact_receivers().await?;
|
||||
let mut observations = 0u64;
|
||||
for (endpoint, (disk, expected)) in &receivers {
|
||||
if lifecycle.is_lock_lost() || attempt.is_lock_lost() || repair_owner.is_lock_lost() {
|
||||
return Err(Error::other("artifact inventory fence was lost"));
|
||||
}
|
||||
let (sender, mut frames) = mpsc::channel(2);
|
||||
let scan = async {
|
||||
match disk.as_ref() {
|
||||
Disk::Local(local) => local.get_disk().inventory_artifact_attempt(*expected, &bucket, &prefix, put.identity(), deadline, sender).await,
|
||||
Disk::Remote(remote) => remote.inventory_artifact_attempt(*expected, (&bucket, incarnation), &prefix, ArtifactInventoryPhase::ArtifactInventoryScan, deadline, sender).await,
|
||||
}
|
||||
};
|
||||
tokio::pin!(scan);
|
||||
let mut sequence = 0;
|
||||
let mut count = 0u64;
|
||||
let mut terminal = false;
|
||||
let mut ended = false;
|
||||
let mut done = false;
|
||||
while !ended || !done {
|
||||
tokio::select! {
|
||||
result = &mut scan, if !done => { result.map_err(Error::other)?; done = true; },
|
||||
frame = frames.recv(), if !ended => match frame {
|
||||
Some(ArtifactInventoryFrame::Entries { sequence: actual, versions }) if !terminal && actual == sequence => {
|
||||
count = count.checked_add(u64::try_from(versions.len()).map_err(Error::other)?).ok_or_else(|| Error::other("artifact inventory count overflow"))?;
|
||||
output.send(ArtifactAttemptInventoryFrame::Observations { endpoint: endpoint.clone(), disk_id: *expected, versions }).await
|
||||
.map_err(|_| Error::other("artifact inventory consumer closed"))?;
|
||||
sequence += 1;
|
||||
}
|
||||
Some(ArtifactInventoryFrame::Complete { disk_id, incarnation: actual, attempt_prefix, entries })
|
||||
if !terminal && disk_id == *expected && actual == incarnation && attempt_prefix == prefix && entries == count => terminal = true,
|
||||
Some(_) => return Err(Error::other("artifact inventory member stream is invalid")),
|
||||
None => ended = true,
|
||||
},
|
||||
}
|
||||
}
|
||||
if !terminal { return Err(Error::other("artifact inventory member is incomplete")); }
|
||||
observations = observations.checked_add(count).ok_or_else(|| Error::other("artifact inventory count overflow"))?;
|
||||
}
|
||||
store.verify_inventory_members(&put, &receivers, deadline).await?;
|
||||
let current = store.artifact_inventory_receiver_identity(&bucket, incarnation, &prefix).await?;
|
||||
if current.receiver_key != put.identity().receiver_key || lifecycle.is_lock_lost() || attempt.is_lock_lost() || repair_owner.is_lock_lost() {
|
||||
return Err(Error::other("artifact inventory authority changed"));
|
||||
}
|
||||
output.send(ArtifactAttemptInventoryFrame::Complete { members: receivers.len(), observations }).await
|
||||
.map_err(|_| Error::other("artifact inventory consumer closed"))?;
|
||||
Ok(())
|
||||
}).await.map_err(|_| Error::other("artifact inventory deadline exceeded"))?
|
||||
}).await?
|
||||
})).await.map_err(|_| Error::other("artifact inventory deadline exceeded"))?
|
||||
}
|
||||
|
||||
pub(super) async fn inventory_artifact_record(&self, bucket: &str, expected: Uuid, prefix: &str) -> Result<PutSealRecord> {
|
||||
use crate::bucket::metadata::BUCKET_INCARNATION_FILE;
|
||||
use crate::config::com::read_config_observation;
|
||||
if expected.is_nil()
|
||||
|| crate::bucket::utils::has_bad_path_component(bucket)
|
||||
|| bucket.is_empty()
|
||||
|| crate::bucket::utils::is_meta_bucketname(bucket)
|
||||
{
|
||||
return Err(Error::other("artifact inventory authority is invalid"));
|
||||
}
|
||||
let store = metadata_sys::object_store_in(&self.ctx).await?;
|
||||
if store.id != self.id {
|
||||
return Err(Error::other("artifact inventory authority belongs to another store"));
|
||||
}
|
||||
// Do not enter metadata_sys's lazy bucket READ/migration path: the
|
||||
// coordinator already owns lifecycle READ and a writer may be queued.
|
||||
// Observation reads bypass cache and suppress physical read repair.
|
||||
let path = BucketMetadata::new(bucket).save_file_path();
|
||||
let (bytes, _) = read_config_observation(store.clone(), &path, 64 * 1024 * 1024).await?;
|
||||
let mut metadata = BucketMetadata::decode_file(&bytes)?;
|
||||
let (sidecar, _) =
|
||||
read_config_observation(store, &format!("{BUCKET_META_PREFIX}/{bucket}/{BUCKET_INCARNATION_FILE}"), 16).await?;
|
||||
if metadata.name != bucket
|
||||
|| !metadata.bucket_creation_committed
|
||||
|| metadata.bucket_incarnation_id != expected
|
||||
|| Uuid::from_slice(&sidecar).ok() != Some(expected)
|
||||
{
|
||||
return Err(Error::other("artifact inventory incarnation mismatch"));
|
||||
}
|
||||
metadata.parse_all_configs()?;
|
||||
if !metadata.versioning_config.as_ref().is_some_and(|config| {
|
||||
config.enabled()
|
||||
&& !config.exclude_folders.unwrap_or(false)
|
||||
&& config.excluded_prefixes.as_ref().is_none_or(Vec::is_empty)
|
||||
}) {
|
||||
return Err(Error::other("artifact inventory requires managed versioning"));
|
||||
}
|
||||
PutSealRecord::new(&metadata, prefix)
|
||||
}
|
||||
|
||||
pub(crate) async fn artifact_inventory_receiver_identity(
|
||||
&self,
|
||||
bucket: &str,
|
||||
expected: Uuid,
|
||||
prefix: &str,
|
||||
) -> Result<ArtifactPutIdentity> {
|
||||
Ok(self.inventory_artifact_record(bucket, expected, prefix).await?.identity())
|
||||
}
|
||||
|
||||
async fn verify_inventory_members(
|
||||
&self,
|
||||
record: &PutSealRecord,
|
||||
captured: &ArtifactReceiverSnapshot,
|
||||
deadline: Instant,
|
||||
) -> Result<()> {
|
||||
for final_pass in [false, true] {
|
||||
let current = self.artifact_receiver_inventory().await?;
|
||||
if current.len() != captured.len()
|
||||
|| current
|
||||
.iter()
|
||||
.any(|(endpoint, disk)| !captured.get(endpoint).is_some_and(|(old, _)| Arc::ptr_eq(old, disk)))
|
||||
{
|
||||
return Err(Error::other("artifact inventory membership changed"));
|
||||
}
|
||||
if final_pass {
|
||||
for (disk, expected) in captured.values() {
|
||||
if disk.cached_disk_id().await != Some(*expected) {
|
||||
return Err(Error::other("artifact inventory expected identity changed"));
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
for (disk, expected) in captured.values() {
|
||||
if disk.cached_disk_id().await != Some(*expected) {
|
||||
return Err(Error::other("artifact inventory expected identity changed"));
|
||||
}
|
||||
match disk.as_ref() {
|
||||
Disk::Local(local) => {
|
||||
let _verification = local
|
||||
.get_disk()
|
||||
.verify_artifact_inventory_closed(*expected, record.identity(), deadline)
|
||||
.await
|
||||
.map_err(Error::other)?;
|
||||
}
|
||||
Disk::Remote(remote) => {
|
||||
let (output, _unused) = mpsc::channel(1);
|
||||
remote
|
||||
.inventory_artifact_attempt(
|
||||
*expected,
|
||||
(&record.bucket, record.incarnation_id),
|
||||
&record.attempt_prefix,
|
||||
ArtifactInventoryPhase::ArtifactInventoryVerify,
|
||||
deadline,
|
||||
output,
|
||||
)
|
||||
.await
|
||||
.map_err(Error::other)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::future::Future;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex, Weak};
|
||||
|
||||
/// Shares only reads started after a caller arrives; retains no idle results.
|
||||
#[derive(Default)]
|
||||
pub(crate) struct OrdinaryClassificationRegistry {
|
||||
gates: Mutex<HashMap<String, Weak<Gate>>>,
|
||||
#[cfg(test)]
|
||||
after_unregister: Option<std::sync::Barrier>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Gate {
|
||||
started_generation: AtomicU64,
|
||||
completed: tokio::sync::Mutex<Option<CompletedRead>>,
|
||||
}
|
||||
|
||||
struct CompletedRead {
|
||||
generation: u64,
|
||||
allowed: bool,
|
||||
}
|
||||
|
||||
struct Registration<'a> {
|
||||
registry: &'a OrdinaryClassificationRegistry,
|
||||
bucket: &'a str,
|
||||
gate: Option<Arc<Gate>>,
|
||||
observed_generation: u64,
|
||||
}
|
||||
|
||||
impl OrdinaryClassificationRegistry {
|
||||
pub(crate) async fn classify(&self, bucket: &str, read: impl Future<Output = bool>) -> bool {
|
||||
self.register(bucket).classify(read).await
|
||||
}
|
||||
|
||||
fn register<'a>(&'a self, bucket: &'a str) -> Registration<'a> {
|
||||
// Callers requiring physical admission must register after acquiring it.
|
||||
// Release the registry lock before awaiting the gate, then acquire the
|
||||
// existing metadata locks.
|
||||
let mut gates = self.gates.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let gate = gates.get(bucket).and_then(Weak::upgrade).unwrap_or_else(|| {
|
||||
let gate = Arc::new(Gate::default());
|
||||
gates.insert(bucket.to_owned(), Arc::downgrade(&gate));
|
||||
gate
|
||||
});
|
||||
let observed_generation = gate.started_generation.load(Ordering::Acquire);
|
||||
Registration {
|
||||
registry: self,
|
||||
bucket,
|
||||
gate: Some(gate),
|
||||
observed_generation,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Registration<'_> {
|
||||
async fn classify(self, read: impl Future<Output = bool>) -> bool {
|
||||
let Some(gate) = self.gate.as_ref() else {
|
||||
return false;
|
||||
};
|
||||
let mut completed = gate.completed.lock().await;
|
||||
// Only a read started after this caller registered can authorize it.
|
||||
// Arrivals during a read therefore wait for the next fresh read.
|
||||
if let Some(result) = completed
|
||||
.as_ref()
|
||||
.filter(|result| result.generation > self.observed_generation)
|
||||
{
|
||||
return result.allowed;
|
||||
}
|
||||
let Ok(previous) = gate
|
||||
.started_generation
|
||||
.try_update(Ordering::AcqRel, Ordering::Acquire, |generation| generation.checked_add(1))
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
// Keep the gate through the original bounded metadata read. Cancellation
|
||||
// releases it without publishing a result; another caller can read again.
|
||||
let allowed = read.await;
|
||||
*completed = Some(CompletedRead {
|
||||
generation: previous + 1,
|
||||
allowed,
|
||||
});
|
||||
allowed
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Registration<'_> {
|
||||
fn drop(&mut self) {
|
||||
let Some(gate) = self.gate.take() else {
|
||||
return;
|
||||
};
|
||||
let mut gates = self.registry.gates.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
// Strong references are acquired and released under this registry lock,
|
||||
// so concurrent final callers cannot both miss the last reference.
|
||||
if Arc::strong_count(&gate) == 1
|
||||
&& gates
|
||||
.get(self.bucket)
|
||||
.is_some_and(|registered| registered.ptr_eq(&Arc::downgrade(&gate)))
|
||||
{
|
||||
gates.remove(self.bucket);
|
||||
}
|
||||
drop(gate);
|
||||
drop(gates);
|
||||
#[cfg(test)]
|
||||
if let Some(barrier) = &self.registry.after_unregister {
|
||||
barrier.wait();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use futures::poll;
|
||||
use std::task::Poll;
|
||||
use tokio::sync::oneshot;
|
||||
|
||||
fn assert_empty(registry: &OrdinaryClassificationRegistry) {
|
||||
assert!(registry.gates.lock().expect("classification registry").is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn late_arrivals_share_only_the_next_fail_closed_read() {
|
||||
let registry = OrdinaryClassificationRegistry::default();
|
||||
let reads = AtomicU64::new(0);
|
||||
let (release, waiting) = oneshot::channel();
|
||||
let mut first = Box::pin(registry.classify("bucket", async {
|
||||
reads.fetch_add(1, Ordering::Relaxed);
|
||||
waiting.await.expect("release first read");
|
||||
true
|
||||
}));
|
||||
assert!(poll!(first.as_mut()).is_pending());
|
||||
let mut followers = (0..16)
|
||||
.map(|_| {
|
||||
Box::pin(registry.classify("bucket", async {
|
||||
reads.fetch_add(1, Ordering::Relaxed);
|
||||
false
|
||||
}))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
for follower in &mut followers {
|
||||
assert!(poll!(follower.as_mut()).is_pending());
|
||||
}
|
||||
release.send(()).expect("first reader still alive");
|
||||
assert_eq!(poll!(first.as_mut()), Poll::Ready(true));
|
||||
for follower in &mut followers {
|
||||
assert_eq!(poll!(follower.as_mut()), Poll::Ready(false));
|
||||
}
|
||||
assert_eq!(reads.load(Ordering::Relaxed), 2);
|
||||
assert_empty(®istry);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn successive_waves_always_read_again() {
|
||||
let registry = OrdinaryClassificationRegistry::default();
|
||||
let reads = AtomicU64::new(0);
|
||||
for expected in [true, false, true] {
|
||||
assert_eq!(
|
||||
registry
|
||||
.classify("bucket", async {
|
||||
reads.fetch_add(1, Ordering::Relaxed);
|
||||
expected
|
||||
})
|
||||
.await,
|
||||
expected
|
||||
);
|
||||
assert_empty(®istry);
|
||||
}
|
||||
assert_eq!(reads.load(Ordering::Relaxed), 3);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn canceled_leader_does_not_publish_or_strand_waiters() {
|
||||
let registry = OrdinaryClassificationRegistry::default();
|
||||
let reads = AtomicU64::new(0);
|
||||
let old_registration = registry.register("bucket");
|
||||
assert!(
|
||||
registry
|
||||
.classify("bucket", async {
|
||||
reads.fetch_add(1, Ordering::Relaxed);
|
||||
true
|
||||
})
|
||||
.await
|
||||
);
|
||||
let mut leader = Box::pin(registry.classify("bucket", async {
|
||||
reads.fetch_add(1, Ordering::Relaxed);
|
||||
std::future::pending::<bool>().await
|
||||
}));
|
||||
assert!(poll!(leader.as_mut()).is_pending());
|
||||
let mut old_waiter =
|
||||
Box::pin(old_registration.classify(async { panic!("old waiter must reuse the completed post-registration read") }));
|
||||
assert!(poll!(old_waiter.as_mut()).is_pending());
|
||||
let mut late_waiter = Box::pin(registry.classify("bucket", async {
|
||||
reads.fetch_add(1, Ordering::Relaxed);
|
||||
false
|
||||
}));
|
||||
assert!(poll!(late_waiter.as_mut()).is_pending());
|
||||
drop(leader);
|
||||
assert_eq!(poll!(old_waiter.as_mut()), Poll::Ready(true));
|
||||
assert_eq!(poll!(late_waiter.as_mut()), Poll::Ready(false));
|
||||
assert_eq!(reads.load(Ordering::Relaxed), 3);
|
||||
assert_empty(®istry);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn canceled_follower_does_not_retain_the_gate() {
|
||||
let registry = OrdinaryClassificationRegistry::default();
|
||||
let (release, waiting) = oneshot::channel();
|
||||
let mut leader = Box::pin(registry.classify("bucket", async {
|
||||
waiting.await.expect("release leader");
|
||||
true
|
||||
}));
|
||||
assert!(poll!(leader.as_mut()).is_pending());
|
||||
let mut follower = Box::pin(registry.classify("bucket", async { panic!("canceled follower must not read") }));
|
||||
assert!(poll!(follower.as_mut()).is_pending());
|
||||
drop(follower);
|
||||
assert_eq!(registry.gates.lock().expect("classification registry").len(), 1);
|
||||
release.send(()).expect("leader still alive");
|
||||
assert_eq!(poll!(leader.as_mut()), Poll::Ready(true));
|
||||
assert_empty(®istry);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn concurrent_final_registrations_remove_the_gate() {
|
||||
let registry = OrdinaryClassificationRegistry {
|
||||
after_unregister: Some(std::sync::Barrier::new(2)),
|
||||
..Default::default()
|
||||
};
|
||||
let first = registry.register("bucket");
|
||||
let second = registry.register("bucket");
|
||||
// Both destructors reach the unlocked barrier before either returns.
|
||||
// Moving the Arc release after this barrier leaves a stale registry key.
|
||||
std::thread::scope(|scope| {
|
||||
scope.spawn(move || drop(first));
|
||||
scope.spawn(move || drop(second));
|
||||
});
|
||||
assert_empty(®istry);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn buckets_and_stores_do_not_share_gates() {
|
||||
let first_store = OrdinaryClassificationRegistry::default();
|
||||
let second_store = OrdinaryClassificationRegistry::default();
|
||||
let mut blocked = Box::pin(first_store.classify("first", std::future::pending()));
|
||||
assert!(poll!(blocked.as_mut()).is_pending());
|
||||
assert_eq!(poll!(Box::pin(first_store.classify("second", async { true }))), Poll::Ready(true));
|
||||
assert_eq!(poll!(Box::pin(second_store.classify("first", async { false }))), Poll::Ready(false));
|
||||
assert_empty(&second_store);
|
||||
drop(blocked);
|
||||
assert_empty(&first_store);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn generation_exhaustion_fails_closed_without_reading() {
|
||||
let registry = OrdinaryClassificationRegistry::default();
|
||||
let registration = registry.register("bucket");
|
||||
let gate = registration.gate.as_ref().expect("registered gate");
|
||||
gate.started_generation.store(u64::MAX, Ordering::Release);
|
||||
*gate.completed.lock().await = Some(CompletedRead {
|
||||
generation: u64::MAX,
|
||||
allowed: true,
|
||||
});
|
||||
assert!(
|
||||
!registry
|
||||
.classify("bucket", async { panic!("exhausted generation must deny without reading") })
|
||||
.await
|
||||
);
|
||||
assert_eq!(gate.started_generation.load(Ordering::Acquire), u64::MAX);
|
||||
drop(registration);
|
||||
assert_empty(®istry);
|
||||
}
|
||||
}
|
||||
@@ -890,6 +890,7 @@ pub(crate) async fn cleanup_free_version_exact(
|
||||
let bucket_guard = api
|
||||
.acquire_bucket_lifecycle_read_lock(&oi.bucket)
|
||||
.await
|
||||
.map(Arc::new)
|
||||
.map_err(std::io::Error::other)?;
|
||||
let (lease, delete_plan) = acquire_free_version_tier_lease(oi, &api.tier_config_mgr()).await?;
|
||||
let local_object = encode_dir_object(&oi.name);
|
||||
@@ -958,7 +959,7 @@ pub(crate) async fn cleanup_free_version_exact(
|
||||
};
|
||||
delete_request.set_tier_free_version();
|
||||
if let Err(err) = set
|
||||
.delete_object_version(&oi.bucket, &local_object, &delete_request, false)
|
||||
.delete_object_version_with_lifecycle_owner(&oi.bucket, &local_object, &delete_request, Arc::clone(&bucket_guard))
|
||||
.await
|
||||
&& first_error.is_none()
|
||||
{
|
||||
@@ -8496,6 +8497,104 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn artifact_delete_entrypoints_free_version_cleaner_completes_with_queued_lifecycle_writer() {
|
||||
let (disk_paths, ecstore) = setup_test_env().await;
|
||||
let bucket = format!("free-version-queued-writer-{}", Uuid::new_v4());
|
||||
let object = "free-version";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
let (backend, identity) = register_recovery_mock_tier(&ecstore).await;
|
||||
seed_recoverable_free_version(&disk_paths, &bucket, object, None, Some(identity)).await;
|
||||
let oi = list_tier_free_versions(Arc::clone(&ecstore), 100, None, None, CancellationToken::new())
|
||||
.await
|
||||
.expect("list real persisted free version")
|
||||
.items
|
||||
.into_iter()
|
||||
.find(|oi| oi.bucket == bucket)
|
||||
.expect("free-version task must address this fixture");
|
||||
assert!(oi.transitioned_object.free_version);
|
||||
backend
|
||||
.set_put_remote_version(Some(oi.transitioned_object.version_id.clone()))
|
||||
.await;
|
||||
let remote = TierConfigMgr::acquire_operation_lease(&ecstore.tier_config_mgr(), "WARM")
|
||||
.await
|
||||
.expect("remote fixture lease");
|
||||
remote
|
||||
.put(&oi.transitioned_object.name, ReaderImpl::Body(Bytes::from_static(b"body")), 4)
|
||||
.await
|
||||
.expect("seed exact remote object");
|
||||
drop(remote);
|
||||
|
||||
let barrier = Arc::new(super::FreeVersionPostRemoteDeleteTestBarrier::default());
|
||||
let cleanup_barrier = Arc::clone(&barrier);
|
||||
let cleanup_store = Arc::clone(&ecstore);
|
||||
let cleanup_oi = oi.clone();
|
||||
let mut cleanup = tokio::spawn(async move {
|
||||
super::FREE_VERSION_POST_REMOTE_DELETE_TEST_BARRIER
|
||||
.scope(cleanup_barrier, async move {
|
||||
super::cleanup_free_version_exact(cleanup_store, &cleanup_oi, &CancellationToken::new()).await
|
||||
})
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(StdDuration::from_secs(10), barrier.arrived.notified())
|
||||
.await
|
||||
.expect("cleaner owns lifecycle READ and has completed remote DELETE");
|
||||
assert!(!backend.contains(&oi.transitioned_object.name).await);
|
||||
for disk in &disk_paths {
|
||||
assert!(
|
||||
disk.join(&bucket).join(object).join(STORAGE_FORMAT_FILE).is_file(),
|
||||
"local exact cleanup must still be pending at the boundary"
|
||||
);
|
||||
}
|
||||
|
||||
// Resolve first so the pending poll is the actual local lock wait.
|
||||
// A later READ would normally be admitted alongside the cleaner's READ;
|
||||
// its pending state independently proves this WRITE is now queued.
|
||||
let lifecycle = ecstore
|
||||
.new_ns_lock(&bucket, crate::storage_api_contracts::bucket::BUCKET_LIFECYCLE_LOCK_OBJECT)
|
||||
.await
|
||||
.expect("lifecycle namespace");
|
||||
let mut writer = Box::pin(lifecycle.get_write_lock(StdDuration::from_secs(30)));
|
||||
let writer_queued = futures::poll!(&mut writer).is_pending();
|
||||
let mut later_reader = Box::pin(lifecycle.get_read_lock(StdDuration::from_secs(30)));
|
||||
let later_reader_blocked = futures::poll!(&mut later_reader).is_pending();
|
||||
drop(later_reader);
|
||||
barrier.release.notify_one();
|
||||
assert!(writer_queued, "cleaner must retain the real lifecycle READ owner");
|
||||
assert!(later_reader_blocked, "queued lifecycle WRITE must block a later READ");
|
||||
|
||||
// Keep the same WRITE future queued, without cancelling/retrying it.
|
||||
// Re-entering READ here would deadlock behind this waiting writer.
|
||||
let finished = tokio::time::timeout(StdDuration::from_secs(10), &mut cleanup).await;
|
||||
if finished.is_err() {
|
||||
cleanup.abort();
|
||||
drop(writer);
|
||||
let _ = tokio::time::timeout(StdDuration::from_secs(5), &mut cleanup).await;
|
||||
panic!("exact local free-version cleanup re-entered lifecycle READ behind the queued writer");
|
||||
}
|
||||
assert!(
|
||||
finished
|
||||
.expect("bounded cleaner completion")
|
||||
.expect("cleaner task")
|
||||
.expect("exact local cleanup")
|
||||
);
|
||||
let writer = tokio::time::timeout(StdDuration::from_secs(10), writer)
|
||||
.await
|
||||
.expect("queued lifecycle WRITE proceeds after real cleanup drains")
|
||||
.expect("lifecycle WRITE acquired");
|
||||
for disk in &disk_paths {
|
||||
assert!(!disk.join(&bucket).join(object).exists(), "free version remains on a real local disk");
|
||||
}
|
||||
assert_eq!(backend.remove_count().await, 1, "cleaner must delete the exact remote tuple once");
|
||||
drop(writer);
|
||||
ecstore
|
||||
.delete_bucket(&bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("remove empty fixture bucket");
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
|
||||
@@ -437,6 +437,8 @@ pub struct BucketMetadata {
|
||||
pub created: OffsetDateTime,
|
||||
pub lock_enabled: bool, // While marked as unused, it may need to be retained
|
||||
pub bucket_incarnation_id: Uuid,
|
||||
pub artifact_base_prefix: Option<String>,
|
||||
pub artifact_backup_retained: bool,
|
||||
pub(crate) bucket_incarnation_sidecar: bool,
|
||||
pub(crate) bucket_creation_committed: bool,
|
||||
pub policy_config_json: Vec<u8>,
|
||||
@@ -512,6 +514,8 @@ impl Default for BucketMetadata {
|
||||
created: OffsetDateTime::UNIX_EPOCH,
|
||||
lock_enabled: Default::default(),
|
||||
bucket_incarnation_id: Uuid::nil(),
|
||||
artifact_base_prefix: None,
|
||||
artifact_backup_retained: false,
|
||||
bucket_incarnation_sidecar: false,
|
||||
bucket_creation_committed: false,
|
||||
policy_config_json: Default::default(),
|
||||
@@ -607,7 +611,7 @@ impl BucketMetadata {
|
||||
/// legacy migration path, which mints the sidecar first; it must not be
|
||||
/// rewritten by this migration.
|
||||
pub(crate) fn needs_bucket_creation_commit(&self) -> bool {
|
||||
self.lock_enabled
|
||||
(self.lock_enabled || self.artifact_base_prefix.is_some())
|
||||
&& !self.bucket_creation_committed
|
||||
&& self.bucket_incarnation_sidecar
|
||||
&& !self.bucket_incarnation_id.is_nil()
|
||||
@@ -704,6 +708,22 @@ impl BucketMetadata {
|
||||
self.bucket_incarnation_id =
|
||||
Uuid::from_slice(&bytes).map_err(|err| Error::other(format!("invalid BucketIncarnationID: {err}")))?;
|
||||
}
|
||||
"ArtifactBackupRetained" => {
|
||||
if self.artifact_backup_retained || !rmp::decode::read_bool(rd)? {
|
||||
return Err(Error::other("invalid artifact backup retention marker"));
|
||||
}
|
||||
self.artifact_backup_retained = true;
|
||||
}
|
||||
"ArtifactBasePrefix" => {
|
||||
if self.artifact_base_prefix.is_some() {
|
||||
return Err(Error::other("duplicate artifact namespace"));
|
||||
}
|
||||
let prefix = read_msgp_str(rd)?;
|
||||
if !crate::storage_api_contracts::bucket::valid_artifact_base_prefix(&prefix) {
|
||||
return Err(Error::other("invalid artifact namespace"));
|
||||
}
|
||||
self.artifact_base_prefix = Some(prefix);
|
||||
}
|
||||
"PolicyConfigJSON" | "PolicyConfigJson" => self.policy_config_json = read_msgp_bin(rd)?,
|
||||
"NotificationConfigXML" | "NotificationConfigXml" => self.notification_config_xml = read_msgp_bin(rd)?,
|
||||
"LifecycleConfigXML" | "LifecycleConfigXml" => self.lifecycle_config_xml = read_msgp_bin(rd)?,
|
||||
@@ -766,7 +786,7 @@ impl BucketMetadata {
|
||||
/// Encode to msgp bytes. Field order follows MinIO BucketMetadata for compatibility.
|
||||
pub fn encode_to<W: Write>(&self, wr: &mut W) -> Result<()> {
|
||||
// Map size: MinIO fields (25) + RustFS extensions (22)
|
||||
let map_len: u32 = 47;
|
||||
let map_len: u32 = 47 + u32::from(self.artifact_base_prefix.is_some()) + u32::from(self.artifact_backup_retained);
|
||||
rmp::encode::write_map_len(wr, map_len)?;
|
||||
|
||||
// MinIO field order (same as Go struct)
|
||||
@@ -780,6 +800,15 @@ impl BucketMetadata {
|
||||
rmp::encode::write_bool(wr, self.lock_enabled)?;
|
||||
|
||||
write_bin_field(wr, "BucketIncarnationID", self.bucket_incarnation_id.as_bytes())?;
|
||||
if let Some(prefix) = &self.artifact_base_prefix {
|
||||
rmp::encode::write_str(wr, "ArtifactBasePrefix")?;
|
||||
rmp::encode::write_str(wr, prefix)?;
|
||||
}
|
||||
|
||||
if self.artifact_backup_retained {
|
||||
rmp::encode::write_str(wr, "ArtifactBackupRetained")?;
|
||||
rmp::encode::write_bool(wr, true)?;
|
||||
}
|
||||
|
||||
write_bin_field(wr, "PolicyConfigJSON", &self.policy_config_json)?;
|
||||
write_bin_field(wr, "NotificationConfigXML", &self.notification_config_xml)?;
|
||||
@@ -880,13 +909,28 @@ impl BucketMetadata {
|
||||
}
|
||||
|
||||
match version {
|
||||
BUCKET_METADATA_VERSION => {}
|
||||
BUCKET_METADATA_VERSION | 2 => {}
|
||||
_ => return Err(Error::other("read_bucket_metadata: version invalid")),
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Version two is reserved for permanent artifact backup retention. Older
|
||||
/// readers reject its header instead of skipping the protection field.
|
||||
pub(crate) fn decode_file(bytes: &[u8]) -> Result<Self> {
|
||||
Self::check_header(bytes)?;
|
||||
let metadata = Self::unmarshal(&bytes[4..])?;
|
||||
let retained_header = LittleEndian::read_u16(&bytes[2..4]) == 2;
|
||||
if retained_header != metadata.artifact_backup_retained {
|
||||
return Err(Error::other("artifact backup retention header mismatch"));
|
||||
}
|
||||
if metadata.artifact_backup_retained {
|
||||
metadata.validate_artifact_configuration()?;
|
||||
}
|
||||
Ok(metadata)
|
||||
}
|
||||
|
||||
pub(crate) fn default_timestamps(&mut self) {
|
||||
if self.policy_config_updated_at == OffsetDateTime::UNIX_EPOCH {
|
||||
self.policy_config_updated_at = self.created
|
||||
@@ -1025,6 +1069,9 @@ impl BucketMetadata {
|
||||
}
|
||||
Some(config)
|
||||
};
|
||||
if self.artifact_base_prefix.is_some() && !config.as_ref().is_some_and(artifact_versioning_enabled) {
|
||||
return Err(Error::other("artifact bucket versioning must remain enabled"));
|
||||
}
|
||||
self.versioning_config_xml = data;
|
||||
self.versioning_config = config;
|
||||
self.versioning_config_updated_at = updated;
|
||||
@@ -1127,13 +1174,34 @@ impl BucketMetadata {
|
||||
&mut self,
|
||||
store: std::sync::Arc<crate::store::ECStore>,
|
||||
write_completion: WriteCompletion,
|
||||
) -> Result<()> {
|
||||
let opts = crate::object_api::ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion,
|
||||
..Default::default()
|
||||
};
|
||||
self.save_with_store_options(store, &opts).await
|
||||
}
|
||||
|
||||
pub(crate) async fn save_with_store_options(
|
||||
&mut self,
|
||||
store: Arc<ECStore>,
|
||||
opts: &crate::object_api::ObjectOptions,
|
||||
) -> Result<()> {
|
||||
self.parse_all_configs()?;
|
||||
self.validate_artifact_update(store.clone()).await?;
|
||||
let mut buf: Vec<u8> = vec![0; 4];
|
||||
|
||||
LittleEndian::write_u16(&mut buf[0..2], BUCKET_METADATA_FORMAT);
|
||||
|
||||
LittleEndian::write_u16(&mut buf[2..4], BUCKET_METADATA_VERSION);
|
||||
LittleEndian::write_u16(
|
||||
&mut buf[2..4],
|
||||
if self.artifact_backup_retained {
|
||||
2
|
||||
} else {
|
||||
BUCKET_METADATA_VERSION
|
||||
},
|
||||
);
|
||||
|
||||
let data = self
|
||||
.marshal_msg()
|
||||
@@ -1141,21 +1209,52 @@ impl BucketMetadata {
|
||||
|
||||
buf.extend_from_slice(&data);
|
||||
|
||||
save_config_with_opts(
|
||||
store,
|
||||
self.save_file_path().as_str(),
|
||||
buf,
|
||||
&crate::object_api::ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
save_config_with_opts(store, self.save_file_path().as_str(), buf, opts).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_artifact_configuration(&self) -> Result<()> {
|
||||
if self.artifact_backup_retained && (self.artifact_base_prefix.is_none() || !self.bucket_creation_committed) {
|
||||
return Err(Error::other("artifact backup retention requires a committed managed bucket"));
|
||||
}
|
||||
if let Some(prefix) = &self.artifact_base_prefix {
|
||||
if !crate::storage_api_contracts::bucket::valid_artifact_base_prefix(prefix) || self.bucket_incarnation_id.is_nil() {
|
||||
return Err(Error::other("invalid artifact bucket identity"));
|
||||
}
|
||||
let versioning = deserialize::<VersioningConfiguration>(&self.versioning_config_xml)?;
|
||||
if !artifact_versioning_enabled(&versioning) {
|
||||
return Err(Error::other("artifact bucket versioning must remain enabled"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Full metadata replacement must not change a generation's creation identity.
|
||||
pub(crate) async fn validate_artifact_update(&self, store: Arc<ECStore>) -> Result<()> {
|
||||
self.validate_artifact_configuration()?;
|
||||
// An existing empty/corrupt metadata object is not permission to create an identity.
|
||||
match read_config_preserve_empty(store, &self.save_file_path()).await {
|
||||
Ok(data) => {
|
||||
let previous = Self::decode_file(&data)?;
|
||||
previous.validate_artifact_configuration()?;
|
||||
if previous.artifact_backup_retained && !self.artifact_backup_retained {
|
||||
return Err(Error::other("artifact backup retention cannot be removed"));
|
||||
}
|
||||
// Deletion removes this record before recreation. A caller-supplied
|
||||
// incarnation cannot turn an existing bucket into a new generation.
|
||||
if previous.artifact_base_prefix != self.artifact_base_prefix
|
||||
|| (previous.artifact_base_prefix.is_some() && previous.bucket_incarnation_id != self.bucket_incarnation_id)
|
||||
{
|
||||
return Err(Error::other("artifact bucket identity is immutable"));
|
||||
}
|
||||
}
|
||||
Err(Error::ConfigNotFound) => {}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_policy_config(&mut self) -> Result<()> {
|
||||
if !self.policy_config_json.is_empty() {
|
||||
self.policy_config = Some(serde_json::from_slice(&self.policy_config_json)?);
|
||||
@@ -1199,6 +1298,7 @@ impl BucketMetadata {
|
||||
/// `rustfs_bucket_metadata_parse_failed_total`, reflected in
|
||||
/// `rustfs_bucket_metadata_unparsable_current`, and logged at error level.
|
||||
pub(super) fn parse_all_configs(&mut self) -> Result<()> {
|
||||
self.validate_artifact_configuration()?;
|
||||
if let Err(e) = self.parse_policy_config() {
|
||||
tracing::warn!(
|
||||
event = "bucket_metadata_parse_failed",
|
||||
@@ -1464,6 +1564,10 @@ impl BucketMetadata {
|
||||
}
|
||||
}
|
||||
|
||||
fn artifact_versioning_enabled(config: &VersioningConfiguration) -> bool {
|
||||
config.enabled() && !config.exclude_folders.unwrap_or(false) && config.excluded_prefixes.as_ref().is_none_or(Vec::is_empty)
|
||||
}
|
||||
|
||||
pub(crate) async fn load_bucket_incarnation(api: Arc<ECStore>, bucket: &str) -> Result<Option<Uuid>> {
|
||||
let path = format!("{BUCKET_META_PREFIX}/{bucket}/{BUCKET_INCARNATION_FILE}");
|
||||
let data = match read_config_preserve_empty(api, &path).await {
|
||||
@@ -1560,9 +1664,7 @@ async fn read_bucket_metadata(api: Arc<ECStore>, bucket: &str) -> Result<BucketM
|
||||
|
||||
let data = read_config(api, &file_path).await?;
|
||||
|
||||
BucketMetadata::check_header(&data)?;
|
||||
|
||||
let bm = BucketMetadata::unmarshal(&data[4..])?;
|
||||
let bm = BucketMetadata::decode_file(&data)?;
|
||||
|
||||
Ok(bm)
|
||||
}
|
||||
@@ -1586,6 +1688,98 @@ where
|
||||
mod test {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn artifact_metadata_roundtrip_and_corrupt_prefix_rejection() {
|
||||
let mut metadata = BucketMetadata::new("artifacts");
|
||||
metadata.artifact_base_prefix = Some("images/".to_owned());
|
||||
let encoded = metadata.marshal_msg().expect("encode artifact metadata");
|
||||
let decoded = BucketMetadata::unmarshal(&encoded).expect("decode artifact metadata");
|
||||
assert_eq!(decoded.artifact_base_prefix, metadata.artifact_base_prefix, "ARTIFACT_METADATA_ROUNDTRIP");
|
||||
assert_eq!(decoded.bucket_incarnation_id, metadata.bucket_incarnation_id);
|
||||
let legacy = BucketMetadata::new("ordinary")
|
||||
.marshal_msg()
|
||||
.expect("encode ordinary metadata");
|
||||
assert!(
|
||||
BucketMetadata::unmarshal(&legacy)
|
||||
.expect("decode ordinary metadata")
|
||||
.artifact_base_prefix
|
||||
.is_none()
|
||||
);
|
||||
for prefix in ["", "../", "outputs", "a b/"] {
|
||||
let mut invalid = Vec::new();
|
||||
rmp::encode::write_map_len(&mut invalid, 1).expect("encode map");
|
||||
rmp::encode::write_str(&mut invalid, "ArtifactBasePrefix").expect("encode field");
|
||||
rmp::encode::write_str(&mut invalid, prefix).expect("encode invalid prefix");
|
||||
assert!(BucketMetadata::unmarshal(&invalid).is_err(), "ARTIFACT_CORRUPT_PREFIX");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_backup_retention_marker_rejects_false_and_duplicates() {
|
||||
for count in [1, 2] {
|
||||
let mut bytes = Vec::new();
|
||||
rmp::encode::write_map_len(&mut bytes, count).unwrap();
|
||||
for _ in 0..count {
|
||||
rmp::encode::write_str(&mut bytes, "ArtifactBackupRetained").unwrap();
|
||||
rmp::encode::write_bool(&mut bytes, count == 2).unwrap();
|
||||
}
|
||||
assert!(BucketMetadata::unmarshal(&bytes).is_err(), "O02P3_INVALID_RETENTION_MARKER");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_backup_retention_requires_matching_header_and_identity() {
|
||||
let mut metadata = BucketMetadata::new("retained");
|
||||
metadata.bucket_incarnation_id = Uuid::new_v4();
|
||||
metadata.bucket_creation_committed = true;
|
||||
metadata.artifact_base_prefix = Some("outputs/".to_owned());
|
||||
metadata.versioning_config_xml = b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec();
|
||||
let file = |metadata: &BucketMetadata, version: u16| {
|
||||
let mut bytes = Vec::new();
|
||||
bytes.extend_from_slice(&BUCKET_METADATA_FORMAT.to_le_bytes());
|
||||
bytes.extend_from_slice(&version.to_le_bytes());
|
||||
bytes.extend_from_slice(&metadata.marshal_msg().unwrap());
|
||||
bytes
|
||||
};
|
||||
assert!(BucketMetadata::decode_file(&file(&metadata, 1)).is_ok(), "O02P3_ORDINARY_V1_COMPATIBLE");
|
||||
assert!(BucketMetadata::decode_file(&file(&metadata, 2)).is_err(), "O02P3_MARKER_REQUIRED");
|
||||
metadata.artifact_backup_retained = true;
|
||||
assert!(
|
||||
BucketMetadata::decode_file(&file(&metadata, 1)).is_err(),
|
||||
"O02P3_PROTECTION_CANNOT_DOWNGRADE_HEADER"
|
||||
);
|
||||
assert!(
|
||||
BucketMetadata::decode_file(&file(&metadata, 2))
|
||||
.unwrap()
|
||||
.artifact_backup_retained,
|
||||
"O02P3_PROTECTED_V2_ROUNDTRIP"
|
||||
);
|
||||
metadata.artifact_base_prefix = None;
|
||||
assert!(
|
||||
BucketMetadata::decode_file(&file(&metadata, 2)).is_err(),
|
||||
"O02P3_PROTECTION_REQUIRES_MANAGED_BUCKET"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_versioning_cannot_be_disabled_or_excluded() {
|
||||
let mut metadata = BucketMetadata::new("artifacts");
|
||||
metadata.artifact_base_prefix = Some("images/".to_owned());
|
||||
for xml in [
|
||||
"",
|
||||
"<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>",
|
||||
"<VersioningConfiguration><Status>Enabled</Status><ExcludeFolders>true</ExcludeFolders></VersioningConfiguration>",
|
||||
"<VersioningConfiguration><Status>Enabled</Status><ExcludedPrefixes><Prefix>images/</Prefix></ExcludedPrefixes></VersioningConfiguration>",
|
||||
] {
|
||||
assert!(
|
||||
metadata
|
||||
.update_config(BUCKET_VERSIONING_CONFIG, xml.as_bytes().to_vec())
|
||||
.is_err(),
|
||||
"ARTIFACT_VERSIONING_REQUIRED"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// rustfs/backlog#1734: `StorageError::clone` rebuilds I/O errors from
|
||||
/// their text. The unreadable-config refusal must stay typed across that
|
||||
/// clone, or a cloned error stops mapping to its retryable S3 response.
|
||||
|
||||
@@ -293,8 +293,10 @@ pub(super) fn get_bucket_metadata_sys() -> Result<Arc<RwLock<BucketMetadataSys>>
|
||||
pub async fn set_bucket_metadata(bucket: String, bm: BucketMetadata) -> Result<()> {
|
||||
let sys = get_bucket_metadata_sys()?;
|
||||
let lock = sys.write().await;
|
||||
lock.set(bucket, Arc::new(bm)).await;
|
||||
Ok(())
|
||||
if bucket != bm.name {
|
||||
return Err(Error::other("bucket metadata identity mismatch"));
|
||||
}
|
||||
lock.set_validated(bm).await
|
||||
}
|
||||
|
||||
pub async fn reload_bucket_metadata(api: Arc<ECStore>, bucket: &str) -> Result<()> {
|
||||
@@ -585,6 +587,15 @@ pub(crate) async fn get_cached_bucket_incarnation_id_in(
|
||||
sys.get_bucket_incarnation_id(bucket).await
|
||||
}
|
||||
|
||||
pub(crate) async fn get_bucket_incarnation_metadata_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
) -> Result<Arc<BucketMetadata>> {
|
||||
let sys = bucket_metadata_sys_of(ctx)?;
|
||||
let sys = sys.read().await.clone();
|
||||
sys.get_bucket_incarnation_metadata(bucket).await
|
||||
}
|
||||
|
||||
pub(crate) async fn set_bucket_metadata_in(ctx: &crate::runtime::instance::InstanceContext, bm: BucketMetadata) -> Result<()> {
|
||||
let sys = bucket_metadata_sys_of(ctx)?;
|
||||
let lock = sys.read().await;
|
||||
@@ -2132,7 +2143,16 @@ impl BucketMetadataSys {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn set(&self, bucket: String, bm: Arc<BucketMetadata>) {
|
||||
pub(crate) async fn set_validated(&self, mut bm: BucketMetadata) -> Result<()> {
|
||||
if bm.artifact_base_prefix.is_some() {
|
||||
bm.parse_all_configs()?;
|
||||
}
|
||||
bm.validate_artifact_update(self.object_store()).await?;
|
||||
self.set(bm.name.clone(), Arc::new(bm)).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn set(&self, bucket: String, bm: Arc<BucketMetadata>) {
|
||||
if !is_meta_bucketname(&bucket) {
|
||||
let publish_lock = self.metadata_publish_lock(&bucket);
|
||||
let _publish_guard = publish_lock.lock().await;
|
||||
@@ -2147,6 +2167,11 @@ impl BucketMetadataSys {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub async fn set_for_test(&self, bucket: String, bm: Arc<BucketMetadata>) {
|
||||
self.set(bucket, bm).await;
|
||||
}
|
||||
|
||||
/// Remove a bucket's cached metadata from the in-memory map.
|
||||
///
|
||||
/// Returns `true` if an entry was present. Reserved meta buckets are ignored.
|
||||
@@ -2651,12 +2676,16 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
async fn get_bucket_incarnation_id(&self, bucket: &str) -> Result<Uuid> {
|
||||
Ok(self.get_bucket_incarnation_metadata(bucket).await?.bucket_incarnation_id)
|
||||
}
|
||||
|
||||
async fn get_bucket_incarnation_metadata(&self, bucket: &str) -> Result<Arc<BucketMetadata>> {
|
||||
let authority = self.get_metadata_authority(bucket).await?;
|
||||
if let BucketMetadataAuthority::Authoritative(metadata) = &authority
|
||||
&& metadata.bucket_incarnation_sidecar
|
||||
&& !metadata.bucket_incarnation_id.is_nil()
|
||||
{
|
||||
return Ok(metadata.bucket_incarnation_id);
|
||||
return Ok(Arc::clone(metadata));
|
||||
}
|
||||
|
||||
match authority {
|
||||
@@ -2665,7 +2694,7 @@ impl BucketMetadataSys {
|
||||
BucketMetadataAuthority::Authoritative(metadata)
|
||||
if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
|
||||
{
|
||||
Ok(metadata.bucket_incarnation_id)
|
||||
Ok(metadata)
|
||||
}
|
||||
BucketMetadataAuthority::Authoritative(_) | BucketMetadataAuthority::Fabricated => {
|
||||
Err(Error::other(format!("bucket incarnation metadata is not authoritative: {bucket}")))
|
||||
@@ -2993,7 +3022,8 @@ impl BucketMetadataSys {
|
||||
load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
|
||||
)
|
||||
.await?;
|
||||
if persisted && metadata.lock_enabled && !metadata.bucket_creation_committed {
|
||||
if persisted && (metadata.lock_enabled || metadata.artifact_base_prefix.is_some()) && !metadata.bucket_creation_committed
|
||||
{
|
||||
return Err(Error::ErasureWriteQuorum);
|
||||
}
|
||||
if persisted {
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
// #730: bucket subsystems still contain staged ECStore migration code.
|
||||
|
||||
pub mod artifact;
|
||||
pub mod bandwidth;
|
||||
pub mod bucket_target_sys;
|
||||
pub mod config_parse_mode;
|
||||
|
||||
@@ -235,6 +235,8 @@ impl QuotaChecker {
|
||||
Ok(updated_at)
|
||||
}
|
||||
|
||||
/// Return quota configuration with optional scanner usage, independently
|
||||
/// of the usage requirements enforced by write admission.
|
||||
pub async fn get_quota_stats(&self, bucket: &str) -> Result<(BucketQuota, Option<u64>), QuotaError> {
|
||||
// If bucket doesn't exist, return ConfigNotFound error
|
||||
if !self.bucket_exists(bucket).await {
|
||||
@@ -244,18 +246,13 @@ impl QuotaChecker {
|
||||
}
|
||||
|
||||
let quota = self.get_quota_config(bucket).await?;
|
||||
let current_usage = if quota.uses_durable_reservations() {
|
||||
self.report_only_usage(bucket).await
|
||||
} else {
|
||||
Some(self.get_real_time_usage(bucket).await?)
|
||||
};
|
||||
let current_usage = self.report_only_usage(bucket).await;
|
||||
|
||||
Ok((quota, current_usage))
|
||||
}
|
||||
|
||||
/// Scanner usage for a bucket the reservation ledger admits is reporting
|
||||
/// input only, so unavailable memory usage is reported as "unknown" and counted
|
||||
/// instead of becoming an error the write path would fail on.
|
||||
/// Best-effort usage for statistics and durable admission reports.
|
||||
/// Missing scanner coverage is unknown, not an error or a zero-sized bucket.
|
||||
async fn report_only_usage(&self, bucket: &str) -> Option<u64> {
|
||||
// The plain lookup: a report must not wait for an early snapshot reload.
|
||||
let usage = get_bucket_usage_memory(bucket).await;
|
||||
@@ -267,7 +264,7 @@ impl QuotaChecker {
|
||||
subsystem = LOG_SUBSYSTEM_QUOTA,
|
||||
state = "usage_unknown",
|
||||
bucket = %bucket,
|
||||
"durable quota usage report degraded"
|
||||
"quota usage report degraded"
|
||||
);
|
||||
}
|
||||
usage
|
||||
@@ -742,8 +739,12 @@ mod tests {
|
||||
1,
|
||||
"fail-closed admission takes the early-reload lookup"
|
||||
);
|
||||
assert!(matches!(checker.get_quota_stats(&legacy).await, Err(QuotaError::UsageUnavailable { .. })));
|
||||
assert_eq!(crate::data_usage::reload_unknown_bucket_usage_calls_for_test(&legacy), 2);
|
||||
let (_, usage) = checker
|
||||
.get_quota_stats(&legacy)
|
||||
.await
|
||||
.expect("legacy quota statistics are reporting only");
|
||||
assert_eq!(usage, None);
|
||||
assert_eq!(crate::data_usage::reload_unknown_bucket_usage_calls_for_test(&legacy), 1);
|
||||
assert_eq!(
|
||||
crate::data_usage::reload_unknown_bucket_usage_calls_for_test(&bucket),
|
||||
0,
|
||||
@@ -894,7 +895,7 @@ mod tests {
|
||||
.update_bucket_metadata_config(&bucket, QUOTA_CONFIG_FILE, legacy_quota_json(8192))
|
||||
.await
|
||||
.expect("newer quota save");
|
||||
sys.read().await.set(bucket.clone(), stale).await;
|
||||
sys.read().await.set_for_test(bucket.clone(), stale).await;
|
||||
assert_eq!(
|
||||
checker
|
||||
.migrate_reservation_protocol(&bucket)
|
||||
|
||||
@@ -245,3 +245,86 @@ async fn quota_without_limit_keeps_the_zero_usage_io_fast_path() {
|
||||
assert_eq!(crate::data_usage::reload_unknown_bucket_usage_calls_for_test(&bucket), 0);
|
||||
stop_reporting_fixture(&store).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_statistics_remain_available_without_a_complete_scanner_snapshot() {
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
|
||||
for configuration in ["unconfigured", "unlimited", "snapshot"] {
|
||||
for source in ["missing", "legacy", "incomplete", "observed", "complete", "empty"] {
|
||||
let (_dirs, store, _) = isolated_durable_quota_store().await;
|
||||
let bucket = format!("quota-statistics-{}", Uuid::new_v4().simple());
|
||||
match configuration {
|
||||
"unconfigured" => store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket without a quota configuration"),
|
||||
"unlimited" => {
|
||||
create_bucket_with_quota_json(
|
||||
&store,
|
||||
&bucket,
|
||||
serde_json::to_vec(&BucketQuota::new(None)).expect("unlimited quota encoding"),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
"snapshot" => create_bucket_with_quota_json(&store, &bucket, legacy_quota_json(4096)).await,
|
||||
_ => unreachable!(),
|
||||
}
|
||||
// An unregistered checker keeps legacy migration out of the
|
||||
// assertion: reporting must work before a protocol upgrade.
|
||||
let metadata_sys = BucketMetadataSys::new(store.clone());
|
||||
let (metadata, _) = metadata_sys.get_config(&bucket).await.expect("load existing bucket metadata");
|
||||
metadata_sys.set_for_test(bucket.clone(), metadata).await;
|
||||
let checker = QuotaChecker::new(Arc::new(RwLock::new(metadata_sys)));
|
||||
let (snapshot_source, size) = if source == "empty" { ("complete", 0) } else { (source, 1000) };
|
||||
install_reporting_snapshot(store.clone(), &bucket, snapshot_source, size).await;
|
||||
crate::data_usage::invalidate_data_usage_snapshot_cache().await;
|
||||
|
||||
let statistics = checker.get_quota_stats(&bucket).await;
|
||||
let early_reloads = crate::data_usage::reload_unknown_bucket_usage_calls_for_test(&bucket);
|
||||
let missing_bucket = format!("missing-{}", Uuid::new_v4().simple());
|
||||
let missing_statistics = checker.get_quota_stats(&missing_bucket).await;
|
||||
crate::data_usage::prepare_bucket_usage_for_namespace_change(&bucket, None)
|
||||
.await
|
||||
.expect("fixture cache cleanup");
|
||||
crate::data_usage::invalidate_data_usage_snapshot_cache().await;
|
||||
stop_reporting_fixture(&store).await;
|
||||
|
||||
let (quota, usage) = statistics.unwrap_or_else(|error| {
|
||||
panic!("configuration={configuration}, source={source}: quota statistics must remain available: {error}")
|
||||
});
|
||||
assert_eq!(quota.quota, (configuration == "snapshot").then_some(4096));
|
||||
assert_eq!(
|
||||
usage,
|
||||
(snapshot_source == "complete").then_some(size),
|
||||
"configuration={configuration}, source={source}: only known usage is numeric"
|
||||
);
|
||||
assert_eq!(early_reloads, 0, "read-only statistics must not force an early snapshot reload");
|
||||
assert!(
|
||||
matches!(missing_statistics, Err(QuotaError::ConfigNotFound { bucket }) if bucket == missing_bucket),
|
||||
"reporting must still reject a missing bucket"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quota_statistics_preserve_configuration_errors() {
|
||||
let (_dirs, store, _) = isolated_durable_quota_store().await;
|
||||
let bucket = format!("quota-invalid-{}", Uuid::new_v4().simple());
|
||||
create_bucket_with_quota_json(&store, &bucket, legacy_quota_json(4096)).await;
|
||||
let metadata_sys = BucketMetadataSys::new(store.clone());
|
||||
let (mut metadata, _) = metadata_sys.get_config(&bucket).await.expect("load existing bucket metadata");
|
||||
Arc::make_mut(&mut metadata).quota_config_json = b"{not-json".to_vec();
|
||||
metadata_sys.set_for_test(bucket.clone(), metadata).await;
|
||||
let checker = QuotaChecker::new(Arc::new(RwLock::new(metadata_sys)));
|
||||
|
||||
let statistics = checker.get_quota_stats(&bucket).await;
|
||||
stop_reporting_fixture(&store).await;
|
||||
|
||||
assert!(
|
||||
matches!(statistics, Err(QuotaError::InvalidConfig { .. })),
|
||||
"unavailable reporting must not hide malformed quota configuration"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -25,7 +25,7 @@ use std::{
|
||||
future::Future,
|
||||
io::ErrorKind,
|
||||
pin::Pin,
|
||||
sync::{Arc, OnceLock, atomic::AtomicBool},
|
||||
sync::{Arc, OnceLock},
|
||||
task::{Context, Poll},
|
||||
time::Duration,
|
||||
};
|
||||
@@ -40,7 +40,7 @@ const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_METACACHE: &str = "metacache";
|
||||
const EVENT_METACACHE_LISTING: &str = "metacache_listing";
|
||||
|
||||
pub type AgreedFn = Box<dyn Fn(MetaCacheEntry) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
|
||||
pub type AgreedFn = Box<dyn Fn(MetaCacheEntry) -> Pin<Box<dyn Future<Output = Result<(), DiskError>> + Send>> + Send + 'static>;
|
||||
pub type PartialFn =
|
||||
Box<dyn Fn(MetaCacheEntries, &[Option<DiskError>]) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
|
||||
pub(crate) type PartialResultFn = Box<
|
||||
@@ -198,6 +198,8 @@ pub(crate) enum TestReaderBehavior {
|
||||
stopped: Arc<tokio::sync::Notify>,
|
||||
},
|
||||
IgnoreCancel,
|
||||
/// Entries followed by a terminator trailer saying the producer stopped at its limit.
|
||||
LimitReached(Vec<MetaCacheEntry>),
|
||||
ProducerError(DiskError),
|
||||
PrimaryErrorThenFallback(DiskError),
|
||||
PartialThenTimeout(Vec<MetaCacheEntry>),
|
||||
@@ -236,8 +238,6 @@ pub struct ListPathRawOptions {
|
||||
pub skip_walkdir_total_timeout: bool,
|
||||
pub walkdir_timeout: Option<Duration>,
|
||||
pub walkdir_stall_timeout: Option<Duration>,
|
||||
/// Shared terminal state for bounded producer walks.
|
||||
pub producer_limit_reached: Option<Arc<AtomicBool>>,
|
||||
pub agreed: Option<AgreedFn>,
|
||||
pub partial: Option<PartialFn>,
|
||||
pub finished: Option<FinishedFn>,
|
||||
@@ -248,7 +248,7 @@ pub struct ListPathRawOptions {
|
||||
#[cfg(test)]
|
||||
pub(crate) peek_timeout: Option<Duration>,
|
||||
#[cfg(test)]
|
||||
pub(crate) test_disk_io_context_observed: Option<Arc<AtomicBool>>,
|
||||
pub(crate) test_disk_io_context_observed: Option<Arc<std::sync::atomic::AtomicBool>>,
|
||||
// pub agreed: Option<Arc<dyn Fn(MetaCacheEntry) + Send + Sync>>,
|
||||
// pub partial: Option<Arc<dyn Fn(MetaCacheEntries, &[Option<Error>]) + Send + Sync>>,
|
||||
// pub finished: Option<Arc<dyn Fn(&[Option<Error>]) + Send + Sync>>,
|
||||
@@ -273,7 +273,6 @@ impl Clone for ListPathRawOptions {
|
||||
skip_walkdir_total_timeout: self.skip_walkdir_total_timeout,
|
||||
walkdir_timeout: self.walkdir_timeout,
|
||||
walkdir_stall_timeout: self.walkdir_stall_timeout,
|
||||
producer_limit_reached: self.producer_limit_reached.clone(),
|
||||
#[cfg(test)]
|
||||
test_reader_behaviors: self.test_reader_behaviors.clone(),
|
||||
#[cfg(test)]
|
||||
@@ -301,22 +300,31 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
|
||||
skip_total_timeout: opts.skip_walkdir_total_timeout,
|
||||
timeout_ms: opts.walkdir_timeout.map(duration_millis),
|
||||
stall_timeout_ms: opts.walkdir_stall_timeout.map(duration_millis),
|
||||
producer_limit_reached: opts.producer_limit_reached.clone(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// How a merged raw listing stream ended.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub struct ListPathRawOutcome {
|
||||
/// At least one producer stopped at its per-disk limit instead of at the
|
||||
/// end of its namespace, so the closed stream is not proof of exhaustion.
|
||||
/// Carried in-band by the metacache stream trailer, so remote producers
|
||||
/// report it as well.
|
||||
pub limit_reached: bool,
|
||||
}
|
||||
|
||||
pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> disk::error::Result<()> {
|
||||
let rx = rx.child_token();
|
||||
let _cancel_guard = rx.clone().drop_guard();
|
||||
list_path_raw_inner(rx, opts, None, None).await
|
||||
list_path_raw_inner(rx, opts, None, None).await.map(|_| ())
|
||||
}
|
||||
|
||||
pub(crate) async fn list_path_raw_with_claim_tracker(
|
||||
rx: CancellationToken,
|
||||
opts: ListPathRawOptions,
|
||||
claim_tracker: FallbackClaimTracker,
|
||||
) -> disk::error::Result<()> {
|
||||
) -> disk::error::Result<ListPathRawOutcome> {
|
||||
let rx = rx.child_token();
|
||||
let _cancel_guard = rx.clone().drop_guard();
|
||||
list_path_raw_inner(rx, opts, Some(claim_tracker), None).await
|
||||
@@ -327,7 +335,7 @@ pub(crate) async fn list_path_raw_with_partial_result(
|
||||
opts: ListPathRawOptions,
|
||||
claim_tracker: FallbackClaimTracker,
|
||||
partial_result: PartialResultFn,
|
||||
) -> disk::error::Result<()> {
|
||||
) -> disk::error::Result<ListPathRawOutcome> {
|
||||
let rx = rx.child_token();
|
||||
let _cancel_guard = rx.clone().drop_guard();
|
||||
list_path_raw_inner(rx, opts, Some(claim_tracker), Some(partial_result)).await
|
||||
@@ -338,7 +346,7 @@ async fn list_path_raw_inner(
|
||||
opts: ListPathRawOptions,
|
||||
fallback_claim_tracker: Option<FallbackClaimTracker>,
|
||||
partial_result: Option<PartialResultFn>,
|
||||
) -> disk::error::Result<()> {
|
||||
) -> disk::error::Result<ListPathRawOutcome> {
|
||||
if opts.disks.is_empty() {
|
||||
return Err(DiskError::ErasureReadQuorum);
|
||||
}
|
||||
@@ -393,6 +401,14 @@ async fn list_path_raw_inner(
|
||||
out.close().await.expect("test entries should close");
|
||||
return Ok(());
|
||||
}
|
||||
TestReaderBehavior::LimitReached(entries) => {
|
||||
let mut out = rustfs_filemeta::MetacacheWriter::new(&mut wr);
|
||||
out.write(&entries).await.expect("test entries should be written");
|
||||
out.close_with_limit_reached(true)
|
||||
.await
|
||||
.expect("test entries should close at the limit");
|
||||
return Ok(());
|
||||
}
|
||||
TestReaderBehavior::Stall => {
|
||||
let _held_writer = wr;
|
||||
cancel_rx_clone.cancelled().await;
|
||||
@@ -531,6 +547,16 @@ async fn list_path_raw_inner(
|
||||
last_err = None;
|
||||
continue;
|
||||
}
|
||||
TestReaderBehavior::LimitReached(entries) => {
|
||||
let mut out = rustfs_filemeta::MetacacheWriter::new(&mut wr);
|
||||
out.write(&entries).await.expect("test fallback entries should be written");
|
||||
out.close_with_limit_reached(true)
|
||||
.await
|
||||
.expect("test fallback entries should close at the limit");
|
||||
need_fallback = false;
|
||||
last_err = None;
|
||||
continue;
|
||||
}
|
||||
TestReaderBehavior::ProducerError(err) | TestReaderBehavior::PrimaryErrorThenFallback(err) => {
|
||||
last_err = Some(err);
|
||||
continue;
|
||||
@@ -699,6 +725,9 @@ async fn list_path_raw_inner(
|
||||
}
|
||||
let mut pending_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
|
||||
let mut peek_outcomes: Vec<Option<PeekOutcome>> = std::iter::repeat_with(|| None).take(readers.len()).collect();
|
||||
// Sticky across rounds: once any reader's trailer says its producer
|
||||
// stopped at the per-disk limit, the merged stream may hide more keys.
|
||||
let mut limit_reached = false;
|
||||
|
||||
loop {
|
||||
let mut current = MetaCacheEntry::default();
|
||||
@@ -710,7 +739,7 @@ async fn list_path_raw_inner(
|
||||
// );
|
||||
|
||||
if revjob_rx.is_cancelled() {
|
||||
return Ok(());
|
||||
return Ok(ListPathRawOutcome { limit_reached });
|
||||
}
|
||||
|
||||
let mut top_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
|
||||
@@ -760,6 +789,7 @@ async fn list_path_raw_inner(
|
||||
continue;
|
||||
}
|
||||
// eof
|
||||
limit_reached |= r.limit_reached() == Some(true);
|
||||
at_eof += 1;
|
||||
// warn!("list_path_raw: peek eof, disk: {}", i);
|
||||
continue;
|
||||
@@ -833,7 +863,7 @@ async fn list_path_raw_inner(
|
||||
*r = MetacacheReader::new(detached_rd);
|
||||
continue;
|
||||
}
|
||||
PeekOutcome::Cancelled => return Ok(()),
|
||||
PeekOutcome::Cancelled => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
}
|
||||
};
|
||||
|
||||
@@ -892,7 +922,7 @@ async fn list_path_raw_inner(
|
||||
if let Some(finished_fn) = opts.finished.as_ref() {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = revjob_rx.cancelled() => return Ok(()),
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
_ = finished_fn(&errs) => {}
|
||||
}
|
||||
}
|
||||
@@ -949,7 +979,7 @@ async fn list_path_raw_inner(
|
||||
{
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = revjob_rx.cancelled() => return Ok(()),
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
_ = finished_fn(&errs) => {}
|
||||
}
|
||||
}
|
||||
@@ -969,8 +999,8 @@ async fn list_path_raw_inner(
|
||||
// warn!("list_path_raw: agreed_fn start, current: {:?}", ¤t.name);
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = revjob_rx.cancelled() => return Ok(()),
|
||||
_ = agreed_fn(current) => {}
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
result = agreed_fn(current) => result?
|
||||
}
|
||||
// warn!("list_path_raw: agreed_fn done");
|
||||
}
|
||||
@@ -993,58 +1023,61 @@ async fn list_path_raw_inner(
|
||||
if let Some(partial_fn) = partial_result.as_ref() {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = revjob_rx.cancelled() => return Ok(()),
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
result = partial_fn(MetaCacheEntries(top_entries), &errs) => result?,
|
||||
}
|
||||
} else if let Some(partial_fn) = opts.partial.as_ref() {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = revjob_rx.cancelled() => return Ok(()),
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
_ = partial_fn(MetaCacheEntries(top_entries), &errs) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
Ok(ListPathRawOutcome { limit_reached })
|
||||
});
|
||||
|
||||
let merge_started = std::time::Instant::now();
|
||||
let Some(revjob) = revjobs.join_next().await else {
|
||||
return Err(DiskError::Unexpected);
|
||||
};
|
||||
if let Err(err) = revjob.map_err(std::io::Error::other)? {
|
||||
rustfs_io_metrics::record_stage_duration("metacache_merge_failed", merge_started.elapsed().as_secs_f64() * 1000.0);
|
||||
if is_missing_path_error(&err) {
|
||||
debug!(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %log_bucket,
|
||||
path = %log_path,
|
||||
state = "merge_job_missing_path",
|
||||
error = ?err,
|
||||
"Metacache merge job missing path skipped"
|
||||
);
|
||||
} else {
|
||||
error!(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %log_bucket,
|
||||
path = %log_path,
|
||||
state = "merge_job_failed",
|
||||
error = ?err,
|
||||
"Metacache merge job failed"
|
||||
);
|
||||
}
|
||||
cancel_rx.cancel();
|
||||
jobs.abort_all();
|
||||
let outcome = match revjob.map_err(std::io::Error::other)? {
|
||||
Ok(outcome) => outcome,
|
||||
Err(err) => {
|
||||
rustfs_io_metrics::record_stage_duration("metacache_merge_failed", merge_started.elapsed().as_secs_f64() * 1000.0);
|
||||
if is_missing_path_error(&err) {
|
||||
debug!(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %log_bucket,
|
||||
path = %log_path,
|
||||
state = "merge_job_missing_path",
|
||||
error = ?err,
|
||||
"Metacache merge job missing path skipped"
|
||||
);
|
||||
} else {
|
||||
error!(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %log_bucket,
|
||||
path = %log_path,
|
||||
state = "merge_job_failed",
|
||||
error = ?err,
|
||||
"Metacache merge job failed"
|
||||
);
|
||||
}
|
||||
cancel_rx.cancel();
|
||||
jobs.abort_all();
|
||||
|
||||
return Err(err);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
if rx.is_cancelled() {
|
||||
cancel_rx.cancel();
|
||||
jobs.abort_all();
|
||||
return Ok(());
|
||||
return Ok(outcome);
|
||||
}
|
||||
rustfs_io_metrics::record_stage_duration("metacache_merge", merge_started.elapsed().as_secs_f64() * 1000.0);
|
||||
|
||||
@@ -1113,7 +1146,7 @@ async fn list_path_raw_inner(
|
||||
}
|
||||
|
||||
// warn!("list_path_raw: done");
|
||||
Ok(())
|
||||
Ok(outcome)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
@@ -1132,6 +1165,7 @@ mod tests {
|
||||
use rustfs_filemeta::MetacacheWriter;
|
||||
use rustfs_filemeta::{FileInfo, FileMeta, MetadataResolutionParams};
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -1252,6 +1286,61 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_raw_reports_limit_reached_from_any_reader_trailer() {
|
||||
let first = fallback_test_entry();
|
||||
let second = MetaCacheEntry {
|
||||
name: "bucket/object-2".to_string(),
|
||||
..fallback_test_entry()
|
||||
};
|
||||
// One bounded producer, one that ends naturally, one legacy empty
|
||||
// stream without any trailer.
|
||||
let outcome = list_path_raw_inner(
|
||||
CancellationToken::new(),
|
||||
ListPathRawOptions {
|
||||
disks: vec![None, None, None],
|
||||
min_disks: 1,
|
||||
test_reader_behaviors: vec![
|
||||
TestReaderBehavior::LimitReached(vec![first.clone()]),
|
||||
TestReaderBehavior::Entries(vec![first, second]),
|
||||
TestReaderBehavior::Eof,
|
||||
],
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("listing should complete");
|
||||
|
||||
assert!(
|
||||
outcome.limit_reached,
|
||||
"a reader whose trailer reports a limit stop must mark the merged stream as possibly incomplete"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_raw_reports_natural_end_when_no_reader_trailer_says_limit() {
|
||||
let outcome = list_path_raw_inner(
|
||||
CancellationToken::new(),
|
||||
ListPathRawOptions {
|
||||
disks: vec![None, None],
|
||||
min_disks: 1,
|
||||
test_reader_behaviors: vec![
|
||||
TestReaderBehavior::Entries(vec![fallback_test_entry()]),
|
||||
TestReaderBehavior::Eof,
|
||||
],
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("listing should complete");
|
||||
|
||||
assert!(!outcome.limit_reached, "natural and legacy stream ends must not claim a limit stop");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_raw_returns_typed_quorum_error_for_multiple_drive_failures() {
|
||||
let err = list_path_raw(
|
||||
@@ -1710,6 +1799,34 @@ mod tests {
|
||||
.expect("aborting the listing should drop the blocked producer");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_raw_propagates_agreed_callback_failure() {
|
||||
let entry = MetaCacheEntry {
|
||||
name: "bucket/object".to_owned(),
|
||||
metadata: vec![1],
|
||||
..Default::default()
|
||||
};
|
||||
let result = timeout(
|
||||
Duration::from_secs(1),
|
||||
list_path_raw(
|
||||
CancellationToken::new(),
|
||||
ListPathRawOptions {
|
||||
disks: vec![None],
|
||||
min_disks: 1,
|
||||
test_reader_behaviors: vec![TestReaderBehavior::Entries(vec![entry])],
|
||||
agreed: Some(Box::new(|_| Box::pin(async { Err(DiskError::ErasureReadQuorum) }))),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("callback failure must terminate the walk");
|
||||
assert_eq!(
|
||||
result.expect_err("agreed resolution must not silently omit the key"),
|
||||
DiskError::ErasureReadQuorum
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_raw_propagates_partial_callback_failure() {
|
||||
let first = MetaCacheEntry {
|
||||
@@ -1811,6 +1928,7 @@ mod tests {
|
||||
Box::pin(async move {
|
||||
callback_started.notify_one();
|
||||
std::future::pending::<()>().await;
|
||||
Ok(())
|
||||
})
|
||||
})),
|
||||
..Default::default()
|
||||
@@ -1962,6 +2080,7 @@ mod tests {
|
||||
let seen = seen_clone.clone();
|
||||
Box::pin(async move {
|
||||
seen.lock().expect("seen mutex poisoned").push(entry.name);
|
||||
Ok(())
|
||||
})
|
||||
})),
|
||||
..Default::default()
|
||||
@@ -2004,6 +2123,7 @@ mod tests {
|
||||
let seen = seen_clone.clone();
|
||||
Box::pin(async move {
|
||||
seen.lock().expect("seen mutex poisoned").push(entry.name);
|
||||
Ok(())
|
||||
})
|
||||
})),
|
||||
..Default::default()
|
||||
@@ -2076,6 +2196,7 @@ mod tests {
|
||||
let seen = agreed_seen.clone();
|
||||
Box::pin(async move {
|
||||
seen.lock().expect("seen mutex poisoned").push(entry.name);
|
||||
Ok(())
|
||||
})
|
||||
})),
|
||||
partial: Some(Box::new(move |entries, _| {
|
||||
@@ -2241,6 +2362,7 @@ mod tests {
|
||||
let agreed_events = agreed_events.clone();
|
||||
Box::pin(async move {
|
||||
agreed_events.lock().expect("events lock").push((entry.name, 4usize, true));
|
||||
Ok(())
|
||||
})
|
||||
})),
|
||||
partial: Some(Box::new(move |entries, _| {
|
||||
@@ -2397,6 +2519,7 @@ mod tests {
|
||||
let name = entry.name;
|
||||
Box::pin(async move {
|
||||
agreed_obs.lock().expect("obs lock").push((name, vec![Some(desc); 4], true));
|
||||
Ok(())
|
||||
})
|
||||
})),
|
||||
partial: Some(Box::new(move |entries, _| {
|
||||
|
||||
@@ -631,6 +631,17 @@ fn decode_cross_pool_fence_capability(expected_member: &str, result: &[u8]) -> R
|
||||
Ok((version, epoch))
|
||||
}
|
||||
|
||||
fn decode_mrf_lifecycle_reader_capability(expected_member: &str, result: &[u8]) -> Result<(u8, Uuid)> {
|
||||
let (&maximum_format_version, identity) = result
|
||||
.split_first()
|
||||
.ok_or_else(|| Error::other("peer returned a truncated MRF lifecycle reader capability"))?;
|
||||
if maximum_format_version == 0 || expected_member.trim().is_empty() {
|
||||
return Err(Error::other("peer returned an invalid MRF lifecycle reader capability"));
|
||||
}
|
||||
let epoch = decode_remote_version_state_capability(expected_member, identity)?;
|
||||
Ok((maximum_format_version, epoch))
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct PeerLiveEventsBatch {
|
||||
pub events: Vec<u8>,
|
||||
@@ -1964,6 +1975,16 @@ impl PeerRestClient {
|
||||
Ok((self.topology_member.clone(), supported_version, epoch))
|
||||
}
|
||||
|
||||
/// Reports one peer's reader support without authorizing a lifecycle writer or rollback.
|
||||
pub async fn probe_mrf_lifecycle_reader(&self, topology_fingerprint: String) -> Result<(String, u8, Uuid)> {
|
||||
let probe = rustfs_protos::mrf_lifecycle_reader_capability_probe(Uuid::new_v4().as_bytes());
|
||||
let result = self
|
||||
.heal_control(rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION, topology_fingerprint, probe)
|
||||
.await?;
|
||||
let (maximum_format_version, epoch) = decode_mrf_lifecycle_reader_capability(&self.topology_member, &result)?;
|
||||
Ok((self.topology_member.clone(), maximum_format_version, epoch))
|
||||
}
|
||||
|
||||
pub async fn probe_ilm_recovery_export(&self, topology_fingerprint: String) -> Result<(String, Uuid)> {
|
||||
let probe = rustfs_protos::ilm_recovery_export_capability_probe(Uuid::new_v4().as_bytes());
|
||||
let result = self
|
||||
@@ -4333,6 +4354,49 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_lifecycle_reader_capability_decoder_preserves_the_advertised_version() {
|
||||
let epoch = Uuid::new_v4();
|
||||
for version in [1, 2, u8::MAX] {
|
||||
let result = rustfs_protos::encode_mrf_lifecycle_reader_capability(version, "node-a:9000", epoch.as_bytes())
|
||||
.expect("capability frame");
|
||||
assert_eq!(
|
||||
decode_mrf_lifecycle_reader_capability("node-a:9000", &result).expect("reported capability"),
|
||||
(version, epoch)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_lifecycle_reader_capability_decoder_rejects_unbound_or_malformed_identity() {
|
||||
let epoch = Uuid::new_v4();
|
||||
let valid =
|
||||
rustfs_protos::encode_mrf_lifecycle_reader_capability(2, "node-a:9000", epoch.as_bytes()).expect("capability frame");
|
||||
let mut invalid = vec![Vec::new(), vec![2], valid[..valid.len() - 1].to_vec()];
|
||||
let mut zero = valid.clone();
|
||||
zero[0] = 0;
|
||||
invalid.push(zero);
|
||||
let mut trailing = valid.clone();
|
||||
trailing.push(0);
|
||||
invalid.push(trailing);
|
||||
let mut length_overflow = valid.clone();
|
||||
length_overflow[1..9].copy_from_slice(&u64::MAX.to_be_bytes());
|
||||
invalid.push(length_overflow);
|
||||
let mut invalid_utf8 = valid.clone();
|
||||
invalid_utf8[9] = 0xff;
|
||||
invalid.push(invalid_utf8);
|
||||
invalid.push(
|
||||
rustfs_protos::encode_mrf_lifecycle_reader_capability(2, "node-a:9000", Uuid::nil().as_bytes())
|
||||
.expect("nil epoch fixture"),
|
||||
);
|
||||
for result in invalid {
|
||||
assert!(decode_mrf_lifecycle_reader_capability("node-a:9000", &result).is_err());
|
||||
}
|
||||
assert!(decode_mrf_lifecycle_reader_capability("node-b:9000", &valid).is_err());
|
||||
let empty = rustfs_protos::encode_mrf_lifecycle_reader_capability(2, "", epoch.as_bytes()).expect("empty member fixture");
|
||||
assert!(decode_mrf_lifecycle_reader_capability("", &empty).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_capability_decoder_fails_closed() {
|
||||
let epoch = Uuid::new_v4();
|
||||
|
||||
@@ -2375,7 +2375,300 @@ fn validate_decoded_file_info(file_info: &FileInfo) -> Result<()> {
|
||||
file_info.validate_for_metadata_read().map_err(Into::into)
|
||||
}
|
||||
|
||||
fn validate_artifact_repair_seal_response(
|
||||
request: &rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsRequest,
|
||||
response: &rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsResponse,
|
||||
) -> Result<()> {
|
||||
let canonical = rustfs_protos::canonical_seal_artifact_attempt_repairs_response_body(request, response)
|
||||
.map_err(|_| Error::other("artifact repair seal response is too large"))?;
|
||||
crate::cluster::rpc::verify_tonic_rpc_response_proof(&canonical, &response.response_proof)
|
||||
.map_err(|_| Error::other("artifact repair seal response proof is invalid"))?;
|
||||
if !response.success {
|
||||
return Err(response.error.clone().unwrap_or_default().into());
|
||||
}
|
||||
let expected_disk = Uuid::from_slice(&request.expected_disk_id).ok().filter(|id| !id.is_nil());
|
||||
if response.error.is_some() || expected_disk.is_none() || Uuid::from_slice(&response.disk_id).ok() != expected_disk {
|
||||
return Err(Error::other("artifact repair seal response disk identity mismatch"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl RemoteDisk {
|
||||
pub(crate) async fn rename_artifact_repair_data(
|
||||
&self,
|
||||
expected_disk: Uuid,
|
||||
source: (&str, &str),
|
||||
fi: &FileInfo,
|
||||
destination: (&str, &str),
|
||||
expected_incarnation: Uuid,
|
||||
) -> Result<RenameDataResp> {
|
||||
use rustfs_protos::proto_gen::node_service::ArtifactRepairRenameDataRequest;
|
||||
self.execute_with_timeout_for_op(
|
||||
"artifact_repair_rename_data",
|
||||
|| async {
|
||||
let body = ArtifactRepairRenameDataRequest {
|
||||
rename: Some(RenameDataRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
src_volume: source.0.to_owned(),
|
||||
src_path: source.1.to_owned(),
|
||||
dst_volume: destination.0.to_owned(),
|
||||
dst_path: destination.1.to_owned(),
|
||||
file_info: compat_json(fi)?,
|
||||
file_info_bin: encode_file_info_msgpack(fi)?.into(),
|
||||
scanner_publication_lease_token: Bytes::new(),
|
||||
bucket_incarnation_id: expected_incarnation.as_bytes().to_vec().into(),
|
||||
}),
|
||||
expected_disk_id: expected_disk.as_bytes().to_vec().into(),
|
||||
};
|
||||
let canonical = rustfs_protos::canonical_artifact_repair_rename_data_request_body(&body)
|
||||
.map_err(|_| Error::other("artifact repair rename request is too large"))?;
|
||||
let mut request = Request::new(body);
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
|
||||
// Unimplemented is terminal: never downgrade repair publication to PUT or legacy RPCs.
|
||||
let response = self
|
||||
.get_client()
|
||||
.await?
|
||||
.artifact_repair_rename_data(request)
|
||||
.await?
|
||||
.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
decode_msgpack_or_json::<RenameDataResp>(
|
||||
&response.rename_data_resp_bin,
|
||||
&response.rename_data_resp,
|
||||
"RenameDataResp",
|
||||
)
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn write_artifact_repair_metadata(
|
||||
&self,
|
||||
expected_disk: Uuid,
|
||||
target: (&str, &str),
|
||||
fi: &FileInfo,
|
||||
expected_incarnation: Uuid,
|
||||
) -> Result<()> {
|
||||
use rustfs_protos::proto_gen::node_service::ArtifactRepairWriteMetadataRequest;
|
||||
self.execute_with_timeout_for_op(
|
||||
"artifact_repair_write_metadata",
|
||||
|| async {
|
||||
let body = ArtifactRepairWriteMetadataRequest {
|
||||
metadata: Some(WriteMetadataRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volume: target.0.to_owned(),
|
||||
path: target.1.to_owned(),
|
||||
file_info: compat_json(fi)?,
|
||||
file_info_bin: encode_file_info_msgpack(fi)?.into(),
|
||||
bucket_incarnation_id: expected_incarnation.as_bytes().to_vec().into(),
|
||||
}),
|
||||
expected_disk_id: expected_disk.as_bytes().to_vec().into(),
|
||||
};
|
||||
let canonical = rustfs_protos::canonical_artifact_repair_write_metadata_request_body(&body)
|
||||
.map_err(|_| Error::other("artifact repair metadata request is too large"))?;
|
||||
let mut request = Request::new(body);
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
|
||||
let response = self
|
||||
.get_client()
|
||||
.await?
|
||||
.artifact_repair_write_metadata(request)
|
||||
.await?
|
||||
.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// One authenticated scan (or explicitly distinct read-only identity probe).
|
||||
/// Partial observations are never retried after output has begun.
|
||||
pub async fn inventory_artifact_attempt(
|
||||
&self,
|
||||
expected_disk: Uuid,
|
||||
target: (&str, Uuid),
|
||||
prefix: &str,
|
||||
phase: rustfs_protos::proto_gen::node_service::ArtifactInventoryPhase,
|
||||
deadline: tokio::time::Instant,
|
||||
output: tokio::sync::mpsc::Sender<crate::disk::ArtifactInventoryFrame>,
|
||||
) -> Result<()> {
|
||||
use crate::disk::{ArtifactInventoryFrame, ArtifactInventoryVersion};
|
||||
use rustfs_protos::{
|
||||
artifact_inventory as protocol,
|
||||
proto_gen::node_service::{ArtifactInventoryRequest, artifact_inventory_frame::Payload},
|
||||
};
|
||||
let (bucket, incarnation) = target;
|
||||
let duration = deadline
|
||||
.checked_duration_since(tokio::time::Instant::now())
|
||||
.ok_or(Error::Timeout)?;
|
||||
let body = ArtifactInventoryRequest {
|
||||
protocol_version: protocol::VERSION,
|
||||
invocation_id: Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
disk: self.endpoint.to_string(),
|
||||
expected_disk_id: expected_disk.as_bytes().to_vec().into(),
|
||||
bucket: bucket.to_owned(),
|
||||
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
|
||||
attempt_prefix: prefix.to_owned(),
|
||||
timeout_millis: u64::try_from(duration.as_millis().min(u128::from(protocol::MAX_TIMEOUT_MILLIS)))
|
||||
.map_err(Error::other)?,
|
||||
phase: phase.into(),
|
||||
};
|
||||
protocol::validate_request(&body).map_err(Error::other)?;
|
||||
tokio::time::timeout_at(deadline, async {
|
||||
let canonical = protocol::canonical_request(&body).map_err(Error::other)?;
|
||||
let mut request = Request::new(body.clone());
|
||||
request.set_timeout(duration);
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
|
||||
let mut stream = self
|
||||
.get_client()
|
||||
.await?
|
||||
.max_decoding_message_size(protocol::MAX_FRAME_BYTES)
|
||||
.inventory_artifact_attempt(request)
|
||||
.await?
|
||||
.into_inner();
|
||||
let mut state = protocol::StreamState::new();
|
||||
let mut terminal = None;
|
||||
while let Some(frame) = stream.message().await? {
|
||||
state
|
||||
.accept(&body, &frame, |canonical, proof| {
|
||||
crate::cluster::rpc::verify_tonic_rpc_response_proof(canonical, proof).is_ok()
|
||||
})
|
||||
.map_err(Error::other)?;
|
||||
match frame.payload {
|
||||
Some(Payload::Entries(entries)) => {
|
||||
let versions = entries
|
||||
.versions
|
||||
.into_iter()
|
||||
.map(|entry| {
|
||||
Ok(ArtifactInventoryVersion {
|
||||
key: entry.key,
|
||||
version_id: Uuid::from_slice(&entry.version_id).map_err(Error::other)?,
|
||||
delete_marker: entry.delete_marker,
|
||||
})
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
output
|
||||
.send(ArtifactInventoryFrame::Entries {
|
||||
sequence: frame.sequence,
|
||||
versions,
|
||||
})
|
||||
.await
|
||||
.map_err(|_| Error::other("artifact inventory consumer closed"))?;
|
||||
}
|
||||
Some(Payload::Complete(complete)) => terminal = Some(complete.entries),
|
||||
Some(Payload::Verified(_)) => {}
|
||||
None => return Err(Error::other("artifact inventory payload is absent")),
|
||||
}
|
||||
}
|
||||
// A valid Complete followed by a transport failure or extra frame is
|
||||
// still incomplete. Publish the local terminal only after clean EOF.
|
||||
state.finish().map_err(Error::other)?;
|
||||
if let Some(entries) = terminal {
|
||||
output
|
||||
.send(ArtifactInventoryFrame::Complete {
|
||||
disk_id: expected_disk,
|
||||
incarnation,
|
||||
attempt_prefix: prefix.to_owned(),
|
||||
entries,
|
||||
})
|
||||
.await
|
||||
.map_err(|_| Error::other("artifact inventory consumer closed"))?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.map_err(|_| Error::Timeout)?
|
||||
}
|
||||
|
||||
pub(crate) async fn seal_artifact_puts(
|
||||
&self,
|
||||
expected_disk: Uuid,
|
||||
bucket: &str,
|
||||
incarnation: Uuid,
|
||||
attempt_prefix: &str,
|
||||
) -> Result<()> {
|
||||
use rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsRequest;
|
||||
self.execute_with_timeout_for_op(
|
||||
"seal_artifact_puts",
|
||||
|| async {
|
||||
let body = SealArtifactAttemptPutsRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
bucket: bucket.to_owned(),
|
||||
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
|
||||
attempt_prefix: attempt_prefix.to_owned(),
|
||||
expected_disk_id: expected_disk.as_bytes().to_vec().into(),
|
||||
};
|
||||
let canonical = rustfs_protos::canonical_seal_artifact_attempt_puts_request_body(&body)
|
||||
.map_err(|_| Error::other("artifact seal request is too large"))?;
|
||||
let mut request = Request::new(body.clone());
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
|
||||
let response = self
|
||||
.get_client()
|
||||
.await?
|
||||
.seal_artifact_attempt_puts(request)
|
||||
.await?
|
||||
.into_inner();
|
||||
let canonical = rustfs_protos::canonical_seal_artifact_attempt_puts_response_body(&body, &response)
|
||||
.map_err(|_| Error::other("artifact seal response is too large"))?;
|
||||
crate::cluster::rpc::verify_tonic_rpc_response_proof(&canonical, &response.response_proof)
|
||||
.map_err(|_| Error::other("artifact seal response proof is invalid"))?;
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
if response.error.is_some() || Uuid::from_slice(&response.disk_id).ok() != Some(expected_disk) {
|
||||
return Err(Error::other("artifact seal response disk identity mismatch"));
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn seal_artifact_repairs(
|
||||
&self,
|
||||
expected_disk: Uuid,
|
||||
bucket: &str,
|
||||
incarnation: Uuid,
|
||||
attempt_prefix: &str,
|
||||
) -> Result<()> {
|
||||
use rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsRequest;
|
||||
if expected_disk.is_nil() || incarnation.is_nil() {
|
||||
return Err(Error::other("artifact repair seal requires non-nil disk and incarnation identities"));
|
||||
}
|
||||
self.execute_with_timeout_for_op(
|
||||
"seal_artifact_repairs",
|
||||
|| async {
|
||||
let body = SealArtifactAttemptPutsRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
bucket: bucket.to_owned(),
|
||||
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
|
||||
attempt_prefix: attempt_prefix.to_owned(),
|
||||
expected_disk_id: expected_disk.as_bytes().to_vec().into(),
|
||||
};
|
||||
let canonical = rustfs_protos::canonical_seal_artifact_attempt_repairs_request_body(&body)
|
||||
.map_err(|_| Error::other("artifact repair seal request is too large"))?;
|
||||
let mut request = Request::new(body.clone());
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
|
||||
let response = self
|
||||
.get_client()
|
||||
.await?
|
||||
.seal_artifact_attempt_repairs(request)
|
||||
.await?
|
||||
.into_inner();
|
||||
validate_artifact_repair_seal_response(&body, &response)
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub(crate) async fn rename_data_borrowed(
|
||||
&self,
|
||||
@@ -2385,20 +2678,39 @@ impl RemoteDisk {
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
) -> Result<RenameDataResp> {
|
||||
self.rename_data_borrowed_with_fence(src_volume, src_path, fi, dst_volume, dst_path, None)
|
||||
self.rename_data_borrowed_with_fence((src_volume, src_path), fi, (dst_volume, dst_path), None, None)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub(crate) async fn rename_data_borrowed_with_fence(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
source: (&str, &str),
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
target: (&str, &str),
|
||||
scanner_publication_lease_token: Option<Uuid>,
|
||||
artifact_incarnation: Option<Uuid>,
|
||||
) -> Result<RenameDataResp> {
|
||||
let (src_volume, src_path) = source;
|
||||
let (dst_volume, dst_path) = target;
|
||||
if let Some(scope) = crate::bucket::artifact::artifact_repair_scope(dst_volume, dst_path)
|
||||
&& let Some(identity) = scope.receiver_identity()
|
||||
{
|
||||
scope.check().map_err(Error::other)?;
|
||||
if artifact_incarnation.is_some() || scanner_publication_lease_token.is_some() {
|
||||
return Err(Error::other("artifact repair cannot use a publication lease"));
|
||||
}
|
||||
let expected = self
|
||||
.get_disk_id()
|
||||
.await?
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or_else(|| Error::other("artifact repair disk identity is unavailable"))?;
|
||||
let result = self
|
||||
.rename_artifact_repair_data(expected, (src_volume, src_path), fi, (dst_volume, dst_path), identity.incarnation)
|
||||
.await;
|
||||
scope.check().map_err(Error::other)?;
|
||||
return result;
|
||||
}
|
||||
trace!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -2421,8 +2733,11 @@ impl RemoteDisk {
|
||||
let file_info_bin = encode_file_info_msgpack(fi)?;
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(RenameDataRequest {
|
||||
bucket_incarnation_id: crate::store::bucket_heal_scope_for_object(dst_volume, dst_path)
|
||||
.map(|scope| scope.incarnation.as_bytes().to_vec().into())
|
||||
bucket_incarnation_id: artifact_incarnation
|
||||
.or_else(|| {
|
||||
crate::store::bucket_heal_scope_for_object(dst_volume, dst_path).map(|scope| scope.incarnation)
|
||||
})
|
||||
.map(|incarnation| incarnation.as_bytes().to_vec().into())
|
||||
.unwrap_or_default(),
|
||||
disk: self.endpoint.to_string(),
|
||||
src_volume: src_volume.to_string(),
|
||||
@@ -2435,7 +2750,11 @@ impl RemoteDisk {
|
||||
.map(|token| token.as_bytes().to_vec().into())
|
||||
.unwrap_or_default(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_rename_data_request_body(request.get_ref());
|
||||
let canonical_body = if artifact_incarnation.is_some() {
|
||||
rustfs_protos::canonical_artifact_rename_data_request_body(request.get_ref())
|
||||
} else {
|
||||
rustfs_protos::canonical_rename_data_request_body(request.get_ref())
|
||||
};
|
||||
let incarnation_bound = !request.get_ref().bucket_incarnation_id.is_empty();
|
||||
let quota_fenced = rustfs_utils::http::metadata_compat::contains_key_str(
|
||||
&fi.metadata,
|
||||
@@ -2458,7 +2777,9 @@ impl RemoteDisk {
|
||||
let rpc_started = rustfs_io_metrics::put_stage_timer();
|
||||
let response = rustfs_io_metrics::observe_put_stage_future(
|
||||
async {
|
||||
if quota_fenced {
|
||||
if artifact_incarnation.is_some() {
|
||||
client.artifact_rename_data(request).await
|
||||
} else if quota_fenced {
|
||||
// The method itself proves quota-token enforcement; an old peer must fail closed.
|
||||
client.rename_data_with_quota_fence(request).await
|
||||
} else if incarnation_bound {
|
||||
@@ -3204,6 +3525,21 @@ impl DiskAPI for RemoteDisk {
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
|
||||
if let Some(scope) = crate::bucket::artifact::artifact_repair_scope(volume, path)
|
||||
&& let Some(identity) = scope.receiver_identity()
|
||||
{
|
||||
scope.check().map_err(Error::other)?;
|
||||
let expected = self
|
||||
.get_disk_id()
|
||||
.await?
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or_else(|| Error::other("artifact repair disk identity is unavailable"))?;
|
||||
let result = self
|
||||
.write_artifact_repair_metadata(expected, (volume, path), &fi, identity.incarnation)
|
||||
.await;
|
||||
scope.check().map_err(Error::other)?;
|
||||
return result;
|
||||
}
|
||||
trace!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -4553,6 +4889,83 @@ mod tests {
|
||||
|
||||
static INIT: Once = Once::new();
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_repair_old_peer_unimplemented_never_downgrades_mutations() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
|
||||
.await
|
||||
.expect("loopback test peer");
|
||||
let endpoint = Endpoint {
|
||||
url: url::Url::parse(&format!("{}/data/rustfs0", peer.addr)).expect("test disk endpoint"),
|
||||
is_local: false,
|
||||
pool_idx: 0,
|
||||
set_idx: 0,
|
||||
disk_idx: 0,
|
||||
};
|
||||
let disk = RemoteDisk::new(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
Arc::new(TcpHttpInternodeDataTransport),
|
||||
)
|
||||
.await
|
||||
.expect("remote test disk");
|
||||
let expected_disk = Uuid::new_v4();
|
||||
let incarnation = Uuid::new_v4();
|
||||
let info = FileInfo::default();
|
||||
let rename = disk
|
||||
.rename_artifact_repair_data(
|
||||
expected_disk,
|
||||
("tmp", "staged"),
|
||||
&info,
|
||||
("artifacts", "outputs/tenant/run/attempt/image"),
|
||||
incarnation,
|
||||
)
|
||||
.await
|
||||
.expect_err("old peer cannot publish repair data");
|
||||
let metadata = disk
|
||||
.write_artifact_repair_metadata(expected_disk, ("artifacts", "outputs/tenant/run/attempt/image"), &info, incarnation)
|
||||
.await
|
||||
.expect_err("old peer cannot publish repair metadata");
|
||||
let seal = disk
|
||||
.seal_artifact_repairs(expected_disk, "artifacts", incarnation, "outputs/tenant/run/attempt")
|
||||
.await
|
||||
.expect_err("old peer cannot confirm repair closure");
|
||||
for error in [rename, metadata, seal] {
|
||||
let DiskError::Io(error) = error else {
|
||||
panic!("expected gRPC status error");
|
||||
};
|
||||
let status = error
|
||||
.get_ref()
|
||||
.and_then(|error| error.downcast_ref::<crate::disk::error::RpcStatusError>())
|
||||
.expect("preserved gRPC status");
|
||||
assert_eq!(status.status().code(), Code::Unimplemented);
|
||||
}
|
||||
let paths = peer.peer.rpc_paths.lock().expect("RPC inventory").clone();
|
||||
for method in [
|
||||
"ArtifactRepairRenameData",
|
||||
"ArtifactRepairWriteMetadata",
|
||||
"SealArtifactAttemptRepairs",
|
||||
] {
|
||||
assert_eq!(
|
||||
paths.iter().filter(|path| path.ends_with(&format!("/{method}"))).count(),
|
||||
1,
|
||||
"each repair method is sent once"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
paths.iter().all(|path| matches!(
|
||||
path.rsplit('/').next(),
|
||||
Some("ArtifactRepairRenameData" | "ArtifactRepairWriteMetadata" | "SealArtifactAttemptRepairs" | "Ping")
|
||||
)),
|
||||
"repair cannot downgrade to a legacy mutation: {paths:?}"
|
||||
);
|
||||
disk.cancel_token.cancel();
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quota_fence_old_peer_rejects_without_legacy_fallback() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
@@ -4648,6 +5061,93 @@ mod tests {
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_repair_seal_response_requires_proof_full_request_and_physical_identity() {
|
||||
use rustfs_protos::proto_gen::node_service::{
|
||||
Error as WireError, SealArtifactAttemptPutsRequest, SealArtifactAttemptPutsResponse,
|
||||
};
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let request = SealArtifactAttemptPutsRequest {
|
||||
disk: "disk-a".into(),
|
||||
bucket: "artifacts".into(),
|
||||
bucket_incarnation_id: vec![1; 16].into(),
|
||||
attempt_prefix: "outputs/tenant/run/attempt".into(),
|
||||
expected_disk_id: vec![2; 16].into(),
|
||||
};
|
||||
let mut response = SealArtifactAttemptPutsResponse {
|
||||
success: true,
|
||||
disk_id: request.expected_disk_id.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
let sign = |request: &SealArtifactAttemptPutsRequest, response: &mut SealArtifactAttemptPutsResponse| {
|
||||
let canonical = rustfs_protos::canonical_seal_artifact_attempt_repairs_response_body(request, response)
|
||||
.expect("repair response canonical");
|
||||
response.response_proof = crate::cluster::rpc::http_auth::sign_tonic_rpc_response_proof(&canonical)
|
||||
.expect("repair response proof")
|
||||
.into();
|
||||
};
|
||||
assert!(validate_artifact_repair_seal_response(&request, &response).is_err(), "proof is mandatory");
|
||||
sign(&request, &mut response);
|
||||
validate_artifact_repair_seal_response(&request, &response).expect("signed exact physical receiver");
|
||||
for mutate in [
|
||||
|r: &mut SealArtifactAttemptPutsRequest| r.disk = "other-disk".into(),
|
||||
|r: &mut SealArtifactAttemptPutsRequest| r.bucket = "other-bucket".into(),
|
||||
|r: &mut SealArtifactAttemptPutsRequest| r.bucket_incarnation_id = vec![3; 16].into(),
|
||||
|r: &mut SealArtifactAttemptPutsRequest| r.attempt_prefix = "other-attempt".into(),
|
||||
|r: &mut SealArtifactAttemptPutsRequest| r.expected_disk_id = vec![4; 16].into(),
|
||||
] {
|
||||
let mut changed = request.clone();
|
||||
mutate(&mut changed);
|
||||
assert!(
|
||||
validate_artifact_repair_seal_response(&changed, &response).is_err(),
|
||||
"another request cannot reuse the proof"
|
||||
);
|
||||
}
|
||||
let legacy = rustfs_protos::canonical_seal_artifact_attempt_puts_response_body(&request, &response).expect("PUT domain");
|
||||
let mut legacy_response = response.clone();
|
||||
legacy_response.response_proof = crate::cluster::rpc::http_auth::sign_tonic_rpc_response_proof(&legacy)
|
||||
.expect("PUT proof")
|
||||
.into();
|
||||
assert!(
|
||||
validate_artifact_repair_seal_response(&request, &legacy_response).is_err(),
|
||||
"PUT proof cannot acknowledge repair seal"
|
||||
);
|
||||
for disk in [vec![], vec![0; 16], vec![3; 16], vec![1; 15]] {
|
||||
let mut changed = response.clone();
|
||||
changed.disk_id = disk.into();
|
||||
sign(&request, &mut changed);
|
||||
assert!(
|
||||
validate_artifact_repair_seal_response(&request, &changed).is_err(),
|
||||
"signed wrong or absent physical identity is invalid"
|
||||
);
|
||||
}
|
||||
for mutate in [
|
||||
|r: &mut SealArtifactAttemptPutsResponse| r.success = false,
|
||||
|r: &mut SealArtifactAttemptPutsResponse| r.error = Some(WireError::default()),
|
||||
] {
|
||||
let mut changed = response.clone();
|
||||
mutate(&mut changed);
|
||||
assert!(
|
||||
validate_artifact_repair_seal_response(&request, &changed).is_err(),
|
||||
"response fields are bound"
|
||||
);
|
||||
sign(&request, &mut changed);
|
||||
assert!(
|
||||
validate_artifact_repair_seal_response(&request, &changed).is_err(),
|
||||
"signed failure or inconsistent success is invalid"
|
||||
);
|
||||
}
|
||||
let mut nil_request = request;
|
||||
nil_request.expected_disk_id = vec![0; 16].into();
|
||||
let mut nil_response = response;
|
||||
nil_response.disk_id = nil_request.expected_disk_id.clone();
|
||||
sign(&nil_request, &mut nil_response);
|
||||
assert!(
|
||||
validate_artifact_repair_seal_response(&nil_request, &nil_response).is_err(),
|
||||
"nil physical identity is never a receipt"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recoverable_batch_delete_rejects_invalid_error_codes_and_length() {
|
||||
let errors = decode_delete_recovery_batch_errors(vec![None, Some(DiskError::FileNotFound.to_u32())], 2)
|
||||
@@ -5038,6 +5538,7 @@ mod tests {
|
||||
format_data: Bytes,
|
||||
read_all_data: Bytes,
|
||||
next_format_read_gate: Arc<StdMutex<Option<Arc<TestFormatReadGate>>>>,
|
||||
rpc_paths: Arc<StdMutex<Vec<String>>>,
|
||||
unsupported_methods: Arc<StdMutex<Vec<String>>>,
|
||||
}
|
||||
|
||||
@@ -5057,6 +5558,7 @@ mod tests {
|
||||
format_data,
|
||||
read_all_data,
|
||||
next_format_read_gate: Arc::default(),
|
||||
rpc_paths: Arc::default(),
|
||||
unsupported_methods: Arc::default(),
|
||||
}
|
||||
}
|
||||
@@ -5112,6 +5614,11 @@ mod tests {
|
||||
}
|
||||
|
||||
fn call(&mut self, request: http::Request<B>) -> Self::Future {
|
||||
self.peer
|
||||
.rpc_paths
|
||||
.lock()
|
||||
.expect("RPC path inventory")
|
||||
.push(request.uri().path().to_owned());
|
||||
match request.uri().path() {
|
||||
"/node_service.NodeService/DiskInfo" => {
|
||||
#[derive(Clone)]
|
||||
|
||||
@@ -438,6 +438,64 @@ where
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_observation<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<(Vec<u8>, ObjectInfo)>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let opts = ObjectOptions {
|
||||
suppress_read_repair: true,
|
||||
no_lock: false,
|
||||
skip_verify_bitrot: false,
|
||||
..Default::default()
|
||||
};
|
||||
crate::object_api::without_get_object_body_cache_hook(read_config_limited_preserve_empty_with_metadata_opts(
|
||||
api, file, &opts, max_bytes,
|
||||
))
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_observation_no_lock<S>(
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
max_bytes: usize,
|
||||
guard: &crate::object_api::WriteCommitGuard,
|
||||
) -> Result<(Vec<u8>, ObjectInfo)>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
if !guard.matches(RUSTFS_META_BUCKET, file) {
|
||||
return Err(Error::InvalidArgument(
|
||||
RUSTFS_META_BUCKET.to_owned(),
|
||||
file.to_owned(),
|
||||
"locked observation requires a matching write owner".to_owned(),
|
||||
));
|
||||
}
|
||||
let lock_lost_error = || Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "locked_config_observation",
|
||||
bucket: RUSTFS_META_BUCKET.to_owned(),
|
||||
object: file.to_owned(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
};
|
||||
if guard.is_lock_lost() {
|
||||
return Err(lock_lost_error());
|
||||
}
|
||||
let opts = ObjectOptions {
|
||||
suppress_read_repair: true,
|
||||
no_lock: true,
|
||||
skip_verify_bitrot: false,
|
||||
..Default::default()
|
||||
};
|
||||
let result = crate::object_api::without_get_object_body_cache_hook(read_config_limited_preserve_empty_with_metadata_opts(
|
||||
api, file, &opts, max_bytes,
|
||||
))
|
||||
.await;
|
||||
if guard.is_lock_lost() {
|
||||
return Err(lock_lost_error());
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_limited_preserve_empty<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<Vec<u8>>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
@@ -484,10 +542,22 @@ pub(crate) async fn read_config_preserve_empty<S>(api: Arc<S>, file: &str) -> Re
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let (data, _obj) = read_config_with_metadata_inner(api, file, &ObjectOptions::default(), true, None).await?;
|
||||
let (data, _obj) = read_config_preserve_empty_with_metadata_opts(api, file, &ObjectOptions::default()).await?;
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
/// Preserve an empty config payload, its metadata, and caller-supplied read options.
|
||||
pub(crate) async fn read_config_preserve_empty_with_metadata_opts<S>(
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(Vec<u8>, ObjectInfo)>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
read_config_with_metadata_inner(api, file, opts, true, None).await
|
||||
}
|
||||
|
||||
pub async fn read_config_no_lock<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
|
||||
where
|
||||
S: ObjectIO<
|
||||
@@ -516,15 +586,13 @@ pub(crate) async fn read_config_no_lock_preserve_empty_with_metadata<S>(api: Arc
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
read_config_with_metadata_inner(
|
||||
read_config_preserve_empty_with_metadata_opts(
|
||||
api,
|
||||
file,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
true,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -2919,9 +2987,9 @@ mod tests {
|
||||
config_task_join_error, configs_semantically_equal, decode_server_config_blob, encode_server_config_blob,
|
||||
heal_config_descriptor, is_standard_object_server_config, lookup_configs, map_system_metadata_write_error,
|
||||
new_and_save_server_config, read_config, read_config_no_lock_preserve_empty_with_metadata, read_config_preserve_empty,
|
||||
read_config_with_metadata, read_config_without_migrate, read_server_config_snapshot, save_config_with_opts_inner,
|
||||
save_server_config, save_server_config_snapshot, save_server_config_snapshot_with_generation,
|
||||
server_config_transaction_lock_path, storage_class_kvs_mut,
|
||||
read_config_preserve_empty_with_metadata_opts, read_config_with_metadata, read_config_without_migrate,
|
||||
read_server_config_snapshot, save_config_with_opts_inner, save_server_config, save_server_config_snapshot,
|
||||
save_server_config_snapshot_with_generation, server_config_transaction_lock_path, storage_class_kvs_mut,
|
||||
};
|
||||
use crate::config::{audit, heal, notify, oidc, scanner};
|
||||
use crate::disk::{RUSTFS_META_BUCKET, endpoint::Endpoint};
|
||||
@@ -5396,6 +5464,7 @@ mod tests {
|
||||
heal_replacement: Option<Vec<u8>>,
|
||||
heal_calls: AtomicUsize,
|
||||
write_calls: AtomicUsize,
|
||||
last_get_no_lock: AtomicBool,
|
||||
last_put_no_lock: AtomicBool,
|
||||
last_put_preconditions: Mutex<Option<HTTPPreconditions>>,
|
||||
revision: AtomicUsize,
|
||||
@@ -5411,6 +5480,7 @@ mod tests {
|
||||
heal_replacement,
|
||||
heal_calls: AtomicUsize::new(0),
|
||||
write_calls: AtomicUsize::new(0),
|
||||
last_get_no_lock: AtomicBool::new(false),
|
||||
last_put_no_lock: AtomicBool::new(false),
|
||||
last_put_preconditions: Mutex::new(None),
|
||||
revision: AtomicUsize::new(1),
|
||||
@@ -5466,8 +5536,9 @@ mod tests {
|
||||
_object: &str,
|
||||
_range: Option<HTTPRangeSpec>,
|
||||
_h: HeaderMap,
|
||||
_opts: &ObjectOptions,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<GetObjectReader> {
|
||||
self.last_get_no_lock.store(opts.no_lock, Ordering::SeqCst);
|
||||
let data = match &*self.state.lock().expect("state lock poisoned") {
|
||||
RecoveryReadState::Missing => return Err(Error::ConfigNotFound),
|
||||
RecoveryReadState::Blob(data) => data.clone(),
|
||||
@@ -5779,6 +5850,325 @@ mod tests {
|
||||
assert_eq!(store.write_calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
type ObservationEofGate = (Option<tokio::sync::oneshot::Sender<()>>, tokio::sync::oneshot::Receiver<()>);
|
||||
|
||||
struct ObservationStream {
|
||||
inner: Cursor<Vec<u8>>,
|
||||
consumed: Arc<AtomicUsize>,
|
||||
fail: bool,
|
||||
eof_gate: Option<ObservationEofGate>,
|
||||
}
|
||||
|
||||
impl AsyncRead for ObservationStream {
|
||||
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
|
||||
if self.fail {
|
||||
return Poll::Ready(Err(std::io::Error::other("observation stream failed")));
|
||||
}
|
||||
if self.inner.position() == u64::try_from(self.inner.get_ref().len()).expect("fixture length")
|
||||
&& let Some((ready, release)) = self.eof_gate.as_mut()
|
||||
{
|
||||
if let Some(ready) = ready.take() {
|
||||
ready.send(()).expect("EOF observer remains active");
|
||||
}
|
||||
match std::future::Future::poll(Pin::new(release), cx) {
|
||||
Poll::Pending => return Poll::Pending,
|
||||
Poll::Ready(result) => result.expect("EOF gate released"),
|
||||
}
|
||||
self.eof_gate = None;
|
||||
}
|
||||
let before = buf.filled().len();
|
||||
let result = Pin::new(&mut self.inner).poll_read(cx, buf);
|
||||
self.consumed.fetch_add(buf.filled().len() - before, Ordering::SeqCst);
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
struct ObservationStore {
|
||||
body: Vec<u8>,
|
||||
declared_size: i64,
|
||||
error: Option<Error>,
|
||||
stream_error: bool,
|
||||
gets: AtomicUsize,
|
||||
eof_gate: std::sync::Mutex<Option<ObservationEofGate>>,
|
||||
consumed: Arc<AtomicUsize>,
|
||||
options: std::sync::Mutex<Option<(bool, bool, bool)>>,
|
||||
writes: AtomicUsize,
|
||||
}
|
||||
|
||||
impl Debug for ObservationStore {
|
||||
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("ObservationStore").finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl ObservationStore {
|
||||
fn new(body: Vec<u8>, declared_size: i64) -> Self {
|
||||
Self {
|
||||
body,
|
||||
declared_size,
|
||||
error: None,
|
||||
stream_error: false,
|
||||
gets: AtomicUsize::new(0),
|
||||
eof_gate: std::sync::Mutex::new(None),
|
||||
consumed: Arc::new(AtomicUsize::new(0)),
|
||||
options: std::sync::Mutex::new(None),
|
||||
writes: AtomicUsize::new(0),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::storage_api_contracts::object::ObjectIO for ObservationStore {
|
||||
type Error = Error;
|
||||
type RangeSpec = HTTPRangeSpec;
|
||||
type HeaderMap = HeaderMap;
|
||||
type ObjectOptions = ObjectOptions;
|
||||
type ObjectInfo = ObjectInfo;
|
||||
type GetObjectReader = GetObjectReader;
|
||||
type PutObjectReader = PutObjReader;
|
||||
|
||||
async fn get_object_reader(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
range: Option<HTTPRangeSpec>,
|
||||
_headers: HeaderMap,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<GetObjectReader> {
|
||||
assert_eq!(bucket, RUSTFS_META_BUCKET);
|
||||
assert_eq!(object, "config/observation.bin");
|
||||
self.gets.fetch_add(1, Ordering::SeqCst);
|
||||
assert!(range.is_none());
|
||||
*self.options.lock().expect("observation options") =
|
||||
Some((opts.suppress_read_repair, opts.no_lock, opts.skip_verify_bitrot));
|
||||
if let Some(error) = &self.error {
|
||||
return Err(error.clone());
|
||||
}
|
||||
Ok(GetObjectReader {
|
||||
stream: Box::new(ObservationStream {
|
||||
inner: Cursor::new(self.body.clone()),
|
||||
consumed: self.consumed.clone(),
|
||||
fail: self.stream_error,
|
||||
eof_gate: self.eof_gate.lock().expect("EOF gate").take(),
|
||||
}),
|
||||
object_info: ObjectInfo {
|
||||
size: self.declared_size,
|
||||
etag: Some("observation-etag".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
buffered_body: None,
|
||||
body_source: Default::default(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn put_object(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_object: &str,
|
||||
_data: &mut PutObjReader,
|
||||
_opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
self.writes.fetch_add(1, Ordering::SeqCst);
|
||||
Err(Error::other("observation must not write"))
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn locked_observation_preserves_options_and_typed_errors() {
|
||||
let object = "config/observation.bin";
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let wrapper = rustfs_lock::NamespaceLockWrapper::new(
|
||||
rustfs_lock::NamespaceLock::with_local_manager("locked-observation-options".to_owned(), manager.clone()),
|
||||
rustfs_lock::ObjectKey::new(RUSTFS_META_BUCKET, object),
|
||||
"locked-observation-options".to_owned(),
|
||||
);
|
||||
let guard = crate::object_api::WriteCommitGuard::acquire(&wrapper, std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("matching real owner");
|
||||
let store = Arc::new(ObservationStore::new(b"raw".to_vec(), 3));
|
||||
let result = super::read_config_observation_no_lock(store.clone(), object, 3, &guard).await;
|
||||
assert!(result.is_ok(), "I06L_READ_FLAGS: {result:?}");
|
||||
assert!(*store.options.lock().expect("options") == Some((true, true, false)), "I06L_READ_FLAGS");
|
||||
assert_eq!(result.expect("raw observation").0, b"raw");
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
for (bucket, key) in [(RUSTFS_META_BUCKET, "config/other.bin"), ("wrong-bucket", object)] {
|
||||
let wrapper = rustfs_lock::NamespaceLockWrapper::new(
|
||||
rustfs_lock::NamespaceLock::with_local_manager("locked-observation-target".to_owned(), manager.clone()),
|
||||
rustfs_lock::ObjectKey::new(bucket, key),
|
||||
"locked-observation-target".to_owned(),
|
||||
);
|
||||
let wrong = crate::object_api::WriteCommitGuard::acquire(&wrapper, std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("wrong-target real owner");
|
||||
let store = Arc::new(ObservationStore::new(Vec::new(), 0));
|
||||
let result = super::read_config_observation_no_lock(store.clone(), object, 8, &wrong).await;
|
||||
assert!(matches!(result, Err(Error::InvalidArgument(..))), "I06L_TARGET_REJECTED");
|
||||
assert!(store.gets.load(Ordering::SeqCst) == 0, "I06L_TARGET_REJECTED");
|
||||
}
|
||||
for error in [
|
||||
Error::FileNotFound,
|
||||
Error::ObjectNotFound(RUSTFS_META_BUCKET.to_owned(), object.to_owned()),
|
||||
Error::ErasureReadQuorum,
|
||||
] {
|
||||
let expected = if error == Error::FileNotFound || matches!(error, Error::ObjectNotFound(..)) {
|
||||
Error::ConfigNotFound
|
||||
} else {
|
||||
error.clone()
|
||||
};
|
||||
let mut fixture = ObservationStore::new(Vec::new(), 0);
|
||||
fixture.error = Some(error);
|
||||
let store = Arc::new(fixture);
|
||||
assert_eq!(
|
||||
super::read_config_observation_no_lock(store.clone(), object, 8, &guard)
|
||||
.await
|
||||
.expect_err("typed error"),
|
||||
expected
|
||||
);
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
let mut fixture = ObservationStore::new(Vec::new(), 0);
|
||||
fixture.stream_error = true;
|
||||
let store = Arc::new(fixture);
|
||||
let err = super::read_config_observation_no_lock(store.clone(), object, 8, &guard)
|
||||
.await
|
||||
.expect_err("stream failure");
|
||||
assert!(err.to_string().contains("observation stream failed"), "I06L_TYPED_STREAM_ERROR");
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
let store = Arc::new(ObservationStore::new(vec![7; 64], 0));
|
||||
let result = super::read_config_observation_no_lock(store.clone(), object, 8, &guard).await;
|
||||
assert!(matches!(result, Err(Error::CorruptedFormat)), "I06L_STREAM_BOUND");
|
||||
assert!(store.consumed.load(Ordering::SeqCst) == 9, "I06L_STREAM_BOUND");
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn locked_observation_rejects_actual_owner_loss_before_and_after_read() {
|
||||
let object = "config/observation.bin";
|
||||
for after_read in [false, true] {
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let client = Arc::new(RefreshControlledClient::new(manager));
|
||||
let wrapper = rustfs_lock::NamespaceLockWrapper::new(
|
||||
rustfs_lock::NamespaceLock::new("locked-observation-loss".to_owned(), client.clone()),
|
||||
rustfs_lock::ObjectKey::new(RUSTFS_META_BUCKET, object),
|
||||
"locked-observation-loss".to_owned(),
|
||||
);
|
||||
let guard = crate::object_api::WriteCommitGuard::acquire(&wrapper, std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("real distributed owner");
|
||||
let signal = guard.lock_lost_signal().expect("acquired distributed owner signal");
|
||||
let store = Arc::new(ObservationStore::new(b"raw".to_vec(), 3));
|
||||
if after_read {
|
||||
let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
|
||||
*store.eof_gate.lock().expect("EOF gate") = Some((Some(ready_tx), release_rx));
|
||||
let mut read = Box::pin(super::read_config_observation_no_lock(store.clone(), object, 3, &guard));
|
||||
tokio::time::timeout(std::time::Duration::from_secs(60), async {
|
||||
tokio::select! {
|
||||
ready = ready_rx => ready.expect("read reached EOF"),
|
||||
result = &mut read => assert_eq!(false, true, "I06L_LOSS_AFTER_READ: returned before EOF gate: {result:?}"),
|
||||
}
|
||||
}).await.expect("EOF watchdog");
|
||||
client.reject_refreshes();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(60), signal.notified())
|
||||
.await
|
||||
.expect("actual owner loss watchdog");
|
||||
assert!(guard.is_lock_lost(), "I06L_LOSS_AFTER_READ");
|
||||
release_tx.send(()).expect("release EOF");
|
||||
let result = tokio::time::timeout(std::time::Duration::from_secs(60), read)
|
||||
.await
|
||||
.expect("read watchdog");
|
||||
assert!(
|
||||
matches!(
|
||||
result,
|
||||
Err(Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "locked_config_observation",
|
||||
..
|
||||
})
|
||||
),
|
||||
"I06L_LOSS_AFTER_READ"
|
||||
);
|
||||
assert_eq!(store.gets.load(Ordering::SeqCst), 1);
|
||||
} else {
|
||||
client.reject_refreshes();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(60), signal.notified())
|
||||
.await
|
||||
.expect("actual owner loss watchdog");
|
||||
assert!(guard.is_lock_lost(), "I06L_LOSS_BEFORE_GET");
|
||||
let result = super::read_config_observation_no_lock(store.clone(), object, 3, &guard).await;
|
||||
assert!(
|
||||
matches!(
|
||||
result,
|
||||
Err(Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "locked_config_observation",
|
||||
..
|
||||
})
|
||||
),
|
||||
"I06L_LOSS_BEFORE_GET"
|
||||
);
|
||||
assert!(store.gets.load(Ordering::SeqCst) == 0, "I06L_LOSS_BEFORE_GET");
|
||||
}
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn observation_preserves_options_and_errors() {
|
||||
for body in [Vec::new(), b"raw config".to_vec()] {
|
||||
let store = Arc::new(ObservationStore::new(body.clone(), i64::try_from(body.len()).expect("fixture size")));
|
||||
let result = super::read_config_observation(store.clone(), "config/observation.bin", 32).await;
|
||||
assert!(result.is_ok(), "observation must preserve empty and nonempty objects: {result:?}");
|
||||
let (observed, info) = result.expect("successful observation");
|
||||
assert_eq!(observed, body);
|
||||
assert_eq!(info.etag.as_deref(), Some("observation-etag"));
|
||||
assert_eq!(
|
||||
*store.options.lock().expect("options"),
|
||||
Some((true, false, false)),
|
||||
"I05_REPAIR_SUPPRESSED"
|
||||
);
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
for (source, expected) in [
|
||||
(Error::FileNotFound, Error::ConfigNotFound),
|
||||
(
|
||||
Error::ObjectNotFound(RUSTFS_META_BUCKET.to_string(), "config/observation.bin".to_string()),
|
||||
Error::ConfigNotFound,
|
||||
),
|
||||
(Error::ErasureReadQuorum, Error::ErasureReadQuorum),
|
||||
] {
|
||||
let mut fixture = ObservationStore::new(Vec::new(), 0);
|
||||
fixture.error = Some(source);
|
||||
let store = Arc::new(fixture);
|
||||
let result = super::read_config_observation(store.clone(), "config/observation.bin", 32).await;
|
||||
assert_eq!(result.expect_err("read error must propagate"), expected);
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
let mut fixture = ObservationStore::new(Vec::new(), 0);
|
||||
fixture.stream_error = true;
|
||||
let store = Arc::new(fixture);
|
||||
let result = super::read_config_observation(store.clone(), "config/observation.bin", 32).await;
|
||||
let error = result.expect_err("stream error must propagate");
|
||||
assert!(error.to_string().contains("observation stream failed"));
|
||||
assert_eq!(store.writes.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn observation_enforces_byte_limit() {
|
||||
let store = Arc::new(ObservationStore::new(vec![7; 64], 0));
|
||||
let result = super::read_config_observation(store.clone(), "config/observation.bin", 8).await;
|
||||
assert!(result.is_err(), "I05_BOUND_ENFORCED");
|
||||
assert_eq!(
|
||||
result.expect_err("bounded observation error"),
|
||||
Error::CorruptedFormat,
|
||||
"I05_BOUND_ENFORCED"
|
||||
);
|
||||
assert_eq!(store.consumed.load(Ordering::SeqCst), 9, "I05_BOUND_ENFORCED: drain exactly limit + 1");
|
||||
let store = Arc::new(ObservationStore::new(vec![7; 8], 8));
|
||||
let result = super::read_config_observation(store.clone(), "config/observation.bin", 8).await;
|
||||
assert!(result.is_ok(), "exact byte limit must succeed: {result:?}");
|
||||
assert_eq!(result.expect("exact limit").0, vec![7; 8]);
|
||||
assert_eq!(store.consumed.load(Ordering::SeqCst), 8);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_config_preserve_empty_distinguishes_empty_object() {
|
||||
let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(Vec::new()), None));
|
||||
@@ -5799,6 +6189,60 @@ mod tests {
|
||||
assert!(data.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_config_preserve_empty_metadata_keeps_empty_and_options() {
|
||||
let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(Vec::new()), None));
|
||||
for no_lock in [false, true] {
|
||||
let opts = ObjectOptions {
|
||||
no_lock,
|
||||
..Default::default()
|
||||
};
|
||||
let (data, info) = read_config_preserve_empty_with_metadata_opts(store.clone(), "config/empty.json", &opts)
|
||||
.await
|
||||
.expect("an existing empty config must retain its payload and metadata");
|
||||
assert!(data.is_empty());
|
||||
assert_eq!(info.size, 0);
|
||||
assert_eq!(info.actual_size, 0);
|
||||
assert_eq!(info.etag.as_deref(), Some("config-1"));
|
||||
assert_eq!(info.data_dir, Some(uuid::Uuid::from_u128(1)));
|
||||
assert_eq!(store.last_get_no_lock.load(Ordering::SeqCst), no_lock);
|
||||
}
|
||||
let err = read_config(store, "config/empty.json")
|
||||
.await
|
||||
.expect_err("the existing reader must still treat empty configs as missing");
|
||||
assert!(matches!(err, Error::ConfigNotFound));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_config_preserve_empty_metadata_keeps_payload_and_metadata() {
|
||||
let payload = b"original config bytes".to_vec();
|
||||
let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(payload.clone()), None));
|
||||
let (data, info) =
|
||||
read_config_preserve_empty_with_metadata_opts(store, "config/nonempty.json", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("a nonempty config must retain its payload and metadata");
|
||||
assert_eq!(data, payload);
|
||||
let size = i64::try_from(payload.len()).expect("test payload size fits i64");
|
||||
assert_eq!(info.size, size);
|
||||
assert_eq!(info.actual_size, size);
|
||||
assert_eq!(info.etag.as_deref(), Some("config-1"));
|
||||
assert_eq!(info.data_dir, Some(uuid::Uuid::from_u128(1)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_config_preserve_empty_metadata_preserves_read_errors() {
|
||||
let missing = Arc::new(RecoveryMockStore::new(RecoveryReadState::Missing, None));
|
||||
let err = read_config_preserve_empty_with_metadata_opts(missing, "config/missing.json", &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("physical absence must remain ConfigNotFound");
|
||||
assert!(matches!(err, Error::ConfigNotFound));
|
||||
let quorum = Arc::new(RecoveryMockStore::new(RecoveryReadState::QuorumError, None));
|
||||
let err = read_config_preserve_empty_with_metadata_opts(quorum, "config/quorum.json", &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("a quorum failure must retain its typed read error");
|
||||
assert!(matches!(err, Error::ErasureReadQuorum));
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl StorageAdminApi for RecoveryMockStore {
|
||||
type BackendInfo = rustfs_madmin::BackendInfo;
|
||||
|
||||
@@ -12431,6 +12431,22 @@ fn resolve_data_movement_lifecycle_expiry_result(action: IlmAction, apply_action
|
||||
)))
|
||||
}
|
||||
|
||||
const DATA_MOVEMENT_DELETE_ALL_DEFERRED_REASON: &str = "delete_all_needs_bucket_write_lock";
|
||||
|
||||
fn log_data_movement_lifecycle_expiry_deferred(bucket: &str, object: &str, action: IlmAction, reason: &str) {
|
||||
warn!(
|
||||
event = EVENT_DECOMMISSION_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
state = "lifecycle_expiry_deferred",
|
||||
bucket = %bucket,
|
||||
object = %object,
|
||||
action = ?action,
|
||||
reason,
|
||||
"Data movement deferred a version's lifecycle expiry; moving the version"
|
||||
);
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) async fn should_skip_lifecycle_for_data_movement(
|
||||
store: Arc<ECStore>,
|
||||
@@ -12490,6 +12506,21 @@ pub(crate) async fn should_skip_lifecycle_for_data_movement(
|
||||
}
|
||||
let applied = !apply_actions
|
||||
|| match lock_lost_signal {
|
||||
// A delete-all is a prefix delete that takes the bucket
|
||||
// lifecycle write lock, which the lifecycle read locks held
|
||||
// by the inline apply and by the rebalance entry's bucket
|
||||
// incarnation fence would block until timeout. The version
|
||||
// is still live, so it moves like any other version and
|
||||
// lifecycle expires it on the target.
|
||||
Some(_) if action.delete_all() => {
|
||||
log_data_movement_lifecycle_expiry_deferred(
|
||||
bucket,
|
||||
&version.name,
|
||||
action,
|
||||
DATA_MOVEMENT_DELETE_ALL_DEFERRED_REASON,
|
||||
);
|
||||
return Ok(false);
|
||||
}
|
||||
Some(signal) => {
|
||||
apply_expiry_rule_for_data_movement(store, &event, event_source, &object_info, Some(signal)).await
|
||||
}
|
||||
@@ -12515,16 +12546,15 @@ pub(crate) async fn should_skip_lifecycle_for_data_movement(
|
||||
// The version is still live, so it moves like any
|
||||
// other version and lifecycle expires it on the
|
||||
// target.
|
||||
warn!(
|
||||
event = EVENT_DECOMMISSION_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
state = "lifecycle_expiry_deferred",
|
||||
bucket = %bucket,
|
||||
object = %version.name,
|
||||
action = ?action,
|
||||
reason = if delete_all { "delete_all_needs_bucket_write_lock" } else { "not_applied" },
|
||||
"Data movement deferred a transitioned version's lifecycle expiry; moving the version"
|
||||
log_data_movement_lifecycle_expiry_deferred(
|
||||
bucket,
|
||||
&version.name,
|
||||
action,
|
||||
if delete_all {
|
||||
DATA_MOVEMENT_DELETE_ALL_DEFERRED_REASON
|
||||
} else {
|
||||
"not_applied"
|
||||
},
|
||||
);
|
||||
return Ok(false);
|
||||
}
|
||||
@@ -17046,6 +17076,7 @@ impl ECStore {
|
||||
&[],
|
||||
data_movement::SourceCleanupBucketFence {
|
||||
expected_incarnation_id: None,
|
||||
lifecycle_owner: None,
|
||||
lifecycle_guard: None,
|
||||
namespace_lock_lost_signal: pool_meta_guard.lock_lost_signal(),
|
||||
object_mutation_fence: Some(&object_fence),
|
||||
@@ -17314,7 +17345,7 @@ impl ECStore {
|
||||
let operation_gate = self.ctx.data_movement_operation_gate();
|
||||
|
||||
let bucket_incarnation_fence = match expected_bucket_incarnation_id {
|
||||
Some(expected) => Some(self.acquire_bucket_incarnation_fence(&bucket, expected).await?),
|
||||
Some(expected) => Some(Arc::new(self.acquire_bucket_incarnation_fence(&bucket, expected).await?)),
|
||||
None => None,
|
||||
};
|
||||
|
||||
@@ -18250,9 +18281,8 @@ impl ECStore {
|
||||
&cleanup_transferred_tier_versions,
|
||||
data_movement::SourceCleanupBucketFence {
|
||||
expected_incarnation_id: expected_bucket_incarnation_id,
|
||||
lifecycle_guard: bucket_incarnation_fence
|
||||
.as_ref()
|
||||
.and_then(|guard| guard.namespace_lock_guard()),
|
||||
lifecycle_owner: bucket_incarnation_fence.as_ref(),
|
||||
lifecycle_guard: None,
|
||||
namespace_lock_lost_signal: None,
|
||||
object_mutation_fence: Some(&source_cleanup_mutation_fence),
|
||||
},
|
||||
@@ -28643,7 +28673,13 @@ impl SetDisks {
|
||||
min_disks: listing_quorum,
|
||||
skip_walkdir_total_timeout: true,
|
||||
walkdir_stall_timeout: Some(DECOMMISSION_BACKGROUND_WALKDIR_STALL_TIMEOUT),
|
||||
agreed: Some(Box::new(move |entry: MetaCacheEntry| Box::pin(cb1(entry)))),
|
||||
agreed: Some(Box::new(move |entry: MetaCacheEntry| {
|
||||
let future = cb1(entry);
|
||||
Box::pin(async move {
|
||||
future.await;
|
||||
Ok(())
|
||||
})
|
||||
})),
|
||||
partial: Some(Box::new(move |entries: MetaCacheEntries, errs: &[Option<DiskError>]| {
|
||||
let resolver = resolver.clone();
|
||||
let cb_func = cb_func.clone();
|
||||
@@ -29372,6 +29408,7 @@ mod pools_tests {
|
||||
pool_meta_save_gate: tokio::sync::Mutex::new(super::PoolMetaWriteState::for_test_bootstrap()),
|
||||
ctx,
|
||||
bucket_fence_registry: Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -499,6 +499,7 @@ mod decommission_lock_order_tests {
|
||||
),
|
||||
ctx,
|
||||
bucket_fence_registry: Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
});
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
(store, refresh_calls)
|
||||
|
||||
@@ -99,6 +99,33 @@ pub struct Sets {
|
||||
ctx: Arc<InstanceContext>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Sets {
|
||||
pub(crate) fn from_test_set(
|
||||
set: Arc<SetDisks>,
|
||||
endpoints: PoolEndpoints,
|
||||
format: FormatV3,
|
||||
pool_idx: usize,
|
||||
parity_count: usize,
|
||||
ctx: Arc<InstanceContext>,
|
||||
) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
id: format.id,
|
||||
disk_set: vec![set],
|
||||
pool_idx,
|
||||
set_count: 1,
|
||||
set_drive_count: endpoints.drives_per_set,
|
||||
endpoints,
|
||||
format,
|
||||
parity_count,
|
||||
default_parity_count: parity_count,
|
||||
distribution_algo: DistributionAlgoVersion::V1,
|
||||
exit_signal: None,
|
||||
ctx,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Sets {
|
||||
fn drop(&mut self) {
|
||||
if let Some(exit_signal) = self.exit_signal.take() {
|
||||
@@ -800,6 +827,10 @@ impl crate::storage_api_contracts::object::ObjectOperations for Sets {
|
||||
mod_time: dst_opts.mod_time,
|
||||
http_preconditions: dst_opts.http_preconditions.clone(),
|
||||
quota_admission: dst_opts.quota_admission,
|
||||
expected_bucket_incarnation_id: dst_opts.expected_bucket_incarnation_id,
|
||||
bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
|
||||
bucket_incarnation_fence: dst_opts.bucket_incarnation_fence.clone(),
|
||||
object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
|
||||
@@ -31,7 +31,7 @@ use crate::storage_api_contracts::{
|
||||
namespace::NamespaceLocking as _,
|
||||
object::{HTTPPreconditions, ObjectOperations as _},
|
||||
};
|
||||
use crate::store::{DecommissionFixedReadAnchor, ECStore, SourceCleanupMutationFence};
|
||||
use crate::store::{BucketIncarnationFenceGuard, DecommissionFixedReadAnchor, ECStore, SourceCleanupMutationFence};
|
||||
use bytes::Bytes;
|
||||
use rustfs_filemeta::{FileInfo, FileInfoVersions, ObjectPartInfo};
|
||||
use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
|
||||
@@ -1278,6 +1278,7 @@ pub(crate) enum SourceCleanupError {
|
||||
#[derive(Clone, Default)]
|
||||
pub(crate) struct SourceCleanupBucketFence<'a> {
|
||||
pub(crate) expected_incarnation_id: Option<uuid::Uuid>,
|
||||
pub(crate) lifecycle_owner: Option<&'a Arc<BucketIncarnationFenceGuard>>,
|
||||
pub(crate) lifecycle_guard: Option<&'a rustfs_lock::NamespaceLockGuard>,
|
||||
pub(crate) namespace_lock_lost_signal: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
|
||||
pub(crate) object_mutation_fence: Option<&'a SourceCleanupMutationFence>,
|
||||
@@ -1486,9 +1487,10 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
|
||||
)
|
||||
};
|
||||
|
||||
if bucket_fence
|
||||
.lifecycle_guard
|
||||
.is_some_and(rustfs_lock::NamespaceLockGuard::is_lock_lost)
|
||||
if bucket_fence.lifecycle_owner.is_some_and(|owner| owner.is_lock_lost())
|
||||
|| bucket_fence
|
||||
.lifecycle_guard
|
||||
.is_some_and(rustfs_lock::NamespaceLockGuard::is_lock_lost)
|
||||
{
|
||||
return Err(SourceCleanupError::Storage(Error::other(format!(
|
||||
"{op_label}: bucket incarnation fence was lost before source cleanup"
|
||||
@@ -1527,6 +1529,9 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
|
||||
if let Some(object_mutation_fence) = bucket_fence.object_mutation_fence {
|
||||
object_mutation_fence.add_namespace_lock_fence(&mut opts);
|
||||
}
|
||||
if let Some(owner) = bucket_fence.lifecycle_owner {
|
||||
owner.attach_owner(&mut opts);
|
||||
}
|
||||
if let Some(bucket_lifecycle_guard) = bucket_fence.lifecycle_guard {
|
||||
opts.add_bucket_lifecycle_lock_guard(bucket_lifecycle_guard);
|
||||
}
|
||||
|
||||
@@ -216,7 +216,7 @@ pub async fn lookup_degraded_bucket_usage_baseline(store: Arc<ECStore>, bucket:
|
||||
// Stale or empty cache: refresh through the TTL-bounded loader. A failed
|
||||
// refresh carries the previous baseline forward, so quota admission keeps
|
||||
// its last grounded values through a backend read outage.
|
||||
let _ = load_data_usage_from_backend_cached(store).await;
|
||||
let _ = project_data_usage_from_backend_cached(store, |_| ()).await;
|
||||
let cache = data_usage_snapshot_cache().read().await;
|
||||
cache
|
||||
.as_ref()
|
||||
@@ -237,31 +237,34 @@ struct CachedDataUsageSnapshot {
|
||||
}
|
||||
|
||||
impl CachedDataUsageSnapshot {
|
||||
fn result(&self) -> Result<DataUsageInfo, Error> {
|
||||
fn result<T>(&self, project: impl FnOnce(&DataUsageInfo) -> T) -> Result<T, Error> {
|
||||
self.info
|
||||
.clone()
|
||||
.as_ref()
|
||||
.map(project)
|
||||
.ok_or_else(|| Error::other("data usage snapshot load recently failed"))
|
||||
}
|
||||
}
|
||||
|
||||
fn fresh_cached_data_usage_snapshot(
|
||||
fn fresh_cached_data_usage_snapshot<T>(
|
||||
cache: &Option<CachedDataUsageSnapshot>,
|
||||
now: tokio::time::Instant,
|
||||
ttl: Duration,
|
||||
) -> Option<Result<DataUsageInfo, Error>> {
|
||||
project: impl FnOnce(&DataUsageInfo) -> T,
|
||||
) -> Option<Result<T, Error>> {
|
||||
cache
|
||||
.as_ref()
|
||||
.filter(|cached| now.duration_since(cached.loaded_at) < ttl)
|
||||
.map(CachedDataUsageSnapshot::result)
|
||||
.map(|cached| cached.result(project))
|
||||
}
|
||||
|
||||
fn cache_data_usage_snapshot_result(
|
||||
fn cache_data_usage_snapshot_result<T>(
|
||||
cache: &mut Option<CachedDataUsageSnapshot>,
|
||||
result: Result<LoadedUsageBaseline, Error>,
|
||||
loaded_at: tokio::time::Instant,
|
||||
refresh_generation: u64,
|
||||
current_generation: u64,
|
||||
) -> Option<Result<DataUsageInfo, Error>> {
|
||||
project: impl FnOnce(&DataUsageInfo) -> T,
|
||||
) -> Option<Result<T, Error>> {
|
||||
if current_generation != refresh_generation {
|
||||
return None;
|
||||
}
|
||||
@@ -280,12 +283,13 @@ fn cache_data_usage_snapshot_result(
|
||||
degraded_baseline.entry(bucket.clone()).or_insert(*size);
|
||||
}
|
||||
}
|
||||
let projected = project(&info);
|
||||
*cache = Some(CachedDataUsageSnapshot {
|
||||
info: Some(info.clone()),
|
||||
info: Some(info),
|
||||
loaded_at,
|
||||
degraded_baseline,
|
||||
});
|
||||
Ok(info)
|
||||
Ok(projected)
|
||||
}
|
||||
Err(e) => {
|
||||
// Keep the previous baseline through a failed refresh: quota
|
||||
@@ -1486,12 +1490,21 @@ async fn normalize_loaded_data_usage(
|
||||
/// `save_data_usage_in_backend` invalidates the cache so a fresh scanner save
|
||||
/// is visible immediately in the saving process.
|
||||
pub async fn load_data_usage_from_backend_cached(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
|
||||
project_data_usage_from_backend_cached(store, DataUsageInfo::clone).await
|
||||
}
|
||||
|
||||
// Project under the cache guard so scalar readers never copy the snapshot's
|
||||
// bucket maps. The owned API shares the same TTL, refresh and invalidation path.
|
||||
async fn project_data_usage_from_backend_cached<T>(
|
||||
store: Arc<ECStore>,
|
||||
project: impl Fn(&DataUsageInfo) -> T,
|
||||
) -> Result<T, Error> {
|
||||
let ttl = Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
|
||||
|
||||
loop {
|
||||
{
|
||||
let cache = data_usage_snapshot_cache().read().await;
|
||||
if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
|
||||
if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl, &project) {
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -1503,7 +1516,7 @@ pub async fn load_data_usage_from_backend_cached(store: Arc<ECStore>) -> Result<
|
||||
.await;
|
||||
{
|
||||
let cache = data_usage_snapshot_cache().read().await;
|
||||
if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
|
||||
if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl, &project) {
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -1512,9 +1525,14 @@ pub async fn load_data_usage_from_backend_cached(store: Arc<ECStore>) -> Result<
|
||||
let result = load_data_usage_from_backend_with_baseline(store.clone()).await;
|
||||
let loaded_at = tokio::time::Instant::now();
|
||||
let mut cache = data_usage_snapshot_cache().write().await;
|
||||
if let Some(result) =
|
||||
cache_data_usage_snapshot_result(&mut cache, result, loaded_at, refresh_generation, data_usage_snapshot_generation())
|
||||
{
|
||||
if let Some(result) = cache_data_usage_snapshot_result(
|
||||
&mut cache,
|
||||
result,
|
||||
loaded_at,
|
||||
refresh_generation,
|
||||
data_usage_snapshot_generation(),
|
||||
&project,
|
||||
) {
|
||||
return result;
|
||||
}
|
||||
drop(cache);
|
||||
@@ -1532,7 +1550,8 @@ pub async fn load_admin_data_usage_from_backend_cached(store: Arc<ECStore>) -> R
|
||||
loop {
|
||||
{
|
||||
let cache = admin_data_usage_snapshot_cache().read().await;
|
||||
if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
|
||||
if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl, DataUsageInfo::clone)
|
||||
{
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -1543,7 +1562,8 @@ pub async fn load_admin_data_usage_from_backend_cached(store: Arc<ECStore>) -> R
|
||||
.await;
|
||||
{
|
||||
let cache = admin_data_usage_snapshot_cache().read().await;
|
||||
if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
|
||||
if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl, DataUsageInfo::clone)
|
||||
{
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -1564,6 +1584,7 @@ pub async fn load_admin_data_usage_from_backend_cached(store: Arc<ECStore>) -> R
|
||||
loaded_at,
|
||||
refresh_generation,
|
||||
admin_data_usage_snapshot_generation(),
|
||||
DataUsageInfo::clone,
|
||||
) {
|
||||
return result;
|
||||
}
|
||||
@@ -2277,8 +2298,10 @@ async fn get_persisted_bucket_usage(bucket: &str) -> Option<u64> {
|
||||
}
|
||||
|
||||
async fn persisted_bucket_usage(store: Arc<ECStore>, bucket: &str) -> Option<u64> {
|
||||
let info = load_data_usage_from_backend_cached(store).await.ok()?;
|
||||
info.buckets_usage.get(bucket).map(|usage| usage.size)
|
||||
project_data_usage_from_backend_cached(store, |info| info.buckets_usage.get(bucket).map(|usage| usage.size))
|
||||
.await
|
||||
.ok()
|
||||
.flatten()
|
||||
}
|
||||
|
||||
async fn reload_persisted_bucket_usage(store: Arc<ECStore>, bucket: &str) -> Option<u64> {
|
||||
@@ -2375,6 +2398,7 @@ where
|
||||
tokio::time::Instant::now(),
|
||||
refresh_generation,
|
||||
data_usage_snapshot_generation(),
|
||||
|_| (),
|
||||
)
|
||||
.is_some()
|
||||
}
|
||||
@@ -2891,6 +2915,8 @@ pub async fn init_compression_total_memory_from_backend(store: Arc<ECStore>) {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
mod snapshot_projection;
|
||||
|
||||
use super::*;
|
||||
use crate::layout::endpoints::EndpointServerPools;
|
||||
use crate::runtime::instance::InstanceContext;
|
||||
@@ -3334,6 +3360,7 @@ mod tests {
|
||||
pool_meta_save_gate: TokioMutex::default(),
|
||||
ctx,
|
||||
bucket_fence_registry: Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -4572,15 +4599,29 @@ mod tests {
|
||||
loaded_at,
|
||||
refresh_generation,
|
||||
data_usage_snapshot_generation(),
|
||||
DataUsageInfo::clone,
|
||||
)
|
||||
.expect("an uninterrupted refresh should populate the cache");
|
||||
assert!(matches!(first, Err(Error::ErasureReadQuorum)));
|
||||
|
||||
let cached = fresh_cached_data_usage_snapshot(&cache, loaded_at + Duration::from_secs(1), Duration::from_secs(30))
|
||||
.expect("the failed load must remain cached within the TTL");
|
||||
let cached = fresh_cached_data_usage_snapshot(
|
||||
&cache,
|
||||
loaded_at + Duration::from_secs(1),
|
||||
Duration::from_secs(30),
|
||||
DataUsageInfo::clone,
|
||||
)
|
||||
.expect("the failed load must remain cached within the TTL");
|
||||
assert!(cached.is_err());
|
||||
|
||||
assert!(fresh_cached_data_usage_snapshot(&cache, loaded_at + Duration::from_secs(30), Duration::from_secs(30)).is_none());
|
||||
assert!(
|
||||
fresh_cached_data_usage_snapshot(
|
||||
&cache,
|
||||
loaded_at + Duration::from_secs(30),
|
||||
Duration::from_secs(30),
|
||||
DataUsageInfo::clone
|
||||
)
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4601,14 +4642,20 @@ mod tests {
|
||||
loaded_at,
|
||||
refresh_generation,
|
||||
data_usage_snapshot_generation(),
|
||||
DataUsageInfo::clone,
|
||||
)
|
||||
.expect("an uninterrupted refresh should populate the cache")
|
||||
.expect("successful load must be returned");
|
||||
assert_snapshot_bucket(&first, "bucket");
|
||||
|
||||
let cached = fresh_cached_data_usage_snapshot(&cache, loaded_at + Duration::from_secs(1), Duration::from_secs(30))
|
||||
.expect("successful load must remain cached within the TTL")
|
||||
.expect("cached success must remain successful");
|
||||
let cached = fresh_cached_data_usage_snapshot(
|
||||
&cache,
|
||||
loaded_at + Duration::from_secs(1),
|
||||
Duration::from_secs(30),
|
||||
DataUsageInfo::clone,
|
||||
)
|
||||
.expect("successful load must remain cached within the TTL")
|
||||
.expect("cached success must remain successful");
|
||||
assert_snapshot_bucket(&cached, "bucket");
|
||||
}
|
||||
|
||||
@@ -4633,6 +4680,7 @@ mod tests {
|
||||
loaded_at,
|
||||
refresh_generation,
|
||||
data_usage_snapshot_generation(),
|
||||
DataUsageInfo::clone,
|
||||
)
|
||||
.expect("an uninterrupted refresh should populate the cache")
|
||||
.expect("successful load must be returned");
|
||||
@@ -4666,6 +4714,7 @@ mod tests {
|
||||
loaded_at,
|
||||
refresh_generation,
|
||||
data_usage_snapshot_generation(),
|
||||
DataUsageInfo::clone,
|
||||
);
|
||||
|
||||
assert!(stale_result.is_none());
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn scalar_projection_borrows_snapshot_and_preserves_unknown_and_zero() {
|
||||
let loaded_at = tokio::time::Instant::now();
|
||||
let mut info = data_usage_info_for_test("known", 1, 42, SystemTime::UNIX_EPOCH);
|
||||
info.buckets_usage.insert("empty".to_string(), BucketUsageInfo::default());
|
||||
let cache = Some(CachedDataUsageSnapshot {
|
||||
info: Some(info),
|
||||
loaded_at,
|
||||
degraded_baseline: HashMap::new(),
|
||||
});
|
||||
let stored = cache
|
||||
.as_ref()
|
||||
.expect("cached snapshot")
|
||||
.info
|
||||
.as_ref()
|
||||
.expect("known snapshot");
|
||||
let ttl = Duration::from_secs(30);
|
||||
|
||||
for (bucket, expected) in [("known", Some(42)), ("empty", Some(0)), ("missing", None)] {
|
||||
let value = fresh_cached_data_usage_snapshot(&cache, loaded_at, ttl, |info| {
|
||||
assert!(std::ptr::eq(info, stored), "projection must borrow the cached snapshot");
|
||||
info.buckets_usage.get(bucket).map(|usage| usage.size)
|
||||
})
|
||||
.expect("fresh snapshot")
|
||||
.expect("successful snapshot");
|
||||
let owned = fresh_cached_data_usage_snapshot(&cache, loaded_at, ttl, DataUsageInfo::clone)
|
||||
.expect("fresh snapshot")
|
||||
.expect("successful snapshot");
|
||||
assert_eq!(value, expected);
|
||||
assert_eq!(value, owned.buckets_usage.get(bucket).map(|usage| usage.size));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn projection_is_skipped_for_missing_failed_expired_or_invalidated_snapshots() {
|
||||
let loaded_at = tokio::time::Instant::now();
|
||||
let ttl = Duration::from_secs(30);
|
||||
let never_project = |_: &DataUsageInfo| -> u64 { panic!("unusable snapshots must not be projected") };
|
||||
assert!(fresh_cached_data_usage_snapshot(&None, loaded_at, ttl, never_project).is_none());
|
||||
|
||||
let mut cache = None;
|
||||
let failure = cache_data_usage_snapshot_result(&mut cache, Err(Error::ErasureReadQuorum), loaded_at, 1, 1, never_project)
|
||||
.expect("matching generation");
|
||||
assert!(matches!(failure, Err(Error::ErasureReadQuorum)));
|
||||
assert!(
|
||||
fresh_cached_data_usage_snapshot(&cache, loaded_at, ttl, never_project)
|
||||
.expect("negative cache remains fresh")
|
||||
.is_err()
|
||||
);
|
||||
|
||||
cache = Some(CachedDataUsageSnapshot {
|
||||
info: Some(data_usage_info_for_test("known", 1, 42, SystemTime::UNIX_EPOCH)),
|
||||
loaded_at,
|
||||
degraded_baseline: HashMap::new(),
|
||||
});
|
||||
assert!(fresh_cached_data_usage_snapshot(&cache, loaded_at + ttl, ttl, never_project).is_none());
|
||||
cache = None;
|
||||
let rejected = cache_data_usage_snapshot_result(
|
||||
&mut cache,
|
||||
Ok(LoadedUsageBaseline {
|
||||
info: data_usage_info_for_test("known", 1, 42, SystemTime::UNIX_EPOCH),
|
||||
degraded_baseline: HashMap::new(),
|
||||
observed_unavailable: false,
|
||||
}),
|
||||
loaded_at,
|
||||
1,
|
||||
2,
|
||||
never_project,
|
||||
);
|
||||
assert!(rejected.is_none());
|
||||
assert!(cache.is_none(), "rejected generation must not republish stale data");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn projected_publication_moves_bucket_maps_into_cache() {
|
||||
let info = data_usage_info_for_test("known", 1, 42, SystemTime::UNIX_EPOCH);
|
||||
let original_key = info.buckets_usage.get_key_value("known").expect("source bucket").0.as_ptr();
|
||||
let mut cache = None;
|
||||
let usage = cache_data_usage_snapshot_result(
|
||||
&mut cache,
|
||||
Ok(LoadedUsageBaseline {
|
||||
info,
|
||||
degraded_baseline: HashMap::new(),
|
||||
observed_unavailable: false,
|
||||
}),
|
||||
tokio::time::Instant::now(),
|
||||
1,
|
||||
1,
|
||||
|info| {
|
||||
assert_eq!(
|
||||
info.buckets_usage
|
||||
.get_key_value("known")
|
||||
.expect("projected bucket")
|
||||
.0
|
||||
.as_ptr(),
|
||||
original_key
|
||||
);
|
||||
info.buckets_usage.get("known").map(|usage| usage.size)
|
||||
},
|
||||
)
|
||||
.expect("matching generation")
|
||||
.expect("successful publication");
|
||||
assert_eq!(usage, Some(42));
|
||||
let cached = cache
|
||||
.as_ref()
|
||||
.expect("published cache")
|
||||
.info
|
||||
.as_ref()
|
||||
.expect("published snapshot");
|
||||
assert_eq!(
|
||||
cached.buckets_usage.get_key_value("known").expect("cached bucket").0.as_ptr(),
|
||||
original_key
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn persisted_scalar_projection_shares_cache_refresh_and_invalidation() {
|
||||
use crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks;
|
||||
use crate::config::com::save_config;
|
||||
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
let mut info = data_usage_info_for_test("known", 1, 42, SystemTime::UNIX_EPOCH);
|
||||
info.buckets_usage.insert("empty".to_string(), BucketUsageInfo::default());
|
||||
info.bucket_sizes.insert("empty".to_string(), 0);
|
||||
info.buckets_count = 2;
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
serde_json::to_vec(&info).expect("encode snapshot"),
|
||||
)
|
||||
.await
|
||||
.expect("persist snapshot");
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
|
||||
for (bucket, expected) in [("known", Some(42)), ("empty", Some(0)), ("missing", None)] {
|
||||
assert_eq!(persisted_bucket_usage(store.clone(), bucket).await, expected);
|
||||
let owned = load_data_usage_from_backend_cached(store.clone())
|
||||
.await
|
||||
.expect("owned cached snapshot");
|
||||
assert_eq!(expected, owned.buckets_usage.get(bucket).map(|usage| usage.size));
|
||||
}
|
||||
info.buckets_usage.get_mut("known").expect("known bucket").size = 84;
|
||||
info.bucket_sizes.insert("known".to_string(), 84);
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
serde_json::to_vec(&info).expect("encode new snapshot"),
|
||||
)
|
||||
.await
|
||||
.expect("persist new snapshot without invalidating cache");
|
||||
assert_eq!(
|
||||
persisted_bucket_usage(store.clone(), "known").await,
|
||||
Some(42),
|
||||
"warm reads reuse the TTL cache"
|
||||
);
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
assert_eq!(
|
||||
persisted_bucket_usage(store.clone(), "known").await,
|
||||
Some(84),
|
||||
"invalidation exposes the new snapshot"
|
||||
);
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
if let Some(cancel) = store.ctx.background_cancel_token() {
|
||||
cancel.cancel();
|
||||
}
|
||||
for pool in &store.pools {
|
||||
for set in &pool.disk_set {
|
||||
let disks = set.disks.read().await.clone();
|
||||
for disk in disks.iter().flatten() {
|
||||
disk.close().await.expect("stop fixture disk monitoring");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -159,10 +159,6 @@ pub fn lstat_std(path: impl AsRef<Path>) -> io::Result<Metadata> {
|
||||
std::fs::symlink_metadata(path)
|
||||
}
|
||||
|
||||
pub async fn make_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
fs::create_dir_all(path.as_ref()).await
|
||||
}
|
||||
|
||||
fn is_dir_error(e: &io::Error) -> bool {
|
||||
e.raw_os_error() == Some(libc::EISDIR)
|
||||
|| e.kind() == io::ErrorKind::IsADirectory
|
||||
@@ -210,10 +206,6 @@ pub fn remove_all_std(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn mkdir(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
fs::create_dir(path.as_ref()).await
|
||||
}
|
||||
|
||||
pub async fn rename(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
|
||||
fs::rename(from, to).await
|
||||
}
|
||||
@@ -472,17 +464,6 @@ mod tests {
|
||||
assert!(metadata.file_type().is_symlink());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_make_dir_all() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let nested_path = temp_dir.path().join("level1").join("level2").join("level3");
|
||||
|
||||
// Should create nested directories
|
||||
assert!(make_dir_all(&nested_path).await.is_ok());
|
||||
assert!(nested_path.exists());
|
||||
assert!(nested_path.is_dir());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_remove_file() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
@@ -555,17 +536,6 @@ mod tests {
|
||||
assert!(!dir_path.exists());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mkdir() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let dir_path = temp_dir.path().join("test_mkdir");
|
||||
|
||||
// Create directory
|
||||
assert!(mkdir(&dir_path).await.is_ok());
|
||||
assert!(dir_path.exists());
|
||||
assert!(dir_path.is_dir());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_rename() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
|
||||
+10649
-1437
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,740 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::disk::error::{DiskError, Result};
|
||||
use std::{fs::File, sync::Arc};
|
||||
|
||||
/// A target disk's admitted artifact publication. Physical mutation workers
|
||||
/// retain this Arc until their final syscall and rollback have finished.
|
||||
pub struct ArtifactLease {
|
||||
_active: File,
|
||||
_key_directory: File,
|
||||
_seal_directory: File,
|
||||
_root: File,
|
||||
}
|
||||
|
||||
pub type ArtifactPutLease = ArtifactLease;
|
||||
pub type ArtifactRepairLease = ArtifactLease;
|
||||
|
||||
/// Read-only census owns both already-closed physical mutation domains.
|
||||
pub(super) struct ArtifactInventoryLease {
|
||||
_put: Arc<ArtifactLease>,
|
||||
_repair: Arc<ArtifactLease>,
|
||||
}
|
||||
|
||||
pub(super) async fn drain_closed(root: File, key: [u8; 32], deadline: tokio::time::Instant) -> Result<ArtifactInventoryLease> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
unix::drain_closed(root, key, deadline).await
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let _ = (root, key, deadline);
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub(crate) enum Domain {
|
||||
Put,
|
||||
Repair,
|
||||
}
|
||||
|
||||
impl Domain {
|
||||
fn directory(self) -> &'static str {
|
||||
match self {
|
||||
Self::Put => ".rustfs-artifact-put-seals",
|
||||
Self::Repair => ".rustfs-artifact-repair-seals",
|
||||
}
|
||||
}
|
||||
|
||||
fn closed(self) -> &'static [u8] {
|
||||
match self {
|
||||
Self::Put => b"rustfs-artifact-put-sealed-v1\n",
|
||||
Self::Repair => b"rustfs-artifact-repair-sealed-v1\n",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn pinned_root(disk: &super::LocalDisk) -> Result<File> {
|
||||
#[cfg(target_os = "linux")]
|
||||
let root = disk.mount_lease.clone();
|
||||
#[cfg(all(unix, not(target_os = "linux")))]
|
||||
let root = disk.artifact_identity_root.clone();
|
||||
#[cfg(not(unix))]
|
||||
let _ = disk;
|
||||
tokio::task::spawn_blocking(move || {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
root.try_clone().map_err(DiskError::from)
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
Err(DiskError::other("artifact disk leases require Unix"))
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?
|
||||
}
|
||||
|
||||
pub(super) async fn acquire(root: File, key: [u8; 32]) -> Result<Arc<ArtifactPutLease>> {
|
||||
acquire_in(root, key, Domain::Put).await
|
||||
}
|
||||
|
||||
pub(super) async fn acquire_in(root: File, key: [u8; 32], domain: Domain) -> Result<Arc<ArtifactLease>> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
unix::acquire(root, key, domain).await
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let _ = (root, key, domain);
|
||||
Err(DiskError::other("artifact disk leases require Unix"))
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn seal(root: File, key: [u8; 32]) -> Result<()> {
|
||||
seal_in(root, key, Domain::Put).await
|
||||
}
|
||||
|
||||
pub(super) async fn seal_in(root: File, key: [u8; 32], domain: Domain) -> Result<()> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
unix::seal(root, key, domain).await
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let _ = (root, key, domain);
|
||||
Err(DiskError::other("artifact disk leases require Unix"))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
mod unix {
|
||||
use super::*;
|
||||
use rustix::fs::{FlockOperation, Mode, OFlags, flock, mkdirat, openat};
|
||||
use rustix::io::Errno;
|
||||
use std::{io, io::Read, io::Write, os::unix::fs::MetadataExt, time::Duration};
|
||||
|
||||
const POLL_INTERVAL: Duration = Duration::from_millis(10);
|
||||
const LOCK_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
struct Opened {
|
||||
root: File,
|
||||
seals: File,
|
||||
key: File,
|
||||
admission: File,
|
||||
active: File,
|
||||
domain: Domain,
|
||||
}
|
||||
|
||||
fn invalid() -> io::Error {
|
||||
io::Error::new(io::ErrorKind::InvalidData, "invalid artifact seal filesystem state")
|
||||
}
|
||||
|
||||
fn directory(parent: &File, name: &str, device: u64) -> io::Result<File> {
|
||||
match mkdirat(parent, name, Mode::RWXU) {
|
||||
Ok(()) | Err(Errno::EXIST) => {}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
let file = File::from(openat(
|
||||
parent,
|
||||
name,
|
||||
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
|
||||
Mode::empty(),
|
||||
)?);
|
||||
let metadata = file.metadata()?;
|
||||
if !metadata.is_dir() || metadata.dev() != device || metadata.mode() & 0o077 != 0 {
|
||||
return Err(invalid());
|
||||
}
|
||||
Ok(file)
|
||||
}
|
||||
|
||||
fn regular(file: &File, device: u64) -> io::Result<std::fs::Metadata> {
|
||||
let metadata = file.metadata()?;
|
||||
if !metadata.is_file() || metadata.dev() != device || metadata.nlink() != 1 || metadata.mode() & 0o077 != 0 {
|
||||
return Err(invalid());
|
||||
}
|
||||
Ok(metadata)
|
||||
}
|
||||
|
||||
fn lock_file(parent: &File, name: &str, device: u64) -> io::Result<File> {
|
||||
let file = File::from(openat(
|
||||
parent,
|
||||
name,
|
||||
OFlags::CREATE | OFlags::RDWR | OFlags::NOFOLLOW | OFlags::CLOEXEC | OFlags::NONBLOCK,
|
||||
Mode::RUSR | Mode::WUSR,
|
||||
)?);
|
||||
if regular(&file, device)?.len() != 0 {
|
||||
return Err(invalid());
|
||||
}
|
||||
Ok(file)
|
||||
}
|
||||
|
||||
impl Opened {
|
||||
fn open(root: File, key: [u8; 32], domain: Domain) -> io::Result<Self> {
|
||||
let metadata = root.metadata()?;
|
||||
if !metadata.is_dir() {
|
||||
return Err(invalid());
|
||||
}
|
||||
let seals = directory(&root, domain.directory(), metadata.dev())?;
|
||||
let name = hex_simd::encode_to_string(key, hex_simd::AsciiCase::Lower);
|
||||
let key = directory(&seals, &name, metadata.dev())?;
|
||||
// Neither lock inode is ever removed or replaced, including after
|
||||
// closing. Independent processes must always lock the same inode.
|
||||
let admission = lock_file(&key, "admission.lock", metadata.dev())?;
|
||||
let active = lock_file(&key, "active.lock", metadata.dev())?;
|
||||
Ok(Self {
|
||||
root,
|
||||
seals,
|
||||
key,
|
||||
admission,
|
||||
active,
|
||||
domain,
|
||||
})
|
||||
}
|
||||
|
||||
fn closed(&self) -> io::Result<Option<File>> {
|
||||
self.closed_with_flags(OFlags::RDWR)
|
||||
}
|
||||
|
||||
fn closed_with_flags(&self, access: OFlags) -> io::Result<Option<File>> {
|
||||
let mut file = match openat(
|
||||
&self.key,
|
||||
"closed",
|
||||
access | OFlags::NOFOLLOW | OFlags::CLOEXEC | OFlags::NONBLOCK,
|
||||
Mode::empty(),
|
||||
) {
|
||||
Ok(file) => File::from(file),
|
||||
Err(Errno::NOENT) => return Ok(None),
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
let length = u64::try_from(self.domain.closed().len()).map_err(|_| invalid())?;
|
||||
if regular(&file, self.root.metadata()?.dev())?.len() != length {
|
||||
return Err(invalid());
|
||||
}
|
||||
let mut bytes = vec![0; self.domain.closed().len()];
|
||||
file.read_exact(&mut bytes)?;
|
||||
if bytes != self.domain.closed() {
|
||||
return Err(invalid());
|
||||
}
|
||||
Ok(Some(file))
|
||||
}
|
||||
|
||||
fn admit(self) -> io::Result<Arc<ArtifactLease>> {
|
||||
if self.closed()?.is_some() {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::PermissionDenied,
|
||||
"artifact attempt mutation domain is closed",
|
||||
));
|
||||
}
|
||||
// Admission serializes the closed check with taking this shared
|
||||
// physical owner. A sealer cannot pass between those two steps.
|
||||
flock(&self.active, FlockOperation::NonBlockingLockShared)?;
|
||||
flock(&self.admission, FlockOperation::Unlock)?;
|
||||
Ok(Arc::new(ArtifactLease {
|
||||
_active: self.active,
|
||||
_key_directory: self.key,
|
||||
_seal_directory: self.seals,
|
||||
_root: self.root,
|
||||
}))
|
||||
}
|
||||
|
||||
fn close(self) -> io::Result<Self> {
|
||||
let marker = match self.closed()? {
|
||||
Some(marker) => marker,
|
||||
None => {
|
||||
let mut marker = File::from(openat(
|
||||
&self.key,
|
||||
"closed",
|
||||
OFlags::CREATE | OFlags::EXCL | OFlags::RDWR | OFlags::NOFOLLOW | OFlags::CLOEXEC,
|
||||
Mode::RUSR | Mode::WUSR,
|
||||
)?);
|
||||
regular(&marker, self.root.metadata()?.dev())?;
|
||||
marker.write_all(self.domain.closed())?;
|
||||
marker
|
||||
}
|
||||
};
|
||||
// No admission is released until the permanent denial and every
|
||||
// ancestor entry are durable. A failed/cancelled close never removes it.
|
||||
marker.sync_all()?;
|
||||
self.admission.sync_all()?;
|
||||
self.active.sync_all()?;
|
||||
self.key.sync_all()?;
|
||||
self.seals.sync_all()?;
|
||||
self.root.sync_all()?;
|
||||
flock(&self.admission, FlockOperation::Unlock)?;
|
||||
Ok(self)
|
||||
}
|
||||
}
|
||||
|
||||
async fn lock(file: &File, operation: FlockOperation) -> io::Result<()> {
|
||||
tokio::time::timeout(LOCK_TIMEOUT, async {
|
||||
loop {
|
||||
match flock(file, operation) {
|
||||
Ok(()) => return Ok(()),
|
||||
Err(Errno::WOULDBLOCK | Errno::INTR) => tokio::time::sleep(POLL_INTERVAL).await,
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "artifact disk lease lock deadline"))?
|
||||
}
|
||||
|
||||
async fn open(root: File, key: [u8; 32], domain: Domain) -> Result<Opened> {
|
||||
tokio::task::spawn_blocking(move || Opened::open(root, key, domain))
|
||||
.await
|
||||
.map_err(DiskError::from)?
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
|
||||
fn read_error(error: io::Error) -> DiskError {
|
||||
match error.kind() {
|
||||
io::ErrorKind::NotFound => DiskError::other("artifact inventory requires closed PUT and repair domains"),
|
||||
io::ErrorKind::InvalidData | io::ErrorKind::NotADirectory => DiskError::FileCorrupt,
|
||||
io::ErrorKind::TimedOut => DiskError::Timeout,
|
||||
_ if error.raw_os_error() == Some(Errno::LOOP.raw_os_error()) => DiskError::FileCorrupt,
|
||||
_ => DiskError::from(error),
|
||||
}
|
||||
}
|
||||
|
||||
fn existing_directory(parent: &File, name: &str, device: u64) -> io::Result<File> {
|
||||
let file = File::from(openat(
|
||||
parent,
|
||||
name,
|
||||
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
|
||||
Mode::empty(),
|
||||
)?);
|
||||
let metadata = file.metadata()?;
|
||||
if metadata.dev() != device || metadata.mode() & 0o077 != 0 {
|
||||
return Err(invalid());
|
||||
}
|
||||
Ok(file)
|
||||
}
|
||||
|
||||
fn existing(root: File, key: [u8; 32], domain: Domain) -> io::Result<Opened> {
|
||||
let device = root.metadata()?.dev();
|
||||
let seals = existing_directory(&root, domain.directory(), device)?;
|
||||
let name = hex_simd::encode_to_string(key, hex_simd::AsciiCase::Lower);
|
||||
let key = existing_directory(&seals, &name, device)?;
|
||||
let lock = |name| -> io::Result<File> {
|
||||
let file = File::from(openat(
|
||||
&key,
|
||||
name,
|
||||
OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC | OFlags::NONBLOCK,
|
||||
Mode::empty(),
|
||||
)?);
|
||||
if regular(&file, device)?.len() != 0 {
|
||||
return Err(invalid());
|
||||
}
|
||||
Ok(file)
|
||||
};
|
||||
let admission = lock("admission.lock")?;
|
||||
let active = lock("active.lock")?;
|
||||
let opened = Opened {
|
||||
root,
|
||||
seals,
|
||||
key,
|
||||
admission,
|
||||
active,
|
||||
domain,
|
||||
};
|
||||
if opened.closed_with_flags(OFlags::RDONLY)?.is_none() {
|
||||
return Err(io::Error::from(io::ErrorKind::NotFound));
|
||||
}
|
||||
Ok(opened)
|
||||
}
|
||||
|
||||
fn retained(opened: Opened) -> Arc<ArtifactLease> {
|
||||
Arc::new(ArtifactLease {
|
||||
_active: opened.active,
|
||||
_key_directory: opened.key,
|
||||
_seal_directory: opened.seals,
|
||||
_root: opened.root,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) async fn drain_closed(
|
||||
root: File,
|
||||
key: [u8; 32],
|
||||
deadline: tokio::time::Instant,
|
||||
) -> Result<ArtifactInventoryLease> {
|
||||
let (put, repair) = tokio::task::spawn_blocking(move || {
|
||||
let put = existing(root.try_clone()?, key, Domain::Put)?;
|
||||
let repair = existing(root, key, Domain::Repair)?;
|
||||
Ok::<_, io::Error>((put, repair))
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?
|
||||
.map_err(read_error)?;
|
||||
// Lock order: closed PUT active exclusive -> closed repair active
|
||||
// exclusive. No admission lock is acquired and no seal is created.
|
||||
tokio::time::timeout_at(deadline, async {
|
||||
lock(&put.active, FlockOperation::NonBlockingLockExclusive).await?;
|
||||
lock(&repair.active, FlockOperation::NonBlockingLockExclusive).await
|
||||
})
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?
|
||||
.map_err(read_error)?;
|
||||
Ok(ArtifactInventoryLease {
|
||||
_put: retained(put),
|
||||
_repair: retained(repair),
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) async fn acquire(root: File, key: [u8; 32], domain: Domain) -> Result<Arc<ArtifactLease>> {
|
||||
let opened = open(root, key, domain).await?;
|
||||
// Lock order: admission exclusive -> active shared. Contention sleeps
|
||||
// asynchronously; no blocking executor thread waits for another owner.
|
||||
lock(&opened.admission, FlockOperation::NonBlockingLockExclusive).await?;
|
||||
tokio::task::spawn_blocking(move || opened.admit())
|
||||
.await
|
||||
.map_err(DiskError::from)?
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
|
||||
pub(super) async fn seal(root: File, key: [u8; 32], domain: Domain) -> Result<()> {
|
||||
let opened = open(root, key, domain).await?;
|
||||
lock(&opened.admission, FlockOperation::NonBlockingLockExclusive).await?;
|
||||
let opened = tokio::task::spawn_blocking(move || opened.close())
|
||||
.await
|
||||
.map_err(DiskError::from)??;
|
||||
// Closing is already durable. Cancellation or timeout here only loses
|
||||
// the confirmation; admission stays permanently closed across reopen.
|
||||
lock(&opened.active, FlockOperation::NonBlockingLockExclusive).await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, unix))]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::{path::Path, time::Duration};
|
||||
|
||||
const KEY: [u8; 32] = [7; 32];
|
||||
|
||||
fn root(path: &Path) -> File {
|
||||
use rustix::fs::{Mode, OFlags, open};
|
||||
File::from(
|
||||
open(
|
||||
path,
|
||||
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
|
||||
Mode::empty(),
|
||||
)
|
||||
.expect("open pinned test root"),
|
||||
)
|
||||
}
|
||||
|
||||
fn directory(path: &Path) -> std::path::PathBuf {
|
||||
path.join(Domain::Put.directory())
|
||||
.join(hex_simd::encode_to_string(KEY, hex_simd::AsciiCase::Lower))
|
||||
}
|
||||
|
||||
async fn closed(path: &Path) {
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while !directory(path).join("closed").exists() {
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("sealer reaches its monotonic close point");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_repair_closing_domain_is_independent_and_survives_reopen() {
|
||||
let temp = tempfile::tempdir().expect("disk root");
|
||||
seal(root(temp.path()), KEY).await.expect("PUT closes first");
|
||||
let repair = acquire_in(root(temp.path()), KEY, Domain::Repair)
|
||||
.await
|
||||
.expect("PUT closure permits existing-version repair");
|
||||
assert!(acquire(root(temp.path()), KEY).await.is_err());
|
||||
drop(repair);
|
||||
seal_in(root(temp.path()), KEY, Domain::Repair)
|
||||
.await
|
||||
.expect("repair closes independently");
|
||||
assert!(acquire_in(root(temp.path()), KEY, Domain::Repair).await.is_err());
|
||||
seal_in(root(temp.path()), KEY, Domain::Repair)
|
||||
.await
|
||||
.expect("reopened repair close is durable");
|
||||
|
||||
let other = [8; 32];
|
||||
seal_in(root(temp.path()), other, Domain::Repair)
|
||||
.await
|
||||
.expect("close only repair admission");
|
||||
drop(
|
||||
acquire(root(temp.path()), other)
|
||||
.await
|
||||
.expect("repair closure permits PUT domain"),
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_repair_cancelled_waiter_keeps_physical_owner_and_durable_close() {
|
||||
let temp = tempfile::tempdir().expect("disk root");
|
||||
let writer = acquire_in(root(temp.path()), KEY, Domain::Repair)
|
||||
.await
|
||||
.expect("repair admitted");
|
||||
let (entered, started) = tokio::sync::oneshot::channel();
|
||||
let (release, waiting) = std::sync::mpsc::channel();
|
||||
let owner = tokio::spawn(async move {
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let _owner = writer;
|
||||
let _ = entered.send(());
|
||||
let _ = waiting.recv();
|
||||
})
|
||||
.await
|
||||
.expect("repair worker");
|
||||
});
|
||||
started.await.expect("physical repair entered");
|
||||
owner.abort();
|
||||
assert!(owner.await.expect_err("cancel repair waiter").is_cancelled());
|
||||
let sealer = tokio::spawn(seal_in(root(temp.path()), KEY, Domain::Repair));
|
||||
let marker = temp
|
||||
.path()
|
||||
.join(Domain::Repair.directory())
|
||||
.join(hex_simd::encode_to_string(KEY, hex_simd::AsciiCase::Lower))
|
||||
.join("closed");
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while !marker.exists() {
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("repair admission durably closes");
|
||||
assert!(!sealer.is_finished(), "ARTIFACT_REPAIR_CANCELLED_PHYSICAL_TAIL");
|
||||
sealer.abort();
|
||||
assert!(sealer.await.expect_err("cancel repair close waiter").is_cancelled());
|
||||
assert!(acquire_in(root(temp.path()), KEY, Domain::Repair).await.is_err());
|
||||
let pending = seal_in(root(temp.path()), KEY, Domain::Repair);
|
||||
tokio::pin!(pending);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(50), &mut pending).await.is_err(),
|
||||
"ARTIFACT_REPAIR_REOPEN_MUST_DRAIN_PHYSICAL_TAIL"
|
||||
);
|
||||
release.send(()).expect("release physical repair");
|
||||
tokio::time::timeout(Duration::from_secs(5), &mut pending)
|
||||
.await
|
||||
.expect("physical drain bounded")
|
||||
.expect("reopened close confirms repair drain");
|
||||
assert!(acquire_in(root(temp.path()), KEY, Domain::Repair).await.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_put_seal_waits_for_independent_openers_and_last_clone() {
|
||||
let temp = tempfile::tempdir().expect("disk root");
|
||||
let first = acquire(root(temp.path()), KEY).await.expect("first admitted writer");
|
||||
let tail = first.clone();
|
||||
let second = acquire(root(temp.path()), KEY).await.expect("independent shared writer");
|
||||
let pending = seal(root(temp.path()), KEY);
|
||||
tokio::pin!(pending);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(50), &mut pending).await.is_err(),
|
||||
"ARTIFACT_PUT_ACTIVE_WRITERS_DRAIN"
|
||||
);
|
||||
drop(first);
|
||||
drop(second);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(50), &mut pending).await.is_err(),
|
||||
"ARTIFACT_PUT_PHYSICAL_CLONE_RETAINED"
|
||||
);
|
||||
assert!(acquire(root(temp.path()), KEY).await.is_err(), "ARTIFACT_PUT_CLOSE_REJECTS_ADMISSION");
|
||||
drop(tail);
|
||||
tokio::time::timeout(Duration::from_secs(5), &mut pending)
|
||||
.await
|
||||
.expect("physical tail drain bounded")
|
||||
.expect("seal complete");
|
||||
seal(root(temp.path()), KEY)
|
||||
.await
|
||||
.expect("independent reopen preserves closed state");
|
||||
assert!(acquire(root(temp.path()), KEY).await.is_err(), "ARTIFACT_PUT_CLOSED_AFTER_REOPEN");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_put_close_cancellation_leaves_durable_denial() {
|
||||
let temp = tempfile::tempdir().expect("disk root");
|
||||
let writer = acquire(root(temp.path()), KEY).await.expect("admitted writer");
|
||||
let sealer = tokio::spawn(seal(root(temp.path()), KEY));
|
||||
closed(temp.path()).await;
|
||||
sealer.abort();
|
||||
assert!(sealer.await.expect_err("cancel close waiter").is_cancelled());
|
||||
assert!(
|
||||
acquire(root(temp.path()), KEY).await.is_err(),
|
||||
"ARTIFACT_PUT_CANCELLED_CLOSE_STAYS_CLOSED"
|
||||
);
|
||||
drop(writer);
|
||||
seal(root(temp.path()), KEY).await.expect("reopen finishes physical drain");
|
||||
assert!(acquire(root(temp.path()), KEY).await.is_err(), "ARTIFACT_PUT_CANCELLED_CLOSE_REOPEN");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_put_physical_worker_retains_lease_after_waiter_cancellation() {
|
||||
let temp = tempfile::tempdir().expect("disk root");
|
||||
let writer = acquire(root(temp.path()), KEY).await.expect("physical writer admitted");
|
||||
let (entered, started) = tokio::sync::oneshot::channel();
|
||||
let (release, waiting) = std::sync::mpsc::channel();
|
||||
let owner = tokio::spawn(async move {
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let _physical_owner = writer;
|
||||
let _ = entered.send(());
|
||||
let _ = waiting.recv();
|
||||
})
|
||||
.await
|
||||
.expect("physical worker");
|
||||
});
|
||||
started.await.expect("physical syscall entered");
|
||||
owner.abort();
|
||||
assert!(owner.await.expect_err("cancel async writer").is_cancelled());
|
||||
let pending = seal(root(temp.path()), KEY);
|
||||
tokio::pin!(pending);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(50), &mut pending).await.is_err(),
|
||||
"ARTIFACT_PUT_CANCELLED_PHYSICAL_TAIL"
|
||||
);
|
||||
release.send(()).expect("release physical worker");
|
||||
tokio::time::timeout(Duration::from_secs(5), &mut pending)
|
||||
.await
|
||||
.expect("physical worker drains")
|
||||
.expect("seal confirms real completion");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_put_seal_waits_for_another_process() {
|
||||
const CHILD_ROOT: &str = "RUSTFS_TEST_ARTIFACT_PUT_SEAL_ROOT";
|
||||
if let Some(path) = std::env::var_os(CHILD_ROOT) {
|
||||
seal(root(Path::new(&path)), KEY)
|
||||
.await
|
||||
.expect("child closes and drains parent writer");
|
||||
assert!(acquire(root(Path::new(&path)), KEY).await.is_err(), "ARTIFACT_PUT_CHILD_CLOSED");
|
||||
return;
|
||||
}
|
||||
struct Child(std::process::Child);
|
||||
impl Drop for Child {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.0.kill();
|
||||
let _ = self.0.wait();
|
||||
}
|
||||
}
|
||||
let temp = tempfile::tempdir().expect("disk root");
|
||||
let writer = acquire(root(temp.path()), KEY).await.expect("parent owns physical write");
|
||||
let mut child = Child(
|
||||
std::process::Command::new(std::env::current_exe().expect("test executable"))
|
||||
.args([
|
||||
"--exact",
|
||||
"disk::local::artifact_lease::tests::artifact_put_seal_waits_for_another_process",
|
||||
])
|
||||
.env(CHILD_ROOT, temp.path())
|
||||
.spawn()
|
||||
.expect("start independent sealer"),
|
||||
);
|
||||
closed(temp.path()).await;
|
||||
assert!(child.0.try_wait().expect("probe child").is_none(), "ARTIFACT_PUT_CROSS_PROCESS_DRAIN");
|
||||
assert!(
|
||||
acquire(root(temp.path()), KEY).await.is_err(),
|
||||
"ARTIFACT_PUT_CROSS_PROCESS_ADMISSION_CLOSED"
|
||||
);
|
||||
drop(writer);
|
||||
let status = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
if let Some(status) = child.0.try_wait().expect("child wait") {
|
||||
break status;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("child physically drains");
|
||||
assert!(status.success(), "ARTIFACT_PUT_CROSS_PROCESS_SEAL");
|
||||
assert!(acquire(root(temp.path()), KEY).await.is_err(), "ARTIFACT_PUT_PROCESS_REOPEN_CLOSED");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_put_lease_rejects_symlinks_at_every_seal_component() {
|
||||
use std::os::unix::fs::{PermissionsExt, symlink};
|
||||
for component in ["directory", "key", "admission.lock", "active.lock", "closed"] {
|
||||
let temp = tempfile::tempdir().expect("disk root");
|
||||
let outside = tempfile::tempdir().expect("symlink destination");
|
||||
let sentinel = outside.path().join("untouched");
|
||||
std::fs::write(&sentinel, b"unchanged").expect("sentinel");
|
||||
let target = match component {
|
||||
"directory" => temp.path().join(Domain::Put.directory()),
|
||||
"key" => {
|
||||
std::fs::create_dir(temp.path().join(Domain::Put.directory())).expect("seal directory");
|
||||
std::fs::set_permissions(temp.path().join(Domain::Put.directory()), std::fs::Permissions::from_mode(0o700))
|
||||
.expect("private seal directory");
|
||||
directory(temp.path())
|
||||
}
|
||||
file => {
|
||||
drop(acquire(root(temp.path()), KEY).await.expect("initialize seal files"));
|
||||
let target = directory(temp.path()).join(file);
|
||||
if file != "closed" {
|
||||
std::fs::remove_file(&target).expect("replace fixture lock");
|
||||
}
|
||||
target
|
||||
}
|
||||
};
|
||||
symlink(
|
||||
if matches!(component, "directory" | "key") {
|
||||
outside.path()
|
||||
} else {
|
||||
&sentinel
|
||||
},
|
||||
target,
|
||||
)
|
||||
.expect("symlink fixture");
|
||||
assert!(
|
||||
acquire(root(temp.path()), KEY).await.is_err(),
|
||||
"ARTIFACT_PUT_SYMLINK_REJECTED: {component}"
|
||||
);
|
||||
assert!(
|
||||
seal(root(temp.path()), KEY).await.is_err(),
|
||||
"ARTIFACT_PUT_SYMLINK_SEAL_REJECTED: {component}"
|
||||
);
|
||||
assert_eq!(std::fs::read(&sentinel).expect("sentinel intact"), b"unchanged");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_put_lease_rejects_corrupt_closed_marker() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let temp = tempfile::tempdir().expect("disk root");
|
||||
drop(acquire(root(temp.path()), KEY).await.expect("initialize seal files"));
|
||||
let marker = directory(temp.path()).join("closed");
|
||||
std::fs::write(&marker, b"partial").expect("corrupt marker fixture");
|
||||
std::fs::set_permissions(&marker, std::fs::Permissions::from_mode(0o600)).expect("marker permissions");
|
||||
assert!(acquire(root(temp.path()), KEY).await.is_err(), "ARTIFACT_PUT_CORRUPT_MARKER_FAILS_CLOSED");
|
||||
assert!(seal(root(temp.path()), KEY).await.is_err(), "ARTIFACT_PUT_CORRUPT_MARKER_NOT_CONFIRMED");
|
||||
assert_eq!(std::fs::read(marker).expect("corruption retained"), b"partial");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_put_lease_stays_on_pinned_root_after_path_replacement() {
|
||||
let temp = tempfile::tempdir().expect("disk root parent");
|
||||
let current = temp.path().join("current");
|
||||
let original = temp.path().join("original");
|
||||
std::fs::create_dir(¤t).expect("initial disk root");
|
||||
let pin = root(¤t);
|
||||
drop(
|
||||
acquire(pin.try_clone().expect("root pin clone"), KEY)
|
||||
.await
|
||||
.expect("initialize pinned seal"),
|
||||
);
|
||||
std::fs::rename(¤t, &original).expect("replace configured pathname");
|
||||
std::fs::create_dir(¤t).expect("replacement disk root");
|
||||
seal(pin, KEY).await.expect("close original disk through its descriptor");
|
||||
assert!(directory(&original).join("closed").exists(), "ARTIFACT_PUT_ROOT_PINNED");
|
||||
assert!(!current.join(Domain::Put.directory()).exists(), "ARTIFACT_PUT_REPLACEMENT_UNTOUCHED");
|
||||
assert!(acquire(root(&original), KEY).await.is_err());
|
||||
}
|
||||
}
|
||||
@@ -345,6 +345,8 @@ async fn prepare_rename_commit_and_destination_metadata(
|
||||
let publication_root = publication_root.clone();
|
||||
os::run_blocking_namespace_operation(mutation_lease, move || {
|
||||
if create_source_parent {
|
||||
#[cfg(test)]
|
||||
super::volume_mutation_test_hooks::run(super::volume_mutation_test_hooks::Stage::Create, &source_parent);
|
||||
std::fs::create_dir_all(&source_parent)?;
|
||||
}
|
||||
let guard = match os::prepare_rename_commit_guard(&source_parent, &destination_parent, &base_dir, &publication_root) {
|
||||
@@ -365,9 +367,15 @@ async fn prepare_rename_commit_and_destination_metadata(
|
||||
};
|
||||
#[cfg(not(any(windows, target_os = "linux")))]
|
||||
let result = {
|
||||
let _ = mutation_lease;
|
||||
if create_source_parent {
|
||||
fs::create_dir_all(source_parent).await.map_err(to_file_error)?;
|
||||
let source_parent = source_parent.to_path_buf();
|
||||
os::run_blocking_namespace_operation(mutation_lease, move || {
|
||||
#[cfg(test)]
|
||||
super::volume_mutation_test_hooks::run(super::volume_mutation_test_hooks::Stage::Create, &source_parent);
|
||||
std::fs::create_dir_all(source_parent)
|
||||
})
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
}
|
||||
let guard =
|
||||
os::prepare_rename_commit_guard(source_parent, destination_parent, base_dir, publication_root).map_err(|err| {
|
||||
@@ -398,30 +406,31 @@ pub(in crate::disk) struct LocalRenamePreflightRejection(());
|
||||
pub(super) struct RenameDataState {
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
preflight_rejection: Option<LocalRenamePreflightRejection>,
|
||||
prepared: Option<Arc<os::NamespaceMutationLease>>,
|
||||
}
|
||||
|
||||
pub(in crate::disk) struct PreparedRenameData {
|
||||
file_info: FileInfo,
|
||||
mutation_lease: Arc<os::NamespaceMutationLease>,
|
||||
}
|
||||
|
||||
impl LocalDisk {
|
||||
#[tracing::instrument(name = "rename_data", target = "rustfs_ecstore::disk::local", level = "trace", skip_all)]
|
||||
pub(super) async fn rename_data_inner(
|
||||
async fn prepare_rename_data_admission(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
state: &mut RenameDataState,
|
||||
) -> Result<RenameDataResp> {
|
||||
crate::hp_guard!("LocalDisk::rename_data");
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
if fi.is_healing() {
|
||||
wait_for_heal_commit_test_barrier(&self.root, dst_volume, dst_path).await?;
|
||||
}
|
||||
source: (&str, &str),
|
||||
destination: (&str, &str),
|
||||
source_volume: Option<&str>,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<PreparedRenameData> {
|
||||
let (src_volume, src_path) = source;
|
||||
let (dst_volume, dst_path) = destination;
|
||||
let mut fi = fi;
|
||||
// A non-force DeleteBucket must not remove a directory while a local
|
||||
// object commit is publishing into it. The peer's empty scan remains
|
||||
// optimistic; this lease establishes the local commit/delete order and
|
||||
// remains owned by any blocking syscall that outlives async cancellation.
|
||||
let (dst_volume_dir, destination_object_path) = self.io_get_bucket_and_object_path(dst_volume, dst_path)?;
|
||||
let destination_object_path = self.io_get_object_path(dst_volume, dst_path)?;
|
||||
let quota_fence_token =
|
||||
match rustfs_utils::http::metadata_compat::get_consistent_str(&fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX) {
|
||||
Some(value) => {
|
||||
@@ -455,16 +464,27 @@ impl LocalDisk {
|
||||
.strip_prefix(self.io_root())
|
||||
.map_err(|_| DiskError::InvalidPath)?,
|
||||
);
|
||||
let metadata_lease =
|
||||
os::acquire_metadata_mutation_lease(&root_destination_object_path, state.namespace_owner.take()).await;
|
||||
let mutation_lease = os::acquire_rename_data_mutation_lease_with_owner(
|
||||
&self.root,
|
||||
dst_volume,
|
||||
&destination_object_path,
|
||||
staging_prefix.as_deref(),
|
||||
Some(metadata_lease),
|
||||
)
|
||||
.await;
|
||||
let metadata_lease = os::acquire_metadata_mutation_lease(&root_destination_object_path, namespace_owner).await;
|
||||
let mutation_lease = if let Some(source_volume) = source_volume {
|
||||
os::acquire_rename_data_mutation_lease_for_volumes(
|
||||
&self.root,
|
||||
source_volume,
|
||||
dst_volume,
|
||||
&destination_object_path,
|
||||
staging_prefix.as_deref(),
|
||||
Some(metadata_lease),
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
os::acquire_rename_data_mutation_lease_with_owner(
|
||||
&self.root,
|
||||
dst_volume,
|
||||
&destination_object_path,
|
||||
staging_prefix.as_deref(),
|
||||
Some(metadata_lease),
|
||||
)
|
||||
.await
|
||||
};
|
||||
if let Some(claim) = quota_fence_claim {
|
||||
mutation_lease.attach_external_guard(claim);
|
||||
}
|
||||
@@ -472,6 +492,99 @@ impl LocalDisk {
|
||||
fi.erasure.index = 0;
|
||||
}
|
||||
fi.validate_for_metadata_read()?;
|
||||
Ok(PreparedRenameData {
|
||||
file_info: fi,
|
||||
mutation_lease,
|
||||
})
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn begin_legacy_rename_data(
|
||||
&self,
|
||||
source: (&str, &str),
|
||||
fi: &FileInfo,
|
||||
destination: (&str, &str),
|
||||
permit: crate::disk::io_admission::DiskIoPermit,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<PreparedRenameData> {
|
||||
// Capture the same permit, mount FD and listener-instance owner before
|
||||
// quota -> metadata -> namespace -> sorted volume READ admission.
|
||||
#[cfg(target_os = "linux")]
|
||||
let owner: Arc<dyn Send + Sync> = Arc::new((permit, self.mount_lease.clone(), namespace_owner));
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let owner: Arc<dyn Send + Sync> = Arc::new((permit, namespace_owner));
|
||||
let prepared = self
|
||||
.prepare_rename_data_admission(fi.clone(), source, destination, Some(source.0), Some(owner))
|
||||
.await?;
|
||||
for volume in [Some(source.0), (source.0 != destination.0).then_some(destination.0)]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
if !skip_access_checks(volume) {
|
||||
let path = self.io_get_bucket_path(volume)?;
|
||||
let metadata = fs::symlink_metadata(path)
|
||||
.await
|
||||
.map_err(crate::disk::error_conv::to_volume_error)?;
|
||||
if !metadata.is_dir() {
|
||||
return Err(DiskError::VolumeNotFound);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(prepared)
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn rename_data_prepared(
|
||||
&self,
|
||||
source: (&str, &str),
|
||||
destination: (&str, &str),
|
||||
prepared: PreparedRenameData,
|
||||
) -> Result<RenameDataResp> {
|
||||
let PreparedRenameData {
|
||||
file_info,
|
||||
mutation_lease,
|
||||
} = prepared;
|
||||
let mut state = RenameDataState {
|
||||
prepared: Some(mutation_lease),
|
||||
..Default::default()
|
||||
};
|
||||
self.rename_data_inner(source.0, source.1, file_info, destination.0, destination.1, &mut state)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(name = "rename_data", target = "rustfs_ecstore::disk::local", level = "trace", skip_all)]
|
||||
pub(super) async fn rename_data_inner(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
state: &mut RenameDataState,
|
||||
) -> Result<RenameDataResp> {
|
||||
crate::hp_guard!("LocalDisk::rename_data");
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
if fi.is_healing() {
|
||||
wait_for_heal_commit_test_barrier(&self.root, dst_volume, dst_path).await?;
|
||||
}
|
||||
let PreparedRenameData {
|
||||
file_info: fi,
|
||||
mutation_lease,
|
||||
} = if let Some(mutation_lease) = state.prepared.take() {
|
||||
PreparedRenameData {
|
||||
file_info: fi,
|
||||
mutation_lease,
|
||||
}
|
||||
} else {
|
||||
self.prepare_rename_data_admission(
|
||||
fi,
|
||||
(src_volume, src_path),
|
||||
(dst_volume, dst_path),
|
||||
None,
|
||||
state.namespace_owner.take(),
|
||||
)
|
||||
.await?
|
||||
};
|
||||
let no_inline = fi.data.is_none() && fi.size > 0;
|
||||
let fi_healing = fi.is_healing();
|
||||
// Snapshot the destination part paths before `fi` is consumed below. These
|
||||
// are the descriptors a reader may hold for the version this call is about
|
||||
// to replace (backlog#1145); readers build the identical string in
|
||||
@@ -507,6 +620,7 @@ impl LocalDisk {
|
||||
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
|
||||
}
|
||||
|
||||
let dst_volume_dir = self.io_get_bucket_path(dst_volume)?;
|
||||
if !skip_access_checks(dst_volume)
|
||||
&& let Err(e) = super::super::fs::access_std(&dst_volume_dir)
|
||||
{
|
||||
@@ -524,8 +638,15 @@ impl LocalDisk {
|
||||
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
|
||||
}
|
||||
|
||||
// xl.meta path
|
||||
let src_file_path = self.io_get_object_path(src_volume, format!("{}/{}", src_path, STORAGE_FORMAT_FILE).as_str())?;
|
||||
// Keep recovery authority separate from active upload metadata so a
|
||||
// committed upload cannot reopen when set-level cleanup fails.
|
||||
let preserve_upload_metadata = src_volume == super::super::RUSTFS_META_MULTIPART_BUCKET;
|
||||
let source_metadata_name = if preserve_upload_metadata {
|
||||
super::super::MULTIPART_COMMIT_METADATA_FILE
|
||||
} else {
|
||||
STORAGE_FORMAT_FILE
|
||||
};
|
||||
let src_file_path = self.io_get_object_path(src_volume, format!("{src_path}/{source_metadata_name}").as_str())?;
|
||||
let dst_file_path = self.io_get_object_open_path(dst_volume, format!("{}/{}", dst_path, STORAGE_FORMAT_FILE).as_str())?;
|
||||
|
||||
// data_dir path (only used by the non-inline commit branch)
|
||||
@@ -590,6 +711,35 @@ impl LocalDisk {
|
||||
data_paths: has_data_dir_path.as_ref(),
|
||||
};
|
||||
|
||||
if preserve_upload_metadata {
|
||||
let source_object_path = self.get_object_path(src_volume, src_path)?;
|
||||
let authority_owner: Arc<dyn Send + Sync> = if source_object_path == self.get_object_path(dst_volume, dst_path)? {
|
||||
commit.mutation_lease.clone()
|
||||
} else {
|
||||
os::try_acquire_metadata_mutation_lease_with_owner(&source_object_path, commit.mutation_lease.clone())?
|
||||
};
|
||||
let authority = self.io_get_object_path(src_volume, &format!("{src_path}/{STORAGE_FORMAT_FILE}"))?;
|
||||
let recovery = self.io_get_object_path(
|
||||
src_volume,
|
||||
&format!("{src_path}/{}/{STORAGE_FORMAT_FILE}", super::super::MULTIPART_RECOVERY_DIR),
|
||||
)?;
|
||||
check_path_length(recovery.to_string_lossy().as_ref())?;
|
||||
// Both volume READs are already held. Neither source key may wait
|
||||
// behind a competing owner that is queued behind a volume WRITE.
|
||||
let recovery_lease = os::try_acquire_namespace_mutation_lease_with_owner(&recovery, authority_owner.clone())?;
|
||||
os::rename_all_with_lease(&authority, &recovery, &src_volume_dir, commit.publication_root, recovery_lease).await?;
|
||||
// This is system metadata authority, so use its existing strict
|
||||
// durability policy even when the object payload is relaxed.
|
||||
if effective_durability(src_volume).syncs_commit_metadata() {
|
||||
os::fsync_dir_with_owner(recovery.parent().ok_or(DiskError::InvalidPath)?, Some(authority_owner.clone()))
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
os::fsync_dir_with_owner(src_file_parent, Some(authority_owner.clone()))
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
}
|
||||
}
|
||||
|
||||
if no_inline {
|
||||
// Non-inline: read xl.meta, parse, write, rename data dir, rename xl.meta
|
||||
let mut xlmeta = FileMeta::new();
|
||||
@@ -798,7 +948,7 @@ impl LocalDisk {
|
||||
if let (Some(old_data_dir), Some(dst_buf)) = (rollback_data_dir, has_dst_buf.as_ref()) {
|
||||
let backup_parent = dst_file_parent.join(old_data_dir.to_string());
|
||||
#[cfg(not(windows))]
|
||||
if let Err(err) = os::make_dir_all(&backup_parent, &skip_parent).await {
|
||||
if let Err(err) = os::make_dir_all_with_owner(&backup_parent, &skip_parent, commit.mutation_lease.clone()).await {
|
||||
commit.restore_published_data_source().await?;
|
||||
return Err(err);
|
||||
}
|
||||
@@ -977,14 +1127,8 @@ impl LocalDisk {
|
||||
#[cfg(windows)]
|
||||
drop(commit.directory_guard);
|
||||
|
||||
if let Some(src_file_path_parent) = src_file_path.parent() {
|
||||
if src_volume != super::super::RUSTFS_META_MULTIPART_BUCKET {
|
||||
let _ = std::fs::remove_dir(src_file_path_parent);
|
||||
} else {
|
||||
let _ = self
|
||||
.delete_file(&dst_volume_dir, &src_file_path_parent.to_path_buf(), true, false)
|
||||
.await;
|
||||
}
|
||||
if !preserve_upload_metadata && let Some(parent) = src_file_path.parent() {
|
||||
let _ = std::fs::remove_dir(parent);
|
||||
}
|
||||
|
||||
// Heal reuses a version's `data_dir` and lands the rebuilt shard on
|
||||
@@ -1017,18 +1161,12 @@ impl LocalDisk {
|
||||
// allowing a retry to reuse the deterministic staging path too early.
|
||||
let src = src_file_path.clone();
|
||||
let dst = dst_file_path.clone();
|
||||
let cleanup_path = if src_volume == super::super::RUSTFS_META_MULTIPART_BUCKET {
|
||||
src_file_path.parent().map(|p| p.to_path_buf())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let dst_path_for_failpoint = dst_path.to_string();
|
||||
let dst_volume_for_source = dst_volume.to_string();
|
||||
let dst_path_for_source = dst_path.to_string();
|
||||
let dst_volume_dir_for_commit = dst_volume_dir.clone();
|
||||
let publication_root_for_commit = self.publication_root.clone();
|
||||
#[cfg(not(windows))]
|
||||
let cleanup_src_parent = cleanup_path.is_none();
|
||||
let cleanup_src_parent = !preserve_upload_metadata;
|
||||
#[cfg(windows)]
|
||||
let source_parent = src_file_parent.to_path_buf();
|
||||
let rename_commit_guard_for_preparation = commit.directory_guard.clone();
|
||||
@@ -1228,13 +1366,9 @@ impl LocalDisk {
|
||||
drop(file_sync_admission.take());
|
||||
#[cfg(windows)]
|
||||
drop(commit.directory_guard);
|
||||
if let Some(ref cleanup) = cleanup_path {
|
||||
let _ = self.delete_file(&dst_volume_dir, cleanup, true, false).await;
|
||||
} else {
|
||||
#[cfg(windows)]
|
||||
if let Some(parent) = src_file_path.parent() {
|
||||
let _ = std::fs::remove_dir(parent);
|
||||
}
|
||||
#[cfg(windows)]
|
||||
if cleanup_src_parent && let Some(parent) = src_file_path.parent() {
|
||||
let _ = std::fs::remove_dir(parent);
|
||||
}
|
||||
for part_path in &invalidate_part_paths {
|
||||
self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
|
||||
@@ -1281,18 +1415,25 @@ impl LocalDisk {
|
||||
let backup_path = dst_parent
|
||||
.join(rollback_data_dir.to_string())
|
||||
.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
// rename_all acquires the backup path's namespace lease. Do not
|
||||
// hold a disk admission while acquiring another namespace lock.
|
||||
// Keep backup mutations serialized, but never wait for a second
|
||||
// namespace while holding the volume READ: its owner can be
|
||||
// queued behind a volume DELETE that is waiting for this commit.
|
||||
drop(file_sync_admission.take());
|
||||
if let Err(err) = os::rename_all_with_owner(
|
||||
staged_backup,
|
||||
&backup_path,
|
||||
&dst_volume_dir,
|
||||
commit.publication_root,
|
||||
Some(commit.mutation_lease.clone()),
|
||||
)
|
||||
.await
|
||||
{
|
||||
let backup_result =
|
||||
match os::try_acquire_namespace_mutation_lease_with_owner(&backup_path, commit.mutation_lease.clone()) {
|
||||
Ok(lease) => {
|
||||
os::rename_all_with_lease(
|
||||
staged_backup,
|
||||
&backup_path,
|
||||
&dst_volume_dir,
|
||||
commit.publication_root,
|
||||
lease,
|
||||
)
|
||||
.await
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
};
|
||||
if let Err(err) = backup_result {
|
||||
let _ = remove_file_if_exists(staged_backup);
|
||||
return Err(err);
|
||||
}
|
||||
@@ -1452,9 +1593,7 @@ impl LocalDisk {
|
||||
}
|
||||
|
||||
// Cleanup
|
||||
if let Some(ref cleanup) = cleanup_path {
|
||||
let _ = self.delete_file(&dst_volume_dir, cleanup, true, false).await;
|
||||
} else if let Some(parent) = src_file_path.parent() {
|
||||
if cleanup_src_parent && let Some(parent) = src_file_path.parent() {
|
||||
let _ = std::fs::remove_dir(parent);
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,202 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
use std::cell::RefCell;
|
||||
|
||||
struct Observation {
|
||||
rebuilds: usize,
|
||||
normalizations: usize,
|
||||
before_recheck: Option<Box<dyn FnOnce()>>,
|
||||
}
|
||||
|
||||
thread_local! {
|
||||
static OBSERVATION: RefCell<Option<Observation>> = const { RefCell::new(None) };
|
||||
}
|
||||
|
||||
struct Observe;
|
||||
|
||||
impl Observe {
|
||||
fn new(before_recheck: Option<Box<dyn FnOnce()>>) -> Self {
|
||||
OBSERVATION.with(|state| {
|
||||
assert!(state.borrow().is_none(), "read-path observations must not nest");
|
||||
*state.borrow_mut() = Some(Observation {
|
||||
rebuilds: 0,
|
||||
normalizations: 0,
|
||||
before_recheck,
|
||||
});
|
||||
});
|
||||
Self
|
||||
}
|
||||
|
||||
fn normalizations(&self) -> usize {
|
||||
OBSERVATION.with(|state| state.borrow().as_ref().expect("active observation").normalizations)
|
||||
}
|
||||
|
||||
fn rebuilds(&self) -> usize {
|
||||
OBSERVATION.with(|state| state.borrow().as_ref().expect("active observation").rebuilds)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Observe {
|
||||
fn drop(&mut self) {
|
||||
OBSERVATION.with(|state| *state.borrow_mut() = None);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn record_bucket_rebuild() {
|
||||
OBSERVATION.with(|state| {
|
||||
if let Some(observation) = state.borrow_mut().as_mut() {
|
||||
observation.rebuilds += 1;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
pub(super) fn record_normalization() {
|
||||
OBSERVATION.with(|state| {
|
||||
if let Some(observation) = state.borrow_mut().as_mut() {
|
||||
observation.normalizations += 1;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
pub(super) fn before_bucket_recheck() {
|
||||
let hook = OBSERVATION.with(|state| state.borrow_mut().as_mut().and_then(|value| value.before_recheck.take()));
|
||||
if let Some(hook) = hook {
|
||||
hook();
|
||||
}
|
||||
}
|
||||
|
||||
async fn local_disk(root: &Path) -> LocalDisk {
|
||||
let endpoint = Endpoint::try_from(root.to_string_lossy().as_ref()).expect("local endpoint");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk");
|
||||
disk.make_volume("bucket").await.expect("bucket");
|
||||
disk
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn checked_read_paths_reuse_bucket_construction() {
|
||||
let root = tempfile::tempdir().expect("disk root");
|
||||
let disk = local_disk(root.path()).await;
|
||||
std::fs::create_dir_all(root.path().join("bucket/prefix/object")).expect("object directories");
|
||||
for key in ["prefix/object/xl.meta", "", "prefix/../object/xl.meta"] {
|
||||
let expected = build_local_disk_object_path(disk.io_root(), "bucket", key);
|
||||
let observation = Observe::new(None);
|
||||
let actual = disk.io_get_checked_read_paths("bucket", key).expect("checked read paths");
|
||||
assert_eq!(actual, expected, "returned paths must retain their existing spelling");
|
||||
assert_eq!(observation.rebuilds(), 0, "the validated tuple already owns its bucket path");
|
||||
// path_join_buf already resolves dot components in object keys.
|
||||
let expected_normalizations = if cfg!(windows) { 3 } else { 0 };
|
||||
assert_eq!(observation.normalizations(), expected_normalizations);
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
{
|
||||
assert_eq!(
|
||||
disk.io_get_bucket_and_object_open_path("bucket", key)
|
||||
.expect("read-version paths"),
|
||||
expected
|
||||
);
|
||||
assert_eq!(observation.rebuilds(), 0, "read_version shares the same checked path core");
|
||||
assert_eq!(observation.normalizations(), 2 * expected_normalizations);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn checked_read_paths_preserve_containment_errors() {
|
||||
let root = tempfile::tempdir().expect("disk root");
|
||||
let disk = local_disk(root.path()).await;
|
||||
for (bucket, key) in [("../outside", "object"), ("bucket", "../outside"), ("bucket", "/outside")] {
|
||||
let old = disk
|
||||
.io_get_object_path(bucket, key)
|
||||
.and_then(|path| disk.io_get_bucket_path(bucket).map(|volume| (volume, path)));
|
||||
let new = disk.io_get_checked_read_paths(bucket, key);
|
||||
assert_eq!(new, old, "path and error behavior for {bucket}/{key}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn checked_read_paths_metadata_bytes_and_absence_errors() {
|
||||
let root = tempfile::tempdir().expect("disk root");
|
||||
let disk = local_disk(root.path()).await;
|
||||
let object = root.path().join("bucket/prefix/object");
|
||||
std::fs::create_dir_all(&object).expect("object");
|
||||
let metadata = b"opaque metadata bytes passed through the read API";
|
||||
std::fs::write(object.join(STORAGE_FORMAT_FILE), metadata).expect("metadata");
|
||||
{
|
||||
let observation = Observe::new(None);
|
||||
let actual = disk
|
||||
.read_metadata("bucket", "prefix/object/xl.meta")
|
||||
.await
|
||||
.expect("read metadata");
|
||||
assert_eq!(actual.as_ref(), metadata);
|
||||
assert_eq!(observation.rebuilds(), 0, "scanner metadata reads use the shared path core");
|
||||
assert_eq!(observation.normalizations(), if cfg!(windows) { 3 } else { 0 });
|
||||
}
|
||||
assert!(matches!(
|
||||
disk.read_metadata("bucket", "missing/xl.meta").await,
|
||||
Err(DiskError::FileNotFound)
|
||||
));
|
||||
assert!(matches!(
|
||||
disk.read_metadata("missing", "object/xl.meta").await,
|
||||
Err(DiskError::VolumeNotFound)
|
||||
));
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn checked_read_paths_recheck_bucket_after_object_validation() {
|
||||
let root = tempfile::tempdir().expect("disk root");
|
||||
let outside = tempfile::tempdir().expect("outside root");
|
||||
let disk = local_disk(root.path()).await;
|
||||
std::fs::create_dir_all(root.path().join("bucket/object")).expect("object");
|
||||
std::fs::write(root.path().join("bucket/object/xl.meta"), b"inside").expect("inside metadata");
|
||||
let bucket = root.path().join("bucket");
|
||||
let saved = root.path().join("saved");
|
||||
let target = outside.path().to_path_buf();
|
||||
let _observation = Observe::new(Some(Box::new(move || {
|
||||
std::fs::rename(&bucket, saved).expect("replace bucket after full-path validation");
|
||||
std::os::unix::fs::symlink(target, bucket).expect("bucket replacement symlink");
|
||||
})));
|
||||
assert!(
|
||||
matches!(disk.io_get_checked_read_paths("bucket", "object/xl.meta"), Err(DiskError::InvalidPath)),
|
||||
"the second bucket check must remain effective before queuing native IO"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_path_normalization_matches_component_normalizer() {
|
||||
for value in [
|
||||
"",
|
||||
".",
|
||||
"..",
|
||||
"./a",
|
||||
"a/./b",
|
||||
"a//b/",
|
||||
"a/../b",
|
||||
"../../a",
|
||||
"/",
|
||||
"//a///b/",
|
||||
"/a/../b",
|
||||
"/../../a",
|
||||
r"C:\a\b",
|
||||
r"C:\a\..\b",
|
||||
r"\\server\share\a",
|
||||
r"\\?\C:\a\.\b",
|
||||
] {
|
||||
let path = Path::new(value);
|
||||
let expected = normalize_path_components(path);
|
||||
let observation = Observe::new(None);
|
||||
let actual = normalize_path_cow(path);
|
||||
assert_eq!(
|
||||
actual.components().collect::<Vec<_>>(),
|
||||
expected.components().collect::<Vec<_>>(),
|
||||
"{value}"
|
||||
);
|
||||
let needs_buffer = cfg!(windows)
|
||||
|| path
|
||||
.components()
|
||||
.any(|component| matches!(component, std::path::Component::ParentDir | std::path::Component::CurDir));
|
||||
assert_eq!(matches!(actual, std::borrow::Cow::Owned(_)), needs_buffer, "{value}");
|
||||
assert_eq!(observation.normalizations(), usize::from(needs_buffer), "{value}");
|
||||
}
|
||||
}
|
||||
+158
-49
@@ -27,6 +27,7 @@ mod io_admission;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod lifecycle_test_io;
|
||||
pub mod local;
|
||||
pub use local::artifact_inventory::{ArtifactInventoryFrame, ArtifactInventoryVersion};
|
||||
pub mod os;
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -49,6 +50,9 @@ pub const FORMAT_CONFIG_FILE: &str = "format.json";
|
||||
pub const HEALING_MARKER_PATH: &str = "healing.bin";
|
||||
pub const STORAGE_FORMAT_FILE: &str = "xl.meta";
|
||||
pub const STORAGE_FORMAT_FILE_BACKUP: &str = "xl.meta.bkp";
|
||||
/// Original upload authority after it stops admitting active part mutations.
|
||||
pub(crate) const MULTIPART_RECOVERY_DIR: &str = ".abort-authority";
|
||||
pub(crate) const MULTIPART_COMMIT_METADATA_FILE: &str = "xl.meta.complete";
|
||||
pub const PART_TRANSACTION_NEW_META: &str = "new.meta";
|
||||
pub const PART_TRANSACTION_OLD_META: &str = "old.meta";
|
||||
pub const PART_TRANSACTION_ROLLBACK: &str = "rollback";
|
||||
@@ -70,6 +74,10 @@ use crate::cluster::rpc::RemoteDisk;
|
||||
use crate::cluster::rpc::build_internode_data_transport_from_env;
|
||||
use crate::disk::disk_store::DiskStoreRenameDataExt;
|
||||
use crate::disk::disk_store::LocalDiskWrapper;
|
||||
pub use crate::disk::disk_store::{
|
||||
LegacyFileStream, LegacyFileWrite, LegacyMetadataWrite, LegacyPartMutation, LegacyPartOperation, LegacyPathDelete,
|
||||
LegacyPathDeleteOperation, LegacyRenameData, LegacyRenameFile,
|
||||
};
|
||||
use crate::disk::health_state::RuntimeDriveHealthState;
|
||||
use crate::disk::local::ScanGuard;
|
||||
use bytes::Bytes;
|
||||
@@ -82,12 +90,7 @@ use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
|
||||
use rustfs_madmin::info_commands::DiskMetrics;
|
||||
use rustfs_rio::ChunkReaderBox;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
fmt::Debug,
|
||||
path::PathBuf,
|
||||
sync::{Arc, atomic::AtomicBool},
|
||||
time::Duration,
|
||||
};
|
||||
use std::{fmt::Debug, path::PathBuf, sync::Arc, time::Duration};
|
||||
use time::OffsetDateTime;
|
||||
use tokio::io::{AsyncRead, AsyncWrite};
|
||||
use uuid::Uuid;
|
||||
@@ -95,6 +98,7 @@ use uuid::Uuid;
|
||||
/// Independent admission and physical ownership for one disk rename.
|
||||
#[derive(Default)]
|
||||
pub(crate) struct RenameDataGuards {
|
||||
pub(crate) artifact_put: Option<crate::bucket::artifact::ArtifactPutIdentity>,
|
||||
pub(crate) scanner_publication_lease_token: Option<Uuid>,
|
||||
pub(crate) external_guard: Option<Arc<dyn Send + Sync>>,
|
||||
pub(crate) namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
@@ -236,6 +240,117 @@ pub enum Disk {
|
||||
}
|
||||
|
||||
impl Disk {
|
||||
/// Capture local file-write admission before authoritative classification.
|
||||
pub async fn begin_legacy_file_write(&self, volume: &str) -> Result<LegacyFileWrite> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.begin_legacy_file_write(volume).await,
|
||||
Self::Remote(_) => Err(DiskError::DiskNotFound),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn begin_legacy_file_write_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<LegacyFileWrite> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.begin_legacy_file_write_with_namespace_owner(volume, owner).await,
|
||||
Self::Remote(_) => Err(DiskError::DiskNotFound),
|
||||
}
|
||||
}
|
||||
|
||||
/// Capture metadata publication admission before authoritative classification.
|
||||
pub async fn begin_legacy_metadata_write(&self, volume: &str, path: &str) -> Result<LegacyMetadataWrite> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.begin_legacy_metadata_write(volume, path).await,
|
||||
Self::Remote(_) => Err(DiskError::DiskNotFound),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn begin_legacy_metadata_write_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<LegacyMetadataWrite> {
|
||||
match self {
|
||||
Self::Local(disk) => {
|
||||
disk.begin_legacy_metadata_write_with_namespace_owner(volume, path, namespace_owner)
|
||||
.await
|
||||
}
|
||||
Self::Remote(_) => Err(DiskError::DiskNotFound),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn record_legacy_admission_failure(
|
||||
&self,
|
||||
operation: &'static str,
|
||||
error: &DiskError,
|
||||
started: std::time::Instant,
|
||||
) {
|
||||
if let Self::Local(disk) = self {
|
||||
disk.record_legacy_admission_failure(operation, error, started);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn begin_legacy_path_delete(
|
||||
&self,
|
||||
volume: &str,
|
||||
operation: LegacyPathDeleteOperation,
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
deadline: tokio::time::Instant,
|
||||
) -> Result<LegacyPathDelete> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.begin_legacy_path_delete(volume, operation, owner, deadline).await,
|
||||
Self::Remote(_) => Err(DiskError::DiskNotFound),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn begin_legacy_part_mutation(
|
||||
&self,
|
||||
source: Option<(&str, &str)>,
|
||||
destination: (&str, &str),
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
operation: LegacyPartOperation,
|
||||
) -> Result<LegacyPartMutation> {
|
||||
match self {
|
||||
Self::Local(disk) => {
|
||||
disk.begin_legacy_part_mutation(source, destination, namespace_owner, operation)
|
||||
.await
|
||||
}
|
||||
Self::Remote(_) => Err(DiskError::DiskNotFound),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn begin_legacy_rename_file(
|
||||
&self,
|
||||
source: (&str, &str),
|
||||
destination: (&str, &str),
|
||||
durable: bool,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<LegacyRenameFile> {
|
||||
match self {
|
||||
Self::Local(disk) => {
|
||||
disk.begin_legacy_rename_file(source, destination, durable, namespace_owner)
|
||||
.await
|
||||
}
|
||||
Self::Remote(_) => Err(DiskError::DiskNotFound),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn begin_legacy_rename_data(
|
||||
&self,
|
||||
source: (&str, &str),
|
||||
fi: &FileInfo,
|
||||
destination: (&str, &str),
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<LegacyRenameData> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.begin_legacy_rename_data(source, fi, destination, namespace_owner).await,
|
||||
Self::Remote(_) => Err(DiskError::DiskNotFound),
|
||||
}
|
||||
}
|
||||
|
||||
/// A read-only probe; the physical delete repeats the same leaf checks.
|
||||
pub async fn is_plain_orphan_data_dir(&self, volume: &str, path: &str) -> Result<bool> {
|
||||
match self {
|
||||
@@ -1059,13 +1174,27 @@ impl Disk {
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
) -> Result<RenameDataResp> {
|
||||
let Some(scope) = crate::store::bucket_heal_scope_for_object(dst_volume, dst_path) else {
|
||||
return self
|
||||
.rename_data_borrowed_with_fence(src_volume, src_path, fi, dst_volume, dst_path, None)
|
||||
.await;
|
||||
let bucket_scope = crate::store::bucket_heal_scope_for_object(dst_volume, dst_path);
|
||||
let repair_scope = crate::bucket::artifact::artifact_repair_scope(dst_volume, dst_path);
|
||||
if let Some(scope) = &bucket_scope {
|
||||
scope.check()?;
|
||||
}
|
||||
if let Some(scope) = &repair_scope {
|
||||
scope.check().map_err(DiskError::other)?;
|
||||
}
|
||||
let repair_lease = if let (Self::Local(disk), Some(scope)) = (self, &repair_scope) {
|
||||
if let Some(identity) = scope.receiver_identity() {
|
||||
scope.check_version(fi.version_id).await.map_err(DiskError::other)?;
|
||||
Some(disk.acquire_artifact_repair_lease(identity.receiver_key).await?)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
scope.check()?;
|
||||
self.rename_data_borrowed_with_fence_and_guard(src_volume, src_path, fi, dst_volume, dst_path, None, Some(scope))
|
||||
let owner: Option<Arc<dyn Send + Sync>> = (bucket_scope.is_some() || repair_scope.is_some())
|
||||
.then(|| Arc::new((bucket_scope, repair_scope, repair_lease)) as Arc<dyn Send + Sync>);
|
||||
self.rename_data_borrowed_with_fence_and_guard(src_volume, src_path, fi, dst_volume, dst_path, None, owner)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -1076,10 +1205,20 @@ impl Disk {
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
guards: RenameDataGuards,
|
||||
mut guards: RenameDataGuards,
|
||||
) -> RenameDataObservation {
|
||||
match self {
|
||||
Disk::Local(local_disk) => {
|
||||
if let Some(identity) = guards.artifact_put.take() {
|
||||
let lease = match local_disk.acquire_artifact_put_lease(identity.receiver_key).await {
|
||||
Ok(lease) => lease,
|
||||
Err(error) => return RenameDataObservation::unknown(Err(error)),
|
||||
};
|
||||
// Keep the existing timeout policy; physical workers own the
|
||||
// receiver lease even after the caller's disk deadline.
|
||||
guards.namespace_owner =
|
||||
Some(Arc::new((guards.external_guard.clone(), guards.namespace_owner.take(), lease)));
|
||||
}
|
||||
local_disk
|
||||
.rename_data_observed_with_guards(src_volume, src_path, fi, dst_volume, dst_path, guards)
|
||||
.await
|
||||
@@ -1087,39 +1226,17 @@ impl Disk {
|
||||
Disk::Remote(remote_disk) => RenameDataObservation::unknown(
|
||||
remote_disk
|
||||
.rename_data_borrowed_with_fence(
|
||||
src_volume,
|
||||
src_path,
|
||||
(src_volume, src_path),
|
||||
fi,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
(dst_volume, dst_path),
|
||||
guards.scanner_publication_lease_token,
|
||||
guards.artifact_put.map(|identity| identity.incarnation),
|
||||
)
|
||||
.await,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn rename_data_borrowed_with_fence(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
scanner_publication_lease_token: Option<Uuid>,
|
||||
) -> Result<RenameDataResp> {
|
||||
self.rename_data_borrowed_with_fence_and_guard(
|
||||
src_volume,
|
||||
src_path,
|
||||
fi,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
scanner_publication_lease_token,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn rename_data_borrowed_with_fence_and_guard(
|
||||
&self,
|
||||
@@ -1140,12 +1257,11 @@ impl Disk {
|
||||
Disk::Remote(remote_disk) => {
|
||||
remote_disk
|
||||
.rename_data_borrowed_with_fence(
|
||||
src_volume,
|
||||
src_path,
|
||||
(src_volume, src_path),
|
||||
fi,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
(dst_volume, dst_path),
|
||||
scanner_publication_lease_token,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -1910,12 +2026,6 @@ pub struct WalkDirOptions {
|
||||
// Override the remote stream stall timeout for long background walks.
|
||||
#[serde(default)]
|
||||
pub stall_timeout_ms: Option<u64>,
|
||||
|
||||
/// In-process completion state for bounded local walks. This is skipped
|
||||
/// from RPC serialization; remote peers retain the legacy natural-EOF
|
||||
/// behavior until they support an explicit capability.
|
||||
#[serde(skip)]
|
||||
pub producer_limit_reached: Option<Arc<AtomicBool>>,
|
||||
}
|
||||
|
||||
impl WalkDirOptions {
|
||||
@@ -2760,7 +2870,6 @@ mod tests {
|
||||
skip_total_timeout: false,
|
||||
timeout_ms: Some(10_000),
|
||||
stall_timeout_ms: Some(20_000),
|
||||
producer_limit_reached: None,
|
||||
};
|
||||
|
||||
assert_eq!(opts.bucket, "test-bucket");
|
||||
|
||||
+1212
-146
File diff suppressed because it is too large
Load Diff
@@ -153,10 +153,13 @@ async fn refresh_loss_lock_client_keeps_rejection_observable_for_late_waiters()
|
||||
);
|
||||
}
|
||||
|
||||
/// Returns the backing [`tempfile::TempDir`]s alongside the set so callers keep
|
||||
/// them alive for the test's duration and the directories are removed on drop.
|
||||
pub(crate) async fn make_local_set_disks(drive_count: usize, parity_count: usize) -> (Vec<tempfile::TempDir>, Arc<SetDisks>) {
|
||||
make_local_set_disks_with_ctx(drive_count, parity_count, crate::runtime::instance::bootstrap_ctx()).await
|
||||
/// Keep the real metadata owner and backing directories alive together.
|
||||
pub(crate) async fn make_local_set_disks(
|
||||
drive_count: usize,
|
||||
parity_count: usize,
|
||||
) -> (crate::set_disk::HermeticSetFixture, Arc<SetDisks>) {
|
||||
let (fixture, _, set) = crate::set_disk::hermetic_set_disks_for_pool_with_default_parity(drive_count, 0, parity_count).await;
|
||||
(fixture, set)
|
||||
}
|
||||
|
||||
/// Like [`make_local_set_disks`], but binds the set to an explicit instance
|
||||
@@ -756,6 +759,9 @@ async fn blackbox_range_read_restores_exact_slice_with_one_offline_disk() {
|
||||
#[tokio::test]
|
||||
async fn blackbox_delete_marker_hides_object_body_without_erasing_prior_version_metadata() {
|
||||
let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
|
||||
let store = crate::bucket::metadata_sys::object_store_in(set_disks.instance_ctx())
|
||||
.await
|
||||
.expect("fixture metadata authority should belong to this set");
|
||||
let bucket = "bb-delete-marker-read-negative";
|
||||
let object = "object.bin";
|
||||
let opts = ObjectOptions {
|
||||
@@ -765,13 +771,26 @@ async fn blackbox_delete_marker_hides_object_body_without_erasing_prior_version_
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
set_disks
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut reader = PutObjReader::from_vec(b"body hidden by delete marker".to_vec());
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut reader, &opts)
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
object_lock_config_snapshot: Some(
|
||||
store
|
||||
.object_lock_config_snapshot(bucket)
|
||||
.await
|
||||
.expect("PUT fixture should own the bucket lifecycle and Object Lock snapshot"),
|
||||
),
|
||||
..opts.clone()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("object should be written before delete marker");
|
||||
|
||||
|
||||
@@ -56,7 +56,7 @@ mod storage_api_contracts;
|
||||
mod store;
|
||||
|
||||
pub use disk::{with_background_disk_io, with_background_resource_unit};
|
||||
pub use store::PoolMetaWriteGateStatus;
|
||||
pub use store::{BucketIncarnationFenceGuard, PoolMetaWriteGateStatus};
|
||||
|
||||
// pub mod checksum;
|
||||
mod event;
|
||||
|
||||
@@ -4473,3 +4473,7 @@ mod tests {
|
||||
.expect_err("a lying frame-layout marker must fail closed, not serve plaintext");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "readers/compression_index_compatibility_tests.rs"]
|
||||
mod compression_index_compatibility_tests;
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
use super::*;
|
||||
use rustfs_utils::http::metadata_compat::{SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, insert_str};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::io::Cursor;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
const STREAM: &[u8] = include_bytes!(concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/tests/fixtures/compression-index/legacy-lz4-stream.bin"
|
||||
));
|
||||
const HISTORICAL_INDEX: &[u8] = include_bytes!(concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/tests/fixtures/compression-index/pre-zigzag-index.bin"
|
||||
));
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
const CURRENT_INDEX: &[u8] =
|
||||
include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/compression-index/zigzag-index.bin"));
|
||||
const PLAINTEXT_LEN: usize = 3 * 1024 * 1024 + 123;
|
||||
const PLAINTEXT_SHA256: &str = "68f9edf6f9600395d6d2b5517d97570fecd2a0c6dbe6130617ef415a3b48d89c";
|
||||
|
||||
fn fixture_info(index: &[u8]) -> ObjectInfo {
|
||||
let mut metadata = HashMap::new();
|
||||
insert_str(&mut metadata, SUFFIX_COMPRESSION, "lz4".to_owned());
|
||||
insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, PLAINTEXT_LEN.to_string());
|
||||
ObjectInfo {
|
||||
size: i64::try_from(STREAM.len()).expect("fixed fixture size fits i64"),
|
||||
parts: Arc::new(vec![ObjectPartInfo {
|
||||
number: 1,
|
||||
size: STREAM.len(),
|
||||
actual_size: i64::try_from(PLAINTEXT_LEN).expect("fixed plaintext length fits i64"),
|
||||
index: Some(Bytes::copy_from_slice(index)),
|
||||
..Default::default()
|
||||
}]),
|
||||
user_defined: Arc::new(metadata),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn fixture_plaintext() -> Vec<u8> {
|
||||
(0..PLAINTEXT_LEN)
|
||||
.map(|offset| {
|
||||
let chunk = offset / 1024;
|
||||
let lane = offset % 1024;
|
||||
if lane < 8 {
|
||||
u64::try_from(chunk).expect("fixed chunk number fits u64").to_le_bytes()[lane]
|
||||
} else {
|
||||
b"abcdefghijklmnopqrstuvwxyz"[(offset + chunk) % 26]
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn historical_index_preserves_the_complete_compressed_body() {
|
||||
let info = fixture_info(HISTORICAL_INDEX);
|
||||
let (mut reader, storage_offset, _) = GetObjectReader::new(
|
||||
Box::new(Cursor::new(STREAM.to_vec())),
|
||||
None,
|
||||
&info,
|
||||
&ObjectOptions::default(),
|
||||
&HeaderMap::new(),
|
||||
)
|
||||
.await
|
||||
.expect("construct reader for fixed historical index");
|
||||
let mut actual = Vec::new();
|
||||
reader.read_to_end(&mut actual).await.expect("read complete historical body");
|
||||
assert_eq!(storage_offset, 0);
|
||||
assert_eq!(actual.len(), PLAINTEXT_LEN);
|
||||
assert_eq!(
|
||||
hex_simd::encode_to_string(Sha256::digest(&actual), hex_simd::AsciiCase::Lower),
|
||||
PLAINTEXT_SHA256
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn historical_index_preserves_ranges_across_compression_blocks() {
|
||||
let info = fixture_info(HISTORICAL_INDEX);
|
||||
let expected = fixture_plaintext();
|
||||
for start in [7, 1024 * 1024 - 17, 2 * 1024 * 1024 + 31, PLAINTEXT_LEN - 97] {
|
||||
let range = HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start: i64::try_from(start).expect("fixed range start fits i64"),
|
||||
end: i64::try_from(start + 63).expect("fixed range end fits i64"),
|
||||
};
|
||||
let plan = ReadPlan::build(Some(range.clone()), &info, &ObjectOptions::default(), &HeaderMap::new())
|
||||
.await
|
||||
.expect("plan historical compressed range");
|
||||
let physical = plan.storage_offset;
|
||||
let input = STREAM.get(physical..).expect("planned offset stays within the fixed stream");
|
||||
let (mut reader, _, _) = GetObjectReader::new(
|
||||
Box::new(Cursor::new(input.to_vec())),
|
||||
Some(range),
|
||||
&info,
|
||||
&ObjectOptions::default(),
|
||||
&HeaderMap::new(),
|
||||
)
|
||||
.await
|
||||
.expect("construct historical range reader");
|
||||
let mut actual = Vec::new();
|
||||
reader.read_to_end(&mut actual).await.expect("read historical range");
|
||||
assert_eq!(actual, expected[start..start + 64], "range starting at {start}");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
#[tokio::test]
|
||||
async fn current_index_selects_a_fixed_compressed_block_for_a_range() {
|
||||
let info = fixture_info(CURRENT_INDEX);
|
||||
let start = 2 * 1024 * 1024 + 31;
|
||||
let range = HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start: i64::try_from(start).expect("fixed range start fits i64"),
|
||||
end: i64::try_from(start + 63).expect("fixed range end fits i64"),
|
||||
};
|
||||
let plan = ReadPlan::build(Some(range.clone()), &info, &ObjectOptions::default(), &HeaderMap::new())
|
||||
.await
|
||||
.expect("plan range with fixed current index");
|
||||
assert_eq!(plan.storage_offset, 34_663, "use the recorded second block boundary");
|
||||
let physical = plan.storage_offset;
|
||||
let (mut reader, _, _) = GetObjectReader::new(
|
||||
Box::new(Cursor::new(STREAM[physical..].to_vec())),
|
||||
Some(range),
|
||||
&info,
|
||||
&ObjectOptions::default(),
|
||||
&HeaderMap::new(),
|
||||
)
|
||||
.await
|
||||
.expect("construct indexed range reader");
|
||||
let mut actual = Vec::new();
|
||||
reader.read_to_end(&mut actual).await.expect("read indexed range");
|
||||
assert_eq!(actual, fixture_plaintext()[start..start + 64]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn truncated_index_falls_back_without_truncating_the_range() {
|
||||
let info = fixture_info(&HISTORICAL_INDEX[..HISTORICAL_INDEX.len() - 3]);
|
||||
let start = 2 * 1024 * 1024 + 31;
|
||||
let range = HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start: i64::try_from(start).expect("fixed range start fits i64"),
|
||||
end: i64::try_from(start + 96).expect("fixed range end fits i64"),
|
||||
};
|
||||
let plan = ReadPlan::build(Some(range.clone()), &info, &ObjectOptions::default(), &HeaderMap::new())
|
||||
.await
|
||||
.expect("invalid index still permits a sequential range plan");
|
||||
assert_eq!(plan.storage_offset, 0, "truncated index must not select a storage offset");
|
||||
let (mut reader, _, _) = GetObjectReader::new(
|
||||
Box::new(Cursor::new(STREAM.to_vec())),
|
||||
Some(range),
|
||||
&info,
|
||||
&ObjectOptions::default(),
|
||||
&HeaderMap::new(),
|
||||
)
|
||||
.await
|
||||
.expect("construct sequential fallback reader");
|
||||
let mut actual = Vec::new();
|
||||
reader
|
||||
.read_to_end(&mut actual)
|
||||
.await
|
||||
.expect("read range without the truncated index");
|
||||
assert_eq!(actual, fixture_plaintext()[start..start + 97]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn historical_index_fallback_does_not_hide_a_body_checksum_failure() {
|
||||
let info = fixture_info(HISTORICAL_INDEX);
|
||||
let mut corrupted = STREAM.to_vec();
|
||||
corrupted[4] ^= 1; // Corrupt the first block's stored CRC, leaving its payload intact.
|
||||
let (mut reader, _, _) = GetObjectReader::new(
|
||||
Box::new(Cursor::new(corrupted)),
|
||||
None,
|
||||
&info,
|
||||
&ObjectOptions::default(),
|
||||
&HeaderMap::new(),
|
||||
)
|
||||
.await
|
||||
.expect("construct reader before body checksum verification");
|
||||
let mut actual = Vec::new();
|
||||
let error = reader
|
||||
.read_to_end(&mut actual)
|
||||
.await
|
||||
.expect_err("bad body CRC cannot become successful EOF");
|
||||
assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert!(actual.len() < PLAINTEXT_LEN);
|
||||
}
|
||||
@@ -152,7 +152,7 @@ pub struct ObjectLockConfigSnapshot {
|
||||
config_revision: Option<OffsetDateTime>,
|
||||
state: crate::bucket::metadata_sys::ObjectLockConfigState,
|
||||
lifecycle_fence: NamespaceLockFence,
|
||||
_lifecycle_guard: Option<rustfs_lock::NamespaceLockGuard>,
|
||||
_lifecycle_guard: Option<Arc<rustfs_lock::NamespaceLockGuard>>,
|
||||
metadata_transaction_guard: Option<Arc<rustfs_lock::NamespaceLockGuard>>,
|
||||
}
|
||||
|
||||
@@ -210,11 +210,36 @@ impl ObjectLockConfigSnapshot {
|
||||
config_revision: Some(config_revision),
|
||||
state,
|
||||
lifecycle_fence,
|
||||
_lifecycle_guard: Some(lifecycle_guard),
|
||||
_lifecycle_guard: Some(Arc::new(lifecycle_guard)),
|
||||
metadata_transaction_guard: Some(Arc::new(metadata_transaction_guard)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn for_store_bucket_under_lifecycle_owner(
|
||||
store_id: Uuid,
|
||||
bucket: &str,
|
||||
bucket_incarnation_id: Uuid,
|
||||
config_revision: OffsetDateTime,
|
||||
state: crate::bucket::metadata_sys::ObjectLockConfigState,
|
||||
lifecycle_guard: Arc<rustfs_lock::NamespaceLockGuard>,
|
||||
metadata_transaction_guard: Arc<rustfs_lock::NamespaceLockGuard>,
|
||||
) -> Self {
|
||||
let mut lifecycle_fence = NamespaceLockFence::new();
|
||||
if let Some(signal) = lifecycle_guard.lock_lost_signal() {
|
||||
lifecycle_fence.add_signal(signal);
|
||||
}
|
||||
Self {
|
||||
store_id: Some(store_id),
|
||||
bucket: Some(bucket.to_owned()),
|
||||
bucket_incarnation_id: Some(bucket_incarnation_id),
|
||||
config_revision: Some(config_revision),
|
||||
state,
|
||||
lifecycle_fence,
|
||||
_lifecycle_guard: Some(lifecycle_guard),
|
||||
metadata_transaction_guard: Some(metadata_transaction_guard),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn for_store_bucket_under_lifecycle_fence(
|
||||
store_id: Uuid,
|
||||
bucket: &str,
|
||||
@@ -254,6 +279,15 @@ impl ObjectLockConfigSnapshot {
|
||||
&self.state
|
||||
}
|
||||
|
||||
pub(crate) fn is_unbound(&self) -> bool {
|
||||
self.store_id.is_none()
|
||||
&& self.bucket.is_none()
|
||||
&& self.bucket_incarnation_id.is_none()
|
||||
&& self.config_revision.is_none()
|
||||
&& self._lifecycle_guard.is_none()
|
||||
&& self.metadata_transaction_guard.is_none()
|
||||
}
|
||||
|
||||
pub(crate) fn is_valid_for_destructive_put(&self, store_id: Uuid, bucket: &str, bucket_incarnation_id: Uuid) -> bool {
|
||||
self.store_id == Some(store_id)
|
||||
&& self.bucket.as_deref() == Some(bucket)
|
||||
@@ -266,6 +300,11 @@ impl ObjectLockConfigSnapshot {
|
||||
.is_some_and(|guard| !guard.is_lock_lost())
|
||||
}
|
||||
|
||||
pub(crate) fn owns_bucket_lifecycle_fence(&self, store_id: Uuid, bucket: &str, bucket_incarnation_id: Uuid) -> bool {
|
||||
self.is_valid_for_destructive_put(store_id, bucket, bucket_incarnation_id)
|
||||
&& self._lifecycle_guard.as_ref().is_some_and(|guard| !guard.is_lock_lost())
|
||||
}
|
||||
|
||||
/// Share the held transaction lock without queuing another reader behind
|
||||
/// a metadata writer that is itself waiting for this snapshot to drop.
|
||||
pub(crate) fn metadata_transaction_guard_for(
|
||||
@@ -962,6 +1001,12 @@ pub struct ObjectOptions {
|
||||
/// A genuine target write guard; loss fences alone do not authorize a borrowed write.
|
||||
#[doc(hidden)]
|
||||
pub write_lock_context: Option<WriteLockContext>,
|
||||
/// Ordinary ECStore PUT target bound to its bucket's immutable artifact namespace.
|
||||
#[doc(hidden)]
|
||||
pub artifact_put_target: Option<Arc<crate::bucket::artifact::ArtifactPutTarget>>,
|
||||
/// Owned, non-serializable authority for one fully sealed managed version.
|
||||
#[doc(hidden)]
|
||||
pub artifact_delete_target: Option<Arc<crate::bucket::artifact::ArtifactDeleteTarget>>,
|
||||
/// In-process delete ownership for late filesystem completion. Never
|
||||
/// serialized or copied into object metadata.
|
||||
#[doc(hidden)]
|
||||
@@ -1041,6 +1086,9 @@ pub struct ObjectOptions {
|
||||
/// Proves an upper layer holds the bucket lifecycle sentinel. A separate
|
||||
/// fence avoids recursively acquiring the read lock behind a queued writer.
|
||||
pub bucket_lifecycle_lock_fence: Option<NamespaceLockFence>,
|
||||
/// Actual generation owner retained by storage commits and their physical tails.
|
||||
#[doc(hidden)]
|
||||
pub bucket_incarnation_fence: Option<Arc<crate::store::BucketIncarnationFenceGuard>>,
|
||||
/// Retains the actual lifecycle guard for an owned multipart commit. A
|
||||
/// lock-loss fence alone does not keep a cancelled caller's lock alive. The
|
||||
/// owner also retains the continuous incarnation-validation coverage.
|
||||
|
||||
@@ -71,7 +71,7 @@ impl WriteCommitGuard {
|
||||
|
||||
/// Namespace locks key the disk identity of the name (see `new_ns_lock`),
|
||||
/// so a guard proves every spelling that resolves to the same file.
|
||||
fn matches(&self, bucket: &str, object: &str) -> bool {
|
||||
pub(crate) fn matches(&self, bucket: &str, object: &str) -> bool {
|
||||
self.bucket.as_ref() == bucket && self.object.as_ref() == object_name_disk_identity(object).as_ref()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -814,6 +814,14 @@ impl InstanceContext {
|
||||
self.tier_config_mgr.get_or_init(TierConfigMgr::new).clone()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn set_test_tier_config_mgr(&self, manager: Arc<RwLock<TierConfigMgr>>) {
|
||||
assert!(
|
||||
self.tier_config_mgr.set(manager).is_ok(),
|
||||
"test tier manager must be installed before use"
|
||||
);
|
||||
}
|
||||
|
||||
/// This instance's event notifier, materializing it on first access.
|
||||
/// Shared for the lifetime of the instance.
|
||||
pub fn event_notifier(&self) -> Arc<RwLock<EventNotifier>> {
|
||||
|
||||
@@ -250,7 +250,7 @@ impl ECStore {
|
||||
}
|
||||
|
||||
let bucket_incarnation_fence = match bucket_configs.bucket_incarnation_id {
|
||||
Some(expected) => Some(self.acquire_bucket_incarnation_fence(&bucket, expected).await?),
|
||||
Some(expected) => Some(Arc::new(self.acquire_bucket_incarnation_fence(&bucket, expected).await?)),
|
||||
None => None,
|
||||
};
|
||||
|
||||
@@ -533,9 +533,8 @@ impl ECStore {
|
||||
&cleanup_transferred_tier_versions,
|
||||
data_movement::SourceCleanupBucketFence {
|
||||
expected_incarnation_id: bucket_configs.bucket_incarnation_id,
|
||||
lifecycle_guard: bucket_incarnation_fence
|
||||
.as_ref()
|
||||
.and_then(|guard| guard.namespace_lock_guard()),
|
||||
lifecycle_owner: bucket_incarnation_fence.as_ref(),
|
||||
lifecycle_guard: None,
|
||||
namespace_lock_lost_signal: lock_lost_signal.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -1729,6 +1728,8 @@ mod tests {
|
||||
/// Both versions are on the target with their identities, the null
|
||||
/// version's content and SHA-256 are intact, and the source is gone.
|
||||
async fn assert_moved(&self) {
|
||||
use crate::disk::DiskAPI as _;
|
||||
|
||||
let target = self.store.pools[1]
|
||||
.get_disks_by_key(self.object)
|
||||
.load_file_info_versions_exact(self.bucket, self.object)
|
||||
@@ -1769,6 +1770,16 @@ mod tests {
|
||||
self.source_versions().await.is_none(),
|
||||
"the source entry must be removed only after both versions moved"
|
||||
);
|
||||
let source_disks = self.source_set.disk_inventory().await;
|
||||
assert_eq!(source_disks.iter().flatten().count(), 2, "both source disks must be online");
|
||||
for disk in source_disks.into_iter().flatten() {
|
||||
assert!(
|
||||
!tokio::fs::try_exists(disk.path().join(self.bucket).join(self.object))
|
||||
.await
|
||||
.expect("inspect the physical rebalance source path"),
|
||||
"rebalance must remove source metadata and shards from every disk"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1782,6 +1793,141 @@ mod tests {
|
||||
fixture.assert_moved().await;
|
||||
}
|
||||
|
||||
/// An `ExpiredObjectAllVersions` expiry is a prefix delete that takes the
|
||||
/// bucket lifecycle write lock, while the entry holds that lock for read
|
||||
/// through its bucket incarnation fence. Applied inline, it waited out the
|
||||
/// lock timeout, stalled every lifecycle read in the bucket behind it, and
|
||||
/// failed the entry. The version moves instead and lifecycle expires it on
|
||||
/// the target (rustfs/backlog#2873).
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn real_rebalance_moves_a_version_expired_by_a_delete_all_rule() {
|
||||
run_large_stack_async_test("rebalance-delete-all-expiry", run_delete_all_expiry_case);
|
||||
}
|
||||
|
||||
async fn run_delete_all_expiry_case() {
|
||||
const REBALANCE_ID: &str = "rebalance-delete-all-expiry";
|
||||
let (_temp_dirs, store, _peer) = crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(None).await;
|
||||
let bucket = "rebalance-delete-all";
|
||||
let object = "expired.bin";
|
||||
// An active run blocks the data-usage publication bucket creation needs.
|
||||
Box::pin(store.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
))
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let incarnation = store.bucket_incarnation_id(bucket).await.expect("bucket incarnation");
|
||||
let rebalance_meta = active_rebalance_meta(REBALANCE_ID);
|
||||
rebalance_meta
|
||||
.save(Arc::clone(&store.pools[0]))
|
||||
.await
|
||||
.expect("persist the active rebalance run");
|
||||
*store.rebalance_meta.write().await = Some(rebalance_meta);
|
||||
let source_set = store.pools[0].get_disks_by_key(object);
|
||||
let payload = b"expired by an all-versions rule".to_vec();
|
||||
let written = Box::pin(source_set.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(payload.clone()),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
mod_time: Some(OffsetDateTime::now_utc() - time::Duration::days(10)),
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
..Default::default()
|
||||
},
|
||||
))
|
||||
.await
|
||||
.expect("seed the expired source version");
|
||||
let lifecycle = BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration: Some(LifecycleExpiration {
|
||||
days: Some(1),
|
||||
expired_object_all_versions: Some(true),
|
||||
..Default::default()
|
||||
}),
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
filter: None,
|
||||
id: Some("delete-all".to_string()),
|
||||
noncurrent_version_expiration: None,
|
||||
noncurrent_version_transitions: None,
|
||||
prefix: None,
|
||||
transitions: None,
|
||||
}],
|
||||
};
|
||||
let entry = metacache_entry_from_source(&source_set, bucket, object).await;
|
||||
|
||||
let lock_timeout = crate::set_disk::get_lock_acquire_timeout();
|
||||
let started = std::time::Instant::now();
|
||||
let outcome = tokio::time::timeout(
|
||||
lock_timeout,
|
||||
Box::pin(Arc::clone(&store).rebalance_entry(
|
||||
RebalanceEntryTarget {
|
||||
bucket: bucket.to_string(),
|
||||
pool_index: 0,
|
||||
},
|
||||
entry,
|
||||
Arc::clone(&source_set),
|
||||
Arc::new(RebalanceBucketConfigs {
|
||||
bucket_incarnation_id: Some(incarnation),
|
||||
lifecycle_config: Some(lifecycle),
|
||||
..Default::default()
|
||||
}),
|
||||
Arc::from(REBALANCE_ID),
|
||||
CancellationToken::new(),
|
||||
)),
|
||||
)
|
||||
.await
|
||||
.expect("the entry must not wait out the bucket lifecycle lock timeout")
|
||||
.expect("rebalance entry");
|
||||
let elapsed = started.elapsed();
|
||||
assert_eq!(outcome, RebalanceEntryOutcome::Completed);
|
||||
assert!(
|
||||
elapsed < lock_timeout / 2,
|
||||
"the entry must finish well under the {lock_timeout:?} lock timeout, took {elapsed:?}"
|
||||
);
|
||||
|
||||
assert!(
|
||||
source_set
|
||||
.load_file_info_versions_exact(bucket, object)
|
||||
.await
|
||||
.expect("read source versions")
|
||||
.is_none(),
|
||||
"the source entry must be removed once the version moved"
|
||||
);
|
||||
let target = store.pools[1]
|
||||
.get_disks_by_key(object)
|
||||
.load_file_info_versions_exact(bucket, object)
|
||||
.await
|
||||
.expect("read target versions")
|
||||
.expect("the version must reach the target pool");
|
||||
assert_eq!(target.versions.len(), 1, "{:?}", target.versions);
|
||||
assert_eq!(target.versions[0].version_id, written.version_id);
|
||||
let mut reader = store.pools[1]
|
||||
.get_object_reader(
|
||||
bucket,
|
||||
object,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: written.version_id.map(|version| version.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("the moved version should be readable");
|
||||
let mut body = Vec::new();
|
||||
reader.stream.read_to_end(&mut body).await.expect("drain the moved version");
|
||||
assert_eq!(body, payload);
|
||||
}
|
||||
|
||||
/// Overwriting after versioning was enabled leaves a newer version above
|
||||
/// the null version. The old `None` read selected that newer version, the
|
||||
/// target accepted it as an equivalent copy, and cleanup then deleted the
|
||||
|
||||
@@ -88,6 +88,7 @@ pub async fn test_store_with_persisted_rebalance_meta(
|
||||
pool_meta_save_gate: tokio::sync::Mutex::default(),
|
||||
ctx,
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
});
|
||||
(temp_dirs, store)
|
||||
}
|
||||
@@ -266,6 +267,7 @@ async fn test_pool_stores_with_contexts(
|
||||
pool_meta_save_gate: tokio::sync::Mutex::new(pool_meta_write_state.independent_clone_for_test()),
|
||||
ctx: store_ctx,
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
})
|
||||
};
|
||||
let store = make_store(ctx);
|
||||
|
||||
@@ -3415,6 +3415,7 @@ fn test_store_with_rebalance_meta(meta: RebalanceMeta) -> Arc<crate::store::ECSt
|
||||
pool_meta_save_gate: tokio::sync::Mutex::default(),
|
||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -641,7 +641,13 @@ impl SetDisks {
|
||||
state = "agreed_entry",
|
||||
"Rebalance listing agreed entry"
|
||||
);
|
||||
Box::pin(cb1(entry))
|
||||
{
|
||||
let future = cb1(entry);
|
||||
Box::pin(async move {
|
||||
future.await;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
})),
|
||||
partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
|
||||
let resolver = resolver.clone();
|
||||
|
||||
@@ -3053,7 +3053,7 @@ impl RenameRollbackReceipt {
|
||||
|
||||
struct RenameRollbackOwnership {
|
||||
receipt: Option<RenameRollbackReceipt>,
|
||||
namespace_commit_guard: Option<Arc<crate::runtime::instance::NamespaceCommitGuard>>,
|
||||
namespace_commit_guard: Option<Arc<dyn Send + Sync>>,
|
||||
}
|
||||
|
||||
async fn inspect_incomplete_rename_rollback(
|
||||
@@ -3169,7 +3169,7 @@ async fn rollback_failed_rename(
|
||||
old_data_dir: rollback_dir,
|
||||
..Default::default()
|
||||
},
|
||||
disk_namespace_commit_guard.map(|owner| owner as Arc<dyn Send + Sync>),
|
||||
disk_namespace_commit_guard,
|
||||
)
|
||||
.await
|
||||
});
|
||||
@@ -3281,10 +3281,11 @@ async fn record_rename_rollback_outcomes(
|
||||
/// quorum and optional scanner lease map together avoids widening either
|
||||
/// fanout helper's argument list while preserving the fence semantics.
|
||||
pub(in crate::set_disk) struct RenameDataFenceOptions<'a> {
|
||||
artifact_put: Option<crate::bucket::artifact::ArtifactPutIdentity>,
|
||||
write_quorum: usize,
|
||||
scanner_publication_lease_tokens: Option<&'a HashMap<String, Uuid>>,
|
||||
scanner_publication_commit_scope: Option<crate::object_api::ScannerPublicationCommitScope>,
|
||||
namespace_commit_guard: Option<Arc<crate::runtime::instance::NamespaceCommitGuard>>,
|
||||
namespace_commit_guard: Option<Arc<dyn Send + Sync>>,
|
||||
rollback_receipt: Option<RenameRollbackReceipt>,
|
||||
}
|
||||
|
||||
@@ -3294,6 +3295,7 @@ impl<'a> RenameDataFenceOptions<'a> {
|
||||
scanner_publication_lease_tokens: Option<&'a HashMap<String, Uuid>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
artifact_put: None,
|
||||
write_quorum,
|
||||
scanner_publication_lease_tokens,
|
||||
scanner_publication_commit_scope: None,
|
||||
@@ -3302,6 +3304,14 @@ impl<'a> RenameDataFenceOptions<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn with_artifact_put(
|
||||
mut self,
|
||||
identity: Option<crate::bucket::artifact::ArtifactPutIdentity>,
|
||||
) -> Self {
|
||||
self.artifact_put = identity;
|
||||
self
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn with_rollback_receipt(mut self, receipt: RenameRollbackReceipt) -> Self {
|
||||
self.rollback_receipt = Some(receipt);
|
||||
self
|
||||
@@ -3317,7 +3327,7 @@ impl<'a> RenameDataFenceOptions<'a> {
|
||||
|
||||
pub(in crate::set_disk) fn with_namespace_commit_guard(
|
||||
mut self,
|
||||
namespace_commit_guard: Option<Arc<crate::runtime::instance::NamespaceCommitGuard>>,
|
||||
namespace_commit_guard: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Self {
|
||||
self.namespace_commit_guard = namespace_commit_guard;
|
||||
self
|
||||
@@ -3770,6 +3780,7 @@ impl SetDisks {
|
||||
fence_options: RenameDataFenceOptions<'_>,
|
||||
) -> disk::error::Result<RenameDataCommit> {
|
||||
let RenameDataFenceOptions {
|
||||
artifact_put,
|
||||
write_quorum,
|
||||
scanner_publication_lease_tokens,
|
||||
scanner_publication_commit_scope: _scanner_publication_commit_scope,
|
||||
@@ -3864,11 +3875,10 @@ impl SetDisks {
|
||||
&file_info,
|
||||
&dst_bucket,
|
||||
&dst_object,
|
||||
disk::RenameDataGuards {
|
||||
crate::disk::RenameDataGuards {
|
||||
artifact_put,
|
||||
scanner_publication_lease_token,
|
||||
namespace_owner: namespace_commit_guard
|
||||
.clone()
|
||||
.map(|owner| owner as Arc<dyn Send + Sync>),
|
||||
namespace_owner: namespace_commit_guard.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
@@ -4177,6 +4187,7 @@ impl SetDisks {
|
||||
.await;
|
||||
}
|
||||
let RenameDataFenceOptions {
|
||||
artifact_put,
|
||||
write_quorum,
|
||||
scanner_publication_lease_tokens,
|
||||
scanner_publication_commit_scope,
|
||||
@@ -4297,11 +4308,10 @@ impl SetDisks {
|
||||
file_info,
|
||||
&dst_bucket,
|
||||
&dst_object,
|
||||
disk::RenameDataGuards {
|
||||
crate::disk::RenameDataGuards {
|
||||
artifact_put,
|
||||
scanner_publication_lease_token,
|
||||
namespace_owner: namespace_commit_guard
|
||||
.clone()
|
||||
.map(|owner| owner as Arc<dyn Send + Sync>),
|
||||
namespace_owner: namespace_commit_guard.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
@@ -12815,6 +12825,118 @@ mod tests {
|
||||
assert!(missing.below_quorum, "missing disk slots should remain visible in the quorum lens");
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn legacy_path_delete_flow_prefix_delete_keeps_scanner_owner_until_drain() {
|
||||
use crate::disk::os::{drain_namespace_for_test, prepared_publication_test_hooks as hooks};
|
||||
use crate::object_api::{
|
||||
ScannerPublicationCommitScope, ScannerPublicationCommitScopeGuard, ScannerPublicationCommitState,
|
||||
};
|
||||
use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks_isolated;
|
||||
use crate::storage_api_contracts::bucket::BucketOperations;
|
||||
|
||||
let (_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "path-delete-scanner";
|
||||
crate::bucket::metadata_sys::object_store_in(&set.ctx)
|
||||
.await
|
||||
.expect("fixture object store")
|
||||
.make_bucket(bucket, &Default::default())
|
||||
.await
|
||||
.expect("authoritative bucket creation");
|
||||
let mut paths = Vec::new();
|
||||
for disk in &disks {
|
||||
disk.write_all(bucket, "prefix/part.1", Bytes::from_static(b"old shard"))
|
||||
.await
|
||||
.expect("seed prefix on every disk");
|
||||
paths.push(
|
||||
disk.get_object_path_for_io_if_local(bucket, "prefix")
|
||||
.expect("local disk")
|
||||
.expect("prefix path"),
|
||||
);
|
||||
}
|
||||
let movement = Arc::new(tokio::sync::RwLock::new(()));
|
||||
let scope = ScannerPublicationCommitScope::new_storage_owned(
|
||||
7,
|
||||
tokio::time::Instant::now() + Duration::from_secs(30),
|
||||
Vec::new(),
|
||||
movement.clone().read_owned().await,
|
||||
);
|
||||
scope.try_begin().expect("begin scanner publication");
|
||||
let caller_guard = ScannerPublicationCommitScopeGuard::new(scope.clone());
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
let _hook = hooks::install_at(hooks::Stage::Rename, &paths[0], move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
let owner_set = set.clone();
|
||||
let owner_scope = scope.clone();
|
||||
let caller = tokio::spawn(async move {
|
||||
owner_set
|
||||
.delete_prefix_with_scanner_publication_lease(bucket, "prefix", None, Some(owner_scope))
|
||||
.await
|
||||
});
|
||||
let entered = tokio::time::timeout(Duration::from_secs(5), entered_rx).await;
|
||||
caller.abort();
|
||||
let _ = caller.await;
|
||||
drop(caller_guard);
|
||||
let state_while_blocked = scope.state();
|
||||
let movement_while_blocked = movement.clone().try_write_owned().is_ok();
|
||||
let disk::Disk::Local(local) = disks[0].as_ref() else {
|
||||
unreachable!("fixture contains local disks");
|
||||
};
|
||||
let local = local.get_disk();
|
||||
let volume_read_held = crate::disk::os::disk_volume_mutation_lock(&local.root, bucket)
|
||||
.try_write_owned()
|
||||
.is_err();
|
||||
let mut deletion = Box::pin(local.delete_volume(bucket, true));
|
||||
let early_deletion = futures::poll!(deletion.as_mut());
|
||||
let deletion_was_blocked = early_deletion.is_pending();
|
||||
// Always release and drain physical work before reporting an ownership failure.
|
||||
let _ = release_tx.send(());
|
||||
let mut drained = Vec::new();
|
||||
for path in &paths {
|
||||
drained.push(tokio::time::timeout(Duration::from_secs(5), drain_namespace_for_test(path)).await);
|
||||
}
|
||||
let deletion_result = match early_deletion {
|
||||
std::task::Poll::Ready(result) => result,
|
||||
std::task::Poll::Pending => tokio::time::timeout(Duration::from_secs(5), deletion)
|
||||
.await
|
||||
.expect("volume deletion drains"),
|
||||
};
|
||||
let terminal = tokio::time::timeout(Duration::from_secs(5), scope.wait_for_completion()).await;
|
||||
assert!(
|
||||
entered.is_ok_and(|result| result.is_ok()),
|
||||
"real prefix deletion must reach its filesystem worker"
|
||||
);
|
||||
assert!(drained.into_iter().all(|result| result.is_ok()), "every physical namespace must drain");
|
||||
deletion_result.expect("volume deletion after physical drain");
|
||||
assert!(
|
||||
volume_read_held && deletion_was_blocked,
|
||||
"volume deletion must wait for the physical prefix worker"
|
||||
);
|
||||
assert_eq!(state_while_blocked, ScannerPublicationCommitState::InFlight);
|
||||
assert!(!movement_while_blocked, "scanner movement remains fenced after caller cancellation");
|
||||
assert_eq!(terminal.expect("scanner owner completion"), ScannerPublicationCommitState::Committed);
|
||||
assert!(scope.release_movement_permit().await, "terminal scanner owner releases movement");
|
||||
assert!(movement.try_write_owned().is_ok());
|
||||
for path in paths.iter().skip(1) {
|
||||
assert!(!path.exists(), "successful prefix fanout removes every surviving volume's prefix");
|
||||
}
|
||||
disks[0].make_volume(bucket).await.expect("recreate deleted volume");
|
||||
disks[0]
|
||||
.write_all(bucket, "prefix/sentinel", Bytes::from_static(b"new generation"))
|
||||
.await
|
||||
.expect("new generation sentinel");
|
||||
assert_eq!(
|
||||
disks[0]
|
||||
.read_all(bucket, "prefix/sentinel")
|
||||
.await
|
||||
.expect("sentinel survives old deletion"),
|
||||
b"new generation".as_slice()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_prefix_succeeds_when_present_disks_reach_quorum() {
|
||||
let bucket = "delete-prefix-bucket";
|
||||
|
||||
@@ -906,6 +906,8 @@ pub(crate) use ops::object::body_cache_plaintext_len;
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) use ops::object::cleanup_rejected_transition_upload_durably;
|
||||
#[cfg(test)]
|
||||
pub(crate) use ops::object::hermetic_set_disks_support::{HermeticSetFixture, hermetic_set_disks_for_pool_with_default_parity};
|
||||
#[cfg(test)]
|
||||
pub(crate) use ops::object::{DeleteCleanupBarrier, DeleteObjectCommitBarrier};
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub use ops::object::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
@@ -1145,6 +1147,310 @@ mod prepared_get_object_metadata_tests {
|
||||
use http::HeaderMap;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
const OBSERVATION_HOOK_BODY_LEN: usize = 1024 * 1024 + 1;
|
||||
|
||||
struct ObservationBodyHook {
|
||||
object: &'static str,
|
||||
calls: Arc<AtomicU64>,
|
||||
body: bytes::Bytes,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::object_api::GetObjectBodyCacheHook for ObservationBodyHook {
|
||||
async fn lookup(&self, bucket: &str, object: &str, info: &ObjectInfo) -> Option<bytes::Bytes> {
|
||||
if bucket == crate::disk::RUSTFS_META_BUCKET && object == self.object {
|
||||
assert_eq!(info.size, i64::try_from(self.body.len()).expect("hook fixture size"));
|
||||
self.calls.fetch_add(1, Ordering::SeqCst);
|
||||
Some(self.body.clone())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct ObservationHookGuard;
|
||||
|
||||
impl Drop for ObservationHookGuard {
|
||||
fn drop(&mut self) {
|
||||
crate::object_api::unregister_get_object_body_cache_hook();
|
||||
}
|
||||
}
|
||||
|
||||
async fn ordinary_observation_get(set: &SetDisks, object: &str) -> Vec<u8> {
|
||||
let mut reader = set
|
||||
.get_object_reader(crate::disk::RUSTFS_META_BUCKET, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("ordinary GET positive control");
|
||||
let mut bytes = Vec::new();
|
||||
reader.stream.read_to_end(&mut bytes).await.expect("ordinary GET body");
|
||||
bytes
|
||||
}
|
||||
|
||||
fn observation_register_hook(object: &'static str, body: Vec<u8>) -> (ObservationHookGuard, Arc<AtomicU64>) {
|
||||
let calls = Arc::new(AtomicU64::new(0));
|
||||
crate::object_api::register_get_object_body_cache_hook(Arc::new(ObservationBodyHook {
|
||||
object,
|
||||
calls: calls.clone(),
|
||||
body: bytes::Bytes::from(body),
|
||||
}));
|
||||
(ObservationHookGuard, calls)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn locked_observation_uses_real_outer_guard() {
|
||||
let object = "config/locked-observation-real.bin";
|
||||
let payload = vec![b'p'; OBSERVATION_HOOK_BODY_LEN];
|
||||
let (_dirs, set) = make_local_set_disks(4, 2).await;
|
||||
let mut reader = PutObjReader::from_vec(payload.clone());
|
||||
let written = set
|
||||
.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("persist locked observation fixture");
|
||||
let (_hook, calls) = observation_register_hook(object, vec![b'x'; payload.len()]);
|
||||
assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; payload.len()]);
|
||||
assert_eq!(calls.swap(0, Ordering::SeqCst), 1);
|
||||
let namespace = set
|
||||
.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, object)
|
||||
.await
|
||||
.expect("namespace");
|
||||
let guard = crate::object_api::WriteCommitGuard::acquire(&namespace, std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("matching real owner");
|
||||
let mut contender = Box::pin(crate::object_api::WriteCommitGuard::acquire(
|
||||
&namespace,
|
||||
std::time::Duration::from_secs(30),
|
||||
));
|
||||
assert!(futures::poll!(contender.as_mut()).is_pending(), "I06L_OWNER_RETAINED");
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(10),
|
||||
crate::config::com::read_config_observation_no_lock(set.clone(), object, payload.len(), &guard),
|
||||
)
|
||||
.await
|
||||
.expect("locked observation hang watchdog");
|
||||
assert!(result.is_ok(), "I06L_NO_NESTED_LOCK: {result:?}");
|
||||
let (bytes, info) = result.expect("locked observation");
|
||||
assert!(bytes == payload, "I06L_HOOK_BYPASS");
|
||||
assert!(calls.load(Ordering::SeqCst) == 0, "I06L_HOOK_BYPASS");
|
||||
assert_eq!(info.size, written.size);
|
||||
assert_eq!(info.etag, written.etag);
|
||||
assert_eq!(info.data_dir, written.data_dir);
|
||||
assert_eq!(info.name, object);
|
||||
assert!(futures::poll!(contender.as_mut()).is_pending(), "I06L_OWNER_RETAINED");
|
||||
drop(guard);
|
||||
let next = tokio::time::timeout(std::time::Duration::from_secs(10), contender)
|
||||
.await
|
||||
.expect("owner release watchdog")
|
||||
.expect("competing owner acquired");
|
||||
drop(next);
|
||||
assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; payload.len()]);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn locked_observation_preserves_empty_missing_and_bound() {
|
||||
let (_dirs, set) = make_local_set_disks(4, 2).await;
|
||||
for (object, body) in [
|
||||
("config/locked-empty.bin", Vec::new()),
|
||||
("config/locked-bound.bin", b"raw".to_vec()),
|
||||
] {
|
||||
let mut reader = PutObjReader::from_vec(body.clone());
|
||||
let written = set
|
||||
.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("persist boundary fixture");
|
||||
let namespace = set
|
||||
.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, object)
|
||||
.await
|
||||
.expect("namespace");
|
||||
let guard = crate::object_api::WriteCommitGuard::acquire(&namespace, std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("matching boundary owner");
|
||||
let result = crate::config::com::read_config_observation_no_lock(set.clone(), object, body.len(), &guard).await;
|
||||
assert!(result.is_ok(), "I06L_EMPTY_DISTINCT: {result:?}");
|
||||
let (bytes, info) = result.expect("boundary observation");
|
||||
assert_eq!(bytes, body);
|
||||
assert_eq!(info.size, written.size);
|
||||
if !body.is_empty() {
|
||||
let result =
|
||||
crate::config::com::read_config_observation_no_lock(set.clone(), object, body.len() - 1, &guard).await;
|
||||
assert!(matches!(result, Err(Error::CorruptedFormat)), "I06L_BOUND_ENFORCED");
|
||||
}
|
||||
}
|
||||
let object = "config/locked-absent.bin";
|
||||
let namespace = set
|
||||
.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, object)
|
||||
.await
|
||||
.expect("namespace");
|
||||
let guard = crate::object_api::WriteCommitGuard::acquire(&namespace, std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("matching missing owner");
|
||||
let result = crate::config::com::read_config_observation_no_lock(set, object, 0, &guard).await;
|
||||
assert!(matches!(result, Err(Error::ConfigNotFound)), "I06L_EMPTY_DISTINCT");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn observation_real_backend_boundaries() {
|
||||
let object = "config/observation-real.bin";
|
||||
let payload = b"persisted config";
|
||||
let (_dirs, set) = make_local_set_disks(4, 2).await;
|
||||
let mut reader = PutObjReader::from_vec(payload.to_vec());
|
||||
let written = set
|
||||
.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("persist real config fixture");
|
||||
let result = crate::config::com::read_config_observation(set.clone(), object, payload.len()).await;
|
||||
assert!(result.is_ok(), "I05_OBSERVATION_SUCCESS: {result:?}");
|
||||
let (bytes, info) = result.expect("real observation success");
|
||||
assert_eq!(bytes, payload);
|
||||
assert_eq!(info.size, written.size);
|
||||
assert_eq!(info.etag, written.etag);
|
||||
assert_eq!(info.data_dir, written.data_dir);
|
||||
assert_eq!(info.name, object);
|
||||
let over = crate::config::com::read_config_observation(set.clone(), object, payload.len() - 1).await;
|
||||
assert!(over.is_err(), "I05_BOUND_ENFORCED");
|
||||
assert_eq!(over.expect_err("over limit must fail"), Error::CorruptedFormat, "I05_BOUND_ENFORCED");
|
||||
let absent = crate::config::com::read_config_observation(set.clone(), "config/observation-absent.bin", 32).await;
|
||||
assert_eq!(absent.expect_err("absent config"), Error::ConfigNotFound);
|
||||
let empty_object = "config/observation-empty.bin";
|
||||
let mut empty = PutObjReader::from_vec(Vec::new());
|
||||
set.put_object(crate::disk::RUSTFS_META_BUCKET, empty_object, &mut empty, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("persist empty config");
|
||||
let empty = crate::config::com::read_config_observation(set, empty_object, 0).await;
|
||||
assert!(empty.is_ok(), "existing empty config must succeed: {empty:?}");
|
||||
let (bytes, info) = empty.expect("empty observation");
|
||||
assert!(bytes.is_empty());
|
||||
assert_eq!(info.size, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn observation_bypasses_registered_body_hook() {
|
||||
let object = "config/observation-hook.bin";
|
||||
let (_dirs, set) = make_local_set_disks(4, 2).await;
|
||||
let mut reader = PutObjReader::from_vec(vec![b'p'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
set.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("persist real config fixture");
|
||||
let (_hook, calls) = observation_register_hook(object, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
assert_eq!(calls.swap(0, Ordering::SeqCst), 1);
|
||||
let result = crate::config::com::read_config_observation(set, object, OBSERVATION_HOOK_BODY_LEN).await;
|
||||
assert!(result.is_ok(), "I05_HOOK_BYPASS: {result:?}");
|
||||
assert_eq!(
|
||||
result.expect("observed persisted body").0,
|
||||
vec![b'p'; OBSERVATION_HOOK_BODY_LEN],
|
||||
"I05_HOOK_BYPASS"
|
||||
);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0, "I05_HOOK_BYPASS");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn observation_hook_scope_restores_after_error() {
|
||||
let object = "config/observation-hook-error.bin";
|
||||
let (_dirs, set) = make_local_set_disks(4, 2).await;
|
||||
let mut reader = PutObjReader::from_vec(vec![b'p'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
set.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("persist real config fixture");
|
||||
let (_hook, calls) = observation_register_hook(object, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
let result = crate::config::com::read_config_observation(set.clone(), object, OBSERVATION_HOOK_BODY_LEN - 1).await;
|
||||
assert_eq!(result.expect_err("bounded observation error"), Error::CorruptedFormat);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
struct PendingObservationStore {
|
||||
set: Arc<SetDisks>,
|
||||
ready: std::sync::Mutex<Option<tokio::sync::oneshot::Sender<()>>>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for PendingObservationStore {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("PendingObservationStore").finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::storage_api_contracts::object::ObjectIO for PendingObservationStore {
|
||||
type Error = Error;
|
||||
type RangeSpec = HTTPRangeSpec;
|
||||
type HeaderMap = HeaderMap;
|
||||
type ObjectOptions = ObjectOptions;
|
||||
type ObjectInfo = ObjectInfo;
|
||||
type GetObjectReader = GetObjectReader;
|
||||
type PutObjectReader = PutObjReader;
|
||||
|
||||
async fn get_object_reader(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
range: Option<HTTPRangeSpec>,
|
||||
headers: HeaderMap,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<GetObjectReader> {
|
||||
let mut reader = self.set.get_object_reader(bucket, object, range, headers, opts).await?;
|
||||
let mut body = Vec::new();
|
||||
reader.stream.read_to_end(&mut body).await.expect("pending producer real GET");
|
||||
assert_eq!(
|
||||
body,
|
||||
vec![b'p'; OBSERVATION_HOOK_BODY_LEN],
|
||||
"I05_HOOK_BYPASS: producer must enter its own scope"
|
||||
);
|
||||
self.ready
|
||||
.lock()
|
||||
.expect("pending signal")
|
||||
.take()
|
||||
.expect("one GET")
|
||||
.send(())
|
||||
.expect("producer entry signal");
|
||||
std::future::pending::<Result<GetObjectReader>>().await
|
||||
}
|
||||
|
||||
async fn put_object(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_object: &str,
|
||||
_data: &mut PutObjReader,
|
||||
_opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
panic!("observation must not PUT");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn observation_hook_scope_restores_after_cancel() {
|
||||
let object = "config/observation-hook-cancel.bin";
|
||||
let (_dirs, set) = make_local_set_disks(4, 2).await;
|
||||
let mut reader = PutObjReader::from_vec(vec![b'p'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
set.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("persist real config fixture");
|
||||
let (_hook, calls) = observation_register_hook(object, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
assert_eq!(calls.swap(0, Ordering::SeqCst), 1);
|
||||
let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
|
||||
let adapter = Arc::new(PendingObservationStore {
|
||||
set: set.clone(),
|
||||
ready: std::sync::Mutex::new(Some(ready_tx)),
|
||||
});
|
||||
let mut scope = Box::pin(crate::config::com::read_config_observation(adapter, object, OBSERVATION_HOOK_BODY_LEN));
|
||||
tokio::select! {
|
||||
result = ready_rx => { result.expect("producer reached pending GET"); }
|
||||
result = &mut scope => { panic!("producer must remain pending until cancelled: {result:?}"); }
|
||||
}
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
drop(scope);
|
||||
assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
const READ_VERSION_BARRIER_GUARD: std::time::Duration = std::time::Duration::from_secs(10);
|
||||
|
||||
fn object_with_initial_data_shards(bucket: &str, prefix: &str) -> String {
|
||||
@@ -12753,7 +13059,7 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[tokio::test]
|
||||
async fn test_get_disks_info_probes_drives_concurrently() {
|
||||
use crate::disk::disk_store::DISK_INFO_PROBE_DELAY_FOR_TEST;
|
||||
|
||||
@@ -12768,6 +13074,8 @@ mod tests {
|
||||
disks.push(Some(disk));
|
||||
}
|
||||
|
||||
// Complete real filesystem setup before measuring virtual probe delays.
|
||||
tokio::time::pause();
|
||||
let probe_delay = std::time::Duration::from_secs(2);
|
||||
let started = tokio::time::Instant::now();
|
||||
let info = DISK_INFO_PROBE_DELAY_FOR_TEST
|
||||
@@ -16614,7 +16922,14 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn set_level_versioned_delete_marker_hides_object_without_corrupting_version_metadata() {
|
||||
let set_disks = make_local_bucket_test_set_disks_with_drive_count(4).await;
|
||||
let (_fixture, _, set_disks) = hermetic_set_disks_for_pool_with_default_parity(4, 0, 2).await;
|
||||
set_disks.set_test_storage_class_config(
|
||||
storageclass::lookup_config_for_pools_without_env(&rustfs_config::server_config::KVS::new(), &[4])
|
||||
.expect("test storage class should resolve for the local drive count"),
|
||||
);
|
||||
let store = crate::bucket::metadata_sys::object_store_in(&set_disks.ctx)
|
||||
.await
|
||||
.expect("fixture metadata authority should belong to this set");
|
||||
let bucket = "bucket-versioned-delete";
|
||||
let object = "object.txt";
|
||||
let opts = ObjectOptions {
|
||||
@@ -16623,13 +16938,26 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
set_disks
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut reader = PutObjReader::from_vec(b"versioned object body".to_vec());
|
||||
let written = set_disks
|
||||
.put_object(bucket, object, &mut reader, &opts)
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
object_lock_config_snapshot: Some(
|
||||
store
|
||||
.object_lock_config_snapshot(bucket)
|
||||
.await
|
||||
.expect("PUT fixture should own the bucket lifecycle and Object Lock snapshot"),
|
||||
),
|
||||
..opts.clone()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("versioned object should be written");
|
||||
assert!(written.version_id.is_some());
|
||||
@@ -16946,7 +17274,14 @@ mod tests {
|
||||
/// `BucketNotEmpty` for a bucket the client has already emptied.
|
||||
#[tokio::test]
|
||||
async fn set_level_explicit_null_version_delete_purges_the_null_delete_marker() {
|
||||
let set_disks = make_local_bucket_test_set_disks().await;
|
||||
let (_fixture, _, set_disks) = hermetic_set_disks_for_pool_with_default_parity(2, 0, 1).await;
|
||||
set_disks.set_test_storage_class_config(
|
||||
storageclass::lookup_config_for_pools_without_env(&rustfs_config::server_config::KVS::new(), &[2])
|
||||
.expect("test storage class should resolve for the local drive count"),
|
||||
);
|
||||
let store = crate::bucket::metadata_sys::object_store_in(&set_disks.ctx)
|
||||
.await
|
||||
.expect("fixture metadata authority should belong to this set");
|
||||
let bucket = "bucket-null-marker-purge";
|
||||
let object = "object.txt";
|
||||
// The delete path reads versioned/suspended from the bucket-config
|
||||
@@ -16968,13 +17303,26 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
set_disks
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut reader = PutObjReader::from_vec(b"suspended version body".to_vec());
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut reader, &opts)
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
object_lock_config_snapshot: Some(
|
||||
store
|
||||
.object_lock_config_snapshot(bucket)
|
||||
.await
|
||||
.expect("PUT fixture should own the bucket lifecycle and Object Lock snapshot"),
|
||||
),
|
||||
..opts.clone()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("suspended-version object should be written");
|
||||
|
||||
@@ -16999,7 +17347,20 @@ mod tests {
|
||||
|
||||
let mut replacement_reader = PutObjReader::from_vec(b"replacement suspended version body".to_vec());
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut replacement_reader, &opts)
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut replacement_reader,
|
||||
&ObjectOptions {
|
||||
object_lock_config_snapshot: Some(
|
||||
store
|
||||
.object_lock_config_snapshot(bucket)
|
||||
.await
|
||||
.expect("PUT fixture should own the bucket lifecycle and Object Lock snapshot"),
|
||||
),
|
||||
..opts.clone()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("a suspended-version PUT should replace the first null marker");
|
||||
let replacement_marker = set_disks
|
||||
@@ -17059,7 +17420,14 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn set_level_version_suspended_delete_creates_null_delete_marker() {
|
||||
let set_disks = make_local_bucket_test_set_disks().await;
|
||||
let (_fixture, _, set_disks) = hermetic_set_disks_for_pool_with_default_parity(2, 0, 1).await;
|
||||
set_disks.set_test_storage_class_config(
|
||||
storageclass::lookup_config_for_pools_without_env(&rustfs_config::server_config::KVS::new(), &[2])
|
||||
.expect("test storage class should resolve for the local drive count"),
|
||||
);
|
||||
let store = crate::bucket::metadata_sys::object_store_in(&set_disks.ctx)
|
||||
.await
|
||||
.expect("fixture metadata authority should belong to this set");
|
||||
let bucket = "bucket-version-suspended-delete";
|
||||
let object = "object.txt";
|
||||
let opts = ObjectOptions {
|
||||
@@ -17069,13 +17437,26 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
set_disks
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut reader = PutObjReader::from_vec(b"suspended version body".to_vec());
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut reader, &opts)
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
object_lock_config_snapshot: Some(
|
||||
store
|
||||
.object_lock_config_snapshot(bucket)
|
||||
.await
|
||||
.expect("PUT fixture should own the bucket lifecycle and Object Lock snapshot"),
|
||||
),
|
||||
..opts.clone()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("suspended-version object should be written");
|
||||
|
||||
|
||||
@@ -113,6 +113,7 @@ pub(crate) struct DeleteRecoveryContext<'a> {
|
||||
pub repair_marker_source: bool,
|
||||
pub namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
pub recovery_ticket: Option<crate::store::delete_recovery_runtime::DeleteRecoveryTicket>,
|
||||
pub published_hint: Option<(Uuid, &'a [Result<DeleteRecoveryInspection>])>,
|
||||
pub can_commit: &'a (dyn Fn() -> bool + Sync),
|
||||
}
|
||||
|
||||
@@ -125,6 +126,7 @@ impl<'a> DeleteRecoveryContext<'a> {
|
||||
repair_marker_source: false,
|
||||
namespace_owner: None,
|
||||
recovery_ticket: None,
|
||||
published_hint: None,
|
||||
can_commit,
|
||||
}
|
||||
}
|
||||
@@ -421,7 +423,9 @@ impl SetDisks {
|
||||
let cleanup = loop {
|
||||
#[cfg(test)]
|
||||
test_hooks::record_cleanup_attempt(bucket, object);
|
||||
match self.recover_delete_key(bucket, object, context.clone()).await {
|
||||
let mut attempt_context = context.clone();
|
||||
attempt_context.published_hint = (attempt == 1).then_some((transaction, published.as_slice()));
|
||||
match self.recover_delete_key(bucket, object, attempt_context).await {
|
||||
// A stale settlement fingerprint only means the snapshot raced
|
||||
// another writer; each attempt re-inspects every participant.
|
||||
Err(DiskError::OutdatedXLMeta) if attempt < PUBLISHED_CLEANUP_ATTEMPTS && can_commit() => attempt += 1,
|
||||
@@ -551,6 +555,7 @@ impl SetDisks {
|
||||
explicit_delete,
|
||||
repair_marker_source,
|
||||
namespace_owner,
|
||||
published_hint,
|
||||
can_commit,
|
||||
recovery_ticket: _,
|
||||
} = context;
|
||||
@@ -589,7 +594,18 @@ impl SetDisks {
|
||||
let mut all_states = BTreeMap::new();
|
||||
for transaction in transactions {
|
||||
let mut complete = Vec::with_capacity(disks.len());
|
||||
for state in inspect(&disks, bucket, object, transaction).await {
|
||||
let inspections = if let Some((published_tx, published_states)) = published_hint
|
||||
&& !repair_marker_source
|
||||
&& !dry_run
|
||||
&& published_tx == transaction
|
||||
&& published_states.len() == disks.len()
|
||||
&& published_states.iter().all(Result::is_ok)
|
||||
{
|
||||
published_states.to_vec()
|
||||
} else {
|
||||
inspect(&disks, bucket, object, transaction).await
|
||||
};
|
||||
for state in inspections {
|
||||
match state {
|
||||
Ok(state) => {
|
||||
if let Some(journal) = &state.journal {
|
||||
@@ -832,6 +848,9 @@ impl SetDisks {
|
||||
bucket,
|
||||
copies,
|
||||
|copy| {
|
||||
if receipt(copy.state) && copy.commit_data_safe {
|
||||
return None;
|
||||
}
|
||||
Some(
|
||||
if copy.state == DeleteRecoveryState::Absent
|
||||
|| (certified_same_transaction
|
||||
@@ -857,7 +876,10 @@ impl SetDisks {
|
||||
&disks,
|
||||
bucket,
|
||||
copies,
|
||||
|_| Some(DeleteRecoveryAction::FinishCommit),
|
||||
|copy| {
|
||||
(copy.state != DeleteRecoveryState::CommitFinished || !copy.commit_data_safe)
|
||||
.then_some(DeleteRecoveryAction::FinishCommit)
|
||||
},
|
||||
can_commit,
|
||||
namespace_owner.as_ref(),
|
||||
&mut summary.removed_versions,
|
||||
@@ -922,13 +944,25 @@ impl SetDisks {
|
||||
// GC is a key-wide barrier: every predecessor must have settled
|
||||
// before the last surviving commit witness can disappear.
|
||||
let mut settled = Vec::with_capacity(all_states.len());
|
||||
for transaction in all_states.keys() {
|
||||
let fresh = inspect(&disks, bucket, object, *transaction).await;
|
||||
if fresh.iter().any(|state| !matches!(state, Ok(state) if matches!(state.state, DeleteRecoveryState::Absent | DeleteRecoveryState::CommitFinished))) {
|
||||
if all_states.len() == 1 {
|
||||
let copies = all_states.into_values().next().ok_or(DiskError::FileCorrupt)?;
|
||||
if copies
|
||||
.iter()
|
||||
.any(|state| !matches!(state.state, DeleteRecoveryState::Absent | DeleteRecoveryState::CommitFinished))
|
||||
{
|
||||
summary.pending += 1;
|
||||
return Ok(summary);
|
||||
}
|
||||
settled.push(fresh.into_iter().collect::<Result<Vec<_>>>()?);
|
||||
settled.push(copies);
|
||||
} else {
|
||||
for transaction in all_states.keys() {
|
||||
let fresh = inspect(&disks, bucket, object, *transaction).await;
|
||||
if fresh.iter().any(|state| !matches!(state, Ok(state) if matches!(state.state, DeleteRecoveryState::Absent | DeleteRecoveryState::CommitFinished))) {
|
||||
summary.pending += 1;
|
||||
return Ok(summary);
|
||||
}
|
||||
settled.push(fresh.into_iter().collect::<Result<Vec<_>>>()?);
|
||||
}
|
||||
}
|
||||
for (index, mut copies) in settled.into_iter().enumerate() {
|
||||
if index > 0 {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -15,7 +15,9 @@
|
||||
use crate::disk::{DiskAPI as _, ReadOptions};
|
||||
use crate::object_api::{ObjectOptions, PutObjReader, ShardIntegrityWriteMode};
|
||||
use crate::set_disk::CompletePart;
|
||||
use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks_for_pool_with_default_parity_isolated;
|
||||
use crate::set_disk::ops::object::hermetic_set_disks_support::{
|
||||
hermetic_set_disks_for_pool_with_default_parity_isolated, lifecycle_owned_put_options,
|
||||
};
|
||||
use crate::storage_api_contracts::multipart::MultipartOperations as _;
|
||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
||||
use rustfs_heal_contracts::heal_channel::{HealOpts, HealScanMode};
|
||||
@@ -39,18 +41,19 @@ async fn remote_metadata_heal_does_not_certify_unread_tier_payload() {
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("fixture bucket");
|
||||
}
|
||||
set.put_object(
|
||||
let opts = lifecycle_owned_put_options(
|
||||
&set,
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![0x31; 1024 * 1024 + 17]),
|
||||
&ObjectOptions {
|
||||
ObjectOptions {
|
||||
no_lock: true,
|
||||
shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Protected),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("protected source");
|
||||
.await;
|
||||
set.put_object(bucket, object, &mut PutObjReader::from_vec(vec![0x31; 1024 * 1024 + 17]), &opts)
|
||||
.await
|
||||
.expect("protected source");
|
||||
for (disk, dir) in disks.iter().zip(&dirs) {
|
||||
let mut info = disk
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
@@ -94,10 +97,15 @@ async fn rollout_requires_both_flags_for_new_puts() {
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("fixture bucket");
|
||||
}
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
let opts = lifecycle_owned_put_options(
|
||||
&set,
|
||||
bucket,
|
||||
ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
for (requested, confirmed, protected) in [
|
||||
(false, false, false),
|
||||
@@ -159,12 +167,17 @@ async fn legacy_shard_repair_and_explicit_version_metadata_recovery_remain_avail
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("fixture bucket");
|
||||
}
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
versioned: true,
|
||||
shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Legacy),
|
||||
..Default::default()
|
||||
};
|
||||
let opts = lifecycle_owned_put_options(
|
||||
&set,
|
||||
bucket,
|
||||
ObjectOptions {
|
||||
no_lock: true,
|
||||
versioned: true,
|
||||
shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Legacy),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let expected = vec![0x67; 1024 * 1024 + 37];
|
||||
for case in ["missing-part", "bad-bitrot", "missing-version-metadata"] {
|
||||
let object = case;
|
||||
@@ -272,10 +285,15 @@ async fn multipart_mode_survives_rollout_switch_changes() {
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("fixture bucket");
|
||||
}
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
let opts = lifecycle_owned_put_options(
|
||||
&set,
|
||||
bucket,
|
||||
ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
for protected in [false, true] {
|
||||
let object = format!("upload-{protected}");
|
||||
let upload =
|
||||
@@ -354,11 +372,16 @@ async fn physical_copy_inherits_source_mode_instead_of_current_switch() {
|
||||
};
|
||||
let object = format!("source-{protected}");
|
||||
let expected = vec![0x39; 1024 * 1024 + 11];
|
||||
let seed_opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
shard_integrity_write_mode: Some(mode),
|
||||
..Default::default()
|
||||
};
|
||||
let seed_opts = lifecycle_owned_put_options(
|
||||
&set,
|
||||
bucket,
|
||||
ObjectOptions {
|
||||
no_lock: true,
|
||||
shard_integrity_write_mode: Some(mode),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let mut source = set
|
||||
.put_object(bucket, &object, &mut PutObjReader::from_vec(expected.clone()), &seed_opts)
|
||||
.await
|
||||
@@ -366,8 +389,8 @@ async fn physical_copy_inherits_source_mode_instead_of_current_switch() {
|
||||
source.metadata_only = false;
|
||||
source.put_object_reader = Some(PutObjReader::from_vec(expected.clone()));
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
shard_integrity_write_mode: None,
|
||||
..seed_opts.clone()
|
||||
};
|
||||
let copied = temp_env::async_with_vars(
|
||||
rollout_vars(!protected, !protected),
|
||||
|
||||
@@ -358,6 +358,7 @@ impl SetDisks {
|
||||
let collector = agreed_collector.clone();
|
||||
Box::pin(async move {
|
||||
collector.ingest(entry);
|
||||
Ok(())
|
||||
})
|
||||
})),
|
||||
partial: Some(Box::new(move |entries: MetaCacheEntries, _errs: &[Option<DiskError>]| {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+1505
-1008
File diff suppressed because it is too large
Load Diff
@@ -16,7 +16,7 @@
|
||||
//! another object (`a/b` and `a/b/c`). Cleanup of the parent must only remove
|
||||
//! the parent's own protocol data and never touch the child's xl.meta or data.
|
||||
|
||||
use super::hermetic_set_disks_support::hermetic_set_disks_isolated;
|
||||
use super::hermetic_set_disks_support::{HermeticSetFixture, hermetic_set_disks_isolated};
|
||||
use super::*;
|
||||
use crate::data_movement::{SourceCleanupBucketFence, SourceCleanupError, cleanup_source_entry_if_unchanged};
|
||||
use rustfs_filemeta::FileInfoVersions;
|
||||
@@ -100,7 +100,7 @@ fn child_tree(temp_dirs: &[TempDir], bucket: &str) -> Vec<Vec<String>> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn setup(payload_len: usize) -> (Vec<TempDir>, Arc<SetDisks>, &'static str, Vec<u8>, Vec<u8>) {
|
||||
async fn setup(payload_len: usize) -> (HermeticSetFixture, Arc<SetDisks>, &'static str, Vec<u8>, Vec<u8>) {
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "data-movement-prefix-cleanup";
|
||||
for disk in &disk_stores {
|
||||
|
||||
@@ -3885,8 +3885,7 @@ mod metadata_cache_tests {
|
||||
|
||||
let bucket = "late-metadata-identity";
|
||||
let object = "late-metadata-identity-object";
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let (_dirs, set) = crate::ecstore_validation_blackbox::make_local_set_disks_with_ctx(4, 2, ctx).await;
|
||||
let (_fixture, _disks, set) = crate::set_disk::hermetic_set_disks_isolated(4).await;
|
||||
set.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create real late-read bucket");
|
||||
|
||||
@@ -8,6 +8,7 @@ pub(crate) mod admin {
|
||||
}
|
||||
|
||||
pub(crate) mod bucket {
|
||||
pub(crate) use rustfs_storage_api::bucket::valid_artifact_base_prefix;
|
||||
pub(crate) use rustfs_storage_api::{
|
||||
BUCKET_LIFECYCLE_LOCK_OBJECT, BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions,
|
||||
SRBucketDeleteOp,
|
||||
|
||||
+274
-233
@@ -848,255 +848,293 @@ impl ECStore {
|
||||
|
||||
#[instrument(skip(self))]
|
||||
pub(super) async fn handle_make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
|
||||
if !is_meta_bucketname(bucket)
|
||||
&& let Err(err) = check_valid_bucket_name_strict(bucket)
|
||||
{
|
||||
return Err(StorageError::BucketNameInvalid(err.to_string()));
|
||||
}
|
||||
self.handle_make_bucket_inner(bucket, opts).await
|
||||
}
|
||||
|
||||
// Lock order: bucket lifecycle -> metadata transaction -> exact bucket namespace.
|
||||
let bucket_lifecycle_guard = if !opts.no_lock {
|
||||
Some(self.acquire_bucket_lifecycle_write_lock(bucket).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let metadata_transaction_guard = if !opts.no_lock && !is_meta_bucketname(bucket) {
|
||||
Some(metadata_sys::acquire_bucket_metadata_transaction_lock_in(&self.ctx, bucket).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let ns_guard = if !opts.no_lock {
|
||||
let ns_lock = self.new_ns_lock(bucket, bucket).await?;
|
||||
Some(
|
||||
await_bucket_namespace_operation(
|
||||
bucket_lifecycle_guard.as_ref(),
|
||||
bucket,
|
||||
"bucket namespace lock acquisition",
|
||||
async {
|
||||
ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| match e {
|
||||
rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => {
|
||||
StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "write",
|
||||
bucket: bucket.to_string(),
|
||||
object: bucket.to_string(),
|
||||
required,
|
||||
achieved,
|
||||
}
|
||||
}
|
||||
other => StorageError::Lock(other),
|
||||
})
|
||||
},
|
||||
)
|
||||
.await?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let existing_bucket_info = match self.get_bucket_info_from_sets(bucket, &BucketOptions::default()).await {
|
||||
Ok(info) => Some(info),
|
||||
Err(err) => {
|
||||
if is_err_bucket_not_found(&err) {
|
||||
None
|
||||
} else {
|
||||
return Err(to_object_err(err, vec![bucket]));
|
||||
}
|
||||
// Keep the large creation future out of the instrumented handler's poll
|
||||
// frame. Guards remain owned by the same task.
|
||||
fn handle_make_bucket_inner<'a>(
|
||||
&'a self,
|
||||
bucket: &'a str,
|
||||
opts: &'a MakeBucketOptions,
|
||||
) -> futures::future::BoxFuture<'a, Result<()>> {
|
||||
Box::pin(async move {
|
||||
if opts.artifact_base_prefix.as_deref().is_some_and(|prefix| {
|
||||
is_meta_bucketname(bucket)
|
||||
|| opts.no_lock
|
||||
|| !crate::storage_api_contracts::bucket::valid_artifact_base_prefix(prefix)
|
||||
}) {
|
||||
return Err(Error::other("invalid artifact bucket creation options"));
|
||||
}
|
||||
};
|
||||
let confirmed_missing = existing_bucket_info.is_none();
|
||||
let mut pending_creation_metadata = None;
|
||||
if confirmed_missing
|
||||
&& !opts.no_lock
|
||||
&& !is_meta_bucketname(bucket)
|
||||
&& let Some(metadata_store) = metadata_sys::object_store_if_initialized_in(&self.ctx).await
|
||||
&& let Some(old_incarnation) = crate::bucket::metadata::load_bucket_incarnation(metadata_store, bucket).await?
|
||||
{
|
||||
// A quorum-level absence is not proof that every disk lost the old
|
||||
// generation. Require the explicit recovery path to verify all disks
|
||||
// and retire this incarnation before a same-name create can mutate
|
||||
// any volume.
|
||||
self.ensure_bucket_volume_absent_on_all_sets(bucket)
|
||||
.await
|
||||
.map_err(|error| match error {
|
||||
StorageError::BucketExists(_) => StorageError::InvalidArgument(
|
||||
if !is_meta_bucketname(bucket)
|
||||
&& let Err(err) = check_valid_bucket_name_strict(bucket)
|
||||
{
|
||||
return Err(StorageError::BucketNameInvalid(err.to_string()));
|
||||
}
|
||||
|
||||
// Lock order: bucket lifecycle -> metadata transaction -> exact bucket namespace.
|
||||
let bucket_lifecycle_guard = if !opts.no_lock {
|
||||
Some(self.acquire_bucket_lifecycle_write_lock(bucket).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let metadata_transaction_guard = if !opts.no_lock && !is_meta_bucketname(bucket) {
|
||||
Some(metadata_sys::acquire_bucket_metadata_transaction_lock_in(&self.ctx, bucket).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let ns_guard = if !opts.no_lock {
|
||||
let ns_lock = self.new_ns_lock(bucket, bucket).await?;
|
||||
Some(
|
||||
await_bucket_namespace_operation(
|
||||
bucket_lifecycle_guard.as_ref(),
|
||||
bucket,
|
||||
"bucket namespace lock acquisition",
|
||||
async {
|
||||
ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| match e {
|
||||
rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => {
|
||||
StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "write",
|
||||
bucket: bucket.to_string(),
|
||||
object: bucket.to_string(),
|
||||
required,
|
||||
achieved,
|
||||
}
|
||||
}
|
||||
other => StorageError::Lock(other),
|
||||
})
|
||||
},
|
||||
)
|
||||
.await?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let existing_bucket_info = match self.get_bucket_info_from_sets(bucket, &BucketOptions::default()).await {
|
||||
Ok(info) => Some(info),
|
||||
Err(err) => {
|
||||
if is_err_bucket_not_found(&err) {
|
||||
None
|
||||
} else {
|
||||
return Err(to_object_err(err, vec![bucket]));
|
||||
}
|
||||
}
|
||||
};
|
||||
let confirmed_missing = existing_bucket_info.is_none();
|
||||
let mut pending_creation_metadata = None;
|
||||
if confirmed_missing
|
||||
&& !opts.no_lock
|
||||
&& !is_meta_bucketname(bucket)
|
||||
&& let Some(metadata_store) = metadata_sys::object_store_if_initialized_in(&self.ctx).await
|
||||
&& let Some(old_incarnation) = crate::bucket::metadata::load_bucket_incarnation(metadata_store, bucket).await?
|
||||
{
|
||||
// A quorum-level absence is not proof that every disk lost the old
|
||||
// generation. Require the explicit recovery path to verify all disks
|
||||
// and retire this incarnation before a same-name create can mutate
|
||||
// any volume.
|
||||
self.ensure_bucket_volume_absent_on_all_sets(bucket)
|
||||
.await
|
||||
.map_err(|error| match error {
|
||||
StorageError::BucketExists(_) => StorageError::InvalidArgument(
|
||||
"CreateBucket".to_string(),
|
||||
"bucket".to_string(),
|
||||
"an old bucket volume remains on disk; administrator recovery refused".to_string(),
|
||||
),
|
||||
other => other,
|
||||
})?;
|
||||
let (metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await?;
|
||||
let metadata_store = metadata_sys::object_store_in(&self.ctx).await?;
|
||||
let pending_intent = persisted
|
||||
&& metadata.bucket_incarnation_id == old_incarnation
|
||||
&& metadata.needs_bucket_creation_commit()
|
||||
&& !crate::bucket::retirement::is_retired(metadata_store, bucket, old_incarnation).await?;
|
||||
if pending_intent {
|
||||
pending_creation_metadata = Some(metadata);
|
||||
} else {
|
||||
return Err(StorageError::InvalidArgument(
|
||||
"CreateBucket".to_string(),
|
||||
"bucket".to_string(),
|
||||
"an old bucket volume remains on disk; administrator recovery refused".to_string(),
|
||||
),
|
||||
other => other,
|
||||
})?;
|
||||
let (metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await?;
|
||||
let metadata_store = metadata_sys::object_store_in(&self.ctx).await?;
|
||||
let pending_intent = persisted
|
||||
&& metadata.bucket_incarnation_id == old_incarnation
|
||||
&& metadata.needs_bucket_creation_commit()
|
||||
&& !crate::bucket::retirement::is_retired(metadata_store, bucket, old_incarnation).await?;
|
||||
if pending_intent {
|
||||
pending_creation_metadata = Some(metadata);
|
||||
} else {
|
||||
return Err(StorageError::InvalidArgument(
|
||||
"CreateBucket".to_string(),
|
||||
"bucket".to_string(),
|
||||
format!("orphaned bucket generation {old_incarnation} requires administrator recovery before recreation"),
|
||||
));
|
||||
}
|
||||
}
|
||||
let existing_metadata = if let Some(metadata) = pending_creation_metadata {
|
||||
Some(metadata)
|
||||
} else if opts.force_create && !confirmed_missing && !is_meta_bucketname(bucket) {
|
||||
let (mut metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await?;
|
||||
if !persisted {
|
||||
metadata = BucketMetadata::new(bucket);
|
||||
metadata.created = existing_bucket_info
|
||||
.as_ref()
|
||||
.and_then(|info| info.created)
|
||||
.unwrap_or(OffsetDateTime::UNIX_EPOCH);
|
||||
// A sidecar left by an interrupted legacy migration is the
|
||||
// bucket's published incarnation; keep it rather than minting
|
||||
// one that would invalidate fences taken on the stored value.
|
||||
// A retired incarnation is residue of a deleted bucket and
|
||||
// must not come back.
|
||||
let store = metadata_sys::object_store_in(&self.ctx).await?;
|
||||
if let Some(stored) = crate::bucket::metadata::load_bucket_incarnation(store.clone(), bucket).await?
|
||||
&& !crate::bucket::retirement::is_retired(store, bucket, stored).await?
|
||||
{
|
||||
metadata.bucket_incarnation_id = stored;
|
||||
format!("orphaned bucket generation {old_incarnation} requires administrator recovery before recreation"),
|
||||
));
|
||||
}
|
||||
} else if !metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() {
|
||||
return Err(Error::other(format!(
|
||||
"bucket incarnation sidecar is missing for new-format metadata: {bucket}"
|
||||
)));
|
||||
} else if !metadata.bucket_incarnation_sidecar {
|
||||
metadata.bucket_incarnation_id = Uuid::new_v4();
|
||||
}
|
||||
Some(metadata)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let mut meta = existing_metadata.unwrap_or_else(|| {
|
||||
if confirmed_missing && !is_meta_bucketname(bucket) {
|
||||
BucketMetadata::new_with_default_durability(bucket)
|
||||
let existing_metadata = if let Some(metadata) = pending_creation_metadata {
|
||||
Some(metadata)
|
||||
} else if (opts.force_create || opts.artifact_base_prefix.is_some())
|
||||
&& !confirmed_missing
|
||||
&& !is_meta_bucketname(bucket)
|
||||
{
|
||||
let (mut metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await?;
|
||||
if !persisted {
|
||||
metadata = BucketMetadata::new(bucket);
|
||||
metadata.created = existing_bucket_info
|
||||
.as_ref()
|
||||
.and_then(|info| info.created)
|
||||
.unwrap_or(OffsetDateTime::UNIX_EPOCH);
|
||||
// A sidecar left by an interrupted legacy migration is the
|
||||
// bucket's published incarnation; keep it rather than minting
|
||||
// one that would invalidate fences taken on the stored value.
|
||||
// A retired incarnation is residue of a deleted bucket and
|
||||
// must not come back.
|
||||
let store = metadata_sys::object_store_in(&self.ctx).await?;
|
||||
if let Some(stored) = crate::bucket::metadata::load_bucket_incarnation(store.clone(), bucket).await?
|
||||
&& !crate::bucket::retirement::is_retired(store, bucket, stored).await?
|
||||
{
|
||||
metadata.bucket_incarnation_id = stored;
|
||||
}
|
||||
} else if !metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() {
|
||||
return Err(Error::other(format!(
|
||||
"bucket incarnation sidecar is missing for new-format metadata: {bucket}"
|
||||
)));
|
||||
} else if !metadata.bucket_incarnation_sidecar {
|
||||
metadata.bucket_incarnation_id = Uuid::new_v4();
|
||||
}
|
||||
Some(metadata)
|
||||
} else {
|
||||
BucketMetadata::new(bucket)
|
||||
}
|
||||
});
|
||||
let existing_incarnation_is_authoritative = meta.bucket_incarnation_sidecar;
|
||||
if confirmed_missing || is_meta_bucketname(bucket) {
|
||||
meta.set_created(opts.created_at);
|
||||
None
|
||||
};
|
||||
|
||||
if opts.versioning_enabled {
|
||||
let has_creation_identity = existing_metadata.is_some();
|
||||
let mut meta = existing_metadata.unwrap_or_else(|| {
|
||||
if confirmed_missing && !is_meta_bucketname(bucket) {
|
||||
BucketMetadata::new_with_default_durability(bucket)
|
||||
} else {
|
||||
BucketMetadata::new(bucket)
|
||||
}
|
||||
});
|
||||
if let Some(prefix) = &opts.artifact_base_prefix {
|
||||
if has_creation_identity {
|
||||
if meta.artifact_base_prefix.as_ref() != Some(prefix) {
|
||||
return Err(Error::other("artifact namespace is immutable"));
|
||||
}
|
||||
} else if confirmed_missing {
|
||||
meta.artifact_base_prefix = Some(prefix.clone());
|
||||
} else {
|
||||
return Err(Error::other("artifact namespace requires a new bucket"));
|
||||
}
|
||||
}
|
||||
let existing_incarnation_is_authoritative = meta.bucket_incarnation_sidecar;
|
||||
if confirmed_missing || is_meta_bucketname(bucket) {
|
||||
meta.set_created(opts.created_at);
|
||||
|
||||
if opts.versioning_enabled || meta.artifact_base_prefix.is_some() {
|
||||
meta.versioning_config_xml =
|
||||
crate::bucket::utils::serialize::<VersioningConfiguration>(&ENABLED_VERSIONING_CONFIG)?;
|
||||
}
|
||||
}
|
||||
|
||||
// Object Lock enable is one-way, and it must apply to an existing bucket
|
||||
// too. Site replication replays make-with-versioning carrying the
|
||||
// source's lockEnabled against a destination bucket that already exists;
|
||||
// gating this on `confirmed_missing` returned success while leaving the
|
||||
// replica unlocked, so replicated versions could be deleted without the
|
||||
// retention the source enforces.
|
||||
let lock_newly_enabled = opts.lock_enabled && !meta.lock_enabled;
|
||||
if opts.lock_enabled {
|
||||
meta.lock_enabled = true;
|
||||
meta.object_lock_config_xml = crate::bucket::utils::serialize(&*crate::store::ENABLED_OBJECT_LOCK_CONFIG)?;
|
||||
meta.versioning_config_xml =
|
||||
crate::bucket::utils::serialize::<VersioningConfiguration>(&ENABLED_VERSIONING_CONFIG)?;
|
||||
}
|
||||
}
|
||||
|
||||
// Object Lock enable is one-way, and it must apply to an existing bucket
|
||||
// too. Site replication replays make-with-versioning carrying the
|
||||
// source's lockEnabled against a destination bucket that already exists;
|
||||
// gating this on `confirmed_missing` returned success while leaving the
|
||||
// replica unlocked, so replicated versions could be deleted without the
|
||||
// retention the source enforces.
|
||||
let lock_newly_enabled = opts.lock_enabled && !meta.lock_enabled;
|
||||
if opts.lock_enabled {
|
||||
meta.lock_enabled = true;
|
||||
meta.object_lock_config_xml = crate::bucket::utils::serialize(&*crate::store::ENABLED_OBJECT_LOCK_CONFIG)?;
|
||||
meta.versioning_config_xml = crate::bucket::utils::serialize::<VersioningConfiguration>(&ENABLED_VERSIONING_CONFIG)?;
|
||||
}
|
||||
|
||||
let metadata_persisted_before_physical = confirmed_missing && !is_meta_bucketname(bucket) && opts.lock_enabled;
|
||||
if metadata_persisted_before_physical {
|
||||
metadata_sys::set_new_bucket_metadata_intent_in(&self.ctx, meta.clone()).await?;
|
||||
if bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| metadata_transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| ns_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
{
|
||||
return Err(Error::other(format!("bucket metadata creation intent lock was lost: {bucket}")));
|
||||
}
|
||||
}
|
||||
|
||||
if confirmed_missing && !is_meta_bucketname(bucket) {
|
||||
// Fence every scanner cycle that could have observed the namespace
|
||||
// before physical creation. Creation may become visible even when a
|
||||
// later metadata write or namespace-lock check fails.
|
||||
crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
|
||||
self.cleanup_bucket_usage(bucket, ns_guard.as_ref()).await?;
|
||||
}
|
||||
|
||||
if let Err(err) = await_bucket_lifecycle_operation(
|
||||
bucket_lifecycle_guard.as_ref(),
|
||||
ns_guard.as_ref(),
|
||||
bucket,
|
||||
"physical bucket creation",
|
||||
await_bucket_namespace_operation(
|
||||
metadata_transaction_guard.as_ref(),
|
||||
bucket,
|
||||
"bucket creation metadata transaction",
|
||||
async {
|
||||
self.make_bucket_on_sets(bucket, opts)
|
||||
.await
|
||||
.map_err(|err| to_object_err(err, vec![bucket]))
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
{
|
||||
if is_err_bucket_exists(&err)
|
||||
&& let Err(heal_err) = self
|
||||
.handle_heal_bucket(
|
||||
bucket,
|
||||
&HealOpts {
|
||||
recreate: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
warn!("best-effort bucket heal after BucketExists failed: {heal_err}");
|
||||
}
|
||||
if confirmed_missing && !is_err_bucket_exists(&err) && ns_guard.as_ref().is_none_or(|guard| !guard.is_lock_lost()) {
|
||||
error!("make bucket failed: {err}");
|
||||
self.rollback_failed_bucket_creation(bucket, bucket_lifecycle_guard.as_ref(), ns_guard.as_ref())
|
||||
.await;
|
||||
}
|
||||
return Err(err);
|
||||
};
|
||||
|
||||
let metadata_result = async {
|
||||
let metadata_persisted_before_physical =
|
||||
confirmed_missing && !is_meta_bucketname(bucket) && (opts.lock_enabled || meta.artifact_base_prefix.is_some());
|
||||
if metadata_persisted_before_physical {
|
||||
return metadata_sys::commit_bucket_metadata_in(&self.ctx, meta).await;
|
||||
metadata_sys::set_new_bucket_metadata_intent_in(&self.ctx, meta.clone()).await?;
|
||||
if bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| metadata_transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| ns_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
{
|
||||
return Err(Error::other(format!("bucket metadata creation intent lock was lost: {bucket}")));
|
||||
}
|
||||
}
|
||||
if is_meta_bucketname(bucket) {
|
||||
metadata_sys::set_bucket_metadata_in(&self.ctx, meta).await
|
||||
} else if existing_incarnation_is_authoritative && !lock_newly_enabled {
|
||||
metadata_sys::commit_bucket_metadata_in(&self.ctx, meta).await
|
||||
} else {
|
||||
metadata_sys::set_new_bucket_metadata_in(&self.ctx, meta).await
|
||||
}
|
||||
}
|
||||
.await;
|
||||
let metadata_lock_lost = bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| metadata_transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| ns_guard.as_ref().is_some_and(|guard| guard.is_lock_lost());
|
||||
if let Err(err) = metadata_result {
|
||||
if !metadata_lock_lost {
|
||||
self.rollback_failed_bucket_creation(bucket, bucket_lifecycle_guard.as_ref(), ns_guard.as_ref())
|
||||
.await;
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
if metadata_lock_lost {
|
||||
return Err(Error::other(format!("bucket metadata initialization lock was lost: {bucket}")));
|
||||
}
|
||||
|
||||
if confirmed_missing && !is_meta_bucketname(bucket) {
|
||||
// A scanner may have sampled the first fence before the bucket
|
||||
// became visible. Fence again after metadata initialization so
|
||||
// that snapshot cannot publish as complete.
|
||||
crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
|
||||
}
|
||||
if confirmed_missing && !is_meta_bucketname(bucket) {
|
||||
// Fence every scanner cycle that could have observed the namespace
|
||||
// before physical creation. Creation may become visible even when a
|
||||
// later metadata write or namespace-lock check fails.
|
||||
crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
|
||||
// Keep cleanup state out of the bucket-creation request future.
|
||||
Box::pin(self.cleanup_bucket_usage(bucket, ns_guard.as_ref())).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
if let Err(err) = await_bucket_lifecycle_operation(
|
||||
bucket_lifecycle_guard.as_ref(),
|
||||
ns_guard.as_ref(),
|
||||
bucket,
|
||||
"physical bucket creation",
|
||||
await_bucket_namespace_operation(
|
||||
metadata_transaction_guard.as_ref(),
|
||||
bucket,
|
||||
"bucket creation metadata transaction",
|
||||
async {
|
||||
self.make_bucket_on_sets(bucket, opts)
|
||||
.await
|
||||
.map_err(|err| to_object_err(err, vec![bucket]))
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
{
|
||||
if is_err_bucket_exists(&err)
|
||||
&& let Err(heal_err) = self
|
||||
.handle_heal_bucket(
|
||||
bucket,
|
||||
&HealOpts {
|
||||
recreate: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
warn!("best-effort bucket heal after BucketExists failed: {heal_err}");
|
||||
}
|
||||
if confirmed_missing && !is_err_bucket_exists(&err) && ns_guard.as_ref().is_none_or(|guard| !guard.is_lock_lost())
|
||||
{
|
||||
error!("make bucket failed: {err}");
|
||||
self.rollback_failed_bucket_creation(bucket, bucket_lifecycle_guard.as_ref(), ns_guard.as_ref())
|
||||
.await;
|
||||
}
|
||||
return Err(err);
|
||||
};
|
||||
|
||||
let metadata_result = async {
|
||||
if metadata_persisted_before_physical {
|
||||
return metadata_sys::commit_bucket_metadata_in(&self.ctx, meta).await;
|
||||
}
|
||||
if is_meta_bucketname(bucket) {
|
||||
metadata_sys::set_bucket_metadata_in(&self.ctx, meta).await
|
||||
} else if existing_incarnation_is_authoritative && !lock_newly_enabled {
|
||||
metadata_sys::commit_bucket_metadata_in(&self.ctx, meta).await
|
||||
} else {
|
||||
metadata_sys::set_new_bucket_metadata_in(&self.ctx, meta).await
|
||||
}
|
||||
}
|
||||
.await;
|
||||
let metadata_lock_lost = bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| metadata_transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| ns_guard.as_ref().is_some_and(|guard| guard.is_lock_lost());
|
||||
if let Err(err) = metadata_result {
|
||||
if !metadata_lock_lost {
|
||||
self.rollback_failed_bucket_creation(bucket, bucket_lifecycle_guard.as_ref(), ns_guard.as_ref())
|
||||
.await;
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
if metadata_lock_lost {
|
||||
return Err(Error::other(format!("bucket metadata initialization lock was lost: {bucket}")));
|
||||
}
|
||||
|
||||
if confirmed_missing && !is_meta_bucketname(bucket) {
|
||||
// A scanner may have sampled the first fence before the bucket
|
||||
// became visible. Fence again after metadata initialization so
|
||||
// that snapshot cannot publish as complete.
|
||||
crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
/// Prove a live bucket generation before repairing missing expansion volumes.
|
||||
@@ -1461,6 +1499,9 @@ impl ECStore {
|
||||
}
|
||||
};
|
||||
if bucket_exists {
|
||||
if !is_meta_bucketname(bucket) {
|
||||
self.require_no_artifact_backup_retention(bucket).await?;
|
||||
}
|
||||
validate_table_bucket_delete_guard(&self.ctx, bucket).await?;
|
||||
|
||||
if !opts.force {
|
||||
|
||||
@@ -160,6 +160,7 @@ pub struct BucketIncarnationFenceGuard {
|
||||
bucket: String,
|
||||
token: Option<u64>,
|
||||
memoized: Option<Uuid>,
|
||||
validated_incarnation: Option<Uuid>,
|
||||
}
|
||||
|
||||
impl BucketIncarnationFenceGuard {
|
||||
@@ -173,6 +174,7 @@ impl BucketIncarnationFenceGuard {
|
||||
bucket: bucket.to_owned(),
|
||||
token,
|
||||
memoized,
|
||||
validated_incarnation: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -183,6 +185,7 @@ impl BucketIncarnationFenceGuard {
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: Future<Output = Result<Uuid>>,
|
||||
{
|
||||
self.validated_incarnation = None;
|
||||
let cacheable = reuse_memo && self.token.is_some();
|
||||
let current = match self.memoized.filter(|_| cacheable) {
|
||||
Some(current) => current,
|
||||
@@ -197,6 +200,7 @@ impl BucketIncarnationFenceGuard {
|
||||
}
|
||||
};
|
||||
self.ensure_held()?;
|
||||
self.validated_incarnation = Some(current);
|
||||
Ok(current)
|
||||
}
|
||||
|
||||
@@ -213,6 +217,19 @@ impl BucketIncarnationFenceGuard {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn matches(&self, store: &super::ECStore, bucket: &str, incarnation: Uuid) -> bool {
|
||||
Arc::ptr_eq(&self.registry, &store.bucket_fence_registry)
|
||||
&& self.bucket == bucket
|
||||
&& self.validated_incarnation == Some(incarnation)
|
||||
&& !self.is_lock_lost()
|
||||
}
|
||||
|
||||
/// Retain this validated lifecycle owner through the storage write tail.
|
||||
pub fn attach_owner(self: &Arc<Self>, opts: &mut crate::object_api::ObjectOptions) {
|
||||
self.attach_to_object_options(opts);
|
||||
opts.bucket_incarnation_fence = Some(Arc::clone(self));
|
||||
}
|
||||
|
||||
/// Propagate lifecycle lock loss into the storage commit checks.
|
||||
/// The caller still owns this guard until the complete write tail drains.
|
||||
pub fn attach_to_object_options(&self, opts: &mut crate::object_api::ObjectOptions) {
|
||||
@@ -221,7 +238,8 @@ impl BucketIncarnationFenceGuard {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn is_lock_lost(&self) -> bool {
|
||||
/// Report whether the lifecycle lock no longer covers this operation.
|
||||
pub fn is_lock_lost(&self) -> bool {
|
||||
self.inner.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost)
|
||||
}
|
||||
|
||||
@@ -401,6 +419,30 @@ mod tests {
|
||||
BucketIncarnationFenceGuard::new(registry, "b", inner, cacheable)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_owner_tracks_validation_instead_of_cached_incarnation() {
|
||||
let registry = Arc::new(BucketFenceRegistry::default());
|
||||
let lock = NamespaceLock::new("artifact-owner-validation".to_owned(), Arc::new(LocalClient::new()));
|
||||
let mut first = test_fence(registry.clone(), &lock, true).await;
|
||||
first.validate(true, || async { Ok(uuid(7)) }).await.expect("seed memo");
|
||||
let mut snapshot = test_fence(registry, &lock, true).await;
|
||||
assert_eq!(snapshot.memoized, Some(uuid(7)));
|
||||
assert_eq!(snapshot.validated_incarnation, None, "registration alone grants no artifact ownership");
|
||||
snapshot
|
||||
.validate(false, || async { Ok(uuid(8)) })
|
||||
.await
|
||||
.expect("validate snapshot");
|
||||
assert_eq!(snapshot.validated_incarnation, Some(uuid(8)));
|
||||
assert_eq!(snapshot.memoized, Some(uuid(7)), "independent snapshots do not update the shared memo");
|
||||
assert!(
|
||||
snapshot
|
||||
.validate(false, || async { Err(StorageError::ErasureReadQuorum) })
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(snapshot.validated_incarnation, None, "failed validation cannot retain earlier ownership");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn overlapping_validations_reuse_one_success_until_coverage_gap() {
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
@@ -39,6 +39,10 @@ pub(crate) struct BucketHealScope {
|
||||
}
|
||||
|
||||
impl BucketHealScope {
|
||||
pub(crate) fn lifecycle_guard(&self) -> Option<&NamespaceLockGuard> {
|
||||
self.fence.namespace_lock_guard()
|
||||
}
|
||||
|
||||
pub(crate) fn check(&self) -> disk::error::Result<()> {
|
||||
if self.fence.is_lock_lost() {
|
||||
return Err(DiskError::other("bucket heal incarnation fence was lost"));
|
||||
@@ -1070,16 +1074,25 @@ impl ECStore {
|
||||
version_id: &str,
|
||||
opts: &HealOpts,
|
||||
) -> Result<(HealResultItem, Option<Error>)> {
|
||||
self.handle_heal_object_with_absence(
|
||||
crate::bucket::artifact::with_artifact_repair_scope(
|
||||
&self.ctx,
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
opts,
|
||||
&mut None,
|
||||
crate::set_disk::AbsenceProofRequest {
|
||||
retirement: None,
|
||||
allow_unversioned: false,
|
||||
},
|
||||
Box::pin(async {
|
||||
self.handle_heal_object_with_absence(
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
opts,
|
||||
&mut None,
|
||||
crate::set_disk::AbsenceProofRequest {
|
||||
retirement: None,
|
||||
allow_unversioned: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
}),
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -1113,6 +1126,24 @@ impl ECStore {
|
||||
version_id: &str,
|
||||
opts: &HealOpts,
|
||||
allow_unversioned_absence: bool,
|
||||
) -> Result<HealObjectStorageResult> {
|
||||
crate::bucket::artifact::with_artifact_repair_scope(
|
||||
&self.ctx,
|
||||
bucket,
|
||||
object,
|
||||
opts,
|
||||
Box::pin(self.heal_object_with_absence_proof_inner(bucket, object, version_id, opts, allow_unversioned_absence)),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn heal_object_with_absence_proof_inner(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: &str,
|
||||
opts: &HealOpts,
|
||||
allow_unversioned_absence: bool,
|
||||
) -> Result<HealObjectStorageResult> {
|
||||
// Object heal may recreate a missing bucket volume before committing a
|
||||
// shard. Keep the current bucket generation fenced for the whole
|
||||
@@ -1131,7 +1162,11 @@ impl ECStore {
|
||||
// Reuse the admission's owner: reacquiring a read lock behind a queued
|
||||
// lifecycle writer would deadlock with the read lock we already hold.
|
||||
let scope = bucket_heal_scope(bucket).filter(|scope| std::ptr::eq(scope.store.as_ref(), self));
|
||||
let guard = if let Some(scope) = &scope {
|
||||
let repair = crate::bucket::artifact::artifact_repair_scope(bucket, object);
|
||||
let guard = if let Some(repair) = &repair {
|
||||
repair.check()?;
|
||||
None
|
||||
} else if let Some(scope) = &scope {
|
||||
scope.check().map_err(Error::from)?;
|
||||
None
|
||||
} else {
|
||||
@@ -1139,11 +1174,13 @@ impl ECStore {
|
||||
};
|
||||
let fence_is_lost = || {
|
||||
scope.as_ref().is_some_and(|scope| scope.fence.is_lock_lost())
|
||||
|| repair.as_ref().is_some_and(|scope| scope.check().is_err())
|
||||
|| guard.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost)
|
||||
};
|
||||
let lifecycle_guard = scope
|
||||
.as_ref()
|
||||
.and_then(|scope| scope.fence.namespace_lock_guard())
|
||||
.or_else(|| repair.as_ref().and_then(|scope| scope.lifecycle_guard()))
|
||||
.or(guard.as_ref())
|
||||
.ok_or_else(|| Error::other("bucket heal requires a held lifecycle guard"))?;
|
||||
let retirement = crate::bucket::retirement::MarkerRetirementContext {
|
||||
@@ -2045,6 +2082,7 @@ mod tests {
|
||||
pool_meta_save_gate: Mutex::default(),
|
||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4148,6 +4186,7 @@ mod tests {
|
||||
pool_meta_save_gate: Mutex::default(),
|
||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
};
|
||||
|
||||
let err = store
|
||||
|
||||
@@ -428,6 +428,7 @@ impl ECStore {
|
||||
|
||||
let mut deployment_id = None;
|
||||
let mut pool_meta_bootstrap_authorities = Vec::new();
|
||||
let mut pending_format_disks = Vec::new();
|
||||
|
||||
// let (endpoint_pools, _) = EndpointServerPools::create_server_endpoints(address.as_str(), &layouts)?;
|
||||
|
||||
@@ -553,6 +554,7 @@ impl ECStore {
|
||||
// formatted it, `LegacyAdoption` when it verified the migration, and
|
||||
// `None` when it merely read a format another node created.
|
||||
pool_meta_bootstrap_authorities.push(loaded_format.pool_meta_bootstrap_authority);
|
||||
pending_format_disks.extend(loaded_format.pending_disks);
|
||||
let fm = loaded_format.format;
|
||||
|
||||
// Format loading succeeded, enable health monitoring on all disks
|
||||
@@ -618,6 +620,7 @@ impl ECStore {
|
||||
// this point) and later reads share one cell.
|
||||
ctx: instance_ctx.clone(),
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
});
|
||||
|
||||
// Only set it when this instance's deployment ID is not yet configured
|
||||
@@ -627,8 +630,21 @@ impl ECStore {
|
||||
|
||||
let wait_sec = 5;
|
||||
let mut exit_count = 0;
|
||||
let disk_sets = ec
|
||||
.pools
|
||||
.iter()
|
||||
.flat_map(|pool| pool.disk_set.iter().cloned())
|
||||
.collect::<Vec<_>>();
|
||||
loop {
|
||||
if let Err(err) = ec.init(ctx.clone()).await {
|
||||
let initialization = async {
|
||||
ec.init(ctx.clone()).await?;
|
||||
for pending in &pending_format_disks {
|
||||
pending.finalize(&instance_ctx, &disk_sets).await?;
|
||||
}
|
||||
Ok::<(), Error>(())
|
||||
}
|
||||
.await;
|
||||
if let Err(err) = initialization {
|
||||
error!("init err: {}", err);
|
||||
error!("retry after {} second", wait_sec);
|
||||
sleep(Duration::from_secs(wait_sec)).await;
|
||||
@@ -652,6 +668,11 @@ impl ECStore {
|
||||
|
||||
#[instrument(level = "debug", skip(self, rx))]
|
||||
pub async fn init(self: &Arc<Self>, rx: CancellationToken) -> Result<()> {
|
||||
// Keep startup state out of callers and the instrumented wrapper.
|
||||
Box::pin(self.init_inner(rx)).await
|
||||
}
|
||||
|
||||
async fn init_inner(self: &Arc<Self>, rx: CancellationToken) -> Result<()> {
|
||||
runtime_sources::ensure_boot_time().await;
|
||||
|
||||
let should_persist_pool_meta = self
|
||||
@@ -869,6 +890,23 @@ impl ECStore {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[cfg(feature = "test-util")]
|
||||
mod artifact_bucket_tests;
|
||||
#[cfg(unix)]
|
||||
mod artifact_inventory_tests;
|
||||
#[cfg(feature = "test-util")]
|
||||
mod artifact_put_seal_tests;
|
||||
#[cfg(all(feature = "test-util", unix))]
|
||||
mod artifact_receiver_tests;
|
||||
#[cfg(feature = "test-util")]
|
||||
mod artifact_repair_tests;
|
||||
#[cfg(feature = "test-util")]
|
||||
mod artifact_retirement_tests;
|
||||
#[cfg(feature = "test-util")]
|
||||
mod artifact_write_entrypoint_tests;
|
||||
#[cfg(all(feature = "test-util", unix))]
|
||||
mod artifact_writes_seal_tests;
|
||||
|
||||
use super::{
|
||||
LOCAL_DECOMMISSION_RESUME_RETRY_DELAY, LOCAL_DECOMMISSION_WATCHDOG_MAX_RETRY_DELAY, PoolMetaWriteState,
|
||||
establish_pool_meta_bootstrap_identity_if_proven, load_pool_meta_for_startup, local_decommission_watchdog_retry_delay,
|
||||
@@ -1781,7 +1819,7 @@ mod tests {
|
||||
let mut pools = Vec::new();
|
||||
for _ in 0..4 {
|
||||
let (pool_dirs, pool_disks, pool) = hermetic_set_disks_isolated(4).await;
|
||||
dirs.extend(pool_dirs);
|
||||
dirs.push(pool_dirs);
|
||||
disks.push(pool_disks);
|
||||
pools.push(pool);
|
||||
}
|
||||
@@ -7298,8 +7336,8 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("seed older target metadata on all disks");
|
||||
let source = store.pools[0]
|
||||
.get_disks_by_key(object)
|
||||
let source_set = store.pools[0].get_disks_by_key(object);
|
||||
let mut source = source_set
|
||||
.load_file_info_versions_exact(&bucket, object)
|
||||
.await
|
||||
.expect("read source marker metadata")
|
||||
@@ -7308,6 +7346,30 @@ mod tests {
|
||||
.into_iter()
|
||||
.find(|version| version.version_id.unwrap_or_default() == marker_id)
|
||||
.expect("find source delete marker");
|
||||
if marker_incarnation.is_none() {
|
||||
// Public writes bind new markers to the current bucket generation.
|
||||
// Seed legacy disk metadata explicitly without weakening admission.
|
||||
rustfs_utils::http::metadata_compat::remove_str(
|
||||
&mut source.metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_BUCKET_INCARNATION_ID,
|
||||
);
|
||||
for disk in source_set.disk_inventory().await.into_iter().flatten() {
|
||||
disk.write_metadata("", &bucket, object, source.clone())
|
||||
.await
|
||||
.expect("seed legacy source marker metadata");
|
||||
}
|
||||
source_set.invalidate_get_object_metadata_cache(&bucket, object).await;
|
||||
}
|
||||
let source = source_set
|
||||
.load_file_info_versions_exact(&bucket, object)
|
||||
.await
|
||||
.expect("reload source marker metadata")
|
||||
.expect("source marker must remain present")
|
||||
.versions
|
||||
.into_iter()
|
||||
.find(|version| version.version_id.unwrap_or_default() == marker_id)
|
||||
.expect("find reloaded source delete marker");
|
||||
assert_eq!(source.delete_marker_incarnation(), marker_incarnation, "source fixture identity");
|
||||
let target_set = store.pools[1].get_disks_by_key(object);
|
||||
for disk in target_set.disk_inventory().await.into_iter().flatten().take(2) {
|
||||
disk.write_metadata("", &bucket, object, source.clone())
|
||||
@@ -7733,6 +7795,103 @@ mod tests {
|
||||
.expect_err("suspended delete must remove the requested UUID version");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn decommission_entry_retains_lifecycle_owner_through_source_cleanup() {
|
||||
use crate::disk::DiskAPI as _;
|
||||
|
||||
let temp_dir = tempfile::tempdir().expect("create decommission lifecycle-owner store");
|
||||
let (_ctx, store, shutdown) = Box::pin(without_storage_class_env(build_isolated_test_store(
|
||||
temp_dir.path(),
|
||||
"decommission-lifecycle-owner",
|
||||
&[4, 4],
|
||||
)))
|
||||
.await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = format!("decom-lifecycle-owner-{}", uuid::Uuid::new_v4());
|
||||
let object = "source.bin";
|
||||
let payload = vec![b'o'; 1_300_000];
|
||||
Box::pin(store.make_bucket(&bucket, &MakeBucketOptions::default()))
|
||||
.await
|
||||
.expect("create ordinary source bucket");
|
||||
let incarnation = store.bucket_incarnation_id(&bucket).await.expect("source bucket incarnation");
|
||||
let opts = ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
..Default::default()
|
||||
};
|
||||
Box::pin(seed_decommission_source(&store, &bucket, object, payload.clone(), &opts)).await;
|
||||
let source_set = store.pools[0].get_disks_by_key(object);
|
||||
let expected = source_set
|
||||
.load_file_info_versions_exact(&bucket, object)
|
||||
.await
|
||||
.expect("load the exact cleanup snapshot")
|
||||
.expect("source object must exist");
|
||||
let source_disks = source_set.disk_inventory().await;
|
||||
assert_eq!(source_disks.iter().flatten().count(), 4, "all source disks must be online");
|
||||
let owner = store
|
||||
.acquire_bucket_incarnation_fence(&bucket, incarnation)
|
||||
.await
|
||||
.expect("hold source bucket lifecycle");
|
||||
let rejected = Box::pin(crate::data_movement::cleanup_source_entry_if_unchanged(
|
||||
Arc::clone(&source_set),
|
||||
&bucket,
|
||||
object,
|
||||
&expected,
|
||||
&[],
|
||||
&[],
|
||||
crate::data_movement::SourceCleanupBucketFence {
|
||||
expected_incarnation_id: Some(incarnation),
|
||||
lifecycle_guard: owner.namespace_lock_guard(),
|
||||
..Default::default()
|
||||
},
|
||||
"test_unowned_source_cleanup",
|
||||
))
|
||||
.await
|
||||
.expect_err("a bare lifecycle signal must not authorize source cleanup");
|
||||
assert!(
|
||||
rejected
|
||||
.to_string()
|
||||
.contains("write requires an owned bucket lifecycle fence"),
|
||||
"cleanup must reject the missing owner: {rejected}"
|
||||
);
|
||||
let mut source = source_set
|
||||
.get_object_reader(&bucket, object, None, HeaderMap::new(), &opts)
|
||||
.await
|
||||
.expect("rejected cleanup must preserve the readable source");
|
||||
let mut remaining = Vec::new();
|
||||
source
|
||||
.stream
|
||||
.read_to_end(&mut remaining)
|
||||
.await
|
||||
.expect("drain preserved source");
|
||||
assert_eq!(remaining, payload);
|
||||
drop(source);
|
||||
drop(owner);
|
||||
|
||||
mark_test_pool_decommissioning(&store, 0).await;
|
||||
Box::pin(run_decommission_entry_retry_test(
|
||||
&store,
|
||||
CancellationToken::new(),
|
||||
&bucket,
|
||||
object,
|
||||
Some(incarnation),
|
||||
Arc::new(AtomicUsize::new(0)),
|
||||
))
|
||||
.await
|
||||
.expect("decommission must copy and clean the ordinary bucket entry with its real owner");
|
||||
assert_eq!(read_decommission_target_body(&store, &bucket, object, &opts).await, payload);
|
||||
assert_decommission_source_absent(&store, &bucket, object, &opts).await;
|
||||
for disk in source_disks.into_iter().flatten() {
|
||||
assert!(
|
||||
!tokio::fs::try_exists(disk.path().join(&bucket).join(object))
|
||||
.await
|
||||
.expect("inspect the physical source path"),
|
||||
"source metadata and shards must be removed from every disk"
|
||||
);
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn decommission_entry_carries_migration_and_cleanup_mutation_fences() {
|
||||
@@ -11861,6 +12020,90 @@ mod tests {
|
||||
.len()
|
||||
}
|
||||
|
||||
// Recovery removes metadata replicas disk by disk. An unreadable prefix
|
||||
// is incomplete evidence and must reset, never advance, the empty streak.
|
||||
#[cfg(feature = "test-util")]
|
||||
fn tier_delete_recovery_listing_is_empty(prefix: &str, count: crate::error::Result<usize>) -> bool {
|
||||
match count {
|
||||
Ok(count) => count == 0,
|
||||
Err(StorageError::InsufficientReadQuorum(bucket, object)) if bucket == RUSTFS_META_BUCKET && object == prefix => {
|
||||
false
|
||||
}
|
||||
Err(error) => panic!("{prefix} recovery inventory should be listable: {error:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn tier_delete_recovery_prefix_is_empty(store: Arc<crate::store::ECStore>, prefix: &str) -> bool {
|
||||
let store_id = store.id;
|
||||
let started = std::time::Instant::now();
|
||||
let count = store
|
||||
.list_objects_v2(RUSTFS_META_BUCKET, prefix, None, None, 100, false, None, false)
|
||||
.await
|
||||
.map(|listing| listing.objects.len());
|
||||
if let Err(error) = &count {
|
||||
eprintln!(
|
||||
"tier-delete recovery inventory unavailable: store={store_id} prefix={prefix} elapsed={:?} error={error:?}",
|
||||
started.elapsed()
|
||||
);
|
||||
}
|
||||
tier_delete_recovery_listing_is_empty(prefix, count)
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[test]
|
||||
fn tier_delete_recovery_observations_require_readable_empty_prefixes() {
|
||||
for prefix in [TIER_DELETE_JOURNAL_PREFIX, TIER_DELETE_DISPATCH_MANIFEST_PREFIX] {
|
||||
let observations = [
|
||||
Ok(0),
|
||||
Err(StorageError::InsufficientReadQuorum(RUSTFS_META_BUCKET.to_string(), prefix.to_string())),
|
||||
Ok(0),
|
||||
Ok(1),
|
||||
Ok(0),
|
||||
];
|
||||
let mut stable_empty = 0;
|
||||
let streaks = observations
|
||||
.into_iter()
|
||||
.map(|count| {
|
||||
if tier_delete_recovery_listing_is_empty(prefix, count) {
|
||||
stable_empty += 1;
|
||||
} else {
|
||||
stable_empty = 0;
|
||||
}
|
||||
stable_empty
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(streaks, [1, 0, 1, 0, 1], "unreadable or nonempty {prefix} cannot certify convergence");
|
||||
for _ in 0..10 {
|
||||
assert!(!tier_delete_recovery_listing_is_empty(
|
||||
prefix,
|
||||
Err(StorageError::InsufficientReadQuorum(RUSTFS_META_BUCKET.to_string(), prefix.to_string())),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[test]
|
||||
fn tier_delete_recovery_observations_reject_unrelated_errors() {
|
||||
for prefix in [TIER_DELETE_JOURNAL_PREFIX, TIER_DELETE_DISPATCH_MANIFEST_PREFIX] {
|
||||
for error in [
|
||||
StorageError::InsufficientReadQuorum("other-bucket".to_string(), prefix.to_string()),
|
||||
StorageError::InsufficientReadQuorum(RUSTFS_META_BUCKET.to_string(), "other-prefix".to_string()),
|
||||
StorageError::FileNotFound,
|
||||
StorageError::ErasureReadQuorum,
|
||||
] {
|
||||
assert!(
|
||||
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
||||
tier_delete_recovery_listing_is_empty(prefix, Err(error))
|
||||
}))
|
||||
.is_err(),
|
||||
"only the exact recovery prefix's InsufficientReadQuorum may defer an observation"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn install_committed_tier_delete_journals(
|
||||
store: Arc<crate::store::ECStore>,
|
||||
@@ -11968,8 +12211,8 @@ mod tests {
|
||||
recover_tier_delete_journal_entries(store.clone(), 100, None)
|
||||
.await
|
||||
.expect("restarted journal recovery pass should finish");
|
||||
if tier_delete_journal_count(store.clone()).await == 0
|
||||
&& tier_delete_dispatch_manifest_count(store.clone()).await == 0
|
||||
if tier_delete_recovery_prefix_is_empty(store.clone(), TIER_DELETE_JOURNAL_PREFIX).await
|
||||
&& tier_delete_recovery_prefix_is_empty(store.clone(), TIER_DELETE_DISPATCH_MANIFEST_PREFIX).await
|
||||
&& tier_delete_dispatch_manifest_recovery_count_for_test(&store) == 0
|
||||
{
|
||||
stable_empty_observations += 1;
|
||||
@@ -13397,8 +13640,8 @@ mod tests {
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
let mut stable_empty_observations = 0;
|
||||
loop {
|
||||
if tier_delete_journal_count(restarted_store.clone()).await == 0
|
||||
&& tier_delete_dispatch_manifest_count(restarted_store.clone()).await == 0
|
||||
if tier_delete_recovery_prefix_is_empty(restarted_store.clone(), TIER_DELETE_JOURNAL_PREFIX).await
|
||||
&& tier_delete_recovery_prefix_is_empty(restarted_store.clone(), TIER_DELETE_DISPATCH_MANIFEST_PREFIX).await
|
||||
&& tier_delete_dispatch_manifest_recovery_count_for_test(&restarted_store) == 0
|
||||
{
|
||||
stable_empty_observations += 1;
|
||||
@@ -13522,8 +13765,8 @@ mod tests {
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
let mut stable_empty_observations = 0;
|
||||
loop {
|
||||
if tier_delete_journal_count(restarted_store.clone()).await == 0
|
||||
&& tier_delete_dispatch_manifest_count(restarted_store.clone()).await == 0
|
||||
if tier_delete_recovery_prefix_is_empty(restarted_store.clone(), TIER_DELETE_JOURNAL_PREFIX).await
|
||||
&& tier_delete_recovery_prefix_is_empty(restarted_store.clone(), TIER_DELETE_DISPATCH_MANIFEST_PREFIX).await
|
||||
&& tier_delete_dispatch_manifest_recovery_count_for_test(&restarted_store) == 0
|
||||
{
|
||||
stable_empty_observations += 1;
|
||||
@@ -20989,10 +21232,13 @@ mod tests {
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn legacy_recovery_disposition_removes_only_local_journals_and_replays() {
|
||||
Box::pin(legacy_recovery_disposition_removes_only_local_journals_and_replays_case()).await;
|
||||
fn legacy_recovery_disposition_removes_only_local_journals_and_replays() {
|
||||
run_large_stack_async_test(
|
||||
"legacy-recovery-disposition",
|
||||
legacy_recovery_disposition_removes_only_local_journals_and_replays_case,
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
|
||||
@@ -0,0 +1,757 @@
|
||||
use super::{build_isolated_test_store, without_storage_class_env};
|
||||
use crate::{
|
||||
bucket::{
|
||||
metadata::{
|
||||
BUCKET_INCARNATION_FILE, BUCKET_METADATA_FILE, BUCKET_TAGGING_CONFIG, BUCKET_VERSIONING_CONFIG, BucketMetadata,
|
||||
load_bucket_metadata_parse_with_presence,
|
||||
},
|
||||
metadata_sys,
|
||||
versioning::VersioningApi,
|
||||
},
|
||||
disk::{BUCKET_META_PREFIX, DiskAPI as _, RUSTFS_META_BUCKET},
|
||||
runtime::instance::InstanceContext,
|
||||
storage_api_contracts::{
|
||||
bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions},
|
||||
namespace::NamespaceLocking as _,
|
||||
},
|
||||
store::ECStore,
|
||||
};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use uuid::Uuid;
|
||||
|
||||
const PREFIX: &str = "outputs/";
|
||||
const ENABLED: &[u8] = b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>";
|
||||
const SUSPENDED: &[u8] = b"<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>";
|
||||
const EXCLUDED_PREFIX: &[u8] = b"<VersioningConfiguration><Status>Enabled</Status><ExcludedPrefixes><Prefix>outputs/</Prefix></ExcludedPrefixes></VersioningConfiguration>";
|
||||
const EXCLUDED_FOLDERS: &[u8] =
|
||||
b"<VersioningConfiguration><Status>Enabled</Status><ExcludeFolders>true</ExcludeFolders></VersioningConfiguration>";
|
||||
const TAGGING: &[u8] = b"<Tagging><TagSet><Tag><Key>purpose</Key><Value>artifact-test</Value></Tag></TagSet></Tagging>";
|
||||
|
||||
async fn open_store(root: &std::path::Path) -> (Arc<InstanceContext>, Arc<ECStore>, CancellationToken) {
|
||||
let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(root, "artifact-bucket", &[4])).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
(ctx, store, shutdown)
|
||||
}
|
||||
|
||||
async fn drain_before_reopen(store: &ECStore, shutdown: &CancellationToken, bucket: &str) {
|
||||
shutdown.cancel();
|
||||
assert!(
|
||||
store.ctx.wait_for_detached_mutations(Duration::from_secs(30)).await,
|
||||
"ARTIFACT_PREVIOUS_STORE_MUTATIONS_DRAINED"
|
||||
);
|
||||
// Internal metadata PUTs can acknowledge quorum before their disk tails finish.
|
||||
// Acquire each key separately to wait for its actual writer to release the lock.
|
||||
for file in [BUCKET_METADATA_FILE, BUCKET_INCARNATION_FILE] {
|
||||
let key = format!("{BUCKET_META_PREFIX}/{bucket}/{file}");
|
||||
let namespace = store.pools[0].disk_set[0]
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &key)
|
||||
.await
|
||||
.expect("resolve persisted identity namespace");
|
||||
drop(
|
||||
namespace
|
||||
.get_write_lock(Duration::from_secs(30))
|
||||
.await
|
||||
.expect("identity write tail drained"),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn dedicated_bucket(store: &ECStore, bucket: &str) {
|
||||
store
|
||||
.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
artifact_base_prefix: Some(PREFIX.to_owned()),
|
||||
// Creation must enforce versioning without relying on the request flag.
|
||||
versioning_enabled: false,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create a dedicated artifact bucket on real disks");
|
||||
}
|
||||
|
||||
async fn persisted(store: &Arc<ECStore>, bucket: &str) -> BucketMetadata {
|
||||
let (metadata, exists) = load_bucket_metadata_parse_with_presence(store.clone(), bucket, true)
|
||||
.await
|
||||
.expect("read actual persisted bucket metadata");
|
||||
assert!(exists, "ARTIFACT_METADATA_PERSISTED");
|
||||
metadata
|
||||
}
|
||||
|
||||
fn assert_namespace(metadata: &BucketMetadata, incarnation: Uuid) {
|
||||
assert_eq!(metadata.artifact_base_prefix.as_deref(), Some(PREFIX), "ARTIFACT_NAMESPACE_IMMUTABLE");
|
||||
assert_eq!(metadata.bucket_incarnation_id, incarnation, "ARTIFACT_INCARNATION_IMMUTABLE");
|
||||
assert!(
|
||||
metadata.versioning_config.as_ref().is_some_and(VersioningApi::enabled),
|
||||
"ARTIFACT_VERSIONING_REQUIRED"
|
||||
);
|
||||
}
|
||||
|
||||
async fn assert_persisted_and_cached(store: &Arc<ECStore>, bucket: &str, incarnation: Uuid) {
|
||||
assert_namespace(&persisted(store, bucket).await, incarnation);
|
||||
let cached = metadata_sys::get_in(&store.ctx, bucket)
|
||||
.await
|
||||
.expect("read cached bucket identity");
|
||||
assert_namespace(&cached, incarnation);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_creation_enables_versioning_and_survives_store_reopen() {
|
||||
let root = tempfile::tempdir().expect("artifact bucket store directory");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-created-reopen";
|
||||
dedicated_bucket(&store, bucket).await;
|
||||
let created = persisted(&store, bucket).await;
|
||||
let incarnation = created.bucket_incarnation_id;
|
||||
assert!(!incarnation.is_nil(), "ARTIFACT_INCARNATION_REQUIRED");
|
||||
assert!(created.bucket_creation_committed, "ARTIFACT_CREATION_COMMITTED");
|
||||
assert_namespace(&created, incarnation);
|
||||
drain_before_reopen(&store, &shutdown, bucket).await;
|
||||
drop(store);
|
||||
drop(ctx);
|
||||
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
metadata_sys::reload_bucket_metadata(store.clone(), bucket)
|
||||
.await
|
||||
.expect("load the reopened bucket from persisted metadata");
|
||||
assert_persisted_and_cached(&store, bucket, incarnation).await;
|
||||
store
|
||||
.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
force_create: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("ordinary force-create replay preserves the dedicated namespace");
|
||||
assert_persisted_and_cached(&store, bucket, incarnation).await;
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_ordinary_configuration_update_preserves_namespace() {
|
||||
let root = tempfile::tempdir().expect("artifact configuration store directory");
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-config-update";
|
||||
dedicated_bucket(&store, bucket).await;
|
||||
let incarnation = persisted(&store, bucket).await.bucket_incarnation_id;
|
||||
metadata_sys::update_in(&store.ctx, bucket, BUCKET_TAGGING_CONFIG, TAGGING.to_vec())
|
||||
.await
|
||||
.expect("ordinary tagging update remains supported");
|
||||
let changed = persisted(&store, bucket).await;
|
||||
assert_eq!(changed.tagging_config_xml, TAGGING, "ARTIFACT_ORDINARY_CONFIG_UPDATED");
|
||||
assert_persisted_and_cached(&store, bucket, incarnation).await;
|
||||
let sys = store.ctx.bucket_metadata_sys().expect("instance metadata system");
|
||||
let sys = sys.read().await.clone();
|
||||
let suspended = crate::bucket::utils::deserialize(SUSPENDED).expect("valid suspended versioning XML");
|
||||
for stale_typed_config in [None, Some(suspended)] {
|
||||
let mut replacement = changed.clone();
|
||||
replacement.versioning_config = stale_typed_config;
|
||||
sys.set_validated(replacement)
|
||||
.await
|
||||
.expect("validated cache setter reparses authoritative XML");
|
||||
assert_persisted_and_cached(&store, bucket, incarnation).await;
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_create_rejects_existing_bucket_type_and_prefix_changes() {
|
||||
let root = tempfile::tempdir().expect("artifact immutable creation store directory");
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let ordinary = "artifact-existing-ordinary";
|
||||
store
|
||||
.make_bucket(ordinary, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create ordinary bucket");
|
||||
let original = persisted(&store, ordinary).await;
|
||||
assert!(original.artifact_base_prefix.is_none());
|
||||
for force_create in [false, true] {
|
||||
let result = store
|
||||
.make_bucket(
|
||||
ordinary,
|
||||
&MakeBucketOptions {
|
||||
artifact_base_prefix: Some(PREFIX.to_owned()),
|
||||
force_create,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_err(), "ARTIFACT_NAMESPACE_IMMUTABLE: ordinary bucket cannot change type");
|
||||
let unchanged = persisted(&store, ordinary).await;
|
||||
assert_eq!(unchanged.artifact_base_prefix, None);
|
||||
assert_eq!(unchanged.bucket_incarnation_id, original.bucket_incarnation_id);
|
||||
assert_eq!(unchanged.versioning_config_xml, original.versioning_config_xml);
|
||||
}
|
||||
let dedicated = "artifact-existing-dedicated";
|
||||
dedicated_bucket(&store, dedicated).await;
|
||||
let incarnation = persisted(&store, dedicated).await.bucket_incarnation_id;
|
||||
for force_create in [false, true] {
|
||||
let result = store
|
||||
.make_bucket(
|
||||
dedicated,
|
||||
&MakeBucketOptions {
|
||||
artifact_base_prefix: Some("another/".to_owned()),
|
||||
force_create,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_err(), "ARTIFACT_NAMESPACE_IMMUTABLE: create cannot replace an existing prefix");
|
||||
assert_persisted_and_cached(&store, dedicated, incarnation).await;
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_full_metadata_replacement_cannot_change_namespace() {
|
||||
let root = tempfile::tempdir().expect("artifact metadata replacement store directory");
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let dedicated = "artifact-replacement-dedicated";
|
||||
let ordinary = "artifact-replacement-ordinary";
|
||||
dedicated_bucket(&store, dedicated).await;
|
||||
store
|
||||
.make_bucket(ordinary, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create ordinary replacement fixture");
|
||||
let original = persisted(&store, dedicated).await;
|
||||
let incarnation = original.bucket_incarnation_id;
|
||||
let sys = store.ctx.bucket_metadata_sys().expect("instance metadata system");
|
||||
let sys = sys.read().await.clone();
|
||||
for route in 0..3 {
|
||||
for prefix in [None, Some("replacement/".to_owned())] {
|
||||
let mut replacement = original.clone();
|
||||
replacement.artifact_base_prefix = prefix;
|
||||
let result = match route {
|
||||
0 => metadata_sys::set_bucket_metadata_in(&store.ctx, replacement).await,
|
||||
1 => sys.persist_and_set(replacement).await,
|
||||
_ => sys.set_validated(replacement).await,
|
||||
};
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"ARTIFACT_NAMESPACE_IMMUTABLE: complete replacement cannot remove or change prefix"
|
||||
);
|
||||
assert_persisted_and_cached(&store, dedicated, incarnation).await;
|
||||
}
|
||||
let mut replacement = persisted(&store, ordinary).await;
|
||||
let ordinary_incarnation = replacement.bucket_incarnation_id;
|
||||
replacement.artifact_base_prefix = Some(PREFIX.to_owned());
|
||||
replacement.versioning_config_xml = ENABLED.to_vec();
|
||||
let result = match route {
|
||||
0 => metadata_sys::set_bucket_metadata_in(&store.ctx, replacement).await,
|
||||
1 => sys.persist_and_set(replacement).await,
|
||||
_ => sys.set_validated(replacement).await,
|
||||
};
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"ARTIFACT_NAMESPACE_IMMUTABLE: complete replacement cannot promote ordinary bucket"
|
||||
);
|
||||
let unchanged = persisted(&store, ordinary).await;
|
||||
assert!(unchanged.artifact_base_prefix.is_none());
|
||||
assert_eq!(unchanged.bucket_incarnation_id, ordinary_incarnation);
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_versioning_cannot_be_suspended_or_removed() {
|
||||
let root = tempfile::tempdir().expect("artifact mandatory versioning store directory");
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-required-versioning";
|
||||
dedicated_bucket(&store, bucket).await;
|
||||
let original = persisted(&store, bucket).await;
|
||||
let incarnation = original.bucket_incarnation_id;
|
||||
let sys = store.ctx.bucket_metadata_sys().expect("instance metadata system");
|
||||
let sys = sys.read().await.clone();
|
||||
for xml in [SUSPENDED, b"".as_slice(), EXCLUDED_PREFIX, EXCLUDED_FOLDERS] {
|
||||
let result = metadata_sys::update_in(&store.ctx, bucket, BUCKET_VERSIONING_CONFIG, xml.to_vec()).await;
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"ARTIFACT_VERSIONING_REQUIRED: configuration update cannot disable versioning"
|
||||
);
|
||||
assert_persisted_and_cached(&store, bucket, incarnation).await;
|
||||
for route in 0..3 {
|
||||
let mut replacement = original.clone();
|
||||
replacement.versioning_config_xml = xml.to_vec();
|
||||
replacement.versioning_config = None;
|
||||
let result = match route {
|
||||
0 => metadata_sys::set_bucket_metadata_in(&store.ctx, replacement).await,
|
||||
1 => sys.persist_and_set(replacement).await,
|
||||
_ => sys.set_validated(replacement).await,
|
||||
};
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"ARTIFACT_VERSIONING_REQUIRED: complete replacement cannot disable versioning"
|
||||
);
|
||||
assert_persisted_and_cached(&store, bucket, incarnation).await;
|
||||
}
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_pending_creation_reopen_retry_preserves_namespace() {
|
||||
let root = tempfile::tempdir().expect("artifact pending creation store directory");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-pending-retry";
|
||||
let mut intent = BucketMetadata::new_with_default_durability(bucket);
|
||||
intent.artifact_base_prefix = Some(PREFIX.to_owned());
|
||||
intent.versioning_config_xml = ENABLED.to_vec();
|
||||
let incarnation = intent.bucket_incarnation_id;
|
||||
// Persist through the production intent writer, before any physical bucket exists.
|
||||
metadata_sys::set_new_bucket_metadata_intent_in(&ctx, intent)
|
||||
.await
|
||||
.expect("persist real creation intent");
|
||||
let pending = persisted(&store, bucket).await;
|
||||
assert!(!pending.bucket_creation_committed, "ARTIFACT_CREATION_PENDING");
|
||||
assert_namespace(&pending, incarnation);
|
||||
for disk in 0..4 {
|
||||
assert!(
|
||||
!root.path().join(format!("pool0/set0/disk{disk}/{bucket}")).exists(),
|
||||
"ARTIFACT_PHYSICAL_BUCKET_NOT_CREATED"
|
||||
);
|
||||
}
|
||||
drain_before_reopen(&store, &shutdown, bucket).await;
|
||||
drop(store);
|
||||
drop(ctx);
|
||||
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let result = store
|
||||
.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
artifact_base_prefix: Some("replacement/".to_owned()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_err(), "ARTIFACT_NAMESPACE_IMMUTABLE: pending intent rejects a changed request");
|
||||
let pending = persisted(&store, bucket).await;
|
||||
assert_namespace(&pending, incarnation);
|
||||
assert!(!pending.bucket_creation_committed);
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("retry resumes persisted creation identity");
|
||||
let completed = persisted(&store, bucket).await;
|
||||
assert!(completed.bucket_creation_committed, "ARTIFACT_CREATION_COMMITTED");
|
||||
assert_persisted_and_cached(&store, bucket, incarnation).await;
|
||||
for disk in 0..4 {
|
||||
assert!(
|
||||
root.path().join(format!("pool0/set0/disk{disk}/{bucket}")).is_dir(),
|
||||
"ARTIFACT_PHYSICAL_BUCKET_CREATED"
|
||||
);
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_partial_volume_retry_preserves_pending_identity() {
|
||||
let root = tempfile::tempdir().expect("artifact partial volume store directory");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-partial-volume";
|
||||
let mut intent = BucketMetadata::new_with_default_durability(bucket);
|
||||
intent.artifact_base_prefix = Some(PREFIX.to_owned());
|
||||
intent.versioning_config_xml = ENABLED.to_vec();
|
||||
let incarnation = intent.bucket_incarnation_id;
|
||||
metadata_sys::set_new_bucket_metadata_intent_in(&ctx, intent)
|
||||
.await
|
||||
.expect("persist real creation intent before a partial physical creation");
|
||||
let disks = store.pools[0].disk_set[0].disk_inventory().await;
|
||||
disks[0]
|
||||
.as_ref()
|
||||
.expect("first real disk")
|
||||
.make_volume(bucket)
|
||||
.await
|
||||
.expect("create only one physical volume");
|
||||
assert!(root.path().join(format!("pool0/set0/disk0/{bucket}")).is_dir());
|
||||
for disk in 1..4 {
|
||||
assert!(!root.path().join(format!("pool0/set0/disk{disk}/{bucket}")).exists());
|
||||
}
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect_err("partial physical creation requires explicit recovery");
|
||||
let pending = persisted(&store, bucket).await;
|
||||
assert_namespace(&pending, incarnation);
|
||||
assert!(!pending.bucket_creation_committed, "ARTIFACT_PARTIAL_CREATION_STAYS_PENDING");
|
||||
for disk in 1..4 {
|
||||
assert!(
|
||||
!root.path().join(format!("pool0/set0/disk{disk}/{bucket}")).exists(),
|
||||
"ARTIFACT_PARTIAL_CREATION_NOT_SILENTLY_COMPLETED"
|
||||
);
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_metadata_replacement_cannot_spoof_incarnation() {
|
||||
let root = tempfile::tempdir().expect("artifact spoofed incarnation store directory");
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-spoofed-incarnation";
|
||||
dedicated_bucket(&store, bucket).await;
|
||||
let original = persisted(&store, bucket).await;
|
||||
let incarnation = original.bucket_incarnation_id;
|
||||
let sys = store.ctx.bucket_metadata_sys().expect("instance metadata system");
|
||||
let sys = sys.read().await.clone();
|
||||
for route in 0..4 {
|
||||
for spoofed in [Uuid::nil(), Uuid::new_v4()] {
|
||||
for (prefix, xml) in [
|
||||
(None, b"".as_slice()),
|
||||
(None, SUSPENDED),
|
||||
(Some("replacement/"), ENABLED),
|
||||
(Some(PREFIX), ENABLED),
|
||||
] {
|
||||
let mut replacement = original.clone();
|
||||
replacement.bucket_incarnation_id = spoofed;
|
||||
replacement.artifact_base_prefix = prefix.map(str::to_owned);
|
||||
replacement.versioning_config_xml = xml.to_vec();
|
||||
replacement.versioning_config = None;
|
||||
let result = match route {
|
||||
0 => metadata_sys::set_bucket_metadata_in(&store.ctx, replacement).await,
|
||||
1 => sys.persist_and_set(replacement).await,
|
||||
2 => sys.set_validated(replacement).await,
|
||||
_ => replacement.save_with_store(store.clone()).await,
|
||||
};
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"ARTIFACT_INCARNATION_IMMUTABLE: route={route}, spoofed={spoofed}, prefix={prefix:?}"
|
||||
);
|
||||
assert_persisted_and_cached(&store, bucket, incarnation).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_real_delete_recreate_allows_new_namespace_and_incarnation() {
|
||||
let root = tempfile::tempdir().expect("artifact genuine recreation store directory");
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-genuine-recreation";
|
||||
dedicated_bucket(&store, bucket).await;
|
||||
let mut incarnation = persisted(&store, bucket).await.bucket_incarnation_id;
|
||||
for prefix in [Some("replacement/"), None] {
|
||||
store
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete the empty bucket through production storage");
|
||||
store
|
||||
.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
artifact_base_prefix: prefix.map(str::to_owned),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("genuine recreation may choose a new bucket namespace");
|
||||
let recreated = persisted(&store, bucket).await;
|
||||
assert!(!recreated.bucket_incarnation_id.is_nil(), "ARTIFACT_INCARNATION_REQUIRED");
|
||||
assert_ne!(recreated.bucket_incarnation_id, incarnation, "ARTIFACT_RECREATION_NEW_INCARNATION");
|
||||
assert_eq!(recreated.artifact_base_prefix.as_deref(), prefix, "ARTIFACT_RECREATION_NEW_NAMESPACE");
|
||||
assert_eq!(recreated.versioning_config.as_ref().is_some_and(VersioningApi::enabled), prefix.is_some());
|
||||
assert!(recreated.bucket_creation_committed, "ARTIFACT_CREATION_COMMITTED");
|
||||
let cached = metadata_sys::get_in(&store.ctx, bucket)
|
||||
.await
|
||||
.expect("recreated identity cached");
|
||||
assert_eq!(cached.bucket_incarnation_id, recreated.bucket_incarnation_id);
|
||||
assert_eq!(cached.artifact_base_prefix.as_deref(), prefix);
|
||||
incarnation = recreated.bucket_incarnation_id;
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial]
|
||||
async fn artifact_bucket_create_observes_current_generation() {
|
||||
let root = tempfile::tempdir().expect("creation observation storage");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-create-observation";
|
||||
dedicated_bucket(&store, bucket).await;
|
||||
let initial = persisted(&store, bucket).await.bucket_incarnation_id;
|
||||
store
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete before observation");
|
||||
dedicated_bucket(&store, bucket).await;
|
||||
let current = persisted(&store, bucket).await.bucket_incarnation_id;
|
||||
assert_ne!(initial, current);
|
||||
// Invoke the exact observation phase consumed by make_artifact_bucket,
|
||||
// after real replacement between its creation and observation phases.
|
||||
assert_eq!(
|
||||
store
|
||||
.observe_artifact_bucket_incarnation(bucket, PREFIX)
|
||||
.await
|
||||
.expect("observe replacement"),
|
||||
current,
|
||||
"ARTIFACT_CREATE_CURRENT_GENERATION"
|
||||
);
|
||||
assert!(
|
||||
store.observe_artifact_bucket_incarnation(bucket, "other/").await.is_err(),
|
||||
"ARTIFACT_CREATE_CURRENT_GENERATION: mismatched namespace cannot produce a receipt"
|
||||
);
|
||||
let attempt = format!("{PREFIX}61/{}/{}/", Uuid::new_v4(), Uuid::new_v4());
|
||||
assert!(
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(bucket, initial, &attempt)
|
||||
.await
|
||||
.is_err(),
|
||||
"ARTIFACT_CREATE_CURRENT_GENERATION: old incarnation cannot seal same prefix"
|
||||
);
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(bucket, current, &attempt)
|
||||
.await
|
||||
.expect("current identity seals");
|
||||
drain_before_reopen(&store, &shutdown, bucket).await;
|
||||
drop(store);
|
||||
drop(ctx);
|
||||
let (_ctx, reopened, stop) = open_store(root.path()).await;
|
||||
assert_eq!(
|
||||
reopened
|
||||
.observe_artifact_bucket_incarnation(bucket, PREFIX)
|
||||
.await
|
||||
.expect("reopen observed identity"),
|
||||
current,
|
||||
"ARTIFACT_CREATE_CURRENT_GENERATION"
|
||||
);
|
||||
stop.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial]
|
||||
async fn artifact_bucket_create_observation_rejects_corrupt_authority() {
|
||||
use crate::{
|
||||
config::com::{read_config, save_config_with_opts},
|
||||
object_api::{ObjectOptions, WriteCompletion},
|
||||
};
|
||||
let root = tempfile::tempdir().expect("creation observation faults");
|
||||
let (mut ctx, mut store, mut shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-create-corruption";
|
||||
let incarnation = store
|
||||
.clone()
|
||||
.make_artifact_bucket(bucket, PREFIX)
|
||||
.await
|
||||
.expect("actual create and observation");
|
||||
let original = persisted(&store, bucket).await;
|
||||
let sys = ctx.bucket_metadata_sys().expect("actual metadata authority");
|
||||
// These faults target the authoritative cache branch actually read by the
|
||||
// observation. Public metadata updates reject them before publication.
|
||||
for fault in 0..6 {
|
||||
let mut damaged = original.clone();
|
||||
match fault {
|
||||
0 => damaged.name = "wrong-bucket".into(),
|
||||
1 => damaged.artifact_base_prefix = Some("other/".into()),
|
||||
2 => damaged.versioning_config = None,
|
||||
3 => damaged.versioning_config.as_mut().expect("versioning").exclude_folders = Some(true),
|
||||
4 => damaged.versioning_config = Some(crate::bucket::utils::deserialize(SUSPENDED).expect("suspended XML")),
|
||||
_ => {
|
||||
damaged.versioning_config = Some(crate::bucket::utils::deserialize(EXCLUDED_PREFIX).expect("excluded prefix XML"))
|
||||
}
|
||||
}
|
||||
sys.read().await.set_for_test(bucket.into(), Arc::new(damaged)).await;
|
||||
assert!(
|
||||
store.observe_artifact_bucket_incarnation(bucket, PREFIX).await.is_err(),
|
||||
"ARTIFACT_CREATE_CURRENT_GENERATION: corrupt cached authority fault={fault}"
|
||||
);
|
||||
sys.read().await.set_for_test(bucket.into(), Arc::new(original.clone())).await;
|
||||
}
|
||||
drop(sys);
|
||||
for (file, corrupt, expected_error) in [
|
||||
(BUCKET_METADATA_FILE, b"corrupt metadata".to_vec(), "read_bucket_metadata: format invalid"),
|
||||
(
|
||||
BUCKET_INCARNATION_FILE,
|
||||
Uuid::nil().as_bytes().to_vec(),
|
||||
"persisted bucket incarnation is nil",
|
||||
),
|
||||
] {
|
||||
let path = format!("{BUCKET_META_PREFIX}/{bucket}/{file}");
|
||||
let saved = read_config(store.clone(), &path).await.expect("real persisted record");
|
||||
let opts = ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
};
|
||||
save_config_with_opts(store.clone(), &path, corrupt, &opts)
|
||||
.await
|
||||
.expect("inject physical record fault");
|
||||
// Reopen with a fresh metadata authority so observation must read the
|
||||
// actual damaged record, without a cached value or a negative entry.
|
||||
drain_before_reopen(&store, &shutdown, bucket).await;
|
||||
drop(store);
|
||||
drop(ctx);
|
||||
(ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let error = store
|
||||
.observe_artifact_bucket_incarnation(bucket, PREFIX)
|
||||
.await
|
||||
.expect_err("physical corruption cannot produce a receipt");
|
||||
assert!(
|
||||
error.to_string().contains(expected_error),
|
||||
"ARTIFACT_CREATE_CURRENT_GENERATION: corrupt physical authority {file}: {error}"
|
||||
);
|
||||
save_config_with_opts(store.clone(), &path, saved, &opts)
|
||||
.await
|
||||
.expect("restore physical record");
|
||||
drain_before_reopen(&store, &shutdown, bucket).await;
|
||||
drop(store);
|
||||
drop(ctx);
|
||||
(ctx, store, shutdown) = open_store(root.path()).await;
|
||||
assert_eq!(
|
||||
store
|
||||
.observe_artifact_bucket_incarnation(bucket, PREFIX)
|
||||
.await
|
||||
.expect("restored observation"),
|
||||
incarnation
|
||||
);
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_create_requires_namespace_locking_before_creation() {
|
||||
temp_env::async_with_vars([("RUSTFS_LOCK_ENABLED", Some("false"))], async {
|
||||
let root = tempfile::tempdir().expect("disabled lock storage");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
assert!(ctx.lock_manager().is_disabled());
|
||||
let bucket = "artifact-create-no-lock";
|
||||
let error = store
|
||||
.clone()
|
||||
.make_artifact_bucket(bucket, PREFIX)
|
||||
.await
|
||||
.expect_err("creation requires real locks");
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("artifact bucket creation requires namespace locking")
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.get_bucket_info(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
|
||||
.await
|
||||
.is_err(),
|
||||
"ARTIFACT_CREATE_LOCK_REQUIRED: rejection precedes physical creation"
|
||||
);
|
||||
shutdown.cancel();
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_inspect_reopens_without_migrating_other_buckets() {
|
||||
use crate::config::com::read_config;
|
||||
let root = tempfile::tempdir().expect("inspect storage");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-inspect-reopen";
|
||||
let incarnation = store
|
||||
.clone()
|
||||
.make_artifact_bucket(bucket, PREFIX)
|
||||
.await
|
||||
.expect("managed bucket");
|
||||
let pending = "artifact-inspect-pending";
|
||||
let mut intent = BucketMetadata::new_with_default_durability(pending);
|
||||
intent.artifact_base_prefix = Some(PREFIX.to_owned());
|
||||
intent.versioning_config_xml = ENABLED.to_vec();
|
||||
metadata_sys::set_new_bucket_metadata_intent_in(&ctx, intent)
|
||||
.await
|
||||
.expect("real pending intent");
|
||||
// Physical quorum exists: a migrating reader could silently complete this intent.
|
||||
let legacy = "artifact-inspect-legacy";
|
||||
for disk in store.pools[0].disk_set[0].disk_inventory().await.into_iter().flatten() {
|
||||
for name in [legacy, pending] {
|
||||
disk.make_volume(name).await.expect("real preexisting volume");
|
||||
}
|
||||
}
|
||||
let pending_path = format!("{BUCKET_META_PREFIX}/{pending}/{BUCKET_METADATA_FILE}");
|
||||
let before = read_config(store.clone(), &pending_path).await.expect("pending bytes");
|
||||
drain_before_reopen(&store, &shutdown, bucket).await;
|
||||
drain_before_reopen(&store, &shutdown, pending).await;
|
||||
drop(store);
|
||||
drop(ctx);
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
assert_eq!(
|
||||
store
|
||||
.inspect_artifact_bucket(bucket, PREFIX)
|
||||
.await
|
||||
.expect("fresh-instance inspection"),
|
||||
incarnation
|
||||
);
|
||||
for name in [legacy, pending, "artifact-inspect-absent"] {
|
||||
assert!(store.inspect_artifact_bucket(name, PREFIX).await.is_err(), "ARTIFACT_INSPECT_READ_ONLY");
|
||||
}
|
||||
assert_eq!(
|
||||
read_config(store.clone(), &pending_path).await.expect("pending remains"),
|
||||
before,
|
||||
"ARTIFACT_INSPECT_READ_ONLY"
|
||||
);
|
||||
assert!(!persisted(&store, pending).await.bucket_creation_committed);
|
||||
for name in [legacy, "artifact-inspect-absent"] {
|
||||
for file in [BUCKET_METADATA_FILE, BUCKET_INCARNATION_FILE] {
|
||||
let path = format!("{BUCKET_META_PREFIX}/{name}/{file}");
|
||||
assert!(
|
||||
read_config(store.clone(), &path).await.is_err(),
|
||||
"ARTIFACT_INSPECT_READ_ONLY: no migration record"
|
||||
);
|
||||
}
|
||||
}
|
||||
for disk in 0..4 {
|
||||
assert!(
|
||||
!root
|
||||
.path()
|
||||
.join(format!("pool0/set0/disk{disk}/artifact-inspect-absent"))
|
||||
.exists()
|
||||
);
|
||||
}
|
||||
assert!(store.inspect_artifact_bucket(bucket, "wrong/").await.is_err());
|
||||
assert_eq!(store.inspect_artifact_bucket(bucket, PREFIX).await.unwrap(), incarnation);
|
||||
store
|
||||
.ctx
|
||||
.update_erasure_type(crate::layout::endpoints::SetupType::DistErasure)
|
||||
.await;
|
||||
let error = store
|
||||
.inspect_artifact_bucket(bucket, PREFIX)
|
||||
.await
|
||||
.expect_err("file endpoints cannot use distributed lock mode");
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("artifact PUT sealing requires local storage or a distributed Unix URL topology")
|
||||
);
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_bucket_inspect_requires_namespace_locking() {
|
||||
temp_env::async_with_vars([("RUSTFS_LOCK_ENABLED", Some("false"))], async {
|
||||
let root = tempfile::tempdir().expect("disabled inspect locking");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
assert!(ctx.lock_manager().is_disabled());
|
||||
let error = store
|
||||
.inspect_artifact_bucket("artifact-inspect-no-lock", PREFIX)
|
||||
.await
|
||||
.expect_err("locking required before reading");
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("artifact bucket inspection requires namespace locking")
|
||||
);
|
||||
shutdown.cancel();
|
||||
})
|
||||
.await;
|
||||
}
|
||||
@@ -0,0 +1,398 @@
|
||||
#![cfg(unix)]
|
||||
|
||||
use super::{build_isolated_test_store, run_large_stack_async_test, without_storage_class_env};
|
||||
use crate::{
|
||||
bucket::metadata_sys,
|
||||
disk::{Disk, local::artifact_inventory::ArtifactInventoryFrame},
|
||||
object_api::{ObjectOptions, PutObjReader, ShardIntegrityWriteMode, WriteCompletion},
|
||||
storage_api_contracts::{
|
||||
bucket::{BucketOperations as _, MakeBucketOptions},
|
||||
object::ObjectIO as _,
|
||||
},
|
||||
};
|
||||
use std::time::Duration;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_inventory_managed_put_zero_and_versions_are_complete() {
|
||||
assert_managed_put_inventory_is_complete(ShardIntegrityWriteMode::Legacy);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_inventory_protected_managed_put_versions_are_complete() {
|
||||
assert_managed_put_inventory_is_complete(ShardIntegrityWriteMode::Protected);
|
||||
}
|
||||
|
||||
fn assert_managed_put_inventory_is_complete(mode: ShardIntegrityWriteMode) {
|
||||
run_large_stack_async_test("artifact-inventory-managed", move || async move {
|
||||
let root = tempfile::tempdir().expect("managed inventory disk roots");
|
||||
let (_ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(root.path(), "inventory-managed", &[4])).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "artifact-inventory-managed";
|
||||
store
|
||||
.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
artifact_base_prefix: Some("outputs/".to_owned()),
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("actual managed versioned bucket");
|
||||
let incarnation = store.bucket_incarnation_id(bucket).await.expect("actual incarnation");
|
||||
let prefix = format!("outputs/74656e616e74/{}/{}/", Uuid::new_v4(), Uuid::new_v4());
|
||||
let keys = vec![
|
||||
(format!("{prefix}zero"), Vec::new()),
|
||||
(format!("{prefix}inline"), vec![0xA5; 1024]),
|
||||
(format!("{prefix}payload"), vec![0xA5; 1024 * 1024]),
|
||||
];
|
||||
let mut expected = std::collections::BTreeSet::new();
|
||||
for (key, data) in &keys {
|
||||
let object = store
|
||||
.put_object(
|
||||
bucket,
|
||||
key,
|
||||
&mut PutObjReader::from_vec(data.clone()),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
shard_integrity_write_mode: Some(mode),
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("real managed PUT including empty, inline and sharded payloads");
|
||||
expected.insert((key.clone(), object.version_id.expect("exact managed version")));
|
||||
}
|
||||
let identity = store
|
||||
.artifact_receiver_identity(bucket, incarnation, &prefix, true)
|
||||
.await
|
||||
.expect("metadata-derived identity");
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_writes(bucket, incarnation, &prefix)
|
||||
.await
|
||||
.expect("actual all-writes seal");
|
||||
for member in store.pools[0].disk_set[0].disk_inventory().await {
|
||||
let member = member.expect("configured local member");
|
||||
let Disk::Local(local) = member.as_ref() else {
|
||||
panic!("local fixture");
|
||||
};
|
||||
let local = local.get_disk();
|
||||
let id = local.artifact_disk_id().await.expect("fresh actual disk identity");
|
||||
let (output, mut input) = tokio::sync::mpsc::channel(2);
|
||||
let (result, frames) = tokio::join!(
|
||||
local.inventory_artifact_attempt(
|
||||
id,
|
||||
bucket,
|
||||
&prefix,
|
||||
identity,
|
||||
tokio::time::Instant::now() + Duration::from_secs(30),
|
||||
output
|
||||
),
|
||||
async {
|
||||
let mut frames = Vec::new();
|
||||
while let Some(frame) = input.recv().await {
|
||||
frames.push(frame);
|
||||
}
|
||||
frames
|
||||
}
|
||||
);
|
||||
assert!(result.is_ok(), "A03I1_REAL_MANAGED_PUT_SCAN: {result:?}");
|
||||
let actual = frames
|
||||
.iter()
|
||||
.filter_map(|frame| match frame {
|
||||
ArtifactInventoryFrame::Entries { versions, .. } => Some(versions),
|
||||
_ => None,
|
||||
})
|
||||
.flatten()
|
||||
.map(|version| (version.key.clone(), version.version_id))
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
assert_eq!(actual, expected, "A03I1_REAL_ZERO_INLINE_AND_SHARDED_VERSIONS");
|
||||
assert!(matches!(frames.last(), Some(ArtifactInventoryFrame::Complete { entries: 3, .. })));
|
||||
}
|
||||
shutdown.cancel();
|
||||
});
|
||||
}
|
||||
|
||||
async fn inventory_fixture() -> (
|
||||
tempfile::TempDir,
|
||||
std::sync::Arc<crate::store::ECStore>,
|
||||
tokio_util::sync::CancellationToken,
|
||||
Uuid,
|
||||
String,
|
||||
) {
|
||||
let root = tempfile::tempdir().expect("inventory aggregate roots");
|
||||
let (_ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(root.path(), "inventory-aggregate", &[4])).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
store
|
||||
.make_bucket(
|
||||
"inventory-aggregate",
|
||||
&MakeBucketOptions {
|
||||
artifact_base_prefix: Some("outputs/".into()),
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("managed bucket");
|
||||
let incarnation = store
|
||||
.bucket_incarnation_id("inventory-aggregate")
|
||||
.await
|
||||
.expect("actual incarnation");
|
||||
let prefix = format!("outputs/74656e616e74/{}/{}/", Uuid::new_v4(), Uuid::new_v4());
|
||||
store
|
||||
.put_object(
|
||||
"inventory-aggregate",
|
||||
&format!("{prefix}object"),
|
||||
&mut PutObjReader::from_vec(vec![1; 1024]),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("actual managed object");
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_writes("inventory-aggregate", incarnation, &prefix)
|
||||
.await
|
||||
.expect("actual two-domain seal");
|
||||
(root, store, shutdown, incarnation, prefix)
|
||||
}
|
||||
|
||||
async fn aggregate(
|
||||
store: std::sync::Arc<crate::store::ECStore>,
|
||||
incarnation: Uuid,
|
||||
prefix: &str,
|
||||
) -> (
|
||||
crate::error::Result<()>,
|
||||
Vec<crate::bucket::artifact::inventory::ArtifactAttemptInventoryFrame>,
|
||||
) {
|
||||
let (output, mut receiver) = tokio::sync::mpsc::channel(1);
|
||||
let drain = async {
|
||||
let mut frames = Vec::new();
|
||||
while let Some(frame) = receiver.recv().await {
|
||||
frames.push(frame);
|
||||
}
|
||||
frames
|
||||
};
|
||||
tokio::join!(
|
||||
store.inventory_artifact_attempt(
|
||||
"inventory-aggregate",
|
||||
incarnation,
|
||||
prefix,
|
||||
tokio::time::Instant::now() + Duration::from_secs(30),
|
||||
output
|
||||
),
|
||||
drain
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_inventory_all_members_required() {
|
||||
run_large_stack_async_test("artifact-inventory-all-members", || async {
|
||||
use crate::bucket::artifact::inventory::ArtifactAttemptInventoryFrame;
|
||||
let (_root, store, shutdown, incarnation, prefix) = inventory_fixture().await;
|
||||
let (result, frames) = aggregate(store.clone(), incarnation, &prefix).await;
|
||||
result.expect("all four real sealed disks scanned");
|
||||
let observed: std::collections::BTreeSet<_> = frames
|
||||
.iter()
|
||||
.filter_map(|frame| match frame {
|
||||
ArtifactAttemptInventoryFrame::Observations {
|
||||
endpoint,
|
||||
disk_id,
|
||||
versions,
|
||||
} => {
|
||||
assert_eq!(versions.len(), 1);
|
||||
assert_eq!(versions[0].key, format!("{prefix}object"));
|
||||
Some((endpoint.clone(), *disk_id))
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(observed.len(), 4, "A03I2_ALL_MEMBERS");
|
||||
assert!(
|
||||
matches!(
|
||||
frames.last(),
|
||||
Some(ArtifactAttemptInventoryFrame::Complete {
|
||||
members: 4,
|
||||
observations: 4
|
||||
})
|
||||
),
|
||||
"A03I2_ALL_MEMBERS"
|
||||
);
|
||||
let original = store.pools[0].disk_set[0].disks.write().await[3].take();
|
||||
let (denied, frames) = aggregate(store.clone(), incarnation, &prefix).await;
|
||||
assert!(denied.is_err(), "one missing configured member is incomplete");
|
||||
assert!(
|
||||
!frames
|
||||
.iter()
|
||||
.any(|frame| matches!(frame, ArtifactAttemptInventoryFrame::Complete { .. }))
|
||||
);
|
||||
store.pools[0].disk_set[0].disks.write().await[3] = original;
|
||||
shutdown.cancel();
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_inventory_membership_change_is_incomplete() {
|
||||
run_large_stack_async_test("artifact-inventory-membership", || async {
|
||||
use crate::bucket::artifact::inventory::ArtifactAttemptInventoryFrame;
|
||||
let (_root, store, shutdown, incarnation, prefix) = inventory_fixture().await;
|
||||
let original = store.pools[0].disk_set[0].disk_inventory().await[0]
|
||||
.clone()
|
||||
.expect("first member");
|
||||
let Disk::Local(local) = original.as_ref() else {
|
||||
panic!("local fixture");
|
||||
};
|
||||
let replacement = std::sync::Arc::new(Disk::Local(local.clone()));
|
||||
let (output, mut receiver) = tokio::sync::mpsc::channel(1);
|
||||
let owned = store.clone();
|
||||
let worker = tokio::spawn(async move {
|
||||
owned
|
||||
.inventory_artifact_attempt(
|
||||
"inventory-aggregate",
|
||||
incarnation,
|
||||
&prefix,
|
||||
tokio::time::Instant::now() + Duration::from_secs(30),
|
||||
output,
|
||||
)
|
||||
.await
|
||||
});
|
||||
assert!(matches!(receiver.recv().await, Some(ArtifactAttemptInventoryFrame::Observations { .. })));
|
||||
store.pools[0].disk_set[0].disks.write().await[0] = Some(replacement);
|
||||
let mut complete = false;
|
||||
while let Some(frame) = receiver.recv().await {
|
||||
complete |= matches!(frame, ArtifactAttemptInventoryFrame::Complete { .. });
|
||||
}
|
||||
let error = worker
|
||||
.await
|
||||
.expect("aggregate worker")
|
||||
.expect_err("same endpoint/UUID but changed captured Arc is incomplete");
|
||||
assert!(error.to_string().contains("membership changed"), "{error}");
|
||||
assert!(!complete);
|
||||
store.pools[0].disk_set[0].disks.write().await[0] = Some(original);
|
||||
shutdown.cancel();
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_inventory_readonly_authority_cold_cache_and_queued_writer() {
|
||||
run_large_stack_async_test("artifact-inventory-readonly-authority", || async {
|
||||
use crate::{
|
||||
disk::{BUCKET_META_PREFIX, DiskAPI as _, RUSTFS_META_BUCKET},
|
||||
storage_api_contracts::namespace::NamespaceLocking as _,
|
||||
};
|
||||
let (_root, store, shutdown, incarnation, prefix) = inventory_fixture().await;
|
||||
let member = store.pools[0].disk_set[0].disk_inventory().await[0]
|
||||
.clone()
|
||||
.expect("real receiver");
|
||||
let Disk::Local(local) = member.as_ref() else {
|
||||
panic!("local");
|
||||
};
|
||||
let expected = local.artifact_disk_id().await.expect("actual disk");
|
||||
let _lifecycle = store
|
||||
.acquire_bucket_lifecycle_read_lock("inventory-aggregate")
|
||||
.await
|
||||
.expect("coordinator lifecycle owner");
|
||||
metadata_sys::remove_bucket_metadata_in(&store.ctx, "inventory-aggregate")
|
||||
.await
|
||||
.expect("cold metadata cache");
|
||||
let lock = store
|
||||
.new_ns_lock("inventory-aggregate", "inventory-aggregate")
|
||||
.await
|
||||
.expect("business bucket namespace");
|
||||
let reader = lock
|
||||
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await
|
||||
.expect("existing namespace reader");
|
||||
let mut writer = Box::pin(lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout()));
|
||||
assert!(
|
||||
matches!(futures::poll!(writer.as_mut()), std::task::Poll::Pending),
|
||||
"real queued business bucket WRITE"
|
||||
);
|
||||
let (output, mut frames) = tokio::sync::mpsc::channel(1);
|
||||
let drain = async { while frames.recv().await.is_some() {} };
|
||||
let endpoint = member.endpoint().to_string();
|
||||
let receive = store.inventory_local_artifact_attempt(
|
||||
(&endpoint, expected),
|
||||
("inventory-aggregate", incarnation),
|
||||
&prefix,
|
||||
rustfs_protos::proto_gen::node_service::ArtifactInventoryPhase::ArtifactInventoryScan,
|
||||
tokio::time::Instant::now() + Duration::from_secs(2),
|
||||
output,
|
||||
);
|
||||
let (result, ()) = tokio::join!(receive, drain);
|
||||
result.expect("A03I2_READONLY_AUTHORITY_NO_REENTRY");
|
||||
drop(writer);
|
||||
drop(reader);
|
||||
drop(_lifecycle);
|
||||
// A legacy physical bucket lacking both authoritative records must
|
||||
// stay legacy after rejected inventory; no implicit getter migration.
|
||||
store
|
||||
.make_bucket(
|
||||
"inventory-legacy",
|
||||
&MakeBucketOptions {
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("ordinary legacy fixture");
|
||||
let members = store.pools[0].disk_set[0].disk_inventory().await;
|
||||
for member in members.iter().flatten() {
|
||||
for file in [".metadata.bin", ".bucket-incarnation"] {
|
||||
let path = member
|
||||
.path()
|
||||
.join(RUSTFS_META_BUCKET)
|
||||
.join(BUCKET_META_PREFIX)
|
||||
.join("inventory-legacy")
|
||||
.join(file);
|
||||
if path.exists() {
|
||||
std::fs::remove_dir_all(&path).expect("remove fixture-only creation facts");
|
||||
}
|
||||
}
|
||||
}
|
||||
metadata_sys::remove_bucket_metadata_in(&store.ctx, "inventory-legacy")
|
||||
.await
|
||||
.expect("cold legacy cache");
|
||||
let (output, mut frames) = tokio::sync::mpsc::channel(1);
|
||||
let drain = async { while frames.recv().await.is_some() {} };
|
||||
let (denied, ()) = tokio::join!(
|
||||
store.clone().inventory_artifact_attempt(
|
||||
"inventory-legacy",
|
||||
Uuid::new_v4(),
|
||||
&prefix,
|
||||
tokio::time::Instant::now() + Duration::from_secs(2),
|
||||
output
|
||||
),
|
||||
drain
|
||||
);
|
||||
assert!(denied.is_err());
|
||||
for member in members.iter().flatten() {
|
||||
for file in [".metadata.bin", ".bucket-incarnation"] {
|
||||
assert!(
|
||||
!member
|
||||
.path()
|
||||
.join(RUSTFS_META_BUCKET)
|
||||
.join(BUCKET_META_PREFIX)
|
||||
.join("inventory-legacy")
|
||||
.join(file)
|
||||
.exists(),
|
||||
"A03I2_READONLY_NO_MIGRATION"
|
||||
);
|
||||
}
|
||||
}
|
||||
shutdown.cancel();
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,587 @@
|
||||
use super::{build_isolated_test_store, without_storage_class_env};
|
||||
use crate::{
|
||||
bucket::{artifact::artifact_put_seal_record_path, metadata_sys},
|
||||
config::com,
|
||||
disk::{DiskAPI as _, RUSTFS_META_BUCKET},
|
||||
object_api::{ObjectInfo, ObjectOptions, PutObjReader, WriteCompletion},
|
||||
runtime::instance::InstanceContext,
|
||||
set_disk::{PutObjectCommitBarrier, PutObjectCommitPause, rename_fanout_barrier},
|
||||
storage_api_contracts::{
|
||||
bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions},
|
||||
namespace::NamespaceLocking as _,
|
||||
object::ObjectIO as _,
|
||||
},
|
||||
store::ECStore,
|
||||
};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use uuid::Uuid;
|
||||
|
||||
const DEADLINE: Duration = Duration::from_secs(30);
|
||||
|
||||
fn attempt_prefix() -> String {
|
||||
format!("outputs/74656e616e74/{}/{}/", Uuid::new_v4(), Uuid::new_v4())
|
||||
}
|
||||
|
||||
async fn open_store(root: &std::path::Path) -> (Arc<InstanceContext>, Arc<ECStore>, CancellationToken) {
|
||||
let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(root, "artifact-put-seal", &[4])).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
(ctx, store, shutdown)
|
||||
}
|
||||
|
||||
async fn create_bucket(store: &ECStore, bucket: &str) -> Uuid {
|
||||
store
|
||||
.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
artifact_base_prefix: Some("outputs/".to_owned()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create a real versioned artifact bucket");
|
||||
store
|
||||
.bucket_incarnation_id(bucket)
|
||||
.await
|
||||
.expect("read actual bucket incarnation")
|
||||
}
|
||||
|
||||
async fn put(store: &ECStore, bucket: &str, object: &str) -> ObjectInfo {
|
||||
store
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![0xA5; 273]),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("unsealed PUT must commit on all four disks")
|
||||
}
|
||||
|
||||
async fn disk_metadata(store: &ECStore, bucket: &str, object: &str) -> Vec<Vec<u8>> {
|
||||
let disks = store.pools[0].disk_set[0].disk_inventory().await;
|
||||
assert_eq!(disks.len(), 4);
|
||||
let mut metadata = Vec::new();
|
||||
for disk in disks {
|
||||
metadata.push(
|
||||
disk.expect("real fixture disk is online")
|
||||
.read_xl(bucket, object, false)
|
||||
.await
|
||||
.expect("read actual per-disk versions")
|
||||
.buf,
|
||||
);
|
||||
}
|
||||
metadata
|
||||
}
|
||||
|
||||
async fn assert_rejected_without_new_version(store: &ECStore, bucket: &str, object: &str, before: &[Vec<u8>]) {
|
||||
let result = store
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![0xBB; 379]),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_err(), "ARTIFACT_PUT_SEALED: later PUT must be rejected");
|
||||
assert_eq!(
|
||||
disk_metadata(store, bucket, object).await,
|
||||
before,
|
||||
"ARTIFACT_PUT_SEALED: no disk gains a version"
|
||||
);
|
||||
}
|
||||
|
||||
async fn tail_barrier_case(cancel_put_waiter: bool) {
|
||||
let root = tempfile::tempdir().expect("artifact tail fixture directory");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-put-tail";
|
||||
let incarnation = create_bucket(&store, bucket).await;
|
||||
let prefix = attempt_prefix();
|
||||
let object = format!("{prefix}image.png");
|
||||
temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
|
||||
let tracker = rename_fanout_barrier::observe_tasks(&object);
|
||||
let tail = rename_fanout_barrier::arm(&object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let quorum = PutObjectCommitBarrier::install(bucket, &object, PutObjectCommitPause::AfterRenameQuorum);
|
||||
let handoff = PutObjectCommitBarrier::install(bucket, &object, PutObjectCommitPause::AfterRenameHandoff);
|
||||
let put_store = store.clone();
|
||||
let put_object = object.clone();
|
||||
let put_task = tokio::spawn(async move {
|
||||
put_store
|
||||
.put_object(
|
||||
bucket,
|
||||
&put_object,
|
||||
&mut PutObjReader::from_vec(vec![0xA5; 273]),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(DEADLINE, tail.wait_until_paused())
|
||||
.await
|
||||
.expect("one real disk pauses before rename");
|
||||
quorum.wait_until_paused().await;
|
||||
if cancel_put_waiter {
|
||||
put_task.abort();
|
||||
}
|
||||
quorum.release();
|
||||
handoff.wait_until_paused().await;
|
||||
handoff.release();
|
||||
let result = tokio::time::timeout(DEADLINE, put_task)
|
||||
.await
|
||||
.expect("PUT waiter returns while its tail remains paused");
|
||||
if cancel_put_waiter {
|
||||
assert!(result.expect_err("cancelled PUT waiter").is_cancelled());
|
||||
} else {
|
||||
result
|
||||
.expect("PUT task joins")
|
||||
.expect("PUT quorum ACK succeeds before tail completion");
|
||||
}
|
||||
assert!(tracker.running() > 0, "real rename tail must still be live");
|
||||
let parallel_object = format!("{prefix}parallel.png");
|
||||
let parallel = tokio::time::timeout(DEADLINE, put(&store, bucket, ¶llel_object))
|
||||
.await
|
||||
.expect("ARTIFACT_PUT_SHARED_ADMISSION: another object in the same attempt completes while A is paused");
|
||||
let mut reader = tokio::time::timeout(
|
||||
DEADLINE,
|
||||
store.get_object_reader(
|
||||
bucket,
|
||||
¶llel_object,
|
||||
None,
|
||||
Default::default(),
|
||||
&ObjectOptions {
|
||||
version_id: Some(parallel.version_id.expect("parallel PUT has a real version").to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("parallel object GET completes while A is paused")
|
||||
.expect("parallel object is readable");
|
||||
let mut bytes = Vec::new();
|
||||
tokio::time::timeout(DEADLINE, reader.stream.read_to_end(&mut bytes))
|
||||
.await
|
||||
.expect("parallel object body completes while A is paused")
|
||||
.expect("read parallel object body");
|
||||
assert_eq!(bytes, vec![0xA5; 273], "ARTIFACT_PUT_SHARED_ADMISSION: B publishes exact bytes");
|
||||
drop(reader);
|
||||
assert!(tracker.running() > 0, "A's physical tail remains paused throughout B's PUT and GET");
|
||||
let record = artifact_put_seal_record_path(bucket, incarnation, &prefix);
|
||||
let attempt_lock = store
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &record)
|
||||
.await
|
||||
.expect("resolve the actual PUT and seal coordination namespace");
|
||||
assert!(
|
||||
attempt_lock.get_write_lock(Duration::from_secs(1)).await.is_err(),
|
||||
"ARTIFACT_PUT_TAIL_BARRIER: the real commit owner retains the attempt lock until its disk tail drains"
|
||||
);
|
||||
let seal_store = store.clone();
|
||||
let seal_prefix = prefix.clone();
|
||||
let mut seal =
|
||||
tokio::spawn(async move { seal_store.seal_artifact_attempt_puts(bucket, incarnation, &seal_prefix).await });
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(200), &mut seal).await.is_err(),
|
||||
"ARTIFACT_PUT_TAIL_BARRIER: seal cannot finish while a real rename tail is paused"
|
||||
);
|
||||
assert!(tracker.running() > 0, "tail remains paused throughout the seal attempt");
|
||||
tail.release();
|
||||
tokio::time::timeout(DEADLINE, seal)
|
||||
.await
|
||||
.expect("seal resumes after tail release")
|
||||
.expect("seal task joins")
|
||||
.expect("seal persists");
|
||||
tokio::time::timeout(DEADLINE, async {
|
||||
while tracker.running() != 0 || ctx.namespace_commits_pending() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("actual detached write owners drain");
|
||||
let before = disk_metadata(&store, bucket, &object).await;
|
||||
assert_rejected_without_new_version(&store, bucket, &object, &before).await;
|
||||
})
|
||||
.await;
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_put_seal_waits_for_quorum_ack_rename_tail() {
|
||||
tail_barrier_case(false).await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_put_seal_waits_after_put_waiter_cancellation() {
|
||||
tail_barrier_case(true).await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_put_seal_survives_reopen_and_is_attempt_scoped() {
|
||||
let root = tempfile::tempdir().expect("artifact durable seal directory");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-put-reopen";
|
||||
let incarnation = create_bucket(&store, bucket).await;
|
||||
let prefix = attempt_prefix();
|
||||
let object = format!("{prefix}image.png");
|
||||
put(&store, bucket, &object).await;
|
||||
let before = disk_metadata(&store, bucket, &object).await;
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(bucket, incarnation, &prefix)
|
||||
.await
|
||||
.expect("persist first seal");
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(bucket, incarnation, &prefix)
|
||||
.await
|
||||
.expect("same seal is idempotent");
|
||||
assert_rejected_without_new_version(&store, bucket, &object, &before).await;
|
||||
put(&store, bucket, &format!("{}other.png", attempt_prefix())).await;
|
||||
shutdown.cancel();
|
||||
assert!(
|
||||
ctx.wait_for_detached_mutations(DEADLINE).await,
|
||||
"old instance mutations drain before reopen"
|
||||
);
|
||||
drop(store);
|
||||
drop(ctx);
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
metadata_sys::reload_bucket_metadata(store.clone(), bucket)
|
||||
.await
|
||||
.expect("reload dedicated bucket after restart");
|
||||
assert_eq!(store.bucket_incarnation_id(bucket).await.expect("reopened incarnation"), incarnation);
|
||||
assert_rejected_without_new_version(&store, bucket, &object, &before).await;
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(bucket, incarnation, &prefix)
|
||||
.await
|
||||
.expect("persisted seal remains idempotent");
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_put_seal_rejects_corrupt_and_wrong_incarnation_records() {
|
||||
super::run_large_stack_async_test("artifact-put-corrupt", artifact_put_corrupt_record_case);
|
||||
}
|
||||
|
||||
async fn artifact_put_corrupt_record_case() {
|
||||
let root = tempfile::tempdir().expect("artifact invalid seal record directory");
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-put-corrupt";
|
||||
let prefix = attempt_prefix();
|
||||
let old_incarnation = create_bucket(&store, bucket).await;
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(bucket, old_incarnation, &prefix)
|
||||
.await
|
||||
.expect("produce a real previous-generation seal");
|
||||
let old_record = com::read_config(store.clone(), &artifact_put_seal_record_path(bucket, old_incarnation, &prefix))
|
||||
.await
|
||||
.expect("read actual seal bytes");
|
||||
store
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete empty old-generation bucket");
|
||||
let incarnation = create_bucket(&store, bucket).await;
|
||||
assert_ne!(incarnation, old_incarnation);
|
||||
assert!(
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(bucket, old_incarnation, &prefix)
|
||||
.await
|
||||
.is_err(),
|
||||
"wrong expected incarnation cannot seal current bucket"
|
||||
);
|
||||
let object = format!("{prefix}image.png");
|
||||
put(&store, bucket, &object).await;
|
||||
let before = disk_metadata(&store, bucket, &object).await;
|
||||
let record = artifact_put_seal_record_path(bucket, incarnation, &prefix);
|
||||
for invalid_record in [old_record, Vec::new(), b"not-json".to_vec()] {
|
||||
com::save_config_with_opts(
|
||||
store.clone(),
|
||||
&record,
|
||||
invalid_record,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("persist actual invalid seal record on all disks");
|
||||
assert_rejected_without_new_version(&store, bucket, &object, &before).await;
|
||||
assert!(
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(bucket, incarnation, &prefix)
|
||||
.await
|
||||
.is_err(),
|
||||
"invalid seal cannot be treated as absent or idempotent success"
|
||||
);
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_put_seal_does_not_change_ordinary_bucket_puts() {
|
||||
let root = tempfile::tempdir().expect("ordinary bucket seal regression directory");
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-put-ordinary";
|
||||
store
|
||||
.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create ordinary versioned bucket");
|
||||
let incarnation = store.bucket_incarnation_id(bucket).await.expect("ordinary incarnation");
|
||||
let prefix = attempt_prefix();
|
||||
let object = format!("{prefix}image.png");
|
||||
let first = put(&store, bucket, &object).await;
|
||||
assert!(
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(bucket, incarnation, &prefix)
|
||||
.await
|
||||
.is_err(),
|
||||
"ordinary bucket cannot acquire artifact seal semantics"
|
||||
);
|
||||
let second = put(&store, bucket, &object).await;
|
||||
assert_ne!(first.version_id, second.version_id, "ordinary versioned PUT creates another version");
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_put_seal_lock_survives_physical_publication_timeout() {
|
||||
// Match large ECStore fixtures while retaining the current-thread clock required by pause/advance.
|
||||
std::thread::Builder::new()
|
||||
.name("artifact-put-physical-timeout".to_owned())
|
||||
.stack_size(32 * 1024 * 1024)
|
||||
.spawn(|| {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("physical timeout test runtime should build");
|
||||
runtime.block_on(artifact_put_physical_timeout_case());
|
||||
})
|
||||
.expect("physical timeout test thread should spawn")
|
||||
.join()
|
||||
.expect("physical timeout test thread should complete");
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
async fn artifact_put_physical_timeout_case() {
|
||||
use crate::disk::{Disk, ReadOptions, STORAGE_FORMAT_FILE, os};
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("60")),
|
||||
(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("false")),
|
||||
],
|
||||
async {
|
||||
let root = tempfile::tempdir().expect("artifact physical timeout directory");
|
||||
let (_ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-put-physical-timeout";
|
||||
let incarnation = create_bucket(&store, bucket).await;
|
||||
let prefix = attempt_prefix();
|
||||
let object = format!("{prefix}image.png");
|
||||
put(&store, bucket, &object).await;
|
||||
let disks = store.pools[0].disk_set[0].disk_inventory().await;
|
||||
let mut hooks = Vec::new();
|
||||
let mut entered = Vec::new();
|
||||
let mut releases = Vec::new();
|
||||
let mut destinations = Vec::new();
|
||||
for disk in disks.iter().flatten() {
|
||||
let Disk::Local(local) = disk.as_ref() else {
|
||||
panic!("physical timeout fixture requires local disks");
|
||||
};
|
||||
// Resolve through the mount lease; Linux may use an FD-based path.
|
||||
let destination = local
|
||||
.get_disk()
|
||||
.get_object_path_for_io(bucket, &object)
|
||||
.expect("real publication path");
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
hooks.push(os::prepared_publication_test_hooks::install(
|
||||
&destination.join(STORAGE_FORMAT_FILE),
|
||||
move || {
|
||||
let _ = entered_tx.send(());
|
||||
// Dropping the sender also releases the syscall during panic unwinding.
|
||||
let _ = release_rx.recv();
|
||||
},
|
||||
));
|
||||
entered.push(entered_rx);
|
||||
releases.push(release_tx);
|
||||
destinations.push(destination);
|
||||
}
|
||||
assert_eq!(entered.len(), 4, "all four real disks must be online");
|
||||
let version = Uuid::new_v4();
|
||||
let put_store = store.clone();
|
||||
let put_object = object.clone();
|
||||
let mut replacement = tokio::spawn(async move {
|
||||
put_store
|
||||
.put_object(
|
||||
bucket,
|
||||
&put_object,
|
||||
&mut PutObjReader::from_vec(vec![0xCC; 379]),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(version.to_string()),
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(DEADLINE, async {
|
||||
tokio::select! {
|
||||
signals = futures::future::join_all(entered) => assert!(signals.into_iter().all(|signal| signal.is_ok())),
|
||||
_ = &mut replacement => panic!("PUT cannot finish before its physical publishers are paused"),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("all physical xl.meta publications reach the blocking syscall");
|
||||
// All disk wrapper timers exist, but their physical closures stay blocked.
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(61)).await;
|
||||
tokio::time::resume();
|
||||
let result = tokio::time::timeout(DEADLINE, replacement)
|
||||
.await
|
||||
.expect("PUT returns after disk wrapper timeouts")
|
||||
.expect("PUT task joins");
|
||||
assert!(result.is_err(), "four timed-out publishers cannot satisfy write quorum");
|
||||
let record = artifact_put_seal_record_path(bucket, incarnation, &prefix);
|
||||
let attempt_lock = store
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &record)
|
||||
.await
|
||||
.expect("actual attempt coordination namespace");
|
||||
let physical_owner_kept_lock = attempt_lock.get_write_lock(Duration::from_secs(1)).await.is_err();
|
||||
|
||||
// Drain physical syscalls before the regression assertion, including a RED run.
|
||||
drop(releases);
|
||||
for (disk, destination) in disks.iter().flatten().zip(&destinations) {
|
||||
drop(
|
||||
tokio::time::timeout(DEADLINE, os::acquire_rename_data_mutation_lease(&disk.path(), bucket, destination))
|
||||
.await
|
||||
.expect("released physical publisher drains"),
|
||||
);
|
||||
let published = disk
|
||||
.read_version("", bucket, &object, &version.to_string(), &ReadOptions::default())
|
||||
.await
|
||||
.expect("timed-out physical writer actually publishes the exact new version");
|
||||
assert_eq!(published.size, 379);
|
||||
}
|
||||
assert!(
|
||||
physical_owner_kept_lock,
|
||||
"ARTIFACT_PUT_TAIL_BARRIER: disk workers retain the attempt lock after the PUT coordinator returns its timeout"
|
||||
);
|
||||
tokio::time::timeout(DEADLINE, store.clone().seal_artifact_attempt_puts(bucket, incarnation, &prefix))
|
||||
.await
|
||||
.expect("seal resumes after physical owners drain")
|
||||
.expect("seal persists after timeout recovery");
|
||||
let before = disk_metadata(&store, bucket, &object).await;
|
||||
assert_rejected_without_new_version(&store, bucket, &object, &before).await;
|
||||
shutdown.cancel();
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn artifact_put_seal_rejects_mismatched_topology_and_preserves_ordinary_local_puts() {
|
||||
use crate::layout::endpoints::SetupType;
|
||||
|
||||
const TOPOLOGY_ERROR: &str = "artifact PUT sealing requires local storage or a distributed Unix URL topology";
|
||||
let root = tempfile::tempdir().expect("artifact distributed topology directory");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
ctx.update_erasure_type(SetupType::Erasure).await;
|
||||
let bucket = "artifact-put-distributed";
|
||||
let incarnation = create_bucket(&store, bucket).await;
|
||||
let ordinary = "artifact-put-distributed-ordinary";
|
||||
store
|
||||
.make_bucket(
|
||||
ordinary,
|
||||
&MakeBucketOptions {
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create ordinary versioned control bucket");
|
||||
let ordinary_incarnation = store.bucket_incarnation_id(ordinary).await.expect("ordinary identity");
|
||||
let prefix = attempt_prefix();
|
||||
let object = format!("{prefix}image.png");
|
||||
put(&store, bucket, &object).await;
|
||||
let before = disk_metadata(&store, bucket, &object).await;
|
||||
let first = put(&store, ordinary, &object).await;
|
||||
|
||||
// Keep every physical disk online: the gate must inspect configured topology.
|
||||
ctx.update_erasure_type(SetupType::DistErasure).await;
|
||||
assert!(ctx.is_dist_erasure().await);
|
||||
let rejected = store
|
||||
.put_object(
|
||||
bucket,
|
||||
&object,
|
||||
&mut PutObjReader::from_vec(vec![0xDD; 379]),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("file endpoints with distributed lock mode must be rejected before publication");
|
||||
assert!(rejected.to_string().contains(TOPOLOGY_ERROR), "ARTIFACT_PUT_TOPOLOGY: {rejected}");
|
||||
assert_eq!(
|
||||
disk_metadata(&store, bucket, &object).await,
|
||||
before,
|
||||
"ARTIFACT_PUT_TOPOLOGY: no disk gains a version"
|
||||
);
|
||||
let rejected = store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(bucket, incarnation, &prefix)
|
||||
.await
|
||||
.expect_err("mismatched topology cannot claim a durable PUT seal");
|
||||
assert!(rejected.to_string().contains(TOPOLOGY_ERROR), "ARTIFACT_PUT_TOPOLOGY: {rejected}");
|
||||
// This local fixture has no distributed lock clients; ordinary PUT retains its local regression check.
|
||||
ctx.update_erasure_type(SetupType::Erasure).await;
|
||||
let second = put(&store, ordinary, &object).await;
|
||||
assert_ne!(
|
||||
first.version_id, second.version_id,
|
||||
"ordinary local PUT stays available after artifact rejection"
|
||||
);
|
||||
assert_eq!(
|
||||
store
|
||||
.bucket_incarnation_id(ordinary)
|
||||
.await
|
||||
.expect("ordinary identity remains available"),
|
||||
ordinary_incarnation
|
||||
);
|
||||
assert_eq!(
|
||||
store
|
||||
.bucket_incarnation_id(bucket)
|
||||
.await
|
||||
.expect("artifact identity remains available"),
|
||||
incarnation
|
||||
);
|
||||
shutdown.cancel();
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,689 @@
|
||||
#![cfg(unix)]
|
||||
|
||||
use super::{build_isolated_test_store, run_large_stack_async_test, without_storage_class_env};
|
||||
use crate::{
|
||||
bucket::{artifact::artifact_version_retirement_record_path, metadata_sys},
|
||||
config::com,
|
||||
disk::{DiskAPI as _, DiskStore, RUSTFS_META_BUCKET, ReadOptions, STORAGE_FORMAT_FILE, os},
|
||||
error::Error,
|
||||
object_api::{ObjectOptions, PutObjReader, ShardIntegrityWriteMode, WriteCompletion},
|
||||
runtime::instance::InstanceContext,
|
||||
storage_api_contracts::{
|
||||
bucket::{BucketOperations as _, MakeBucketOptions},
|
||||
heal::HealOperations,
|
||||
object::ObjectIO as _,
|
||||
},
|
||||
store::ECStore,
|
||||
};
|
||||
use rustfs_heal_contracts::heal_channel::{HealOpts, HealScanMode};
|
||||
use rustfs_utils::path::encode_dir_object;
|
||||
use std::{
|
||||
path::{Path, PathBuf},
|
||||
sync::Arc,
|
||||
time::Duration,
|
||||
};
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use uuid::Uuid;
|
||||
|
||||
const DEADLINE: Duration = Duration::from_secs(30);
|
||||
const PAYLOAD_SIZE: usize = 1024 * 1024;
|
||||
|
||||
fn attempt_prefix() -> String {
|
||||
format!("outputs/74656e616e74/{}/{}/", Uuid::new_v4(), Uuid::new_v4())
|
||||
}
|
||||
|
||||
fn put_options() -> ObjectOptions {
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn heal_options() -> HealOpts {
|
||||
HealOpts {
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
async fn open_store(root: &std::path::Path) -> (Arc<InstanceContext>, Arc<ECStore>, CancellationToken) {
|
||||
let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(root, "artifact-retirement", &[4])).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
(ctx, store, shutdown)
|
||||
}
|
||||
|
||||
async fn create_bucket(store: &ECStore, bucket: &str, artifact: bool) -> Uuid {
|
||||
store
|
||||
.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
artifact_base_prefix: artifact.then(|| "outputs/".to_owned()),
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create real four-disk versioned bucket");
|
||||
store
|
||||
.bucket_incarnation_id(bucket)
|
||||
.await
|
||||
.expect("read actual bucket incarnation")
|
||||
}
|
||||
|
||||
async fn disks(store: &ECStore) -> Vec<DiskStore> {
|
||||
let disks = store.pools[0].disk_set[0]
|
||||
.disk_inventory()
|
||||
.await
|
||||
.into_iter()
|
||||
.map(|disk| disk.expect("fixture disk remains online"))
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(disks.len(), 4);
|
||||
disks
|
||||
}
|
||||
|
||||
fn object_path(disk: &DiskStore, bucket: &str, object: &str) -> PathBuf {
|
||||
disk.get_object_path_for_io_if_local(bucket, &encode_dir_object(object))
|
||||
.expect("fixture uses local disks")
|
||||
.expect("resolve object through held mount")
|
||||
}
|
||||
|
||||
async fn file_bytes(path: &Path) -> Option<Vec<u8>> {
|
||||
match tokio::fs::read(path).await {
|
||||
Ok(bytes) => Some(bytes),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
|
||||
Err(error) => panic!("read actual fixture file: {error}"),
|
||||
}
|
||||
}
|
||||
|
||||
async fn metadata(store: &ECStore, bucket: &str, object: &str) -> Vec<Option<Vec<u8>>> {
|
||||
let mut result = Vec::new();
|
||||
for disk in disks(store).await {
|
||||
let path = object_path(&disk, bucket, object).join(STORAGE_FORMAT_FILE);
|
||||
result.push(file_bytes(&path).await);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
async fn put_payload(store: &ECStore, bucket: &str, object: &str, byte: u8, legacy: bool) -> Uuid {
|
||||
let mut options = put_options();
|
||||
if legacy {
|
||||
// Exercise genuine supported minority metadata recovery, without fabricated proofs.
|
||||
options.shard_integrity_write_mode = Some(ShardIntegrityWriteMode::Legacy);
|
||||
}
|
||||
let body = if object.ends_with('/') {
|
||||
Vec::new()
|
||||
} else {
|
||||
vec![byte; PAYLOAD_SIZE]
|
||||
};
|
||||
store
|
||||
.put_object(bucket, object, &mut PutObjReader::from_vec(body), &options)
|
||||
.await
|
||||
.expect("commit exact payload and version to all four disks")
|
||||
.version_id
|
||||
.expect("artifact PUT returns a non-null version")
|
||||
}
|
||||
|
||||
async fn assert_payload(store: &ECStore, bucket: &str, object: &str, version: Uuid, byte: u8) {
|
||||
let mut reader = store
|
||||
.get_object_reader(
|
||||
bucket,
|
||||
object,
|
||||
None,
|
||||
Default::default(),
|
||||
&ObjectOptions {
|
||||
version_id: Some(version.to_string()),
|
||||
..put_options()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("retirement leaves the exact committed version readable");
|
||||
assert_eq!(reader.object_info.version_id, Some(version), "GET preserves exact version identity");
|
||||
let mut body = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("read exact committed bytes");
|
||||
let expected = if object.ends_with('/') {
|
||||
Vec::new()
|
||||
} else {
|
||||
vec![byte; PAYLOAD_SIZE]
|
||||
};
|
||||
assert_eq!(body, expected, "retirement does not delete or replace committed bytes");
|
||||
}
|
||||
|
||||
async fn remove_shard(store: &ECStore, bucket: &str, object: &str, version: Uuid) -> (DiskStore, PathBuf) {
|
||||
let disk = disks(store).await.remove(0);
|
||||
let info = disk
|
||||
.read_version("", bucket, &encode_dir_object(object), &version.to_string(), &ReadOptions::default())
|
||||
.await
|
||||
.expect("read exact version before removing a real shard");
|
||||
assert_eq!(info.version_id, Some(version));
|
||||
let part = object_path(&disk, bucket, object)
|
||||
.join(info.data_dir.expect("non-inline object has a data directory").to_string())
|
||||
.join("part.1");
|
||||
tokio::fs::remove_file(&part).await.expect("remove one real payload shard");
|
||||
assert!(!part.exists());
|
||||
(disk, part)
|
||||
}
|
||||
|
||||
async fn assert_healthy_version(store: &ECStore, bucket: &str, object: &str, version: Uuid) {
|
||||
for disk in disks(store).await {
|
||||
let info = disk
|
||||
.read_version("", bucket, &encode_dir_object(object), &version.to_string(), &ReadOptions::default())
|
||||
.await
|
||||
.expect("repaired disk retains exact committed metadata");
|
||||
assert_eq!(info.version_id, Some(version));
|
||||
assert_eq!(info.size, i64::try_from(PAYLOAD_SIZE).expect("payload size fits i64"));
|
||||
let checked = disk
|
||||
.check_parts(bucket, &encode_dir_object(object), &info)
|
||||
.await
|
||||
.expect("verify genuine repaired shard");
|
||||
assert!(!checked.results.is_empty());
|
||||
assert!(
|
||||
checked
|
||||
.results
|
||||
.iter()
|
||||
.all(|result| *result == crate::disk::CHECK_PART_SUCCESS),
|
||||
"unretired version's real repaired shard passes integrity verification"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_policy_error(error: &Error) {
|
||||
assert!(
|
||||
!matches!(
|
||||
error,
|
||||
Error::FileNotFound | Error::FileVersionNotFound | Error::ObjectNotFound(_, _) | Error::VersionNotFound(_, _, _)
|
||||
),
|
||||
"retirement policy refusal must not claim physical absence: {error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
async fn assert_repair_rejected(store: &ECStore, bucket: &str, object: &str, version: &str) -> Error {
|
||||
let result = HealOperations::heal_object(store, bucket, object, version, &heal_options()).await;
|
||||
let error = match result {
|
||||
Err(error) | Ok((_, Some(error))) => error,
|
||||
Ok((item, None)) => {
|
||||
panic!("ARTIFACT_RETIREMENT_NO_RESURRECTION: retired exact version was accepted for real repair: {item:?}")
|
||||
}
|
||||
};
|
||||
assert_policy_error(&error);
|
||||
// The proof-returning API must preserve the same refusal without converting it
|
||||
// into authoritative absence that a later destructive caller could consume.
|
||||
match store.heal_object_with_proof(bucket, object, version, &heal_options()).await {
|
||||
Err(error) => assert_policy_error(&error),
|
||||
Ok(result) => {
|
||||
assert!(result.absence.is_none(), "retirement never produces a physical absence proof");
|
||||
assert_policy_error(result.error.as_ref().expect("retired repair returns a policy error"));
|
||||
}
|
||||
}
|
||||
error
|
||||
}
|
||||
|
||||
async fn assert_retired_repair_rejected(store: &ECStore, bucket: &str, object: &str, version: &str) {
|
||||
let error = assert_repair_rejected(store, bucket, object, version).await;
|
||||
assert!(
|
||||
error.to_string().contains("artifact object version is retired"),
|
||||
"ARTIFACT_RETIREMENT_NO_RESURRECTION: expected retirement refusal, received {error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
async fn seal(store: &Arc<ECStore>, bucket: &str, incarnation: Uuid, prefix: &str) {
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(bucket, incarnation, prefix)
|
||||
.await
|
||||
.expect("persist the attempt PUT seal before retirement");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_retirement_requires_seal_exact_identity_and_canonical_directory() {
|
||||
run_large_stack_async_test("artifact-retirement-identity", || async {
|
||||
let root = tempfile::tempdir().expect("retirement identity directory");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-retirement-identity";
|
||||
let incarnation = create_bucket(&store, bucket, true).await;
|
||||
let prefix = attempt_prefix();
|
||||
let object = format!("{prefix}image.png");
|
||||
let directory = format!("{prefix}folder/");
|
||||
let version = put_payload(&store, bucket, &object, 0xA5, false).await;
|
||||
let directory_version = put_payload(&store, bucket, &directory, 0, false).await;
|
||||
let before = metadata(&store, bucket, &object).await;
|
||||
assert!(
|
||||
store
|
||||
.clone()
|
||||
.retire_artifact_object_version(bucket, incarnation, &prefix, &object, version)
|
||||
.await
|
||||
.is_err(),
|
||||
"unsealed attempt cannot retire a version"
|
||||
);
|
||||
seal(&store, bucket, incarnation, &prefix).await;
|
||||
let wrong_prefix = attempt_prefix();
|
||||
seal(&store, bucket, incarnation, &wrong_prefix).await;
|
||||
for (expected, attempt, target, selected) in [
|
||||
(Uuid::new_v4(), prefix.clone(), object.clone(), version),
|
||||
(incarnation, wrong_prefix, object.clone(), version),
|
||||
(incarnation, prefix.clone(), object.clone(), Uuid::nil()),
|
||||
(incarnation, prefix.clone(), prefix.clone(), version),
|
||||
] {
|
||||
assert!(
|
||||
store
|
||||
.clone()
|
||||
.retire_artifact_object_version(bucket, expected, &attempt, &target, selected)
|
||||
.await
|
||||
.is_err(),
|
||||
"retirement requires the exact incarnation, attempt, inner object, and non-null version"
|
||||
);
|
||||
}
|
||||
assert_eq!(metadata(&store, bucket, &object).await, before);
|
||||
let ordinary = "artifact-retirement-ordinary";
|
||||
let ordinary_incarnation = create_bucket(&store, ordinary, false).await;
|
||||
assert!(
|
||||
store
|
||||
.clone()
|
||||
.retire_artifact_object_version(ordinary, ordinary_incarnation, &prefix, &object, version)
|
||||
.await
|
||||
.is_err(),
|
||||
"retirement is limited to dedicated artifact buckets"
|
||||
);
|
||||
|
||||
store
|
||||
.clone()
|
||||
.retire_artifact_object_version(bucket, incarnation, &prefix, &directory, directory_version)
|
||||
.await
|
||||
.expect("retire logical directory identity");
|
||||
let record = artifact_version_retirement_record_path(bucket, incarnation, &prefix, &directory, directory_version);
|
||||
assert_eq!(
|
||||
record,
|
||||
artifact_version_retirement_record_path(
|
||||
bucket,
|
||||
incarnation,
|
||||
&prefix,
|
||||
&encode_dir_object(&directory),
|
||||
directory_version
|
||||
)
|
||||
);
|
||||
let bytes = com::read_config(store.clone(), &record)
|
||||
.await
|
||||
.expect("read durable directory retirement");
|
||||
store
|
||||
.clone()
|
||||
.retire_artifact_object_version(bucket, incarnation, &prefix, &encode_dir_object(&directory), directory_version)
|
||||
.await
|
||||
.expect("encoded directory retirement retry is idempotent");
|
||||
assert_eq!(com::read_config(store.clone(), &record).await.expect("read unchanged record"), bytes);
|
||||
assert_retired_repair_rejected(&store, bucket, &directory, &directory_version.to_string()).await;
|
||||
assert_retired_repair_rejected(&store, bucket, &encode_dir_object(&directory), &directory_version.to_string()).await;
|
||||
assert_payload(&store, bucket, &directory, directory_version, 0).await;
|
||||
// Rejected identity requests must not retire the valid regular object.
|
||||
let (_, missing) = remove_shard(&store, bucket, &object, version).await;
|
||||
let (_, error) = HealOperations::heal_object(store.as_ref(), bucket, &object, &version.to_string(), &heal_options())
|
||||
.await
|
||||
.expect("unretired version remains repairable after rejected retirement");
|
||||
assert!(error.is_none(), "unretired repair failed: {error:?}");
|
||||
assert!(missing.exists(), "unretired version's real shard is restored");
|
||||
assert_healthy_version(&store, bucket, &object, version).await;
|
||||
assert_payload(&store, bucket, &object, version, 0xA5).await;
|
||||
shutdown.cancel();
|
||||
assert!(ctx.wait_for_detached_mutations(DEADLINE).await);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_retirement_rejects_explicit_and_latest_repair_but_preserves_other_version_after_reopen() {
|
||||
run_large_stack_async_test("artifact-retirement-versions", || async {
|
||||
let root = tempfile::tempdir().expect("retired version fixture directory");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-retirement-versions";
|
||||
let incarnation = create_bucket(&store, bucket, true).await;
|
||||
let prefix = attempt_prefix();
|
||||
let object = format!("{prefix}image.png");
|
||||
let retained = put_payload(&store, bucket, &object, 0xA5, false).await;
|
||||
let retired = put_payload(&store, bucket, &object, 0xB6, false).await;
|
||||
assert_ne!(retained, retired);
|
||||
seal(&store, bucket, incarnation, &prefix).await;
|
||||
let before = metadata(&store, bucket, &object).await;
|
||||
let mut physical_parts = Vec::new();
|
||||
for disk in disks(&store).await {
|
||||
for version in [retained, retired] {
|
||||
let info = disk
|
||||
.read_version("", bucket, &object, &version.to_string(), &ReadOptions::default())
|
||||
.await
|
||||
.expect("snapshot exact version before retirement");
|
||||
assert_eq!(info.version_id, Some(version));
|
||||
let part = object_path(&disk, bucket, &object)
|
||||
.join(info.data_dir.expect("payload data directory").to_string())
|
||||
.join("part.1");
|
||||
physical_parts.push((part.clone(), tokio::fs::read(part).await.expect("snapshot original shard bytes")));
|
||||
}
|
||||
}
|
||||
store
|
||||
.clone()
|
||||
.retire_artifact_object_version(bucket, incarnation, &prefix, &object, retired)
|
||||
.await
|
||||
.expect("persist exact latest-version retirement");
|
||||
assert_eq!(
|
||||
metadata(&store, bucket, &object).await,
|
||||
before,
|
||||
"retirement itself does not edit object metadata"
|
||||
);
|
||||
for (part, bytes) in physical_parts {
|
||||
assert_eq!(
|
||||
tokio::fs::read(part).await.expect("retirement leaves every physical shard"),
|
||||
bytes,
|
||||
"retirement changes neither version's physical data"
|
||||
);
|
||||
}
|
||||
assert_payload(&store, bucket, &object, retired, 0xB6).await;
|
||||
assert_payload(&store, bucket, &object, retained, 0xA5).await;
|
||||
let (_, retired_part) = remove_shard(&store, bucket, &object, retired).await;
|
||||
let before = metadata(&store, bucket, &object).await;
|
||||
for selected in [retired.to_string(), String::new()] {
|
||||
assert_retired_repair_rejected(&store, bucket, &object, &selected).await;
|
||||
assert!(
|
||||
!retired_part.exists(),
|
||||
"ARTIFACT_RETIREMENT_NO_RESURRECTION: retired shard must stay absent"
|
||||
);
|
||||
assert_eq!(metadata(&store, bucket, &object).await, before, "retired repair publishes no metadata");
|
||||
}
|
||||
let (_, retained_part) = remove_shard(&store, bucket, &object, retained).await;
|
||||
let (_, error) = HealOperations::heal_object(store.as_ref(), bucket, &object, &retained.to_string(), &heal_options())
|
||||
.await
|
||||
.expect("another version on the same object remains repairable");
|
||||
assert!(error.is_none(), "other-version repair failed: {error:?}");
|
||||
assert!(retained_part.exists(), "other version's actual shard is restored");
|
||||
assert_healthy_version(&store, bucket, &object, retained).await;
|
||||
assert!(!retired_part.exists(), "other-version repair must not restore the retired shard");
|
||||
assert_payload(&store, bucket, &object, retained, 0xA5).await;
|
||||
assert_payload(&store, bucket, &object, retired, 0xB6).await;
|
||||
shutdown.cancel();
|
||||
assert!(ctx.wait_for_detached_mutations(DEADLINE).await);
|
||||
drop(store);
|
||||
drop(ctx);
|
||||
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
assert_eq!(store.bucket_incarnation_id(bucket).await.expect("reopened incarnation"), incarnation);
|
||||
assert_retired_repair_rejected(&store, bucket, &object, &retired.to_string()).await;
|
||||
assert!(!retired_part.exists(), "reopen cannot forget exact-version retirement");
|
||||
assert_payload(&store, bucket, &object, retired, 0xB6).await;
|
||||
assert_payload(&store, bucket, &object, retained, 0xA5).await;
|
||||
shutdown.cancel();
|
||||
assert!(ctx.wait_for_detached_mutations(DEADLINE).await);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_retirement_blocks_minority_metadata_regeneration() {
|
||||
run_large_stack_async_test("artifact-retirement-regeneration", || async {
|
||||
let root = tempfile::tempdir().expect("minority metadata fixture directory");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-retirement-regeneration";
|
||||
let incarnation = create_bucket(&store, bucket, true).await;
|
||||
let prefix = attempt_prefix();
|
||||
let object = format!("{prefix}image.png");
|
||||
let version = put_payload(&store, bucket, &object, 0xA5, true).await;
|
||||
seal(&store, bucket, incarnation, &prefix).await;
|
||||
store
|
||||
.clone()
|
||||
.retire_artifact_object_version(bucket, incarnation, &prefix, &object, version)
|
||||
.await
|
||||
.expect("persist version retirement before minority metadata damage");
|
||||
assert_payload(&store, bucket, &object, version, 0xA5).await;
|
||||
let mut parts = Vec::new();
|
||||
for (index, disk) in disks(&store).await.iter().enumerate() {
|
||||
let info = disk
|
||||
.read_version("", bucket, &object, &version.to_string(), &ReadOptions::default())
|
||||
.await
|
||||
.expect("read genuine legacy version before damage");
|
||||
assert_eq!(info.version_id, Some(version));
|
||||
let part = object_path(disk, bucket, &object)
|
||||
.join(info.data_dir.expect("legacy payload has data directory").to_string())
|
||||
.join("part.1");
|
||||
parts.push((part.clone(), tokio::fs::read(&part).await.expect("snapshot real shard bytes")));
|
||||
if index > 0 {
|
||||
tokio::fs::remove_file(object_path(disk, bucket, &object).join(STORAGE_FORMAT_FILE))
|
||||
.await
|
||||
.expect("leave exactly one genuine metadata copy");
|
||||
}
|
||||
}
|
||||
let before = metadata(&store, bucket, &object).await;
|
||||
assert_eq!(before.iter().filter(|entry| entry.is_some()).count(), 1);
|
||||
assert_retired_repair_rejected(&store, bucket, &object, &version.to_string()).await;
|
||||
assert_eq!(
|
||||
metadata(&store, bucket, &object).await,
|
||||
before,
|
||||
"ARTIFACT_RETIREMENT_NO_RESURRECTION: minority regeneration must not publish retired metadata"
|
||||
);
|
||||
for (path, bytes) in parts {
|
||||
assert_eq!(tokio::fs::read(path).await.expect("retired shard remains physically present"), bytes);
|
||||
}
|
||||
shutdown.cancel();
|
||||
assert!(ctx.wait_for_detached_mutations(DEADLINE).await);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_retirement_corrupt_or_wrong_identity_records_fail_closed() {
|
||||
run_large_stack_async_test("artifact-retirement-records", || async {
|
||||
let root = tempfile::tempdir().expect("corrupt retirement fixture directory");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-retirement-records";
|
||||
let incarnation = create_bucket(&store, bucket, true).await;
|
||||
let prefix = attempt_prefix();
|
||||
let object = format!("{prefix}image.png");
|
||||
let version = put_payload(&store, bucket, &object, 0xA5, false).await;
|
||||
seal(&store, bucket, incarnation, &prefix).await;
|
||||
store
|
||||
.clone()
|
||||
.retire_artifact_object_version(bucket, incarnation, &prefix, &object, version)
|
||||
.await
|
||||
.expect("create genuine retirement record");
|
||||
let record = artifact_version_retirement_record_path(bucket, incarnation, &prefix, &object, version);
|
||||
let original = com::read_config(store.clone(), &record)
|
||||
.await
|
||||
.expect("read genuine retirement JSON");
|
||||
let decoded: serde_json::Value = serde_json::from_slice(&original).expect("retirement record is JSON");
|
||||
let mut invalid = vec![Vec::new(), b"not-json".to_vec()];
|
||||
for (field, value) in [
|
||||
("format_version", serde_json::json!(2)),
|
||||
("bucket", serde_json::json!("different-bucket")),
|
||||
("base_prefix", serde_json::json!("other/")),
|
||||
("incarnation_id", serde_json::json!(Uuid::new_v4())),
|
||||
("attempt_prefix", serde_json::json!(attempt_prefix())),
|
||||
("object", serde_json::json!(format!("{prefix}other.png"))),
|
||||
("version_id", serde_json::json!(Uuid::new_v4())),
|
||||
("unexpected_field", serde_json::json!(true)),
|
||||
] {
|
||||
let mut changed = decoded.clone();
|
||||
changed
|
||||
.as_object_mut()
|
||||
.expect("strict identity record is an object")
|
||||
.insert(field.to_owned(), value);
|
||||
invalid.push(serde_json::to_vec(&changed).expect("encode invalid persisted identity"));
|
||||
}
|
||||
let (_, missing) = remove_shard(&store, bucket, &object, version).await;
|
||||
let before = metadata(&store, bucket, &object).await;
|
||||
for bytes in invalid {
|
||||
// Inject corrupt persisted bytes through real guarded config I/O; the
|
||||
// fixture does not emulate the production retirement state machine.
|
||||
com::with_config_object_write_lock(store.clone(), record.clone(), {
|
||||
let store = store.clone();
|
||||
let record = record.clone();
|
||||
move |guard| async move { com::save_config_no_lock(store, &record, bytes, &guard).await }
|
||||
})
|
||||
.await
|
||||
.expect("hold matching record guard")
|
||||
.expect("persist invalid record on real disks");
|
||||
assert!(
|
||||
store
|
||||
.clone()
|
||||
.retire_artifact_object_version(bucket, incarnation, &prefix, &object, version)
|
||||
.await
|
||||
.is_err(),
|
||||
"corrupt or wrong-identity record cannot confirm idempotent retirement"
|
||||
);
|
||||
assert_repair_rejected(&store, bucket, &object, &version.to_string()).await;
|
||||
assert!(!missing.exists(), "invalid record must not authorize real repair publication");
|
||||
assert_eq!(metadata(&store, bucket, &object).await, before);
|
||||
assert_payload(&store, bucket, &object, version, 0xA5).await;
|
||||
}
|
||||
shutdown.cancel();
|
||||
assert!(ctx.wait_for_detached_mutations(DEADLINE).await);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_retirement_waits_for_cancelled_physical_repair_and_retries_cancelled_retirement() {
|
||||
run_large_stack_async_test("artifact-retirement-physical-tail", || async {
|
||||
let root = tempfile::tempdir().expect("physical retirement fixture directory");
|
||||
let (ctx, store, shutdown) = open_store(root.path()).await;
|
||||
let bucket = "artifact-retirement-physical-tail";
|
||||
let incarnation = create_bucket(&store, bucket, true).await;
|
||||
let prefix = attempt_prefix();
|
||||
let object = format!("{prefix}image.png");
|
||||
let version = put_payload(&store, bucket, &object, 0xA5, false).await;
|
||||
seal(&store, bucket, incarnation, &prefix).await;
|
||||
let (disk, part) = remove_shard(&store, bucket, &object, version).await;
|
||||
let destination = object_path(&disk, bucket, &object);
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
let repair_hook = os::prepared_publication_test_hooks::install_at(
|
||||
os::prepared_publication_test_hooks::Stage::PreparedRename,
|
||||
&destination.join(STORAGE_FORMAT_FILE),
|
||||
move || {
|
||||
let _ = entered_tx.send(());
|
||||
// Sender drop also releases physical work when an assertion unwinds.
|
||||
let _ = release_rx.recv();
|
||||
},
|
||||
);
|
||||
let repair_store = store.clone();
|
||||
let repair_object = object.clone();
|
||||
let mut repair = tokio::spawn(async move {
|
||||
HealOperations::heal_object(repair_store.as_ref(), bucket, &repair_object, &version.to_string(), &heal_options())
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(DEADLINE, async {
|
||||
tokio::select! {
|
||||
entered = entered_rx => entered.expect("repair reaches real physical rename syscall"),
|
||||
result = &mut repair => panic!("repair returned before syscall pause: {result:?}"),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("real repair worker is paused");
|
||||
repair.abort();
|
||||
assert!(repair.await.expect_err("cancel repair caller").is_cancelled());
|
||||
|
||||
let retire_store = store.clone();
|
||||
let retire_prefix = prefix.clone();
|
||||
let retire_object = object.clone();
|
||||
let mut retirement = tokio::spawn(async move {
|
||||
retire_store
|
||||
.retire_artifact_object_version(bucket, incarnation, &retire_prefix, &retire_object, version)
|
||||
.await
|
||||
});
|
||||
let blocked = tokio::time::timeout(Duration::from_millis(200), &mut retirement)
|
||||
.await
|
||||
.is_err();
|
||||
drop(release_tx);
|
||||
drop(
|
||||
tokio::time::timeout(DEADLINE, os::acquire_rename_data_mutation_lease(&disk.path(), bucket, &destination))
|
||||
.await
|
||||
.expect("cancelled physical repair drains"),
|
||||
);
|
||||
drop(repair_hook);
|
||||
if blocked {
|
||||
tokio::time::timeout(DEADLINE, retirement)
|
||||
.await
|
||||
.expect("retirement resumes after real repair tail")
|
||||
.expect("retirement task joins")
|
||||
.expect("retirement persists");
|
||||
}
|
||||
assert!(
|
||||
blocked,
|
||||
"ARTIFACT_RETIREMENT_NO_RESURRECTION: retirement must wait for cancelled repair's physical publication tail"
|
||||
);
|
||||
assert_payload(&store, bucket, &object, version, 0xA5).await;
|
||||
// A cancelled repair may roll back its destination; retirement must stop
|
||||
// any later repair regardless of whether that first publication committed.
|
||||
let before = metadata(&store, bucket, &object).await;
|
||||
let part_before = file_bytes(&part).await;
|
||||
assert_retired_repair_rejected(&store, bucket, &object, &version.to_string()).await;
|
||||
assert_eq!(metadata(&store, bucket, &object).await, before);
|
||||
assert_eq!(file_bytes(&part).await, part_before);
|
||||
|
||||
let second_prefix = attempt_prefix();
|
||||
let second_object = format!("{second_prefix}cancelled.bin");
|
||||
let second_version = put_payload(&store, bucket, &second_object, 0xB6, false).await;
|
||||
seal(&store, bucket, incarnation, &second_prefix).await;
|
||||
let record = artifact_version_retirement_record_path(bucket, incarnation, &second_prefix, &second_object, second_version);
|
||||
let record_disk = disks(&store).await.remove(0);
|
||||
let record_path = object_path(&record_disk, RUSTFS_META_BUCKET, &record).join(STORAGE_FORMAT_FILE);
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
let _retirement_hook = os::prepared_publication_test_hooks::install_at(
|
||||
os::prepared_publication_test_hooks::Stage::PreparedRename,
|
||||
&record_path,
|
||||
move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
},
|
||||
);
|
||||
let retire_store = store.clone();
|
||||
let retire_prefix = second_prefix.clone();
|
||||
let retire_object = second_object.clone();
|
||||
let mut retirement = tokio::spawn(async move {
|
||||
retire_store
|
||||
.retire_artifact_object_version(bucket, incarnation, &retire_prefix, &retire_object, second_version)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(DEADLINE, async {
|
||||
tokio::select! {
|
||||
entered = entered_rx => entered.expect("retirement reaches real record rename syscall"),
|
||||
result = &mut retirement => panic!("retirement returned before durable record publication: {result:?}"),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("retirement's real config publication is paused");
|
||||
retirement.abort();
|
||||
assert!(retirement.await.expect_err("cancel retirement caller").is_cancelled());
|
||||
let retry_store = store.clone();
|
||||
let retry_prefix = second_prefix.clone();
|
||||
let retry_object = second_object.clone();
|
||||
let mut retry = tokio::spawn(async move {
|
||||
retry_store
|
||||
.retire_artifact_object_version(bucket, incarnation, &retry_prefix, &retry_object, second_version)
|
||||
.await
|
||||
});
|
||||
let retry_blocked = tokio::time::timeout(Duration::from_millis(200), &mut retry).await.is_err();
|
||||
drop(release_tx);
|
||||
if retry_blocked {
|
||||
tokio::time::timeout(DEADLINE, retry)
|
||||
.await
|
||||
.expect("retry waits for original publication tail")
|
||||
.expect("retry task joins")
|
||||
.expect("cancelled retirement is confirmed idempotently");
|
||||
}
|
||||
assert!(retry_blocked, "retirement retry cannot pass a physically incomplete original publication");
|
||||
let persisted = com::read_config(store.clone(), &record)
|
||||
.await
|
||||
.expect("cancelled retirement left durable record");
|
||||
store
|
||||
.clone()
|
||||
.retire_artifact_object_version(bucket, incarnation, &second_prefix, &second_object, second_version)
|
||||
.await
|
||||
.expect("idempotently confirm cancelled retirement");
|
||||
assert_eq!(com::read_config(store.clone(), &record).await.expect("read confirmed record"), persisted);
|
||||
assert_retired_repair_rejected(&store, bucket, &second_object, &second_version.to_string()).await;
|
||||
assert_payload(&store, bucket, &second_object, second_version, 0xB6).await;
|
||||
shutdown.cancel();
|
||||
assert!(ctx.wait_for_detached_mutations(DEADLINE).await);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,944 @@
|
||||
use super::{build_isolated_test_store, run_large_stack_async_test, without_storage_class_env};
|
||||
use crate::{
|
||||
bucket::metadata_sys,
|
||||
disk::DiskAPI as _,
|
||||
error::Error,
|
||||
object_api::{ObjectInfo, ObjectOptions, PutObjReader, WriteCommitGuard, WriteCompletion},
|
||||
storage_api_contracts::{
|
||||
bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions},
|
||||
multipart::{CompletePart, MultipartOperations as _},
|
||||
namespace::NamespaceLocking as _,
|
||||
object::{ObjectIO as _, ObjectOperations as _},
|
||||
},
|
||||
store::ECStore,
|
||||
};
|
||||
use std::{collections::HashMap, time::Duration};
|
||||
use uuid::Uuid;
|
||||
|
||||
fn object_key() -> String {
|
||||
format!("outputs/74656e616e74/{}/{}/image.png", Uuid::new_v4(), Uuid::new_v4())
|
||||
}
|
||||
|
||||
fn options() -> ObjectOptions {
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn body() -> PutObjReader {
|
||||
PutObjReader::from_vec(vec![0xA5; 273])
|
||||
}
|
||||
|
||||
fn unsupported<T>(result: Result<T, Error>) {
|
||||
let Err(error) = result else {
|
||||
panic!("ARTIFACT_WRITE_UNSUPPORTED: operation unexpectedly succeeded")
|
||||
};
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("artifact bucket does not support this write operation"),
|
||||
"ARTIFACT_WRITE_UNSUPPORTED: {error}"
|
||||
);
|
||||
}
|
||||
|
||||
async fn bucket(store: &ECStore, name: &str, artifact: bool) -> Uuid {
|
||||
store
|
||||
.make_bucket(
|
||||
name,
|
||||
&MakeBucketOptions {
|
||||
artifact_base_prefix: artifact.then(|| "outputs/".to_owned()),
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create real versioned fixture bucket");
|
||||
store.bucket_incarnation_id(name).await.expect("persisted incarnation")
|
||||
}
|
||||
|
||||
async fn snapshot(store: &ECStore, bucket: &str, object: &str) -> Vec<Vec<u8>> {
|
||||
let disks = store.pools[0].disk_set[0].disk_inventory().await;
|
||||
assert_eq!(disks.len(), 4);
|
||||
let mut versions = Vec::new();
|
||||
for disk in disks {
|
||||
versions.push(
|
||||
disk.expect("online fixture disk")
|
||||
.read_xl(bucket, object, false)
|
||||
.await
|
||||
.expect("physical version metadata")
|
||||
.buf,
|
||||
);
|
||||
}
|
||||
versions
|
||||
}
|
||||
|
||||
async fn copy_info(store: &ECStore, bucket: &str, object: &str) -> ObjectInfo {
|
||||
let mut info = store
|
||||
.get_object_info(bucket, object, &options())
|
||||
.await
|
||||
.expect("real source object");
|
||||
info.put_object_reader = Some(body());
|
||||
info
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_write_entrypoint_object_mutations_are_rejected() {
|
||||
run_large_stack_async_test("artifact-object-entrypoints", object_mutations_case);
|
||||
}
|
||||
|
||||
async fn object_mutations_case() {
|
||||
let root = tempfile::tempdir().expect("object entrypoint fixture directory");
|
||||
let (ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(root.path(), "artifact-object-entrypoints", &[4])).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let artifact = "artifact-entrypoint-object";
|
||||
let ordinary = "artifact-entrypoint-control";
|
||||
bucket(&store, artifact, true).await;
|
||||
bucket(&store, ordinary, false).await;
|
||||
let object = object_key();
|
||||
store
|
||||
.put_object(artifact, &object, &mut body(), &options())
|
||||
.await
|
||||
.expect("ordinary owned artifact PUT is supported");
|
||||
store
|
||||
.put_object(ordinary, &object, &mut body(), &options())
|
||||
.await
|
||||
.expect("ordinary control object");
|
||||
let before = snapshot(&store, artifact, &object).await;
|
||||
|
||||
let mut source = copy_info(&store, ordinary, &object).await;
|
||||
unsupported(
|
||||
store
|
||||
.copy_object(ordinary, &object, artifact, &object, &mut source, &options(), &options())
|
||||
.await,
|
||||
);
|
||||
let mut source = copy_info(&store, artifact, &object).await;
|
||||
unsupported(
|
||||
store
|
||||
.copy_object(artifact, &object, artifact, &object, &mut source, &options(), &options())
|
||||
.await,
|
||||
);
|
||||
for opts in [
|
||||
ObjectOptions {
|
||||
replication_request: true,
|
||||
..options()
|
||||
},
|
||||
ObjectOptions {
|
||||
data_movement: true,
|
||||
..options()
|
||||
},
|
||||
ObjectOptions {
|
||||
preserve_ciphertext: true,
|
||||
..options()
|
||||
},
|
||||
ObjectOptions {
|
||||
metadata_chg: true,
|
||||
..options()
|
||||
},
|
||||
ObjectOptions {
|
||||
replication_status_writeback: Some(Box::default()),
|
||||
..options()
|
||||
},
|
||||
] {
|
||||
unsupported(store.put_object(artifact, &object, &mut body(), &opts).await);
|
||||
}
|
||||
unsupported(
|
||||
store
|
||||
.put_object_for_data_movement(
|
||||
artifact,
|
||||
&object,
|
||||
&mut body(),
|
||||
&ObjectOptions {
|
||||
data_movement: true,
|
||||
..options()
|
||||
},
|
||||
None,
|
||||
)
|
||||
.await,
|
||||
);
|
||||
{
|
||||
let lock = store
|
||||
.new_ns_lock(artifact, &object)
|
||||
.await
|
||||
.expect("real borrowed target namespace");
|
||||
let guard = WriteCommitGuard::acquire(&lock, Duration::from_secs(30))
|
||||
.await
|
||||
.expect("real borrowed write owner");
|
||||
let mut opts = options();
|
||||
opts.add_write_commit_guard(&guard);
|
||||
unsupported(store.put_object(artifact, &object, &mut body(), &opts).await);
|
||||
}
|
||||
unsupported(store.transition_object(artifact, &object, &options()).await);
|
||||
unsupported(store.clone().restore_transitioned_object(artifact, &object, &options()).await);
|
||||
unsupported(
|
||||
store
|
||||
.put_object_metadata(
|
||||
artifact,
|
||||
&object,
|
||||
&ObjectOptions {
|
||||
eval_metadata: Some(HashMap::from([("x-amz-meta-test".to_owned(), "changed".to_owned())])),
|
||||
..options()
|
||||
},
|
||||
)
|
||||
.await,
|
||||
);
|
||||
unsupported(store.put_object_tags(artifact, &object, "kind=artifact", &options()).await);
|
||||
unsupported(store.delete_object_tags(artifact, &object, &options()).await);
|
||||
assert_eq!(
|
||||
snapshot(&store, artifact, &object).await,
|
||||
before,
|
||||
"ARTIFACT_WRITE_UNSUPPORTED: every disk retains the exact original versions"
|
||||
);
|
||||
|
||||
// An artifact source remains readable; only the destination write is classified.
|
||||
let mut source = copy_info(&store, artifact, &object).await;
|
||||
store
|
||||
.copy_object(artifact, &object, ordinary, "copied.png", &mut source, &options(), &options())
|
||||
.await
|
||||
.expect("copy into ordinary bucket stays supported");
|
||||
store
|
||||
.put_object(
|
||||
ordinary,
|
||||
&object,
|
||||
&mut body(),
|
||||
&ObjectOptions {
|
||||
replication_request: true,
|
||||
..options()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("ordinary replication PUT stays supported");
|
||||
{
|
||||
let incarnation = store
|
||||
.bucket_incarnation_id_from_disk(ordinary)
|
||||
.await
|
||||
.expect("ordinary incarnation");
|
||||
let lifecycle = std::sync::Arc::new(
|
||||
store
|
||||
.acquire_bucket_incarnation_fence(ordinary, incarnation)
|
||||
.await
|
||||
.expect("ordinary lifecycle owner"),
|
||||
);
|
||||
let lock = store.new_ns_lock(ordinary, &object).await.expect("ordinary target namespace");
|
||||
let guard = WriteCommitGuard::acquire(&lock, Duration::from_secs(30))
|
||||
.await
|
||||
.expect("ordinary borrowed owner");
|
||||
let mut opts = options();
|
||||
lifecycle.attach_owner(&mut opts);
|
||||
opts.add_write_commit_guard(&guard);
|
||||
store
|
||||
.put_object(ordinary, &object, &mut body(), &opts)
|
||||
.await
|
||||
.expect("ordinary borrowed PUT stays supported");
|
||||
}
|
||||
store
|
||||
.put_object_metadata(
|
||||
ordinary,
|
||||
&object,
|
||||
&ObjectOptions {
|
||||
eval_metadata: Some(HashMap::from([("x-amz-meta-test".to_owned(), "changed".to_owned())])),
|
||||
..options()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("ordinary metadata update stays supported");
|
||||
store
|
||||
.put_object_tags(ordinary, &object, "kind=ordinary", &options())
|
||||
.await
|
||||
.expect("ordinary tag update stays supported");
|
||||
store
|
||||
.delete_object_tags(ordinary, &object, &options())
|
||||
.await
|
||||
.expect("ordinary tag removal stays supported");
|
||||
assert_eq!(
|
||||
store
|
||||
.get_object_info(ordinary, &object, &options())
|
||||
.await
|
||||
.expect("ordinary object remains readable")
|
||||
.size,
|
||||
273
|
||||
);
|
||||
assert_eq!(
|
||||
snapshot(&store, artifact, &object).await,
|
||||
before,
|
||||
"ARTIFACT_WRITE_UNSUPPORTED: reading an artifact source cannot mutate its versions"
|
||||
);
|
||||
let tier = "ARTIFACT-ENTRYPOINT-CONTROL";
|
||||
let backend = crate::services::tier::test_util::register_mock_tier(&ctx.tier_config_mgr(), tier).await;
|
||||
let current = store
|
||||
.get_object_info(ordinary, &object, &options())
|
||||
.await
|
||||
.expect("ordinary source before transition");
|
||||
let exact = ObjectOptions {
|
||||
version_id: current.version_id.map(|version| version.to_string()),
|
||||
..options()
|
||||
};
|
||||
store
|
||||
.transition_object(
|
||||
ordinary,
|
||||
&object,
|
||||
&ObjectOptions {
|
||||
transition: crate::bucket::lifecycle::lifecycle::TransitionOptions {
|
||||
status: rustfs_filemeta::TRANSITION_PENDING.to_owned(),
|
||||
tier: tier.to_owned(),
|
||||
etag: current.etag.expect("ordinary source etag"),
|
||||
..Default::default()
|
||||
},
|
||||
mod_time: current.mod_time,
|
||||
..exact.clone()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("ordinary transition stays supported");
|
||||
assert_eq!(backend.object_count().await, 1);
|
||||
let mut restore_transition = crate::bucket::lifecycle::lifecycle::TransitionOptions::default();
|
||||
restore_transition.restore_request.days = Some(1);
|
||||
store
|
||||
.clone()
|
||||
.restore_transitioned_object(
|
||||
ordinary,
|
||||
&object,
|
||||
&ObjectOptions {
|
||||
transition: restore_transition,
|
||||
..exact
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("ordinary restore stays supported");
|
||||
shutdown.cancel();
|
||||
assert!(ctx.wait_for_detached_mutations(Duration::from_secs(30)).await);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_write_entrypoint_multipart_rejects_writes_but_keeps_legacy_cleanup() {
|
||||
run_large_stack_async_test("artifact-multipart-entrypoints", multipart_case);
|
||||
}
|
||||
|
||||
async fn multipart_case() {
|
||||
let root = tempfile::tempdir().expect("multipart entrypoint fixture directory");
|
||||
let (ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(root.path(), "artifact-multipart-entrypoints", &[4])).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let artifact = "artifact-entrypoint-multipart";
|
||||
let ordinary = "artifact-multipart-control";
|
||||
let incarnation = bucket(&store, artifact, true).await;
|
||||
bucket(&store, ordinary, false).await;
|
||||
let object = object_key();
|
||||
store
|
||||
.put_object(artifact, &object, &mut body(), &options())
|
||||
.await
|
||||
.expect("seed the current artifact version");
|
||||
let before = snapshot(&store, artifact, &object).await;
|
||||
let opts = ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
..options()
|
||||
};
|
||||
unsupported(store.new_multipart_upload(artifact, &object, &opts).await);
|
||||
// Seed actual legacy staging while holding the lifecycle owner required by the pool API.
|
||||
let lifecycle = store
|
||||
.acquire_bucket_lifecycle_read_lock(artifact)
|
||||
.await
|
||||
.expect("legacy upload lifecycle owner");
|
||||
let mut staging_opts = opts.clone();
|
||||
staging_opts.add_bucket_lifecycle_lock_guard(&lifecycle);
|
||||
let upload = store.pools[0]
|
||||
.new_multipart_upload(artifact, &object, &staging_opts)
|
||||
.await
|
||||
.expect("real legacy upload staging");
|
||||
let part = store.pools[0]
|
||||
.put_object_part(artifact, &object, &upload.upload_id, 1, &mut body(), &staging_opts)
|
||||
.await
|
||||
.expect("real legacy staged part");
|
||||
drop(staging_opts);
|
||||
drop(lifecycle);
|
||||
let parts = vec![CompletePart {
|
||||
part_num: part.part_num,
|
||||
etag: part.etag.clone(),
|
||||
..Default::default()
|
||||
}];
|
||||
unsupported(
|
||||
store
|
||||
.put_object_part(artifact, &object, &upload.upload_id, 2, &mut body(), &opts)
|
||||
.await,
|
||||
);
|
||||
unsupported(
|
||||
store
|
||||
.clone()
|
||||
.complete_multipart_upload(artifact, &object, &upload.upload_id, parts.clone(), &opts)
|
||||
.await,
|
||||
);
|
||||
let movement = ObjectOptions {
|
||||
data_movement: true,
|
||||
part_number: Some(2),
|
||||
..opts.clone()
|
||||
};
|
||||
unsupported(
|
||||
store
|
||||
.handle_new_multipart_upload_with_pool_idx(artifact, &object, &movement, None)
|
||||
.await,
|
||||
);
|
||||
unsupported(
|
||||
store
|
||||
.put_object_part_for_data_movement(0, artifact, &object, &upload.upload_id, &mut body(), &movement)
|
||||
.await,
|
||||
);
|
||||
unsupported(
|
||||
store
|
||||
.clone()
|
||||
.complete_multipart_upload_for_data_movement((0, None), artifact, &object, &upload.upload_id, parts, &movement)
|
||||
.await,
|
||||
);
|
||||
assert_eq!(
|
||||
snapshot(&store, artifact, &object).await,
|
||||
before,
|
||||
"ARTIFACT_WRITE_UNSUPPORTED: rejected multipart completion publishes no version"
|
||||
);
|
||||
let listing = store
|
||||
.list_multipart_uploads(artifact, &object, None, None, None, 100)
|
||||
.await
|
||||
.expect("list legacy uploads remains supported");
|
||||
assert_eq!(listing.uploads.len(), 1);
|
||||
assert_eq!(listing.uploads[0].upload_id, upload.upload_id);
|
||||
let staged = store
|
||||
.list_object_parts(artifact, &object, &upload.upload_id, None, 100, &opts)
|
||||
.await
|
||||
.expect("list legacy parts remains supported");
|
||||
assert_eq!(staged.parts.len(), 1);
|
||||
assert_eq!(staged.parts[0].etag, part.etag);
|
||||
store
|
||||
.abort_multipart_upload(artifact, &object, &upload.upload_id, &opts)
|
||||
.await
|
||||
.expect("abort legacy upload remains supported");
|
||||
assert!(
|
||||
store
|
||||
.list_multipart_uploads(artifact, &object, None, None, None, 100)
|
||||
.await
|
||||
.expect("list after abort")
|
||||
.uploads
|
||||
.is_empty()
|
||||
);
|
||||
assert_eq!(
|
||||
snapshot(&store, artifact, &object).await,
|
||||
before,
|
||||
"ARTIFACT_WRITE_UNSUPPORTED: legacy abort leaves published versions intact"
|
||||
);
|
||||
|
||||
let upload = store
|
||||
.new_multipart_upload(ordinary, &object, &options())
|
||||
.await
|
||||
.expect("ordinary multipart initialization");
|
||||
let part = store
|
||||
.put_object_part(ordinary, &object, &upload.upload_id, 1, &mut body(), &options())
|
||||
.await
|
||||
.expect("ordinary part write");
|
||||
let completed = store
|
||||
.clone()
|
||||
.complete_multipart_upload(
|
||||
ordinary,
|
||||
&object,
|
||||
&upload.upload_id,
|
||||
vec![CompletePart {
|
||||
part_num: part.part_num,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
}],
|
||||
&options(),
|
||||
)
|
||||
.await
|
||||
.expect("ordinary multipart completion");
|
||||
assert_eq!(completed.size, 273);
|
||||
shutdown.cancel();
|
||||
assert!(ctx.wait_for_detached_mutations(Duration::from_secs(30)).await);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_write_entrypoint_classification_is_bound_to_bucket_incarnation() {
|
||||
run_large_stack_async_test("artifact-entrypoint-recreation", recreation_case);
|
||||
}
|
||||
|
||||
async fn recreation_case() {
|
||||
let root = tempfile::tempdir().expect("entrypoint recreation fixture directory");
|
||||
let (ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(root.path(), "artifact-entrypoint-recreation", &[4])).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let name = "artifact-entrypoint-recreated";
|
||||
let old_incarnation = bucket(&store, name, false).await;
|
||||
let object = object_key();
|
||||
let barrier =
|
||||
crate::set_disk::PutObjectCommitBarrier::install(name, &object, crate::set_disk::PutObjectCommitPause::BeforeNamespace);
|
||||
let put_store = store.clone();
|
||||
let put_object = object.clone();
|
||||
let put = tokio::spawn(async move {
|
||||
put_store
|
||||
.put_object(
|
||||
name,
|
||||
&put_object,
|
||||
&mut body(),
|
||||
&ObjectOptions {
|
||||
replication_request: true,
|
||||
..options()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
tokio::time::timeout(Duration::from_secs(30), store.delete_bucket(name, &DeleteBucketOptions::default()))
|
||||
.await
|
||||
.expect("staged old-generation upload does not prevent bucket deletion")
|
||||
.expect("delete old ordinary bucket");
|
||||
let new_incarnation = bucket(&store, name, true).await;
|
||||
assert_ne!(new_incarnation, old_incarnation);
|
||||
barrier.release();
|
||||
let error = tokio::time::timeout(Duration::from_secs(30), put)
|
||||
.await
|
||||
.expect("stale staged PUT terminates")
|
||||
.expect("PUT task joins")
|
||||
.expect_err("classification of an ordinary bucket cannot authorize the new artifact bucket");
|
||||
assert!(
|
||||
matches!(error, Error::BucketNotFound(ref bucket) if bucket == name),
|
||||
"ARTIFACT_WRITE_INCARNATION: {error}"
|
||||
);
|
||||
for disk in store.pools[0].disk_set[0].disk_inventory().await {
|
||||
let error = disk
|
||||
.expect("online fixture disk")
|
||||
.read_xl(name, &object, false)
|
||||
.await
|
||||
.err()
|
||||
.expect("no stale object metadata may be published");
|
||||
assert!(
|
||||
matches!(error, crate::disk::error::DiskError::FileNotFound),
|
||||
"ARTIFACT_WRITE_INCARNATION: {error}"
|
||||
);
|
||||
}
|
||||
shutdown.cancel();
|
||||
assert!(ctx.wait_for_detached_mutations(Duration::from_secs(30)).await);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_write_entrypoint_lifecycle_fences_require_real_matching_owners() {
|
||||
run_large_stack_async_test("artifact-entrypoint-lifecycle-owner", lifecycle_owner_case);
|
||||
}
|
||||
|
||||
async fn lifecycle_owner_case() {
|
||||
let root = tempfile::tempdir().expect("lifecycle owner fixture directory");
|
||||
let (ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(root.path(), "artifact-lifecycle-owner", &[4])).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let ordinary = "artifact-lifecycle-ordinary";
|
||||
let artifact = "artifact-lifecycle-dedicated";
|
||||
bucket(&store, ordinary, false).await;
|
||||
bucket(&store, artifact, true).await;
|
||||
let object = object_key();
|
||||
for name in [ordinary, artifact] {
|
||||
store
|
||||
.put_object(name, &object, &mut body(), &options())
|
||||
.await
|
||||
.expect("initial real object");
|
||||
}
|
||||
let before = snapshot(&store, ordinary, &object).await;
|
||||
let artifact_before = snapshot(&store, artifact, &object).await;
|
||||
let unrelated = store
|
||||
.new_ns_lock(ordinary, "unrelated-object")
|
||||
.await
|
||||
.expect("unrelated namespace");
|
||||
let unrelated_owner = unrelated
|
||||
.get_read_lock(Duration::from_secs(30))
|
||||
.await
|
||||
.expect("live unrelated owner");
|
||||
let mut foreign_fence = options();
|
||||
foreign_fence.add_bucket_lifecycle_lock_guard(&unrelated_owner);
|
||||
let lifecycle = store
|
||||
.acquire_bucket_lifecycle_read_lock(ordinary)
|
||||
.await
|
||||
.expect("real lifecycle owner");
|
||||
let mut released_fence = options();
|
||||
released_fence.add_bucket_lifecycle_lock_guard(&lifecycle);
|
||||
let borrowed_snapshot = store
|
||||
.object_lock_config_snapshot_under_lifecycle_fence(
|
||||
ordinary,
|
||||
released_fence
|
||||
.bucket_lifecycle_lock_fence
|
||||
.as_ref()
|
||||
.expect("borrowed lifecycle fence"),
|
||||
)
|
||||
.await
|
||||
.expect("real snapshot retaining only its metadata lock");
|
||||
drop(lifecycle);
|
||||
let mut snapshot_without_owner = options();
|
||||
borrowed_snapshot.add_lock_fences(&mut snapshot_without_owner);
|
||||
snapshot_without_owner.object_lock_config_snapshot = Some(borrowed_snapshot);
|
||||
|
||||
for opts in [foreign_fence, released_fence, snapshot_without_owner] {
|
||||
let mut source = copy_info(&store, ordinary, &object).await;
|
||||
for result in [
|
||||
store
|
||||
.put_object(
|
||||
ordinary,
|
||||
&object,
|
||||
&mut body(),
|
||||
&ObjectOptions {
|
||||
replication_request: true,
|
||||
..opts.clone()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map(|_| ()),
|
||||
store
|
||||
.copy_object(ordinary, &object, ordinary, &object, &mut source, &options(), &opts)
|
||||
.await
|
||||
.map(|_| ()),
|
||||
store
|
||||
.put_object_tags(ordinary, &object, "owner=fake", &opts)
|
||||
.await
|
||||
.map(|_| ()),
|
||||
store.put_object_metadata(ordinary, &object, &opts).await.map(|_| ()),
|
||||
store.clone().restore_transitioned_object(ordinary, &object, &opts).await,
|
||||
store.new_multipart_upload(ordinary, &object, &opts).await.map(|_| ()),
|
||||
] {
|
||||
let error = result.expect_err("a fence without its matching lifecycle owner cannot authorize a write");
|
||||
assert!(
|
||||
error.to_string().contains("write requires an owned bucket lifecycle fence"),
|
||||
"ARTIFACT_WRITE_LIFECYCLE_OWNER: {error}"
|
||||
);
|
||||
}
|
||||
let error = store
|
||||
.put_object(artifact, &object, &mut body(), &opts)
|
||||
.await
|
||||
.expect_err("ordinary artifact PUT also requires the lifecycle owner");
|
||||
assert!(
|
||||
error.to_string().contains("write requires an owned bucket lifecycle fence"),
|
||||
"ARTIFACT_WRITE_LIFECYCLE_OWNER: {error}"
|
||||
);
|
||||
assert_eq!(
|
||||
snapshot(&store, ordinary, &object).await,
|
||||
before,
|
||||
"ARTIFACT_WRITE_LIFECYCLE_OWNER: ordinary metadata unchanged"
|
||||
);
|
||||
assert_eq!(
|
||||
snapshot(&store, artifact, &object).await,
|
||||
artifact_before,
|
||||
"ARTIFACT_WRITE_LIFECYCLE_OWNER: artifact versions unchanged"
|
||||
);
|
||||
}
|
||||
drop(unrelated_owner);
|
||||
assert!(
|
||||
store
|
||||
.list_multipart_uploads(ordinary, &object, None, None, None, 100)
|
||||
.await
|
||||
.expect("inspect rejected multipart creation")
|
||||
.uploads
|
||||
.is_empty()
|
||||
);
|
||||
|
||||
let owned_snapshot = store
|
||||
.object_lock_config_snapshot(ordinary)
|
||||
.await
|
||||
.expect("real lifecycle-owning snapshot");
|
||||
let mut owned = options();
|
||||
owned_snapshot.add_lock_fences(&mut owned);
|
||||
owned.object_lock_config_snapshot = Some(owned_snapshot.clone());
|
||||
let lifecycle_namespace = store
|
||||
.new_ns_lock(ordinary, crate::storage_api_contracts::bucket::BUCKET_LIFECYCLE_LOCK_OBJECT)
|
||||
.await
|
||||
.expect("actual bucket deletion namespace");
|
||||
assert!(
|
||||
lifecycle_namespace.get_write_lock(Duration::from_millis(100)).await.is_err(),
|
||||
"matching snapshot actually blocks the bucket deletion lock"
|
||||
);
|
||||
store
|
||||
.put_object_tags(ordinary, &object, "owner=real", &owned)
|
||||
.await
|
||||
.expect("matching owned snapshot permits ordinary tags");
|
||||
let upload = store
|
||||
.new_multipart_upload(ordinary, &object, &owned)
|
||||
.await
|
||||
.expect("matching owned snapshot permits ordinary multipart creation");
|
||||
store
|
||||
.abort_multipart_upload(ordinary, &object, &upload.upload_id, &owned)
|
||||
.await
|
||||
.expect("clean up supported multipart upload");
|
||||
drop(owned);
|
||||
drop(owned_snapshot);
|
||||
let artifact_snapshot = store
|
||||
.object_lock_config_snapshot(artifact)
|
||||
.await
|
||||
.expect("artifact lifecycle-owning snapshot");
|
||||
let mut owned = options();
|
||||
artifact_snapshot.add_lock_fences(&mut owned);
|
||||
owned.object_lock_config_snapshot = Some(artifact_snapshot);
|
||||
store
|
||||
.put_object(artifact, &object, &mut body(), &owned)
|
||||
.await
|
||||
.expect("matching owned snapshot permits ordinary artifact PUT");
|
||||
assert_ne!(
|
||||
snapshot(&store, artifact, &object).await,
|
||||
artifact_before,
|
||||
"supported artifact PUT publishes a new version"
|
||||
);
|
||||
drop(owned);
|
||||
shutdown.cancel();
|
||||
assert!(ctx.wait_for_detached_mutations(Duration::from_secs(30)).await);
|
||||
}
|
||||
|
||||
// Include directories as well as bytes: a rejected delete must not even stage
|
||||
// a rollback directory before another layer can refuse the request.
|
||||
async fn delete_physical_snapshot(
|
||||
store: &ECStore,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
) -> Vec<std::collections::BTreeMap<std::path::PathBuf, Option<Vec<u8>>>> {
|
||||
let mut snapshots = Vec::new();
|
||||
for disk in store.pools[0].disk_set[0].disk_inventory().await {
|
||||
let disk = disk.expect("online delete fixture disk");
|
||||
let root = disk.path().join(bucket).join(object);
|
||||
let mut entries = std::collections::BTreeMap::new();
|
||||
let mut pending = vec![root.clone()];
|
||||
while let Some(directory) = pending.pop() {
|
||||
let mut children = tokio::fs::read_dir(&directory)
|
||||
.await
|
||||
.expect("read physical artifact directory");
|
||||
while let Some(child) = children.next_entry().await.expect("read physical artifact entry") {
|
||||
let path = child.path();
|
||||
let relative = path.strip_prefix(&root).expect("fixture descendant").to_owned();
|
||||
if child.file_type().await.expect("physical entry type").is_dir() {
|
||||
entries.insert(relative, None);
|
||||
pending.push(path);
|
||||
} else {
|
||||
entries.insert(relative, Some(tokio::fs::read(path).await.expect("physical artifact bytes")));
|
||||
}
|
||||
}
|
||||
}
|
||||
snapshots.push(entries);
|
||||
}
|
||||
snapshots
|
||||
}
|
||||
|
||||
async fn seed_delete_versions(store: &ECStore, bucket: &str, object: &str, incarnation: Uuid) -> (Uuid, Uuid) {
|
||||
let first = store
|
||||
.put_object(bucket, object, &mut PutObjReader::from_vec(vec![0xA5; 2 * 1024 * 1024]), &options())
|
||||
.await
|
||||
.expect("publish actual external artifact shards")
|
||||
.version_id
|
||||
.expect("versioned artifact identity");
|
||||
store
|
||||
.put_object(bucket, object, &mut body(), &options())
|
||||
.await
|
||||
.expect("publish another readable artifact version");
|
||||
let marker = Uuid::new_v4();
|
||||
let mut fi = rustfs_filemeta::FileInfo {
|
||||
name: object.to_owned(),
|
||||
version_id: Some(marker),
|
||||
deleted: true,
|
||||
mod_time: Some(time::OffsetDateTime::now_utc()),
|
||||
..Default::default()
|
||||
};
|
||||
fi.set_delete_marker_incarnation(incarnation);
|
||||
// Seed a pre-existing marker through the physical fixture API. Ordinary
|
||||
// managed deletion must never be needed to construct its own refusal case.
|
||||
for disk in store.pools[0].disk_set[0].disk_inventory().await {
|
||||
disk.expect("online marker fixture disk")
|
||||
.write_metadata("", bucket, object, fi.clone())
|
||||
.await
|
||||
.expect("persist pre-existing delete marker");
|
||||
}
|
||||
(first, marker)
|
||||
}
|
||||
|
||||
fn delete_variants(version: Uuid, marker: Uuid) -> Vec<ObjectOptions> {
|
||||
vec![
|
||||
options(),
|
||||
ObjectOptions {
|
||||
version_id: Some(version.to_string()),
|
||||
..options()
|
||||
},
|
||||
ObjectOptions {
|
||||
version_id: Some(marker.to_string()),
|
||||
..options()
|
||||
},
|
||||
ObjectOptions {
|
||||
version_id: Some(Uuid::new_v4().to_string()),
|
||||
delete_marker: true,
|
||||
replication_request: true,
|
||||
delete_replication: Some(crate::bucket::replication::ReplicationState::default()),
|
||||
..options()
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_delete_entrypoints_store_rejects_single_and_batch_before_mutation() {
|
||||
run_large_stack_async_test("artifact-delete-store", || delete_entrypoints_case(false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn artifact_delete_entrypoints_set_disks_rejects_direct_mutations() {
|
||||
run_large_stack_async_test("artifact-delete-set-disks", || delete_entrypoints_case(true));
|
||||
}
|
||||
|
||||
async fn delete_entrypoints_case(direct: bool) {
|
||||
use crate::storage_api_contracts::object::ObjectToDelete;
|
||||
let root = tempfile::tempdir().expect("delete entrypoint fixture root");
|
||||
let (ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(root.path(), "artifact-delete-entrypoints", &[4])).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let managed = "artifact-delete-managed";
|
||||
let ordinary = "artifact-delete-control";
|
||||
let incarnation = bucket(&store, managed, true).await;
|
||||
let ordinary_incarnation = bucket(&store, ordinary, false).await;
|
||||
let object = object_key();
|
||||
let (version, marker) = seed_delete_versions(&store, managed, &object, incarnation).await;
|
||||
let (control_version, control_marker) = seed_delete_versions(&store, ordinary, &object, ordinary_incarnation).await;
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
let before = delete_physical_snapshot(&store, managed, &object).await;
|
||||
assert!(
|
||||
before.iter().all(|disk| disk.keys().any(|path| path.ends_with("part.1"))),
|
||||
"fixture must include external shards"
|
||||
);
|
||||
|
||||
for sealed in [false, true] {
|
||||
if sealed {
|
||||
let attempt = object.rsplit_once('/').expect("attempt/object fixture").0.to_owned() + "/";
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_puts(managed, incarnation, &attempt)
|
||||
.await
|
||||
.expect("seal ordinary PUT domain");
|
||||
store
|
||||
.clone()
|
||||
.seal_artifact_attempt_repairs(managed, incarnation, &attempt)
|
||||
.await
|
||||
.expect("seal repair domain");
|
||||
}
|
||||
for opts in delete_variants(version, marker) {
|
||||
let result = if direct {
|
||||
set.delete_object(managed, &object, opts).await
|
||||
} else {
|
||||
// Hold the object lock to distinguish the ECStore guard from a
|
||||
// later SetDisks refusal, without removing that second defense.
|
||||
let lock = store.new_ns_lock(managed, &object).await.expect("delete namespace");
|
||||
let _held = lock
|
||||
.get_write_lock(Duration::from_secs(5))
|
||||
.await
|
||||
.expect("hold delete namespace");
|
||||
tokio::time::timeout(Duration::from_secs(1), store.delete_object(managed, &object, opts))
|
||||
.await
|
||||
.expect("managed rejection precedes object locking")
|
||||
};
|
||||
unsupported(result);
|
||||
assert_eq!(delete_physical_snapshot(&store, managed, &object).await, before);
|
||||
}
|
||||
let targets = vec![
|
||||
ObjectToDelete {
|
||||
object_name: object.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
ObjectToDelete {
|
||||
object_name: object.clone(),
|
||||
version_id: Some(version),
|
||||
..Default::default()
|
||||
},
|
||||
ObjectToDelete {
|
||||
object_name: object.clone(),
|
||||
version_id: Some(marker),
|
||||
..Default::default()
|
||||
},
|
||||
ObjectToDelete {
|
||||
object_name: object.clone(),
|
||||
version_id: Some(version),
|
||||
..Default::default()
|
||||
},
|
||||
ObjectToDelete {
|
||||
object_name: object.clone(),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
delete_marker_replication_status: Some("REPLICA".to_owned()),
|
||||
..Default::default()
|
||||
},
|
||||
];
|
||||
let (deleted, errors) = if direct {
|
||||
set.delete_objects(managed, targets.clone(), options()).await
|
||||
} else {
|
||||
let lock = store.new_ns_lock(managed, &object).await.expect("batch namespace");
|
||||
let _held = lock
|
||||
.get_write_lock(Duration::from_secs(5))
|
||||
.await
|
||||
.expect("hold batch namespace");
|
||||
tokio::time::timeout(Duration::from_secs(1), store.delete_objects(managed, targets.clone(), options()))
|
||||
.await
|
||||
.expect("managed batch rejection precedes object locking")
|
||||
};
|
||||
assert_eq!(deleted.len(), targets.len());
|
||||
assert_eq!(errors.len(), targets.len());
|
||||
for error in errors {
|
||||
unsupported::<()>(Err(error.expect("each managed delete retains its own refusal")));
|
||||
}
|
||||
assert_eq!(delete_physical_snapshot(&store, managed, &object).await, before);
|
||||
let request = rustfs_filemeta::FileInfo {
|
||||
version_id: Some(version),
|
||||
name: object.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
unsupported(if direct {
|
||||
set.delete_object_version(managed, &object, &request, false).await
|
||||
} else {
|
||||
store.delete_object_version(managed, &object, &request, false).await
|
||||
});
|
||||
assert_eq!(delete_physical_snapshot(&store, managed, &object).await, before);
|
||||
}
|
||||
|
||||
let control_before = delete_physical_snapshot(&store, ordinary, &object).await;
|
||||
let mut control_opts = options();
|
||||
if direct {
|
||||
// An unbound policy snapshot remains supported, but is not lifecycle
|
||||
// authority. The direct entry point must acquire its own real owner.
|
||||
control_opts.object_lock_config_snapshot = Some(std::sync::Arc::new(crate::object_api::ObjectLockConfigSnapshot::new(
|
||||
metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
|
||||
)));
|
||||
}
|
||||
let created = if direct {
|
||||
set.delete_object(ordinary, &object, control_opts.clone()).await
|
||||
} else {
|
||||
store.delete_object(ordinary, &object, control_opts.clone()).await
|
||||
}
|
||||
.expect("ordinary bucket actually creates a delete marker");
|
||||
assert!(created.delete_marker);
|
||||
assert_ne!(delete_physical_snapshot(&store, ordinary, &object).await, control_before);
|
||||
let control_targets = vec![
|
||||
ObjectToDelete {
|
||||
object_name: object.clone(),
|
||||
version_id: Some(control_version),
|
||||
..Default::default()
|
||||
},
|
||||
ObjectToDelete {
|
||||
object_name: object.clone(),
|
||||
version_id: Some(control_marker),
|
||||
..Default::default()
|
||||
},
|
||||
];
|
||||
let (_, errors) = if direct {
|
||||
set.delete_objects(ordinary, control_targets, control_opts).await
|
||||
} else {
|
||||
store.delete_objects(ordinary, control_targets, control_opts).await
|
||||
};
|
||||
assert!(errors.iter().all(Option::is_none), "ordinary batch deletion must commit: {errors:?}");
|
||||
for disk in set.disk_inventory().await {
|
||||
let disk = disk.expect("online control disk");
|
||||
for removed in [control_version, control_marker] {
|
||||
assert!(
|
||||
disk.read_version(ordinary, ordinary, &object, &removed.to_string(), &crate::disk::ReadOptions::default())
|
||||
.await
|
||||
.is_err(),
|
||||
"ordinary deletion removed the exact physical version"
|
||||
);
|
||||
}
|
||||
}
|
||||
shutdown.cancel();
|
||||
assert!(ctx.wait_for_detached_mutations(Duration::from_secs(30)).await);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -86,6 +86,160 @@ pub async fn connect_load_init_formats(
|
||||
pub(crate) struct LoadedFormat {
|
||||
pub(crate) format: FormatV3,
|
||||
pub(crate) pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority,
|
||||
pub(crate) pending_disks: Vec<PendingFormatDisk>,
|
||||
}
|
||||
|
||||
pub(crate) struct PendingFormatDisk {
|
||||
disk: DiskStore,
|
||||
}
|
||||
|
||||
impl PendingFormatDisk {
|
||||
/// Reconcile only after bootstrap metadata has succeeded, before Ready.
|
||||
pub(crate) async fn finalize(
|
||||
&self,
|
||||
instance_ctx: &Arc<InstanceContext>,
|
||||
sets: &[Arc<crate::set_disk::SetDisks>],
|
||||
) -> Result<()> {
|
||||
let endpoint = self.disk.endpoint();
|
||||
let pool_index = usize::try_from(endpoint.pool_idx).map_err(|_| Error::CorruptedFormat)?;
|
||||
let set_index = usize::try_from(endpoint.set_idx).map_err(|_| Error::CorruptedFormat)?;
|
||||
let disk_index = usize::try_from(endpoint.disk_idx).map_err(|_| Error::CorruptedFormat)?;
|
||||
let set = sets
|
||||
.iter()
|
||||
.find(|set| set.pool_index == pool_index && set.set_index == set_index)
|
||||
.ok_or(Error::CorruptedFormat)?;
|
||||
if set.set_endpoints.get(disk_index) != Some(&endpoint) {
|
||||
return Err(Error::CorruptedFormat);
|
||||
}
|
||||
// Capture current handles, including a reconnect that completed since
|
||||
// format selection. No registry guard is held during format I/O.
|
||||
let registry = instance_ctx.local_disk_map();
|
||||
let receivers = registry.read().await;
|
||||
let receiver = if endpoint.is_local {
|
||||
receivers.get(&endpoint.to_string()).and_then(Clone::clone)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let selected = set
|
||||
.disks
|
||||
.read()
|
||||
.await
|
||||
.get(disk_index)
|
||||
.cloned()
|
||||
.ok_or(Error::CorruptedFormat)?;
|
||||
if endpoint.is_local && receiver.is_none() {
|
||||
// An earlier successful pass may already have quarantined this
|
||||
// slot before a different slot raced with reconnect and retried.
|
||||
let topology = instance_ctx.local_disk_set_drives();
|
||||
let local_slots = topology.read().await;
|
||||
let local_slot = local_slots
|
||||
.get(pool_index)
|
||||
.and_then(|sets| sets.get(set_index))
|
||||
.and_then(|disks| disks.get(disk_index))
|
||||
.ok_or(Error::CorruptedFormat)?;
|
||||
return if matches!(receivers.get(&endpoint.to_string()), Some(None)) && local_slot.is_none() && selected.is_none() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(Error::other("format reconciliation receiver changed"))
|
||||
};
|
||||
}
|
||||
drop(receivers);
|
||||
let disk = selected.as_ref().unwrap_or(&self.disk);
|
||||
let observation = load_format_erasure(receiver.as_ref().unwrap_or(disk), false).await;
|
||||
self.publish_observation(instance_ctx, set, selected.as_ref(), receiver.as_ref(), observation)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn publish_observation(
|
||||
&self,
|
||||
instance_ctx: &Arc<InstanceContext>,
|
||||
set: &crate::set_disk::SetDisks,
|
||||
selected: Option<&DiskStore>,
|
||||
receiver: Option<&DiskStore>,
|
||||
observation: disk::error::Result<FormatV3>,
|
||||
) -> Result<()> {
|
||||
let endpoint = self.disk.endpoint();
|
||||
let pool_index = usize::try_from(endpoint.pool_idx).map_err(|_| Error::CorruptedFormat)?;
|
||||
let set_index = usize::try_from(endpoint.set_idx).map_err(|_| Error::CorruptedFormat)?;
|
||||
let disk_index = usize::try_from(endpoint.disk_idx).map_err(|_| Error::CorruptedFormat)?;
|
||||
if set.pool_index != pool_index || set.set_index != set_index || set.set_endpoints.get(disk_index) != Some(&endpoint) {
|
||||
return Err(Error::CorruptedFormat);
|
||||
}
|
||||
let valid = observation.as_ref().is_ok_and(|format| {
|
||||
set.format.check_other(format).is_ok()
|
||||
&& set.format.find_disk_index_by_disk_id(format.erasure.this).ok() == Some((set_index, disk_index))
|
||||
});
|
||||
let disk_id = if valid {
|
||||
Some(observation.map_err(Error::from)?.erasure.this)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let path = endpoint.to_string();
|
||||
// Local sets must share the registered receiver's health and I/O
|
||||
// admission state, just as Sets::new_with_instance_ctx does.
|
||||
let disk = receiver.unwrap_or_else(|| selected.unwrap_or(&self.disk));
|
||||
if disk.endpoint() != endpoint
|
||||
|| endpoint.is_local != receiver.is_some()
|
||||
|| receiver.is_some_and(|receiver| receiver.endpoint() != endpoint)
|
||||
{
|
||||
return Err(Error::CorruptedFormat);
|
||||
}
|
||||
|
||||
// All format I/O is complete. Publish under registry -> ID map -> local
|
||||
// topology -> set slot, rechecking handle identity before any mutation.
|
||||
let registry = instance_ctx.local_disk_map();
|
||||
let mut receivers = registry.write().await;
|
||||
let id_map = instance_ctx.local_disk_id_map();
|
||||
let mut ids = id_map.write().await;
|
||||
let topology = instance_ctx.local_disk_set_drives();
|
||||
let mut local_slots = topology.write().await;
|
||||
let mut slots = set.disks.write().await;
|
||||
let slot = slots.get_mut(disk_index).ok_or(Error::CorruptedFormat)?;
|
||||
let unchanged = match (slot.as_ref(), selected) {
|
||||
(None, None) => true,
|
||||
(Some(current), Some(observed)) => Arc::ptr_eq(current, observed),
|
||||
_ => false,
|
||||
};
|
||||
if !unchanged {
|
||||
return Err(Error::other("format reconciliation slot changed"));
|
||||
}
|
||||
if let Some(receiver) = receiver {
|
||||
if !receivers
|
||||
.get(&path)
|
||||
.and_then(Option::as_ref)
|
||||
.is_some_and(|current| Arc::ptr_eq(current, receiver))
|
||||
{
|
||||
return Err(Error::other("format reconciliation receiver changed"));
|
||||
}
|
||||
let local_slot = local_slots
|
||||
.get_mut(pool_index)
|
||||
.and_then(|sets| sets.get_mut(set_index))
|
||||
.and_then(|disks| disks.get_mut(disk_index))
|
||||
.ok_or(Error::CorruptedFormat)?;
|
||||
if !local_slot.as_ref().is_some_and(|current| Arc::ptr_eq(current, receiver)) {
|
||||
return Err(Error::other("format reconciliation local slot changed"));
|
||||
}
|
||||
// Local ID setters only update memory, without filesystem I/O.
|
||||
receiver.set_disk_id_state(disk_id).await?;
|
||||
ids.retain(|_, registered_path| registered_path != &path);
|
||||
if let Some(id) = disk_id {
|
||||
ids.insert(id, path.clone());
|
||||
} else {
|
||||
receivers.insert(path, None);
|
||||
*local_slot = None;
|
||||
}
|
||||
}
|
||||
if receiver.is_none() {
|
||||
disk.set_disk_id_state(disk_id).await?;
|
||||
}
|
||||
if valid {
|
||||
disk.enable_health_check();
|
||||
*slot = Some(disk.clone());
|
||||
} else {
|
||||
*slot = None;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
@@ -137,13 +291,15 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
if matches!(format_quorum, Some(Err(Error::ErasureReadQuorum)))
|
||||
&& let Some((format, members)) = select_existing_format_read_quorum(&formats, &errs, set_drive_count)
|
||||
{
|
||||
retain_format_quorum_members(instance_ctx, disks, &format, &members, set_drive_count).await?;
|
||||
let pending_disks =
|
||||
retain_format_quorum_members(instance_ctx, disks, &format, &members, set_drive_count, Some(&errs)).await?;
|
||||
// Loading an existing format grants no authority to initialize pool
|
||||
// metadata or migrate formats. The persisted storage classes and each
|
||||
// object's EC metadata still determine service and operation quorums.
|
||||
return Ok(LoadedFormat {
|
||||
format,
|
||||
pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority::None,
|
||||
pending_disks,
|
||||
});
|
||||
}
|
||||
let resumable_partial_migration = formats_present > 0
|
||||
@@ -158,10 +314,11 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
match try_migrate_format(disks, &formats, set_count, set_drive_count).await {
|
||||
Ok(LegacyFormatOutcome::Migrated { format, quorum_members }) => {
|
||||
info!("Migrated format from MinIO config");
|
||||
retain_format_quorum_members(instance_ctx, disks, &format, &quorum_members, set_drive_count).await?;
|
||||
retain_format_quorum_members(instance_ctx, disks, &format, &quorum_members, set_drive_count, None).await?;
|
||||
return Ok(LoadedFormat {
|
||||
format: *format,
|
||||
pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority::LegacyAdoption,
|
||||
pending_disks: Vec::new(),
|
||||
});
|
||||
}
|
||||
Ok(LegacyFormatOutcome::Incompatible) => {
|
||||
@@ -182,6 +339,7 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
return Ok(LoadedFormat {
|
||||
format: fm,
|
||||
pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority::Fresh,
|
||||
pending_disks: Vec::new(),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -206,11 +364,13 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
None => select_format_erasure_in_quorum(&formats, 0)?,
|
||||
};
|
||||
check_format_erasure_value_for_topology(&fm, formats.len(), set_drive_count)?;
|
||||
retain_format_quorum_members(instance_ctx, disks, &fm, &quorum_members, set_drive_count).await?;
|
||||
let pending_disks =
|
||||
retain_format_quorum_members(instance_ctx, disks, &fm, &quorum_members, set_drive_count, Some(&errs)).await?;
|
||||
|
||||
Ok(LoadedFormat {
|
||||
format: fm,
|
||||
pool_meta_bootstrap_authority: verified_legacy_adoption_source(disks, &formats, set_count, set_drive_count).await?,
|
||||
pending_disks,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -239,7 +399,8 @@ async fn retain_format_quorum_members(
|
||||
format: &FormatV3,
|
||||
quorum_members: &[bool],
|
||||
set_drive_count: usize,
|
||||
) -> Result<()> {
|
||||
observed_errors: Option<&[Option<DiskError>]>,
|
||||
) -> Result<Vec<PendingFormatDisk>> {
|
||||
if set_drive_count == 0 || quorum_members.len() != disks.len() {
|
||||
return Err(Error::CorruptedFormat);
|
||||
}
|
||||
@@ -279,10 +440,31 @@ async fn retain_format_quorum_members(
|
||||
let mut local_pool_endpoints = Vec::new();
|
||||
let mut selected_local_disk_ids = Vec::new();
|
||||
let mut quarantined_endpoints = Vec::new();
|
||||
let mut pending_disks = Vec::new();
|
||||
let registry = instance_ctx.local_disk_map();
|
||||
let receivers = registry.read().await;
|
||||
for (index, ((disk, endpoint), belongs_to_quorum)) in disks.iter().zip(&endpoints).zip(quorum_members).enumerate() {
|
||||
let pending = !belongs_to_quorum
|
||||
&& matches!(
|
||||
observed_errors.and_then(|errors| errors.get(index)),
|
||||
Some(Some(DiskError::UnformattedDisk))
|
||||
);
|
||||
let receiver = endpoint
|
||||
.as_ref()
|
||||
.filter(|endpoint| endpoint.is_local)
|
||||
.and_then(|endpoint| receivers.get(&endpoint.to_string()))
|
||||
.and_then(Clone::clone);
|
||||
// A first writer may still be publishing this slot's format. Keep its
|
||||
// receiver reachable, but do not count it as a format quorum member.
|
||||
let defer = pending && disk.as_ref().is_some_and(|disk| !disk.is_local() || receiver.is_some());
|
||||
if defer {
|
||||
pending_disks.push(PendingFormatDisk {
|
||||
disk: disk.as_ref().ok_or(Error::CorruptedFormat)?.clone(),
|
||||
});
|
||||
}
|
||||
if let Some(endpoint) = endpoint.as_ref().filter(|endpoint| endpoint.is_local) {
|
||||
local_pool_endpoints.push(endpoint.to_string());
|
||||
if !belongs_to_quorum {
|
||||
if !belongs_to_quorum && !defer {
|
||||
quarantined_endpoints.push(endpoint.clone());
|
||||
}
|
||||
}
|
||||
@@ -302,6 +484,7 @@ async fn retain_format_quorum_members(
|
||||
selected_local_disk_ids.push((disk_id, disk.endpoint().to_string()));
|
||||
}
|
||||
}
|
||||
drop(receivers);
|
||||
quarantine_local_disks(instance_ctx, &quarantined_endpoints).await?;
|
||||
|
||||
for (disk, disk_id) in disks.iter().zip(member_disk_ids) {
|
||||
@@ -315,7 +498,7 @@ async fn retain_format_quorum_members(
|
||||
*disk = None;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
Ok(pending_disks)
|
||||
}
|
||||
|
||||
pub fn should_init_erasure_disks(errs: &[Option<DiskError>]) -> bool {
|
||||
@@ -369,7 +552,7 @@ async fn init_format_erasure(
|
||||
write_format_file_all(disks, &fms).await?;
|
||||
let format = get_format_erasure_in_quorum(&fms, 0)?;
|
||||
let quorum_members = vec![true; disks.len()];
|
||||
retain_format_quorum_members(instance_ctx, disks, &format, &quorum_members, set_drive_count).await?;
|
||||
retain_format_quorum_members(instance_ctx, disks, &format, &quorum_members, set_drive_count, None).await?;
|
||||
Ok(format)
|
||||
}
|
||||
|
||||
@@ -976,6 +1159,314 @@ mod tests {
|
||||
(temp_dir, disks)
|
||||
}
|
||||
|
||||
struct PendingFormatFixture {
|
||||
ctx: Arc<InstanceContext>,
|
||||
set: Arc<crate::set_disk::SetDisks>,
|
||||
pending: PendingFormatDisk,
|
||||
receiver: DiskStore,
|
||||
format: FormatV3,
|
||||
_temp: tempfile::TempDir,
|
||||
}
|
||||
|
||||
async fn pending_format_fixture() -> PendingFormatFixture {
|
||||
let (temp, mut disks) = local_disks(4).await;
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let format = FormatV3::new(1, 4);
|
||||
let endpoints = disks.iter().flatten().map(|disk| disk.endpoint()).collect::<Vec<_>>();
|
||||
// Bootstrap receivers are distinct from the subsequently opened set handles.
|
||||
let (receivers, errors) = init_disks(
|
||||
&Endpoints::from(endpoints.clone()),
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(errors.iter().all(Option::is_none));
|
||||
record_local_disks(&ctx, receivers.iter().flatten().cloned().collect()).await;
|
||||
*ctx.local_disk_set_drives().write().await = vec![vec![receivers.clone()]];
|
||||
for (index, disk) in disks.iter().enumerate().take(3) {
|
||||
let mut member = format.clone();
|
||||
member.erasure.this = format.erasure.sets[0][index];
|
||||
save_format_file(disk, &Some(member))
|
||||
.await
|
||||
.expect("first three formats should publish");
|
||||
}
|
||||
let receiver = receivers[3].as_ref().expect("pending receiver should exist").clone();
|
||||
let mut last = format.clone();
|
||||
last.erasure.this = format.erasure.sets[0][3];
|
||||
let bootstrap = crate::store::peer::BootstrapLocalTarget::new(ctx.clone());
|
||||
bootstrap
|
||||
.begin_legacy_local_file_write(&receiver, RUSTFS_META_BUCKET, None)
|
||||
.await
|
||||
.expect("pending format writer should be admitted")
|
||||
.write_all("pending-format", last.to_json().expect("format should encode").into_bytes().into())
|
||||
.await
|
||||
.expect("last format temporary file should be durable");
|
||||
let mut loaded = connect_load_init_formats_with_instance_ctx(&ctx, false, &mut disks, 1, 4, None)
|
||||
.await
|
||||
.expect("three existing formats should supply read quorum");
|
||||
assert!(matches!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None));
|
||||
assert_eq!(loaded.pending_disks.len(), 1);
|
||||
assert!(disks[3].is_none(), "unformatted slot must not count as a member");
|
||||
let set = crate::set_disk::SetDisks::new_with_instance_ctx(
|
||||
"pending-format-test".to_string(),
|
||||
Arc::new(tokio::sync::RwLock::new(disks)),
|
||||
4,
|
||||
2,
|
||||
0,
|
||||
0,
|
||||
endpoints,
|
||||
loaded.format,
|
||||
Vec::new(),
|
||||
ctx.clone(),
|
||||
)
|
||||
.await;
|
||||
PendingFormatFixture {
|
||||
ctx,
|
||||
set,
|
||||
pending: loaded.pending_disks.remove(0),
|
||||
receiver,
|
||||
format,
|
||||
_temp: temp,
|
||||
}
|
||||
}
|
||||
|
||||
async fn publish_pending_format(fixture: &PendingFormatFixture) {
|
||||
let bootstrap = crate::store::peer::BootstrapLocalTarget::new(fixture.ctx.clone());
|
||||
// Resolve the endpoint again, just as the writer's next RPC does. A
|
||||
// retained Arc alone would bypass the receiver quarantine regression.
|
||||
let resolved = bootstrap
|
||||
.local_mutation_disk(&fixture.receiver.endpoint().to_string())
|
||||
.await
|
||||
.expect("pending receiver must remain endpoint-addressable");
|
||||
assert!(Arc::ptr_eq(&resolved, &fixture.receiver));
|
||||
bootstrap
|
||||
.begin_legacy_local_rename_file(
|
||||
&fixture.receiver.endpoint().to_string(),
|
||||
(RUSTFS_META_BUCKET, "pending-format"),
|
||||
(RUSTFS_META_BUCKET, FORMAT_CONFIG_FILE),
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("writer's next rename should still be admitted")
|
||||
.commit()
|
||||
.await
|
||||
.expect("writer should complete the last format");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn format_quorum_preserves_pending_bootstrap_receiver_and_restores_completed_slot() {
|
||||
let fixture = pending_format_fixture().await;
|
||||
publish_pending_format(&fixture).await;
|
||||
fixture
|
||||
.pending
|
||||
.finalize(&fixture.ctx, std::slice::from_ref(&fixture.set))
|
||||
.await
|
||||
.expect("completed format should reconcile before Ready");
|
||||
let slots = fixture.set.disks.read().await;
|
||||
assert_eq!(slots.iter().flatten().count(), 4);
|
||||
assert!(Arc::ptr_eq(
|
||||
slots[3].as_ref().expect("restored local member should exist"),
|
||||
&fixture.receiver
|
||||
));
|
||||
for (index, disk) in slots.iter().enumerate() {
|
||||
let disk = disk.as_ref().expect("all four members should be restored");
|
||||
let loaded = load_format_erasure(disk, false)
|
||||
.await
|
||||
.expect("restored member format should read");
|
||||
assert_eq!(loaded.shared_identity(), fixture.format.shared_identity());
|
||||
assert_eq!(loaded.erasure.this, fixture.format.erasure.sets[0][index]);
|
||||
}
|
||||
let id = fixture.format.erasure.sets[0][3];
|
||||
assert_eq!(
|
||||
slots[3]
|
||||
.as_ref()
|
||||
.expect("last member should exist")
|
||||
.get_disk_id()
|
||||
.await
|
||||
.expect("set disk ID should read"),
|
||||
Some(id)
|
||||
);
|
||||
assert_eq!(fixture.receiver.get_disk_id().await.expect("receiver ID should read"), Some(id));
|
||||
assert_eq!(
|
||||
fixture.ctx.local_disk_id_map().read().await.get(&id),
|
||||
Some(&fixture.receiver.endpoint().to_string())
|
||||
);
|
||||
assert!(Arc::ptr_eq(
|
||||
fixture.ctx.local_disk_set_drives().read().await[0][0][3]
|
||||
.as_ref()
|
||||
.expect("local slot should remain active"),
|
||||
&fixture.receiver
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn format_quorum_quarantines_still_unformatted_receiver_before_ready() {
|
||||
let fixture = pending_format_fixture().await;
|
||||
fixture
|
||||
.pending
|
||||
.finalize(&fixture.ctx, std::slice::from_ref(&fixture.set))
|
||||
.await
|
||||
.expect("unformatted minority should be isolated without changing quorum");
|
||||
assert!(fixture.ctx.local_disk_map().read().await[&fixture.receiver.endpoint().to_string()].is_none());
|
||||
assert!(fixture.ctx.local_disk_set_drives().read().await[0][0][3].is_none());
|
||||
assert!(fixture.set.disks.read().await[3].is_none());
|
||||
assert_eq!(fixture.set.disks.read().await.iter().flatten().count(), 3);
|
||||
fixture
|
||||
.pending
|
||||
.finalize(&fixture.ctx, std::slice::from_ref(&fixture.set))
|
||||
.await
|
||||
.expect("a later init retry must preserve completed quarantine");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn format_quorum_quarantines_foreign_completed_receiver_before_ready() {
|
||||
let fixture = pending_format_fixture().await;
|
||||
let mut foreign = FormatV3::new(1, 4);
|
||||
foreign.erasure.this = foreign.erasure.sets[0][3];
|
||||
write_format_file(&fixture.receiver, &foreign)
|
||||
.await
|
||||
.expect("foreign format should publish");
|
||||
fixture
|
||||
.pending
|
||||
.finalize(&fixture.ctx, std::slice::from_ref(&fixture.set))
|
||||
.await
|
||||
.expect("foreign minority should be isolated");
|
||||
assert!(fixture.ctx.local_disk_map().read().await[&fixture.receiver.endpoint().to_string()].is_none());
|
||||
assert!(fixture.ctx.local_disk_set_drives().read().await[0][0][3].is_none());
|
||||
assert!(fixture.set.disks.read().await[3].is_none());
|
||||
assert!(
|
||||
!fixture
|
||||
.ctx
|
||||
.local_disk_id_map()
|
||||
.read()
|
||||
.await
|
||||
.contains_key(&foreign.erasure.this)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn format_quorum_rechecks_replaced_receiver_after_io_and_accepts_current_member_on_retry() {
|
||||
let fixture = pending_format_fixture().await;
|
||||
publish_pending_format(&fixture).await;
|
||||
let observed = load_format_erasure(&fixture.receiver, false)
|
||||
.await
|
||||
.expect("old receiver observation should load");
|
||||
let replacement = new_disk(
|
||||
&fixture.receiver.endpoint(),
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("replacement receiver should open");
|
||||
fixture
|
||||
.ctx
|
||||
.local_disk_map()
|
||||
.write()
|
||||
.await
|
||||
.insert(replacement.endpoint().to_string(), Some(replacement.clone()));
|
||||
fixture.ctx.local_disk_set_drives().write().await[0][0][3] = Some(replacement.clone());
|
||||
*fixture.set.disks.write().await.get_mut(3).expect("last slot should exist") = Some(replacement.clone());
|
||||
assert!(
|
||||
fixture
|
||||
.pending
|
||||
.publish_observation(&fixture.ctx, &fixture.set, None, Some(&fixture.receiver), Ok(observed))
|
||||
.await
|
||||
.is_err(),
|
||||
"stale I/O must not publish over a replacement"
|
||||
);
|
||||
assert!(Arc::ptr_eq(
|
||||
fixture.ctx.local_disk_map().read().await[&replacement.endpoint().to_string()]
|
||||
.as_ref()
|
||||
.expect("replacement must not be quarantined"),
|
||||
&replacement
|
||||
));
|
||||
fixture
|
||||
.pending
|
||||
.finalize(&fixture.ctx, std::slice::from_ref(&fixture.set))
|
||||
.await
|
||||
.expect("existing init retry should validate the current replacement");
|
||||
assert!(Arc::ptr_eq(
|
||||
fixture.set.disks.read().await[3]
|
||||
.as_ref()
|
||||
.expect("validated slot should exist"),
|
||||
&replacement
|
||||
));
|
||||
assert_eq!(
|
||||
replacement.get_disk_id().await.expect("replacement ID should read"),
|
||||
Some(fixture.format.erasure.sets[0][3])
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn format_quorum_remote_pending_publication_checks_shared_identity_and_slot() {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let format = FormatV3::new(1, 4);
|
||||
let mut endpoint = Endpoint::try_from("http://remote-server:9000/data").expect("remote endpoint should parse");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(3);
|
||||
let disk = new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("remote client should initialize without connecting");
|
||||
let mut endpoints = vec![endpoint.clone(); 4];
|
||||
for (index, endpoint) in endpoints.iter_mut().enumerate() {
|
||||
endpoint.set_disk_index(index);
|
||||
}
|
||||
let set = crate::set_disk::SetDisks::new_with_instance_ctx(
|
||||
"remote-pending-test".to_string(),
|
||||
Arc::new(tokio::sync::RwLock::new(vec![None; 4])),
|
||||
4,
|
||||
2,
|
||||
0,
|
||||
0,
|
||||
endpoints,
|
||||
format.clone(),
|
||||
Vec::new(),
|
||||
ctx.clone(),
|
||||
)
|
||||
.await;
|
||||
let pending = PendingFormatDisk { disk: disk.clone() };
|
||||
let mut wrong_slot = format.clone();
|
||||
wrong_slot.erasure.this = format.erasure.sets[0][2];
|
||||
pending
|
||||
.publish_observation(&ctx, &set, None, None, Ok(wrong_slot))
|
||||
.await
|
||||
.expect("wrong-slot format should remain excluded");
|
||||
assert!(set.disks.read().await[3].is_none());
|
||||
let mut foreign = FormatV3::new(1, 4);
|
||||
foreign.erasure.this = foreign.erasure.sets[0][3];
|
||||
pending
|
||||
.publish_observation(&ctx, &set, None, None, Ok(foreign))
|
||||
.await
|
||||
.expect("foreign remote format should remain excluded");
|
||||
assert!(set.disks.read().await[3].is_none());
|
||||
let mut matching = format.clone();
|
||||
matching.erasure.this = format.erasure.sets[0][3];
|
||||
pending
|
||||
.publish_observation(&ctx, &set, None, None, Ok(matching))
|
||||
.await
|
||||
.expect("exact remote slot should restore");
|
||||
assert!(Arc::ptr_eq(set.disks.read().await[3].as_ref().expect("remote slot should exist"), &disk));
|
||||
assert_eq!(
|
||||
disk.get_disk_id().await.expect("remote ID state should read"),
|
||||
Some(format.erasure.sets[0][3])
|
||||
);
|
||||
assert!(
|
||||
ctx.local_disk_map().read().await.is_empty(),
|
||||
"remote restoration must not register a local receiver"
|
||||
);
|
||||
}
|
||||
|
||||
async fn two_local_disks_with_missing_third() -> (tempfile::TempDir, Vec<Option<DiskStore>>) {
|
||||
let (temp_dir, mut disks) = local_disks(2).await;
|
||||
disks.push(None);
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -431,8 +431,8 @@ pub(crate) use bucket::await_bucket_namespace_operation;
|
||||
pub use bucket_fence::BucketIncarnationFenceGuard;
|
||||
pub(crate) mod delete_recovery_runtime;
|
||||
mod heal;
|
||||
pub(crate) use heal::{BucketHealScope, bucket_heal_scope, bucket_heal_scope_for_object};
|
||||
pub use heal::{HealObjectAbsenceProof, HealObjectStorageResult};
|
||||
pub(crate) use heal::{bucket_heal_scope, bucket_heal_scope_for_object};
|
||||
mod heal_walk;
|
||||
pub use heal_walk::HealWalkVersion;
|
||||
mod init;
|
||||
@@ -494,6 +494,7 @@ pub struct ECStore {
|
||||
/// Memoizes bucket-incarnation validation under continuous lifecycle
|
||||
/// read-lock coverage (see [`bucket_fence`]).
|
||||
pub(crate) bucket_fence_registry: Arc<bucket_fence::BucketFenceRegistry>,
|
||||
pub(crate) ordinary_classification: crate::bucket::artifact::OrdinaryClassificationRegistry,
|
||||
}
|
||||
|
||||
const METRIC_SCANNER_DATA_MOVEMENT_PAUSED: &str = "rustfs_scanner_data_movement_paused";
|
||||
@@ -1430,7 +1431,7 @@ impl BucketOperations for ECStore {
|
||||
|
||||
#[instrument(skip(self))]
|
||||
async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
|
||||
Box::pin(self.handle_make_bucket(bucket, opts)).await
|
||||
self.handle_make_bucket(bucket, opts).await
|
||||
}
|
||||
|
||||
#[instrument(skip(self))]
|
||||
@@ -1904,6 +1905,7 @@ mod tests {
|
||||
pool_meta_save_gate: Mutex::default(),
|
||||
ctx,
|
||||
bucket_fence_registry: Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
};
|
||||
|
||||
let rendered = format!("{store:?}");
|
||||
@@ -1919,6 +1921,19 @@ mod tests {
|
||||
assert!(!rendered.contains("DiskHealthTracker"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn make_bucket_handler_future_remains_stack_bounded() {
|
||||
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
|
||||
let opts = MakeBucketOptions::default();
|
||||
let future = store.handle_make_bucket("stack-bounded-bucket", &opts);
|
||||
let future_size = std::mem::size_of_val(&future);
|
||||
|
||||
assert!(
|
||||
future_size <= 4 * 1024,
|
||||
"make bucket handler future must remain stack-bounded; measured {future_size} bytes"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_options_debug_does_not_expand_tier_store_handle() {
|
||||
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
|
||||
@@ -1979,6 +1994,7 @@ mod tests {
|
||||
pool_meta_save_gate: Mutex::default(),
|
||||
ctx,
|
||||
bucket_fence_registry: Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -602,7 +602,7 @@ impl ECStore {
|
||||
mutation_fence: Option<&ObjectLockDiagGuard>,
|
||||
) -> Result<(MultipartUploadResult, usize, Option<Uuid>)> {
|
||||
check_new_object_args(bucket, object, opts.data_movement)?;
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.prepare_unsupported_artifact_write(bucket, opts).await?;
|
||||
|
||||
if self.single_pool() {
|
||||
self.apply_decommission_target_mutation_fence(0, object, &mut opts, mutation_fence)
|
||||
@@ -741,10 +741,7 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<PartInfo> {
|
||||
check_put_object_part_args(bucket, object, upload_id)?;
|
||||
let (mut opts, bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
if let Some(guard) = bucket_lifecycle_guard {
|
||||
opts.bucket_lifecycle_lock_owner = Some(Arc::new(guard));
|
||||
}
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.prepare_unsupported_artifact_write(bucket, opts).await?;
|
||||
if !self.single_pool() {
|
||||
opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
|
||||
}
|
||||
@@ -778,7 +775,7 @@ impl ECStore {
|
||||
if !opts.data_movement {
|
||||
return Err(Error::other("targeted multipart upload requires data_movement options"));
|
||||
}
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.prepare_unsupported_artifact_write(bucket, opts).await?;
|
||||
ensure_decommission_capacity_mutation_id(bucket, object, &mut opts);
|
||||
let pool = self.pools.get(target_pool_idx).ok_or_else(|| {
|
||||
Error::InvalidArgument("data-movement".to_string(), "target-pool".to_string(), target_pool_idx.to_string())
|
||||
@@ -1160,7 +1157,7 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
check_complete_multipart_args(bucket, object, upload_id)?;
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.prepare_unsupported_artifact_write(bucket, opts).await?;
|
||||
if !self.single_pool() {
|
||||
opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
|
||||
}
|
||||
@@ -1261,7 +1258,7 @@ impl ECStore {
|
||||
if !opts.data_movement {
|
||||
return Err(Error::other("targeted multipart completion requires data_movement options"));
|
||||
}
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.prepare_unsupported_artifact_write(bucket, opts).await?;
|
||||
ensure_decommission_capacity_mutation_id(bucket, object, &mut opts);
|
||||
if opts.overwrites_existing_version() && !is_meta_bucketname(bucket) {
|
||||
let expected_incarnation_id = opts
|
||||
@@ -1666,6 +1663,7 @@ mod tests {
|
||||
pool_meta_save_gate: Mutex::default(),
|
||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
ordinary_classification: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+600
-453
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,32 @@
|
||||
# Compression-index compatibility fixtures
|
||||
|
||||
These are synthetic public fixtures, generated with RustFS source and the locked dependencies at `017b85a5bfc3c2c779af942b68e3a7c42b3a9d81`. They contain no captured user object or credentials.
|
||||
|
||||
`legacy-lz4-stream.bin` was emitted by `rustfs_rio::CompressReader` using LZ4 and its default 1 MiB block size. The compressor and utility compression source files are byte-identical between this commit and `17aa75a1759e856e93b8c1df2fd447b5ce92e30d`.
|
||||
|
||||
`pre-zigzag-index.bin` was emitted by compiling the complete, unmodified `crates/rio/src/compress_index.rs` module from `17aa75a1759e856e93b8c1df2fd447b5ce92e30d`, before the signed-varint fix. The compressor's serializable Index state was deserialized into that historical Index type and passed to its writer. Its historical reader verified complete consumption, identical state and two seek results before the bytes were retained. This pins a historical writer module; it is not a full historical release or a captured stored object.
|
||||
|
||||
`zigzag-index.bin` was emitted from the same Index state using the current writer, and serves as an indexed-read control. The old and new index files intentionally differ.
|
||||
|
||||
## Reproducing the input
|
||||
|
||||
The plaintext length is `3 * 1024 * 1024 + 123`. For each byte offset, let `chunk = offset / 1024` and `lane = offset % 1024`. The first eight bytes of each 1024-byte chunk are `u64::to_le_bytes(chunk)`; the remainder is `b"abcdefghijklmnopqrstuvwxyz"[(offset + chunk) % 26]`.
|
||||
|
||||
Drain the legacy LZ4 CompressReader into a byte vector, obtain its Index through TryGetIndex, then serialize that Index state through serde_json. Compile the pinned historical module, deserialize the state into its Index, and call into_vec. Emit the current writer's into_vec result separately. Do not recreate the old index through the reader being tested.
|
||||
|
||||
The recorded state has `est_block_uncomp = 0`, total compressed offset `52137`, total uncompressed offset `3145728`, and these `(compressed, uncompressed)` entries:
|
||||
|
||||
- `(17085, 1048576)`
|
||||
- `(34663, 2097152)`
|
||||
- `(52137, 3145728)`
|
||||
|
||||
The final partial block is part of the stream; the recorded totals are the Index state exposed by this historical compressor, not a replacement for the object's actual length.
|
||||
|
||||
| Bytes | Length | SHA-256 |
|
||||
| --- | ---: | --- |
|
||||
| Plaintext | 3145851 | `68f9edf6f9600395d6d2b5517d97570fecd2a0c6dbe6130617ef415a3b48d89c` |
|
||||
| Legacy LZ4 stream | 52212 | `cbb8ead867a159a6a089a3a09fd424986cc8995bcb384434d0b95d0d9bdfef7e` |
|
||||
| Historical index | 47 | `ff4fb74d14a8ddc0d3c31d565c30b62e142df02ac0d19fa270c666b98a8e7171` |
|
||||
| Current index | 51 | `8a819221f9bdaa2bd05e14eead2833746ea01cb1c68086bffa25a731c252174e` |
|
||||
|
||||
Tests exercise the production read planner and GetObjectReader rather than decoding the expected bytes with a replacement implementation. Historical-body and fallback cases apply to both legacy and rio-v2 builds; the exact legacy indexed-offset control is feature-gated because each backend has its own index representation. No throughput or application-level release qualification is inferred from these fixtures.
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -596,6 +596,191 @@ fn bench_metacache_resolve(c: &mut Criterion) {
|
||||
group.finish();
|
||||
}
|
||||
|
||||
fn listing_history(versions: usize, cached: bool) -> MetaCacheEntry {
|
||||
let mut metadata = FileMeta::new();
|
||||
for index in 0..versions {
|
||||
let (data, parity) = if index + 1 == versions { (12, 4) } else { (8, 8) };
|
||||
let mut info = FileInfo::new("object", data, parity);
|
||||
info.volume = "bucket".to_owned();
|
||||
info.erasure.index = 1;
|
||||
info.data_dir = Some(Uuid::from_u128(0x1000 + u128::try_from(index).expect("data directory index")));
|
||||
info.versioned = true;
|
||||
info.version_id = Some(Uuid::from_u128(u128::try_from(index + 1).expect("version index")));
|
||||
info.mod_time = Some(
|
||||
OffsetDateTime::from_unix_timestamp(1_700_000_000 + i64::try_from(index).expect("timestamp")).expect("fixture time"),
|
||||
);
|
||||
info.size = 1;
|
||||
info.parts = vec![ObjectPartInfo {
|
||||
number: 1,
|
||||
size: 1,
|
||||
actual_size: 1,
|
||||
..Default::default()
|
||||
}];
|
||||
info.metadata.insert("etag".to_owned(), format!("etag-{index}"));
|
||||
info.metadata.insert("x-amz-meta-description".to_owned(), "m".repeat(1024));
|
||||
metadata.add_version(info).expect("valid version history");
|
||||
}
|
||||
MetaCacheEntry {
|
||||
name: "object".to_owned(),
|
||||
metadata: metadata.marshal_msg().expect("serialize history"),
|
||||
cached: cached.then_some(metadata),
|
||||
reusable: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn owned_agreed_latest_reference(
|
||||
mut entry: MetaCacheEntry,
|
||||
replicas: usize,
|
||||
params: &rustfs_filemeta::MetadataResolutionParams,
|
||||
enforce_write_quorum: bool,
|
||||
) -> rustfs_filemeta::MetaCacheEntryResolution {
|
||||
use rustfs_filemeta::MetaCacheEntryResolution::{NeedsSupplement, Rejected, Resolved};
|
||||
if !enforce_write_quorum {
|
||||
return Resolved(entry);
|
||||
}
|
||||
if entry.is_dir() {
|
||||
return if replicas >= params.dir_quorum {
|
||||
Resolved(entry)
|
||||
} else {
|
||||
Rejected
|
||||
};
|
||||
}
|
||||
if replicas < params.obj_quorum {
|
||||
return Rejected;
|
||||
}
|
||||
let metadata = match entry.xl_meta_ref() {
|
||||
Ok(metadata) => metadata,
|
||||
Err(_) if params.obj_quorum <= 1 => return Resolved(entry),
|
||||
Err(_) => return Rejected,
|
||||
};
|
||||
if params.requested_versions == 1
|
||||
&& let Some(first) = metadata.versions.first()
|
||||
&& !first.header.free_version()
|
||||
&& replicas >= first.listing_quorum(params.obj_quorum, params.write_quorum_slack)
|
||||
{
|
||||
return Resolved(entry);
|
||||
}
|
||||
// These fixtures cover the fast prefix, never older-version reconciliation.
|
||||
NeedsSupplement(entry)
|
||||
}
|
||||
|
||||
fn bench_listing_borrowed_metadata(c: &mut Criterion) {
|
||||
use rustfs_filemeta::{FileMetaVersion, MetaCacheEntries, MetadataResolutionParams};
|
||||
let mut group = c.benchmark_group("listing_borrowed_metadata");
|
||||
group
|
||||
.sample_size(20)
|
||||
.warm_up_time(Duration::from_millis(150))
|
||||
.measurement_time(Duration::from_millis(700));
|
||||
for versions in [2, 16, 256] {
|
||||
for cached in [false, true] {
|
||||
let entry = listing_history(versions, cached);
|
||||
let params = MetadataResolutionParams {
|
||||
dir_quorum: 8,
|
||||
obj_quorum: 8,
|
||||
requested_versions: 1,
|
||||
write_quorum_slack: 8,
|
||||
bucket: "bucket".to_owned(),
|
||||
..Default::default()
|
||||
};
|
||||
let slots = vec![Some(&entry); 8];
|
||||
let owned = MetaCacheEntries(vec![Some(entry.clone()); 8])
|
||||
.resolve_with_write_quorum(params.clone())
|
||||
.expect("previous committed version");
|
||||
let borrowed =
|
||||
MetaCacheEntries::resolve_borrowed_with_write_quorum(&slots, params.clone()).expect("borrowed previous version");
|
||||
assert_eq!(owned.metadata, borrowed.metadata);
|
||||
let id = format!("{versions}_versions/{}", if cached { "cached" } else { "cold" });
|
||||
group.throughput(Throughput::Bytes(u64::try_from(entry.metadata.len()).expect("metadata size")));
|
||||
// ABBA order permits an explicit baseline-drift check for each fixture.
|
||||
for phase in ["owned_a1", "borrowed_b1", "borrowed_b2", "owned_a2"] {
|
||||
group.bench_function(BenchmarkId::new("replicas", format!("{id}/{phase}")), |b| {
|
||||
b.iter(|| {
|
||||
if phase.starts_with("owned") {
|
||||
black_box(
|
||||
MetaCacheEntries(vec![Some(black_box(&entry).clone()); 8])
|
||||
.resolve_with_write_quorum(params.clone()),
|
||||
)
|
||||
} else {
|
||||
let slots = vec![Some(black_box(&entry)); 8];
|
||||
black_box(MetaCacheEntries::resolve_borrowed_with_write_quorum(&slots, params.clone()))
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
let mut agreed_metadata = entry
|
||||
.cached
|
||||
.clone()
|
||||
.unwrap_or_else(|| FileMeta::load(&entry.metadata).expect("agreed fixture"));
|
||||
let mut latest = FileMetaVersion::try_from(agreed_metadata.versions[0].meta.as_slice()).expect("latest body");
|
||||
let object = latest.object.as_mut().expect("latest object");
|
||||
object.erasure_m = 8;
|
||||
object.erasure_n = 8;
|
||||
agreed_metadata.versions[0] = latest.try_into().expect("balanced latest header");
|
||||
let agreed = MetaCacheEntry {
|
||||
name: entry.name.clone(),
|
||||
metadata: agreed_metadata.marshal_msg().expect("agreed history"),
|
||||
cached: cached.then_some(agreed_metadata),
|
||||
reusable: false,
|
||||
};
|
||||
for phase in ["owned_a1", "borrowed_b1", "borrowed_b2", "owned_a2"] {
|
||||
group.bench_function(BenchmarkId::new("agreed", format!("{id}/{phase}")), |b| {
|
||||
b.iter_batched(
|
||||
|| agreed.clone(),
|
||||
|candidate| {
|
||||
let resolved = if phase.starts_with("owned") {
|
||||
owned_agreed_latest_reference(candidate, 8, ¶ms, true)
|
||||
} else {
|
||||
rustfs_filemeta::resolve_agreed_entry(candidate, 8, ¶ms, true)
|
||||
};
|
||||
let rustfs_filemeta::MetaCacheEntryResolution::Resolved(candidate) = resolved else {
|
||||
panic!("balanced latest must use agreed fast prefix");
|
||||
};
|
||||
black_box(candidate)
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
);
|
||||
});
|
||||
}
|
||||
let replicas: Vec<_> = (1..=8)
|
||||
.map(|disk| {
|
||||
let mut metadata = entry
|
||||
.cached
|
||||
.clone()
|
||||
.unwrap_or_else(|| FileMeta::load(&entry.metadata).expect("decode fixture"));
|
||||
for version in &mut metadata.versions {
|
||||
let mut payload = FileMetaVersion::try_from(version.meta.as_slice()).expect("version body");
|
||||
payload.object.as_mut().expect("object body").erasure_index = disk;
|
||||
version.meta = payload.marshal_msg().expect("per-disk body");
|
||||
}
|
||||
MetaCacheEntry {
|
||||
name: entry.name.clone(),
|
||||
metadata: metadata.marshal_msg().expect("per-disk metadata"),
|
||||
cached: cached.then_some(metadata),
|
||||
reusable: false,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
assert!(replicas[0].headers_agree(&replicas, false));
|
||||
for phase in ["owned_a1", "borrowed_b1", "borrowed_b2", "owned_a2"] {
|
||||
group.bench_function(BenchmarkId::new("headers", format!("{id}/{phase}")), |b| {
|
||||
b.iter(|| {
|
||||
let agreed = if phase.starts_with("owned") {
|
||||
replicas
|
||||
.iter()
|
||||
.all(|candidate| replicas[0].matches(Some(black_box(candidate)), false).1)
|
||||
} else {
|
||||
replicas[0].headers_agree(black_box(&replicas), false)
|
||||
};
|
||||
black_box(agreed)
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
group.finish();
|
||||
}
|
||||
|
||||
criterion_group!(
|
||||
benches,
|
||||
bench_create_real_xlmeta,
|
||||
@@ -612,7 +797,8 @@ criterion_group!(
|
||||
bench_list_page_to_fileinfo,
|
||||
bench_delete_inline_versions,
|
||||
bench_metacache_stream_page,
|
||||
bench_metacache_resolve
|
||||
bench_metacache_resolve,
|
||||
bench_listing_borrowed_metadata
|
||||
);
|
||||
|
||||
criterion_main!(benches);
|
||||
|
||||
@@ -154,6 +154,7 @@ pub fn is_skip_meta_key(key: &str) -> bool {
|
||||
}
|
||||
|
||||
mod codec;
|
||||
pub(crate) use codec::{BorrowedFileMeta, reconcile_headers};
|
||||
mod inline_data;
|
||||
pub(crate) mod msgp_decode;
|
||||
mod validation;
|
||||
|
||||
@@ -597,7 +597,7 @@ impl FileMeta {
|
||||
return false;
|
||||
}
|
||||
|
||||
let mut is_delete_marker = false;
|
||||
let mut is_delete_marker = true;
|
||||
|
||||
let _ = Self::decode_versions(meta, versions, |_: usize, hdr: &[u8], _: &[u8]| {
|
||||
let mut header = FileMetaVersionHeader::default();
|
||||
@@ -605,6 +605,9 @@ impl FileMeta {
|
||||
return Err(Error::DoneForNow);
|
||||
}
|
||||
|
||||
if header.free_version() {
|
||||
return Ok(());
|
||||
}
|
||||
is_delete_marker = header.version_type == VersionType::Delete;
|
||||
|
||||
Err(Error::DoneForNow)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user