mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-10 14:16:01 +00:00
Compare commits
25 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 7b6e45d372 | |||
| 09c85a5f29 | |||
| 00aeb12914 | |||
| ffb18979f8 | |||
| 97c7b451d2 | |||
| 9ba95cac37 | |||
| cec796328c | |||
| 1aa4fa145f | |||
| d286f3d06c | |||
| 3391528025 | |||
| 0d1490da3c | |||
| 6421a7ff60 | |||
| 27141f6312 | |||
| 940e221988 | |||
| 8b66e9b62a | |||
| 6a879be1fb | |||
| 0b40d47a8a | |||
| e292637ee0 | |||
| 8b6b1e53a3 | |||
| 358bf6832d | |||
| 70fb8bf504 | |||
| 110f630a5c | |||
| 54bd3c8e24 | |||
| 1643cb29f1 | |||
| bf8d5a32b6 |
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=874c881d7b45f12378a5817c7f42c95c4981960a2ec9ce12dcf4af239ae1f9d5
|
||||
sha256-linux=9515861be899ceb10e2e0ef93c34208bb7a7a8a7f8067a02db4cfba23270ebd6
|
||||
sha256-darwin=15cb0cf9909bfbfc5a835fb08bd3675516c641db1b92ecc1e170a1cfa0fa2fd5
|
||||
sha256-linux=3163fdd29df5def86cf511ca7db05880d5c0caaa608a7031a71394327f7c217a
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=6d18f9cce820c51d5589de944e8cc185f73eeca0ea9a9916651943e3759169d0
|
||||
sha256=5fbb230b89212b7c3d7229d6cef3e7e2d16f0ecfec62237ebc770785706f67d9
|
||||
|
||||
@@ -114,7 +114,7 @@
|
||||
"sets": 3,
|
||||
"pools": 3,
|
||||
"outage_target_manifest_required": false,
|
||||
"scope": "Target process crash during partial background rebuild on three single-node EC8+4 pools; exact baseline S3 bodies and replacement-drive shards, with outage object verified through S3 but not forced onto the replaced target drive."
|
||||
"scope": "Target process crash during partial background rebuild on three single-node EC8+4 pools; exact baseline S3 bodies and replacement-drive shards, with down-window outage PUT refusal recorded and a deferred post-rejoin outage object verified through S3 when the full target pool was offline."
|
||||
}
|
||||
},
|
||||
"release_lanes": {
|
||||
|
||||
@@ -335,11 +335,7 @@ jobs:
|
||||
# re-enabled by backlog#1304 (restore accepts serialize on a short CAS
|
||||
# guard; the copy-back no longer holds the #4877 whole-copy-back lock,
|
||||
# so the mid-restore ongoing read and fast 409 rejection it asserts are
|
||||
# the implemented contract). The remaining exclusions each hit a
|
||||
# DIFFERENT, independent issue (all tracked under rustfs/backlog#1148;
|
||||
# they keep #[ignore] with a backlog reference):
|
||||
# - test_noncurrent_{expiry,transition}_still_works_after_immediate_compensation_transition:
|
||||
# noncurrent transition/expiry after an immediate compensation transition.
|
||||
# the implemented contract).
|
||||
- name: Run ignored ILM integration tests serially
|
||||
env:
|
||||
# Match the measured Test and Lint link budget. The default exposed
|
||||
@@ -352,7 +348,7 @@ jobs:
|
||||
NEXTEST_HIDE_PROGRESS_BAR=1 timeout --verbose --signal=TERM --kill-after=30s 80m \
|
||||
cargo nextest run -j1 --run-ignored ignored-only \
|
||||
-p rustfs-scanner -p rustfs \
|
||||
-E '(binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))) and not (test(test_noncurrent_expiry_still_works_after_immediate_compensation_transition) or test(test_noncurrent_transition_still_works_after_immediate_compensation_transition))' \
|
||||
-E 'binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))' \
|
||||
--status-level all --final-status-level all \
|
||||
2>&1 | tee artifacts/ilm-integration/nextest.log
|
||||
status=${PIPESTATUS[0]}
|
||||
|
||||
Generated
+46
-46
@@ -271,7 +271,7 @@ version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -282,7 +282,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"once_cell_polyfill",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1641,9 +1641,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.13.1"
|
||||
version = "2.13.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da"
|
||||
checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
@@ -1906,7 +1906,7 @@ dependencies = [
|
||||
"maybe-owned",
|
||||
"rustix",
|
||||
"rustix-linux-procfs",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
"winx",
|
||||
]
|
||||
|
||||
@@ -2270,9 +2270,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "console"
|
||||
version = "0.16.4"
|
||||
version = "0.16.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4fe5f465a4f6fee88fad41b85d990f84c835335e85b5d9e6e63e0d06d28cba7c"
|
||||
checksum = "e96a4956774c13c126a8b5af4daa79384f4d826534c95a02d76afb39e2ab64e3"
|
||||
dependencies = [
|
||||
"encode_unicode",
|
||||
"libc",
|
||||
@@ -3942,7 +3942,7 @@ dependencies = [
|
||||
"libc",
|
||||
"option-ext",
|
||||
"redox_users",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3951,7 +3951,7 @@ version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"objc2",
|
||||
]
|
||||
|
||||
@@ -4287,7 +4287,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4416,7 +4416,7 @@ version = "25.12.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"rustc_version",
|
||||
]
|
||||
|
||||
@@ -5706,7 +5706,7 @@ version = "0.7.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed3bd0ecfbb87805f538bb7b32e5239ca0763890c623e349860ecba69469f2bb"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"cfg-if",
|
||||
"libc",
|
||||
]
|
||||
@@ -6212,7 +6212,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c9f8ff371890db2cf65a0758dba9a79f9cd965de369f6dbdc6581a22780af45e"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"dashmap",
|
||||
@@ -6794,7 +6794,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0f27695f286b461da077b8c2f72f47feaa04ce3c3f9c0976257410e90e21208a"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"btoi",
|
||||
"byteorder",
|
||||
"bytes",
|
||||
@@ -6826,7 +6826,7 @@ version = "0.7.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "22f9786d56d972959e1408b6a93be6af13b9c1392036c5c1fafa08a1b0c6ee87"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"byteorder",
|
||||
"derive_builder",
|
||||
"getset",
|
||||
@@ -6874,7 +6874,7 @@ version = "0.29.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"cfg-if",
|
||||
"cfg_aliases",
|
||||
"libc",
|
||||
@@ -6887,7 +6887,7 @@ version = "0.30.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"cfg-if",
|
||||
"cfg_aliases",
|
||||
"libc",
|
||||
@@ -6899,7 +6899,7 @@ version = "0.31.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"cfg-if",
|
||||
"cfg_aliases",
|
||||
"libc",
|
||||
@@ -6963,7 +6963,7 @@ version = "0.50.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -7100,7 +7100,7 @@ version = "0.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d164abbde0b3c03edb9edb9cb8d31a7f5b79015c692b7c771f6e0840e9106b9f"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"libloading",
|
||||
"nvml-wrapper-sys",
|
||||
"static_assertions",
|
||||
@@ -7151,7 +7151,7 @@ version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"dispatch2",
|
||||
"objc2",
|
||||
]
|
||||
@@ -7168,7 +7168,7 @@ version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"objc2",
|
||||
]
|
||||
|
||||
@@ -7721,7 +7721,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e33c6dbf1a8fb7f71742cd70f5e9f0986e60b2d19dc0b28d9ca0d1323259274a"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"thiserror 2.0.20",
|
||||
"zerocopy",
|
||||
"zerocopy-derive",
|
||||
@@ -7777,7 +7777,7 @@ version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "575828d9d7d205188048eb1508560607a03d21eafdbba47b8cade1736c1c28e1"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"c-enum",
|
||||
"perf-event-open-sys2",
|
||||
]
|
||||
@@ -8294,7 +8294,7 @@ checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744"
|
||||
dependencies = [
|
||||
"bit-set",
|
||||
"bit-vec 0.8.0",
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"num-traits",
|
||||
"rand 0.9.5",
|
||||
"rand_chacha 0.9.0",
|
||||
@@ -8436,7 +8436,7 @@ version = "0.13.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e9f068eba8e7071c5f9511831b44f32c740d5adf574e990f946ddb53db2f314e"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"memchr",
|
||||
"unicase",
|
||||
]
|
||||
@@ -8712,7 +8712,7 @@ dependencies = [
|
||||
"once_cell",
|
||||
"socket2",
|
||||
"tracing",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -8874,7 +8874,7 @@ version = "11.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -8983,7 +8983,7 @@ version = "0.5.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -9310,7 +9310,7 @@ checksum = "036204edbd199552a5b3832f63c60dcdf395dc44c7f06b4af1c0e8139cc11bce"
|
||||
dependencies = [
|
||||
"aes 0.9.3",
|
||||
"aws-lc-rs",
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"block-padding 0.4.2",
|
||||
"byteorder",
|
||||
"bytes",
|
||||
@@ -9391,7 +9391,7 @@ version = "3.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "093197e526668d92bba562e2bbbe98d1af9831bf080b619c736316ca1fa35101"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"dashmap",
|
||||
@@ -11061,11 +11061,11 @@ version = "1.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11148,7 +11148,7 @@ dependencies = [
|
||||
"security-framework",
|
||||
"security-framework-sys",
|
||||
"webpki-root-certs",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11431,7 +11431,7 @@ version = "3.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"core-foundation 0.10.1",
|
||||
"core-foundation-sys",
|
||||
"libc",
|
||||
@@ -11949,7 +11949,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -12100,7 +12100,7 @@ dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"psm",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -12321,7 +12321,7 @@ version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"core-foundation 0.9.4",
|
||||
"system-configuration-sys",
|
||||
]
|
||||
@@ -12405,10 +12405,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.4.3",
|
||||
"getrandom 0.3.4",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -12876,7 +12876,7 @@ version = "0.6.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"http 1.5.0",
|
||||
@@ -12895,7 +12895,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "08a05a66a4fdd61cbbe0a1d755ffe0ca6aba159dd4820936a0ff8a8278245b9c"
|
||||
dependencies = [
|
||||
"async-compression",
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
@@ -13244,9 +13244,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "uuid"
|
||||
version = "1.26.0"
|
||||
version = "1.26.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812"
|
||||
checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce"
|
||||
dependencies = [
|
||||
"getrandom 0.4.3",
|
||||
"js-sys",
|
||||
@@ -13510,7 +13510,7 @@ version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -13820,7 +13820,7 @@ version = "0.36.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
|
||||
+1
-1
@@ -335,7 +335,7 @@ tracing-subscriber = { version = "0.3.23" }
|
||||
transform-stream = "0.3.1"
|
||||
url = "2.5.8"
|
||||
urlencoding = "2.1.3"
|
||||
uuid = { version = "1.26.0" }
|
||||
uuid = { version = "1.26.1" }
|
||||
vaultrs = { version = "0.8.0" }
|
||||
tar = "0.4.46"
|
||||
walkdir = "2.5.0"
|
||||
|
||||
@@ -22,7 +22,11 @@ mod tests {
|
||||
init_logging, rustfs_binary_path,
|
||||
};
|
||||
use crate::storage_api::RUSTFS_META_BUCKET;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::{
|
||||
error::{ProvideErrorMetadata, SdkError},
|
||||
operation::put_object::PutObjectError,
|
||||
primitives::ByteStream,
|
||||
};
|
||||
use http::Method;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashSet;
|
||||
@@ -39,6 +43,13 @@ mod tests {
|
||||
|
||||
const POOL_METADATA_OBJECT: &str = "pool.bin";
|
||||
|
||||
struct ReplacementDriveSelection {
|
||||
replaced_disk: PathBuf,
|
||||
replacement_format_path: PathBuf,
|
||||
replacement_format: Vec<u8>,
|
||||
expected_pool_metadata: Option<VersionShardCensus>,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct EvidenceBuild {
|
||||
sha256: String,
|
||||
@@ -586,6 +597,54 @@ mod tests {
|
||||
&& operations["activeBySource"]["admin"].as_u64() == Some(1)
|
||||
}
|
||||
|
||||
fn is_service_unavailable_put(error: &SdkError<PutObjectError>) -> bool {
|
||||
error.as_service_error().and_then(ProvideErrorMetadata::code) == Some("ServiceUnavailable")
|
||||
}
|
||||
|
||||
fn select_replacement_drive(
|
||||
cluster: &RustFSTestClusterEnvironment,
|
||||
node_index: usize,
|
||||
require_pool_metadata: bool,
|
||||
) -> Result<ReplacementDriveSelection, Box<dyn Error + Send + Sync>> {
|
||||
let node = cluster
|
||||
.nodes
|
||||
.get(node_index)
|
||||
.ok_or_else(|| format!("replacement node {node_index} is absent"))?;
|
||||
let mut incomplete_pool_metadata = Vec::new();
|
||||
|
||||
for drive in &node.data_dirs {
|
||||
let replaced_disk = PathBuf::from(drive);
|
||||
let replacement_format_path = replaced_disk.join(".rustfs.sys").join("format.json");
|
||||
let replacement_format = std::fs::read(&replacement_format_path).map_err(|err| {
|
||||
format!("failed to capture target format before replacement wipe at {replacement_format_path:?}: {err}")
|
||||
})?;
|
||||
if !require_pool_metadata {
|
||||
return Ok(ReplacementDriveSelection {
|
||||
replaced_disk,
|
||||
replacement_format_path,
|
||||
replacement_format,
|
||||
expected_pool_metadata: None,
|
||||
});
|
||||
}
|
||||
|
||||
let census = census_object_version_on_disk(&replaced_disk, RUSTFS_META_BUCKET, POOL_METADATA_OBJECT, None)?;
|
||||
if census.is_complete() {
|
||||
return Ok(ReplacementDriveSelection {
|
||||
replaced_disk,
|
||||
replacement_format_path,
|
||||
replacement_format,
|
||||
expected_pool_metadata: Some(census),
|
||||
});
|
||||
}
|
||||
incomplete_pool_metadata.push(census);
|
||||
}
|
||||
|
||||
Err(format!(
|
||||
"no replacement drive on node {node_index} held complete pool metadata before the fault: {incomplete_pool_metadata:?}"
|
||||
)
|
||||
.into())
|
||||
}
|
||||
|
||||
async fn replacement_recovery_status(
|
||||
cluster: &RustFSTestClusterEnvironment,
|
||||
) -> Result<serde_json::Value, Box<dyn Error + Send + Sync>> {
|
||||
@@ -1008,7 +1067,7 @@ mod tests {
|
||||
let mut versions_observed = false;
|
||||
let mut observations = Vec::with_capacity(cluster.nodes.len());
|
||||
for (node_index, node) in cluster.nodes.iter().enumerate() {
|
||||
let (status, body) = timeout(
|
||||
let response = timeout(
|
||||
Duration::from_secs(5),
|
||||
admin_request(
|
||||
&node.url,
|
||||
@@ -1019,8 +1078,22 @@ mod tests {
|
||||
&cluster.secret_key,
|
||||
),
|
||||
)
|
||||
.await??;
|
||||
assert_eq!(status, 200, "scanner status must be available: {body}");
|
||||
.await;
|
||||
let (status, body) = match response {
|
||||
Ok(Ok(response)) => response,
|
||||
Ok(Err(error)) => {
|
||||
observations.push(format!("node{node_index}: scanner status request failed: {error}"));
|
||||
continue;
|
||||
}
|
||||
Err(_) => {
|
||||
observations.push(format!("node{node_index}: scanner status request exceeded 5s"));
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if status != 200 {
|
||||
observations.push(format!("node{node_index}: scanner status returned {status}: {body}"));
|
||||
continue;
|
||||
}
|
||||
let status: serde_json::Value = serde_json::from_str(&body)?;
|
||||
assert_eq!(status["enabled"].as_bool(), Some(true), "scanner must stay enabled: {status}");
|
||||
let metrics = &status["metrics"];
|
||||
@@ -1257,16 +1330,18 @@ mod tests {
|
||||
let bucket = "heal-restart-during-rebuild";
|
||||
clients[0].create_bucket().bucket(bucket).send().await?;
|
||||
|
||||
let replaced_disk = PathBuf::from(&cluster.nodes[1].data_dir);
|
||||
let replacement_format_path = replaced_disk.join(".rustfs.sys").join("format.json");
|
||||
let replacement_format = std::fs::read(&replacement_format_path).map_err(|err| {
|
||||
format!("failed to capture target format before replacement wipe at {replacement_format_path:?}: {err}")
|
||||
})?;
|
||||
let ReplacementDriveSelection {
|
||||
replaced_disk,
|
||||
replacement_format_path,
|
||||
replacement_format,
|
||||
expected_pool_metadata,
|
||||
} = select_replacement_drive(&cluster, 1, background_enabled)?;
|
||||
let default_online_object_count = if !outage_target_manifest_required { 64 } else { 24 };
|
||||
let online_object_count = std::env::var("RUSTFS_HEAL_CHAOS_OBJECT_COUNT")
|
||||
.ok()
|
||||
.and_then(|value| value.parse::<usize>().ok())
|
||||
.unwrap_or(24)
|
||||
.clamp(8, 64);
|
||||
.unwrap_or(default_online_object_count)
|
||||
.clamp(8, 128);
|
||||
let object_size_bytes = std::env::var("RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES")
|
||||
.ok()
|
||||
.and_then(|value| value.parse::<usize>().ok())
|
||||
@@ -1317,17 +1392,9 @@ mod tests {
|
||||
attempt_count += 1;
|
||||
}
|
||||
|
||||
let expected_pool_metadata = if background_enabled {
|
||||
let census = census_object_version_on_disk(&replaced_disk, RUSTFS_META_BUCKET, POOL_METADATA_OBJECT, None)?;
|
||||
assert!(
|
||||
census.is_complete(),
|
||||
"target must hold complete pool metadata before the fault: {census:?}"
|
||||
);
|
||||
if background_enabled {
|
||||
wait_for_scanner_cycle_after(&cluster, 0).await?;
|
||||
Some(census)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
}
|
||||
|
||||
cluster.stop_node(1)?;
|
||||
std::fs::remove_dir_all(&replaced_disk)?;
|
||||
@@ -1342,48 +1409,87 @@ mod tests {
|
||||
"replacement target must retain only its preformatted topology identity"
|
||||
);
|
||||
|
||||
let outage_key = "cluster/written-while-node-down.bin";
|
||||
let outage_payload_seed = 0xf1;
|
||||
timeout(
|
||||
Duration::from_secs(30),
|
||||
clients[2]
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(outage_key)
|
||||
.body(ByteStream::from(deterministic_object_body(object_size_bytes, outage_payload_seed)))
|
||||
.send(),
|
||||
)
|
||||
.await??;
|
||||
let max_outage_write_attempts = topology.total_drives().max(1);
|
||||
let mut outage_key = None;
|
||||
let mut outage_write_deferred_until_rejoin = false;
|
||||
let mut service_unavailable_outage_writes = 0usize;
|
||||
let mut last_service_unavailable = None;
|
||||
for attempt in 0..max_outage_write_attempts {
|
||||
let candidate_key = format!("cluster/written-while-node-down-{attempt:04}.bin");
|
||||
let put_result = timeout(
|
||||
Duration::from_secs(30),
|
||||
clients[2]
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(&candidate_key)
|
||||
.body(ByteStream::from(deterministic_object_body(object_size_bytes, outage_payload_seed)))
|
||||
.send(),
|
||||
)
|
||||
.await;
|
||||
match put_result {
|
||||
Ok(Ok(_)) => {
|
||||
outage_key = Some(candidate_key);
|
||||
break;
|
||||
}
|
||||
Ok(Err(error)) if is_service_unavailable_put(&error) => {
|
||||
service_unavailable_outage_writes += 1;
|
||||
last_service_unavailable = Some(format!("{error:?}"));
|
||||
}
|
||||
Ok(Err(error)) => return Err(error.into()),
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
}
|
||||
let outage_key = match outage_key {
|
||||
Some(key) => key,
|
||||
None if !outage_target_manifest_required => {
|
||||
outage_write_deferred_until_rejoin = true;
|
||||
"cluster/written-after-target-rejoin.bin".to_string()
|
||||
}
|
||||
None => {
|
||||
return Err(format!(
|
||||
"no online pool accepted an outage object after {max_outage_write_attempts} candidates; \
|
||||
observed {service_unavailable_outage_writes} ServiceUnavailable responses; \
|
||||
last ServiceUnavailable: {last_service_unavailable:?}"
|
||||
)
|
||||
.into());
|
||||
}
|
||||
};
|
||||
|
||||
let mut outage_peer_erasure_indices = HashSet::new();
|
||||
for (node_index, node) in cluster.nodes.iter().enumerate() {
|
||||
if node_index == 1 {
|
||||
continue;
|
||||
}
|
||||
for (drive_index, drive) in node.data_dirs.iter().enumerate() {
|
||||
let census = census_object_version_on_disk(Path::new(drive), bucket, outage_key, None)?;
|
||||
if !census.has_xl_meta {
|
||||
if !outage_write_deferred_until_rejoin {
|
||||
for (node_index, node) in cluster.nodes.iter().enumerate() {
|
||||
if node_index == 1 {
|
||||
continue;
|
||||
}
|
||||
assert!(
|
||||
census.is_complete(),
|
||||
"online node {node_index} drive {drive_index} must hold a complete outage-object shard: {census:?}"
|
||||
);
|
||||
let erasure_index = census.erasure_index.ok_or_else(|| {
|
||||
format!("online node {node_index} drive {drive_index} outage-object shard has no erasure index: {census:?}")
|
||||
})?;
|
||||
assert!(
|
||||
(1..=erasure_set_drive_count).contains(&erasure_index),
|
||||
"online node {node_index} drive {drive_index} outage-object erasure index is out of range: {census:?}"
|
||||
);
|
||||
assert!(
|
||||
outage_peer_erasure_indices.insert(erasure_index),
|
||||
"outage-object erasure index {erasure_index} is duplicated across online drives"
|
||||
);
|
||||
for (drive_index, drive) in node.data_dirs.iter().enumerate() {
|
||||
let census = census_object_version_on_disk(Path::new(drive), bucket, &outage_key, None)?;
|
||||
if !census.has_xl_meta {
|
||||
continue;
|
||||
}
|
||||
assert!(
|
||||
census.is_complete(),
|
||||
"online node {node_index} drive {drive_index} must hold a complete outage-object shard: {census:?}"
|
||||
);
|
||||
let erasure_index = census.erasure_index.ok_or_else(|| {
|
||||
format!(
|
||||
"online node {node_index} drive {drive_index} outage-object shard has no erasure index: {census:?}"
|
||||
)
|
||||
})?;
|
||||
assert!(
|
||||
(1..=erasure_set_drive_count).contains(&erasure_index),
|
||||
"online node {node_index} drive {drive_index} outage-object erasure index is out of range: {census:?}"
|
||||
);
|
||||
assert!(
|
||||
outage_peer_erasure_indices.insert(erasure_index),
|
||||
"outage-object erasure index {erasure_index} is duplicated across online drives"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
!outage_peer_erasure_indices.is_empty() && outage_peer_erasure_indices.len() <= erasure_set_drive_count,
|
||||
outage_write_deferred_until_rejoin
|
||||
|| (!outage_peer_erasure_indices.is_empty() && outage_peer_erasure_indices.len() <= erasure_set_drive_count),
|
||||
"outage-object must occupy one non-empty erasure set"
|
||||
);
|
||||
if outage_target_manifest_required {
|
||||
@@ -1436,7 +1542,12 @@ mod tests {
|
||||
);
|
||||
let recovered: serde_json::Value = serde_json::from_str(&status_body)
|
||||
.map_err(|err| format!("background heal status is not JSON ({err}): {status_body}"))?;
|
||||
let ready = if background_enabled {
|
||||
let ready = if background_enabled && outage_write_deferred_until_rejoin {
|
||||
// Whole-pool outage writes are deferred, so start the admin
|
||||
// root heal as soon as the target answers instead of waiting
|
||||
// for background convergence to consume the interruption window.
|
||||
true
|
||||
} else if background_enabled {
|
||||
recovered["clusterStatusComplete"] == serde_json::Value::Bool(true)
|
||||
} else {
|
||||
cluster_heal_is_idle(&recovered)
|
||||
@@ -1457,7 +1568,7 @@ mod tests {
|
||||
"non-admin Heal is disabled, so the replacement target must remain empty before the explicit root heal"
|
||||
);
|
||||
assert!(
|
||||
!census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?.has_xl_meta,
|
||||
!census_object_version_on_disk(&replaced_disk, bucket, &outage_key, None)?.has_xl_meta,
|
||||
"the object written during the outage must be absent before the explicit root heal"
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -1467,38 +1578,50 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
let background_rejoin_heal_evidence = background_enabled && outage_write_deferred_until_rejoin;
|
||||
let heal_url = format!("{}/rustfs/admin/v3/heal/?forceStart=true", cluster.nodes[0].url);
|
||||
let heal_start_body = signed_admin_post(&heal_url, Some(heal_body), &cluster.access_key, &cluster.secret_key).await?;
|
||||
let heal_start: serde_json::Value = serde_json::from_str(&heal_start_body)
|
||||
.map_err(|err| format!("heal start response is not JSON ({err}): {heal_start_body}"))?;
|
||||
let client_token = heal_start["clientToken"]
|
||||
.as_str()
|
||||
.filter(|token| !token.is_empty())
|
||||
.ok_or_else(|| format!("heal start response has no client token: {heal_start}"))?;
|
||||
let task_status_url = format!("{}/rustfs/admin/v3/heal/?clientToken={client_token}", cluster.nodes[0].url);
|
||||
let (mut client_token, mut task_status_url) = if background_rejoin_heal_evidence {
|
||||
(String::new(), String::new())
|
||||
} else {
|
||||
let heal_start_body = signed_admin_post(&heal_url, Some(heal_body), &cluster.access_key, &cluster.secret_key).await?;
|
||||
let heal_start: serde_json::Value = serde_json::from_str(&heal_start_body)
|
||||
.map_err(|err| format!("heal start response is not JSON ({err}): {heal_start_body}"))?;
|
||||
let client_token = heal_start["clientToken"]
|
||||
.as_str()
|
||||
.filter(|token| !token.is_empty())
|
||||
.ok_or_else(|| format!("heal start response has no client token: {heal_start}"))?
|
||||
.to_string();
|
||||
let task_status_url = format!("{}/rustfs/admin/v3/heal/?clientToken={client_token}", cluster.nodes[0].url);
|
||||
(client_token, task_status_url)
|
||||
};
|
||||
let restart_recovery_admin_after_failure =
|
||||
scenario == InterruptionScenario::BackgroundTargetCrashEc84MultiPool && !background_rejoin_heal_evidence;
|
||||
let mut recovery_admin_task_restarted = false;
|
||||
|
||||
let partial_timeout_secs = std::env::var("RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS")
|
||||
.ok()
|
||||
.and_then(|value| value.parse::<u64>().ok())
|
||||
.unwrap_or(60);
|
||||
let partial_deadline = Instant::now() + Duration::from_secs(partial_timeout_secs);
|
||||
loop {
|
||||
let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
|
||||
let active_status: serde_json::Value = serde_json::from_str(&status_body)
|
||||
.map_err(|err| format!("background heal status is not JSON ({err}): {status_body}"))?;
|
||||
let active = if background_enabled {
|
||||
active_status["state"].as_str() == Some("active")
|
||||
&& active_status["healOperations"]["activeBySource"]["admin"].as_u64() == Some(1)
|
||||
} else {
|
||||
only_admin_heal_is_active(&active_status)
|
||||
};
|
||||
if active {
|
||||
break;
|
||||
if !background_rejoin_heal_evidence {
|
||||
loop {
|
||||
let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
|
||||
let active_status: serde_json::Value = serde_json::from_str(&status_body)
|
||||
.map_err(|err| format!("background heal status is not JSON ({err}): {status_body}"))?;
|
||||
let active = if background_enabled {
|
||||
active_status["state"].as_str() == Some("active")
|
||||
&& active_status["healOperations"]["activeBySource"]["admin"].as_u64() == Some(1)
|
||||
} else {
|
||||
only_admin_heal_is_active(&active_status)
|
||||
};
|
||||
if active {
|
||||
break;
|
||||
}
|
||||
if Instant::now() >= partial_deadline {
|
||||
return Err(format!("root heal never became active within {partial_timeout_secs}s: {active_status}").into());
|
||||
}
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
if Instant::now() >= partial_deadline {
|
||||
return Err(format!("root heal never became active within {partial_timeout_secs}s: {active_status}").into());
|
||||
}
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
let (partial_count, partial_manifest) = loop {
|
||||
// Hash one committed shard to prove progress without letting a
|
||||
@@ -1516,14 +1639,24 @@ mod tests {
|
||||
}
|
||||
if materialized == expected_manifests.len() {
|
||||
return Err(format!(
|
||||
"root heal rebuilt all {} baseline objects before the target could be interrupted",
|
||||
"{} rebuilt all {} baseline objects before the target could be interrupted",
|
||||
if background_rejoin_heal_evidence {
|
||||
"background rejoin heal"
|
||||
} else {
|
||||
"root heal"
|
||||
},
|
||||
expected_manifests.len()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
if Instant::now() >= partial_deadline {
|
||||
return Err(format!(
|
||||
"root heal made no observable partial progress on the replacement target within {partial_timeout_secs}s"
|
||||
"{} made no observable partial progress on the replacement target within {partial_timeout_secs}s",
|
||||
if background_rejoin_heal_evidence {
|
||||
"background rejoin heal"
|
||||
} else {
|
||||
"root heal"
|
||||
}
|
||||
)
|
||||
.into());
|
||||
}
|
||||
@@ -1534,25 +1667,40 @@ mod tests {
|
||||
let pre_interrupt_status: serde_json::Value = serde_json::from_str(&pre_interrupt_status_body)
|
||||
.map_err(|err| format!("pre-interrupt background heal status is not JSON ({err}): {pre_interrupt_status_body}"))?;
|
||||
let pre_interrupt_replacement = replacement_recovery_status(&cluster).await?;
|
||||
let coordinator_log = std::fs::read_to_string(format!("{log_dir}/node0.log"))?;
|
||||
assert!(
|
||||
coordinator_log
|
||||
.lines()
|
||||
.filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
|
||||
.any(|event| {
|
||||
event["event"] == "heal_task_state"
|
||||
&& event["task_id"] == client_token
|
||||
&& event["heal_type"] == "cluster"
|
||||
&& event["state"] == "started"
|
||||
}),
|
||||
"node 0 must have started the exact admin task before interruption"
|
||||
);
|
||||
if background_rejoin_heal_evidence {
|
||||
let target_log = std::fs::read_to_string(format!("{log_dir}/node1.log"))?;
|
||||
assert!(
|
||||
target_log
|
||||
.lines()
|
||||
.filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
|
||||
.any(|event| {
|
||||
event["event"] == "heal_task_state" && event["heal_type"] == "erasure_set" && event["state"] == "started"
|
||||
}),
|
||||
"node 1 must have started a background erasure-set heal before interruption"
|
||||
);
|
||||
} else {
|
||||
let coordinator_log = std::fs::read_to_string(format!("{log_dir}/node0.log"))?;
|
||||
assert!(
|
||||
coordinator_log
|
||||
.lines()
|
||||
.filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
|
||||
.any(|event| {
|
||||
event["event"] == "heal_task_state"
|
||||
&& event["task_id"].as_str() == Some(client_token.as_str())
|
||||
&& event["heal_type"] == "cluster"
|
||||
&& event["state"] == "started"
|
||||
}),
|
||||
"node 0 must have started the exact admin task before interruption"
|
||||
);
|
||||
}
|
||||
let pre_interrupt_operations = &pre_interrupt_status["healOperations"];
|
||||
assert_eq!(
|
||||
pre_interrupt_operations["activeBySource"]["admin"].as_u64(),
|
||||
Some(1),
|
||||
"interruption must occur while the single admin task is active: {pre_interrupt_status}"
|
||||
);
|
||||
if !background_rejoin_heal_evidence {
|
||||
assert_eq!(
|
||||
pre_interrupt_operations["activeBySource"]["admin"].as_u64(),
|
||||
Some(1),
|
||||
"interruption must occur while the single admin task is active: {pre_interrupt_status}"
|
||||
);
|
||||
}
|
||||
if !background_enabled {
|
||||
assert!(
|
||||
only_admin_heal_is_active(&pre_interrupt_status),
|
||||
@@ -1675,6 +1823,7 @@ mod tests {
|
||||
scenario,
|
||||
InterruptionScenario::BackgroundTargetRestart
|
||||
| InterruptionScenario::BackgroundTargetRestartEc84
|
||||
| InterruptionScenario::BackgroundTargetRestartEc84MultiSet
|
||||
| InterruptionScenario::BackgroundCoordinatorRestart
|
||||
) {
|
||||
cluster.stop_node_gracefully(interruption_node).await?;
|
||||
@@ -1742,12 +1891,43 @@ mod tests {
|
||||
.unwrap_or(180);
|
||||
let heal_deadline = Instant::now() + Duration::from_secs(heal_timeout_secs);
|
||||
loop {
|
||||
if restart_recovery_admin_after_failure && !recovery_admin_task_restarted {
|
||||
let task_state = timeout(
|
||||
Duration::from_secs(2),
|
||||
signed_admin_post(&task_status_url, None, &cluster.access_key, &cluster.secret_key),
|
||||
)
|
||||
.await;
|
||||
if let Ok(Ok(body)) = task_state
|
||||
&& let Ok(status) = serde_json::from_str::<serde_json::Value>(&body)
|
||||
&& status["summary"].as_str() == Some("failed")
|
||||
{
|
||||
let heal_start_body =
|
||||
signed_admin_post(&heal_url, Some(heal_body), &cluster.access_key, &cluster.secret_key).await?;
|
||||
let heal_start: serde_json::Value = serde_json::from_str(&heal_start_body)
|
||||
.map_err(|err| format!("recovery heal start response is not JSON ({err}): {heal_start_body}"))?;
|
||||
client_token = heal_start["clientToken"]
|
||||
.as_str()
|
||||
.filter(|token| !token.is_empty())
|
||||
.ok_or_else(|| format!("recovery heal start response has no client token: {heal_start}"))?
|
||||
.to_string();
|
||||
task_status_url = format!("{}/rustfs/admin/v3/heal/?clientToken={client_token}", cluster.nodes[0].url);
|
||||
recovery_admin_task_restarted = true;
|
||||
info!(
|
||||
event = "heal_interruption_recovery_task_restarted",
|
||||
component = "e2e_test",
|
||||
subsystem = "heal",
|
||||
interruption_kind,
|
||||
recovery_client_token = client_token,
|
||||
"Restarted admin root heal after interrupted target-crash task failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
let baseline_recovered = metadata_count(&replaced_disk, bucket, &expected_manifests) == expected_manifests.len();
|
||||
let outage_recovered =
|
||||
!outage_target_manifest_required || object_metadata_exists_on_disk(&replaced_disk, bucket, outage_key);
|
||||
!outage_target_manifest_required || object_metadata_exists_on_disk(&replaced_disk, bucket, &outage_key);
|
||||
if baseline_recovered && outage_recovered {
|
||||
let matching = matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?;
|
||||
let outage_census = census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?;
|
||||
let outage_census = census_object_version_on_disk(&replaced_disk, bucket, &outage_key, None)?;
|
||||
let pool_metadata_matches = match &expected_pool_metadata {
|
||||
Some(expected) => {
|
||||
census_object_version_on_disk(&replaced_disk, RUSTFS_META_BUCKET, POOL_METADATA_OBJECT, None)?
|
||||
@@ -1764,21 +1944,25 @@ mod tests {
|
||||
}
|
||||
if Instant::now() >= heal_deadline {
|
||||
let matching = matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?;
|
||||
let outage_census = census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?;
|
||||
let outage_census = census_object_version_on_disk(&replaced_disk, bucket, &outage_key, None)?;
|
||||
let pool_metadata =
|
||||
census_object_version_on_disk(&replaced_disk, RUSTFS_META_BUCKET, POOL_METADATA_OBJECT, None)?;
|
||||
let final_status = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key)
|
||||
.await
|
||||
.unwrap_or_else(|err| format!("status request failed: {err}"));
|
||||
let task_status = match timeout(
|
||||
Duration::from_secs(5),
|
||||
signed_admin_post(&task_status_url, None, &cluster.access_key, &cluster.secret_key),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Ok(body)) => heal_task_status_diagnostic(&body),
|
||||
Ok(Err(err)) => format!("task status request failed: {err}"),
|
||||
Err(_) => "task status request exceeded 5s diagnostic budget".to_string(),
|
||||
let task_status = if background_rejoin_heal_evidence {
|
||||
"background erasure-set heal has no admin root-heal task token".to_string()
|
||||
} else {
|
||||
match timeout(
|
||||
Duration::from_secs(5),
|
||||
signed_admin_post(&task_status_url, None, &cluster.access_key, &cluster.secret_key),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Ok(body)) => heal_task_status_diagnostic(&body),
|
||||
Ok(Err(err)) => format!("task status request failed: {err}"),
|
||||
Err(_) => "task status request exceeded 5s diagnostic budget".to_string(),
|
||||
}
|
||||
};
|
||||
let replacement_status = match timeout(Duration::from_secs(5), replacement_recovery_status(&cluster)).await {
|
||||
Ok(Ok(status)) => status.to_string(),
|
||||
@@ -1802,7 +1986,7 @@ mod tests {
|
||||
expected.key
|
||||
);
|
||||
}
|
||||
let outage_census = census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?;
|
||||
let outage_census = census_object_version_on_disk(&replaced_disk, bucket, &outage_key, None)?;
|
||||
if outage_target_manifest_required {
|
||||
assert!(
|
||||
outage_census.is_complete(),
|
||||
@@ -1821,11 +2005,43 @@ mod tests {
|
||||
wait_for_scanner_cycle_after(&cluster, cycle_end).await?;
|
||||
}
|
||||
|
||||
if outage_write_deferred_until_rejoin {
|
||||
let deferred_deadline = Instant::now() + Duration::from_secs(60);
|
||||
loop {
|
||||
let put_result = timeout(
|
||||
Duration::from_secs(30),
|
||||
clients[0]
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(&outage_key)
|
||||
.body(ByteStream::from(deterministic_object_body(object_size_bytes, outage_payload_seed)))
|
||||
.send(),
|
||||
)
|
||||
.await;
|
||||
match put_result {
|
||||
Ok(Ok(_)) => break,
|
||||
Ok(Err(error)) if is_service_unavailable_put(&error) && Instant::now() < deferred_deadline => {
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
Ok(Err(error)) => return Err(error.into()),
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
}
|
||||
info!(
|
||||
event = "heal_interruption_outage_write_deferred",
|
||||
component = "e2e_test",
|
||||
subsystem = "heal",
|
||||
interruption_kind,
|
||||
outage_key,
|
||||
"Deferred whole-pool outage write until the target pool rejoined"
|
||||
);
|
||||
}
|
||||
|
||||
let mut expected_keys = created_online_objects
|
||||
.iter()
|
||||
.map(|(key, _)| key.clone())
|
||||
.collect::<HashSet<_>>();
|
||||
assert!(expected_keys.insert(outage_key.to_string()));
|
||||
assert!(expected_keys.insert(outage_key.clone()));
|
||||
let node_listings = assert_all_nodes_list_exact_keys(&clients, bucket, &expected_keys).await?;
|
||||
|
||||
let target_client = cluster.create_s3_client(1)?;
|
||||
@@ -1849,7 +2065,7 @@ mod tests {
|
||||
}));
|
||||
}
|
||||
}
|
||||
let response = target_client.get_object().bucket(bucket).key(outage_key).send().await?;
|
||||
let response = target_client.get_object().bucket(bucket).key(&outage_key).send().await?;
|
||||
let actual = response.body.collect().await?.into_bytes();
|
||||
let expected_outage_body = deterministic_object_body(object_size_bytes, outage_payload_seed);
|
||||
assert_eq!(actual.as_ref(), expected_outage_body.as_slice(), "object body changed for {outage_key}");
|
||||
@@ -1860,7 +2076,7 @@ mod tests {
|
||||
"expected_sha256": sha256_hex(&expected_outage_body),
|
||||
"actual_sha256": sha256_hex(&actual),
|
||||
"expected_physical": null,
|
||||
"physical": census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?,
|
||||
"physical": census_object_version_on_disk(&replaced_disk, bucket, &outage_key, None)?,
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -1878,11 +2094,13 @@ mod tests {
|
||||
sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
|
||||
let task_status_body = signed_admin_post(&task_status_url, None, &cluster.access_key, &cluster.secret_key).await?;
|
||||
let task_status: serde_json::Value = serde_json::from_str(&task_status_body)
|
||||
.map_err(|err| format!("heal task status is not JSON ({err}): {task_status_body}"))?;
|
||||
if task_status["summary"].as_str() != Some("finished") {
|
||||
return Err(format!("heal data rebuilt but task did not finish successfully: {task_status}").into());
|
||||
if !background_rejoin_heal_evidence {
|
||||
let task_status_body = signed_admin_post(&task_status_url, None, &cluster.access_key, &cluster.secret_key).await?;
|
||||
let task_status: serde_json::Value = serde_json::from_str(&task_status_body)
|
||||
.map_err(|err| format!("heal task status is not JSON ({err}): {task_status_body}"))?;
|
||||
if task_status["summary"].as_str() != Some("finished") {
|
||||
return Err(format!("heal data rebuilt but task did not finish successfully: {task_status}").into());
|
||||
}
|
||||
}
|
||||
if interruption_node == 0 {
|
||||
// Restart recovery must finish the original durable root request.
|
||||
@@ -1906,7 +2124,21 @@ mod tests {
|
||||
evidence_context.run.binary.sha256,
|
||||
"server build changed during restart"
|
||||
);
|
||||
let evidence = serde_json::json!({
|
||||
let outage_write_diagnostic = (!outage_target_manifest_required).then(|| {
|
||||
serde_json::json!({
|
||||
"attempted": true,
|
||||
"required": false,
|
||||
"accepted": true,
|
||||
"attempts": if outage_write_deferred_until_rejoin {
|
||||
max_outage_write_attempts
|
||||
} else {
|
||||
service_unavailable_outage_writes + 1
|
||||
},
|
||||
"service_unavailable": service_unavailable_outage_writes,
|
||||
"deferred_until_rejoin": outage_write_deferred_until_rejoin,
|
||||
})
|
||||
});
|
||||
let mut evidence = serde_json::json!({
|
||||
"schema": 1, "case": evidence_context.case.id, "evidence": evidence_context.case.evidence,
|
||||
"run_id": evidence_context.run.run_id, "source_revision": evidence_context.run.source_revision,
|
||||
"test_build": compiled_test_identity(),
|
||||
@@ -1916,7 +2148,11 @@ mod tests {
|
||||
"erasure_set_drive_count": erasure_set_drive_count,
|
||||
"sets": topology.set_count(evidence_context.case.erasure_set_drive_count),
|
||||
"pools": topology.pool_count(),
|
||||
"rebuild_owner": if background_rejoin_heal_evidence { "background-erasure-set" } else { "admin-root-heal" },
|
||||
"outage_target_manifest_required": outage_target_manifest_required,
|
||||
"outage_write_deferred_until_rejoin": outage_write_deferred_until_rejoin,
|
||||
"admin_root_heal_takeover": !background_rejoin_heal_evidence,
|
||||
"recovery_admin_task_restarted": recovery_admin_task_restarted,
|
||||
"distributed_ec_invalidation": true,
|
||||
"peer_count": cluster.nodes.len(),
|
||||
"same_window_remote_proof": true,
|
||||
@@ -1925,6 +2161,9 @@ mod tests {
|
||||
"unclean_shutdown_marker": unclean_shutdown_marker_observed.unwrap_or(false),
|
||||
"objects": evidence_objects, "node_listings": node_listings,
|
||||
});
|
||||
if let Some(outage_write) = outage_write_diagnostic {
|
||||
evidence["outage_write"] = outage_write;
|
||||
}
|
||||
let data = serde_json::to_vec(&evidence)?;
|
||||
if data.len() > 1024 * 1024 {
|
||||
return Err("scanner/heal oracle exceeds the 1 MiB artifact budget".into());
|
||||
|
||||
@@ -52,8 +52,8 @@ use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
BucketLifecycleConfiguration, BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, ExpirationStatus,
|
||||
LifecycleRule, LifecycleRuleFilter, NoncurrentVersionTransition, RestoreRequest, Transition, TransitionStorageClass,
|
||||
VersioningConfiguration,
|
||||
LifecycleRule, LifecycleRuleFilter, MetadataDirective, NoncurrentVersionTransition, RestoreRequest, Transition,
|
||||
TransitionStorageClass, VersioningConfiguration,
|
||||
};
|
||||
use http::Method;
|
||||
use serde::Deserialize;
|
||||
@@ -963,6 +963,81 @@ async fn test_hermetic_transition_main_path() -> TestResult {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// PUT and materialized self-copy must retain cleanup ownership of a replaced
|
||||
/// transitioned null version while publishing the new bytes and metadata.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn test_hermetic_transition_overwrite_and_self_copy() -> TestResult {
|
||||
let mut cold = RustFSTestEnvironment::new().await?;
|
||||
cold.access_key = "coldtieradmin".to_string();
|
||||
cold.secret_key = "coldtiersecret".to_string();
|
||||
cold.start_rustfs_server_without_cleanup(vec![]).await?;
|
||||
let cold_client = cold.create_s3_client();
|
||||
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
let mut hot = RustFSTestEnvironment::new().await?;
|
||||
start_tier_source(&mut hot, crate::common::FAST_DATA_USAGE_SCANNER_ENV).await?;
|
||||
let hot_client = hot.create_s3_client();
|
||||
add_rustfs_tier(&hot, &cold).await?;
|
||||
hot_client.create_bucket().bucket(SOURCE_BUCKET).send().await?;
|
||||
|
||||
let data = payload();
|
||||
for self_copy in [false, true] {
|
||||
hot_client
|
||||
.put_bucket_lifecycle_configuration()
|
||||
.bucket(SOURCE_BUCKET)
|
||||
.lifecycle_configuration(BucketLifecycleConfiguration::builder().rules(transition_rule()?).build()?)
|
||||
.send()
|
||||
.await?;
|
||||
put_multipart_object(&hot_client, SOURCE_BUCKET, OBJECT_KEY, &data).await?;
|
||||
wait_for_transition(&hot_client, SOURCE_BUCKET, OBJECT_KEY, StdDuration::from_secs(90)).await?;
|
||||
assert_eq!(cold_tier_object_count(&cold_client).await?, 1);
|
||||
// Keep the replacement local so disappearance of the old remote
|
||||
// object cannot be confused with another automatic transition.
|
||||
hot_client.delete_bucket_lifecycle().bucket(SOURCE_BUCKET).send().await?;
|
||||
|
||||
let expected = if self_copy { data.clone() } else { vec![0x73; 513] };
|
||||
if self_copy {
|
||||
hot_client
|
||||
.copy_object()
|
||||
.bucket(SOURCE_BUCKET)
|
||||
.key(OBJECT_KEY)
|
||||
.copy_source(format!("{SOURCE_BUCKET}/{}", urlencoding::encode(OBJECT_KEY)))
|
||||
.metadata_directive(MetadataDirective::Replace)
|
||||
.content_type("text/plain")
|
||||
.metadata("replacement", "kept")
|
||||
.send()
|
||||
.await?;
|
||||
} else {
|
||||
hot_client
|
||||
.put_object()
|
||||
.bucket(SOURCE_BUCKET)
|
||||
.key(OBJECT_KEY)
|
||||
.body(ByteStream::from(expected.clone()))
|
||||
.content_type("text/plain")
|
||||
.metadata("replacement", "kept")
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
wait_for_cold_tier_empty(&cold_client, StdDuration::from_secs(90)).await?;
|
||||
let current = hot_client.get_object().bucket(SOURCE_BUCKET).key(OBJECT_KEY).send().await?;
|
||||
assert_eq!(current.content_type(), Some("text/plain"));
|
||||
assert_eq!(current.metadata().and_then(|m| m.get("replacement")).map(String::as_str), Some("kept"));
|
||||
assert!(
|
||||
current
|
||||
.metadata()
|
||||
.is_none_or(|metadata| !metadata.contains_key(USER_META_KEY))
|
||||
);
|
||||
assert_eq!(current.body.collect().await?.into_bytes().as_ref(), expected.as_slice());
|
||||
hot_client
|
||||
.delete_object()
|
||||
.bucket(SOURCE_BUCKET)
|
||||
.key(OBJECT_KEY)
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Restore a transitioned object through a real RustFS remote tier.
|
||||
///
|
||||
/// The test covers the externally visible copy-back contract that a mock tier
|
||||
|
||||
@@ -27,7 +27,9 @@ mod tests {
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
|
||||
use aws_sdk_s3::types::{
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, VersioningConfiguration,
|
||||
};
|
||||
use tracing::info;
|
||||
|
||||
fn create_s3_client(env: &RustFSTestEnvironment) -> Client {
|
||||
@@ -83,6 +85,156 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_version_tags(client: &Client, bucket: &str, key: &str, version: Option<&str>, value: Option<&str>) {
|
||||
let tags = client
|
||||
.get_object_tagging()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.set_version_id(version.map(str::to_owned))
|
||||
.send()
|
||||
.await
|
||||
.expect("GetObjectTagging must accept the exact version selector");
|
||||
let expected = value
|
||||
.map(|value| vec![Tag::builder().key("generation").value(value).build().expect("valid tag")])
|
||||
.unwrap_or_default();
|
||||
assert_eq!(tags.tag_set(), expected, "version selector: {version:?}");
|
||||
}
|
||||
|
||||
async fn assert_null_tagging_across_versioning_changes(client: &Client, bucket: &str, key: &str) {
|
||||
assert_version_tags(client, bucket, key, None, None).await;
|
||||
assert_version_tags(client, bucket, key, Some("null"), None).await;
|
||||
client
|
||||
.put_object_tagging()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.version_id("null")
|
||||
.tagging(
|
||||
Tagging::builder()
|
||||
.tag_set(
|
||||
Tag::builder()
|
||||
.key("generation")
|
||||
.value("original-null")
|
||||
.build()
|
||||
.expect("valid tag"),
|
||||
)
|
||||
.build()
|
||||
.expect("valid tagging"),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.expect("tag the original null version");
|
||||
|
||||
enable_versioning(client, bucket)
|
||||
.await
|
||||
.expect("enable versioning over a null version");
|
||||
let versioned = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.tagging("generation=versioned")
|
||||
.body(ByteStream::from_static(b"new version"))
|
||||
.send()
|
||||
.await
|
||||
.expect("write a newer UUID version");
|
||||
let version = versioned.version_id().expect("versioned PUT must return a UUID");
|
||||
assert_ne!(version, "null");
|
||||
assert_version_tags(client, bucket, key, None, Some("versioned")).await;
|
||||
assert_version_tags(client, bucket, key, Some(version), Some("versioned")).await;
|
||||
assert_version_tags(client, bucket, key, Some("null"), Some("original-null")).await;
|
||||
|
||||
client
|
||||
.put_object_tagging()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.version_id("null")
|
||||
.tagging(
|
||||
Tagging::builder()
|
||||
.tag_set(
|
||||
Tag::builder()
|
||||
.key("generation")
|
||||
.value("updated-null")
|
||||
.build()
|
||||
.expect("valid tag"),
|
||||
)
|
||||
.build()
|
||||
.expect("valid tagging"),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.expect("update tags on the noncurrent null version");
|
||||
assert_version_tags(client, bucket, key, Some("null"), Some("updated-null")).await;
|
||||
assert_version_tags(client, bucket, key, None, Some("versioned")).await;
|
||||
client
|
||||
.delete_object_tagging()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.version_id("null")
|
||||
.send()
|
||||
.await
|
||||
.expect("delete only the noncurrent null version tags");
|
||||
assert_version_tags(client, bucket, key, Some("null"), None).await;
|
||||
assert_version_tags(client, bucket, key, Some(version), Some("versioned")).await;
|
||||
|
||||
suspend_versioning(client, bucket).await.expect("suspend versioning");
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.tagging("generation=suspended-null")
|
||||
.body(ByteStream::from_static(b"replacement null version"))
|
||||
.send()
|
||||
.await
|
||||
.expect("replace the null version while suspended");
|
||||
assert_version_tags(client, bucket, key, None, Some("suspended-null")).await;
|
||||
assert_version_tags(client, bucket, key, Some("null"), Some("suspended-null")).await;
|
||||
assert_version_tags(client, bucket, key, Some(version), Some("versioned")).await;
|
||||
|
||||
enable_versioning(client, bucket).await.expect("re-enable versioning");
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.tagging("generation=latest")
|
||||
.body(ByteStream::from_static(b"latest version"))
|
||||
.send()
|
||||
.await
|
||||
.expect("write a new latest version");
|
||||
assert_version_tags(client, bucket, key, Some("null"), Some("suspended-null")).await;
|
||||
assert_version_tags(client, bucket, key, None, Some("latest")).await;
|
||||
client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.version_id("null")
|
||||
.send()
|
||||
.await
|
||||
.expect("remove only the null version");
|
||||
assert_version_tags(client, bucket, key, None, Some("latest")).await;
|
||||
let absent_version = uuid::Uuid::new_v4().to_string();
|
||||
for (missing_key, selector, expected_code) in [
|
||||
(key, Some("null"), "NoSuchVersion"),
|
||||
(key, Some(absent_version.as_str()), "NoSuchVersion"),
|
||||
("never-created", None, "NoSuchKey"),
|
||||
("never-created", Some("null"), "NoSuchVersion"),
|
||||
("never-created", Some(absent_version.as_str()), "NoSuchVersion"),
|
||||
] {
|
||||
let missing = client
|
||||
.get_object_tagging()
|
||||
.bucket(bucket)
|
||||
.key(missing_key)
|
||||
.set_version_id(selector.map(str::to_owned))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a missing version must not fall back to latest");
|
||||
assert_eq!(
|
||||
missing.as_service_error().and_then(ProvideErrorMetadata::code),
|
||||
Some(expected_code),
|
||||
"key: {missing_key}, version selector: {selector:?}"
|
||||
);
|
||||
}
|
||||
assert_version_tags(client, bucket, key, None, Some("latest")).await;
|
||||
}
|
||||
|
||||
/// Test 1: PutObject should return version_id when versioning is enabled
|
||||
/// This directly addresses the Veeam issue from #1066
|
||||
#[tokio::test]
|
||||
@@ -262,7 +414,9 @@ mod tests {
|
||||
info!("🧪 TEST: PutObject behavior without versioning (no regression)");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
|
||||
env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
|
||||
env.start_rustfs_server_without_cleanup(vec![])
|
||||
.await
|
||||
.expect("Failed to start isolated RustFS");
|
||||
|
||||
let client = create_s3_client(&env);
|
||||
let bucket = "test-no-versioning";
|
||||
@@ -290,6 +444,9 @@ mod tests {
|
||||
output.version_id().is_none() || output.version_id() == Some("null"),
|
||||
"non-versioned PUT must omit version ID or return the S3 null version"
|
||||
);
|
||||
// Reuse this unversioned fixture to prove explicit null never becomes
|
||||
// an implicit latest-version read after enable/suspend transitions.
|
||||
assert_null_tagging_across_versioning_changes(&client, bucket, key).await;
|
||||
info!("✅ PASSED: PutObject works correctly without versioning");
|
||||
}
|
||||
|
||||
|
||||
@@ -822,7 +822,7 @@ async fn scan_exact_free_version_targets(
|
||||
let mut targets = Vec::new();
|
||||
for pool in &api.pools {
|
||||
for set in &pool.disk_set {
|
||||
let versions = match set.load_file_info_versions_exact(&oi.bucket, &oi.name).await {
|
||||
let versions = match set.load_file_info_versions_for_tier_cleanup(&oi.bucket, &oi.name).await {
|
||||
Ok(Some(versions)) => versions,
|
||||
Ok(None) => continue,
|
||||
Err(err) if is_err_strict_volume_not_found(&err) => continue,
|
||||
@@ -1220,7 +1220,7 @@ impl ExpiryState {
|
||||
|
||||
while state.tasks_tx.len() < n {
|
||||
let (tx, rx) = mpsc::channel(EXPIRY_WORKER_QUEUE_CAPACITY);
|
||||
let api = api.clone();
|
||||
let api = Arc::downgrade(&api);
|
||||
let rx = Arc::new(tokio::sync::Mutex::new(rx));
|
||||
let stats = Arc::clone(&state.stats);
|
||||
let recovery_notify = Arc::clone(&state.recovery_notify);
|
||||
@@ -1248,14 +1248,18 @@ impl ExpiryState {
|
||||
|
||||
async fn worker(
|
||||
rx: &mut Receiver<Option<ExpiryOpType>>,
|
||||
api: Arc<ECStore>,
|
||||
api: Weak<ECStore>,
|
||||
stats: Arc<ExpiryStats>,
|
||||
recovery_notify: Arc<Notify>,
|
||||
) {
|
||||
let cancel_token = api.ctx.background_cancel_token().unwrap_or_else(|| {
|
||||
let Some(initial_api) = api.upgrade() else {
|
||||
return;
|
||||
};
|
||||
let cancel_token = initial_api.ctx.background_cancel_token().unwrap_or_else(|| {
|
||||
static FALLBACK: std::sync::OnceLock<tokio_util::sync::CancellationToken> = std::sync::OnceLock::new();
|
||||
FALLBACK.get_or_init(tokio_util::sync::CancellationToken::new).clone()
|
||||
});
|
||||
drop(initial_api);
|
||||
|
||||
loop {
|
||||
select! {
|
||||
@@ -1284,6 +1288,9 @@ impl ExpiryState {
|
||||
let v = v.expect("received None after None check");
|
||||
stats.decrement_pending_tasks();
|
||||
let _active_task = ExpiryActiveTask::begin(Arc::clone(&stats));
|
||||
let Some(api) = api.upgrade() else {
|
||||
return;
|
||||
};
|
||||
if v.as_any().is::<ExpiryTask>() {
|
||||
let v = v.as_any().downcast_ref::<ExpiryTask>().expect("ExpiryTask downcast failed");
|
||||
//debug!("lifecycle expiry worker received task: {:?}", v.obj_info);
|
||||
@@ -7759,8 +7766,9 @@ mod tests {
|
||||
let (tx, mut rx) = tokio::sync::mpsc::channel(2);
|
||||
let worker_stats = Arc::clone(&stats);
|
||||
let worker_notify = Arc::clone(&recovery_notify);
|
||||
let worker_store = Arc::downgrade(&ecstore);
|
||||
let worker = tokio::spawn(async move {
|
||||
ExpiryState::worker(&mut rx, ecstore, worker_stats, worker_notify).await;
|
||||
ExpiryState::worker(&mut rx, worker_store, worker_stats, worker_notify).await;
|
||||
});
|
||||
let oi = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
@@ -7861,8 +7869,9 @@ mod tests {
|
||||
let (tx, mut rx) = tokio::sync::mpsc::channel(2);
|
||||
let worker_stats = Arc::clone(&stats);
|
||||
let worker_notify = Arc::clone(&recovery_notify);
|
||||
let worker_store = Arc::downgrade(&ecstore);
|
||||
let worker = tokio::spawn(async move {
|
||||
ExpiryState::worker(&mut rx, ecstore, worker_stats, worker_notify).await;
|
||||
ExpiryState::worker(&mut rx, worker_store, worker_stats, worker_notify).await;
|
||||
});
|
||||
|
||||
stats.increment_pending_tasks();
|
||||
@@ -8031,7 +8040,7 @@ mod tests {
|
||||
let (tx, mut rx) = tokio::sync::mpsc::channel(2);
|
||||
let worker_stats = Arc::clone(&stats);
|
||||
let worker_notify = Arc::clone(&recovery_notify);
|
||||
let worker_store = Arc::clone(&ecstore);
|
||||
let worker_store = Arc::downgrade(&ecstore);
|
||||
let worker = tokio::spawn(async move {
|
||||
ExpiryState::worker(&mut rx, worker_store, worker_stats, worker_notify).await;
|
||||
});
|
||||
@@ -8091,6 +8100,75 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn tier_overwrite_cleanup_retains_a_minority_live_remote_reference() {
|
||||
let (disk_paths, ecstore) = setup_test_env().await;
|
||||
let bucket = format!("overwrite-minority-{}", Uuid::new_v4());
|
||||
let object = "still-referenced";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
let (backend, identity) = register_recovery_mock_tier(&ecstore).await;
|
||||
seed_recoverable_free_version(&disk_paths, &bucket, object, None, Some(identity.clone())).await;
|
||||
let page = list_tier_free_versions(Arc::clone(&ecstore), 100, None, None, CancellationToken::new())
|
||||
.await
|
||||
.expect("list persisted cleanup owner");
|
||||
let owner = page.items.into_iter().find(|oi| oi.bucket == bucket).expect("seeded owner");
|
||||
backend
|
||||
.set_put_remote_version(Some(owner.transitioned_object.version_id.clone()))
|
||||
.await;
|
||||
let lease = TierConfigMgr::acquire_operation_lease(&ecstore.tier_config_mgr(), "WARM")
|
||||
.await
|
||||
.expect("remote fixture lease");
|
||||
lease
|
||||
.put(
|
||||
&owner.transitioned_object.name,
|
||||
rustfs_s3_client::transition_api::ReaderImpl::Body(bytes::Bytes::from_static(b"old")),
|
||||
3,
|
||||
)
|
||||
.await
|
||||
.expect("seed referenced remote bytes");
|
||||
drop(lease);
|
||||
let path = disk_paths[0].join(&bucket).join(object).join(STORAGE_FORMAT_FILE);
|
||||
let cleanup_metadata = fs::read(&path).await.expect("save completed replica");
|
||||
let mut live = FileInfo::new(object, 2, 2);
|
||||
live.volume = bucket.clone();
|
||||
live.erasure.index = 1;
|
||||
live.data_dir = Some(Uuid::new_v4());
|
||||
live.mod_time = Some(OffsetDateTime::now_utc());
|
||||
live.size = 3;
|
||||
live.add_object_part(1, "149603e6c03516362a8da23f624db945".to_string(), 3, live.mod_time, 3, None, None);
|
||||
live.transition_status = TRANSITION_COMPLETE.to_string();
|
||||
live.transition_tier = "WARM".to_string();
|
||||
live.transitioned_objname = owner.transitioned_object.name.clone();
|
||||
live.transition_version = Some(owner.transitioned_object.version_id.clone());
|
||||
live.transition_version_state = rustfs_filemeta::TransitionVersionState::Exact;
|
||||
rustfs_utils::http::insert_str(&mut live.metadata, rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID, identity);
|
||||
let mut old_metadata = FileMeta::new();
|
||||
old_metadata.add_version(live).expect("prepare minority live source");
|
||||
fs::write(&path, old_metadata.marshal_msg().expect("encode live source"))
|
||||
.await
|
||||
.expect("model one replica retained by an interrupted overwrite");
|
||||
|
||||
let err = super::cleanup_free_version_exact(Arc::clone(&ecstore), &owner, &CancellationToken::new())
|
||||
.await
|
||||
.expect_err("quorum free versions cannot erase a minority live reference");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::WouldBlock);
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
assert!(backend.contains(&owner.transitioned_object.name).await);
|
||||
|
||||
fs::write(&path, cleanup_metadata)
|
||||
.await
|
||||
.expect("complete replica convergence");
|
||||
assert!(
|
||||
super::cleanup_free_version_exact(Arc::clone(&ecstore), &owner, &CancellationToken::new())
|
||||
.await
|
||||
.expect("converged cleanup can delete the exact remote owner")
|
||||
);
|
||||
assert_eq!(backend.remove_count().await, 1);
|
||||
assert!(!backend.contains(&owner.transitioned_object.name).await);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
@@ -8118,7 +8196,7 @@ mod tests {
|
||||
let (tx, mut rx) = tokio::sync::mpsc::channel(2);
|
||||
let worker_stats = Arc::clone(&stats);
|
||||
let worker_notify = Arc::clone(&recovery_notify);
|
||||
let worker_store = Arc::clone(&ecstore);
|
||||
let worker_store = Arc::downgrade(&ecstore);
|
||||
let worker = tokio::spawn(async move {
|
||||
ExpiryState::worker(&mut rx, worker_store, worker_stats, worker_notify).await;
|
||||
});
|
||||
@@ -8225,7 +8303,7 @@ mod tests {
|
||||
let (tx, mut rx) = tokio::sync::mpsc::channel(2);
|
||||
let worker_stats = Arc::clone(&stats);
|
||||
let worker_notify = Arc::clone(&recovery_notify);
|
||||
let worker_store = Arc::clone(&ecstore);
|
||||
let worker_store = Arc::downgrade(&ecstore);
|
||||
let worker = tokio::spawn(async move {
|
||||
ExpiryState::worker(&mut rx, worker_store, worker_stats, worker_notify).await;
|
||||
});
|
||||
@@ -8279,8 +8357,9 @@ mod tests {
|
||||
let (tx, mut rx) = tokio::sync::mpsc::channel(2);
|
||||
let worker_stats = Arc::clone(&stats);
|
||||
let worker_notify = Arc::clone(&recovery_notify);
|
||||
let worker_store = Arc::downgrade(&ecstore);
|
||||
let worker = tokio::spawn(async move {
|
||||
ExpiryState::worker(&mut rx, ecstore, worker_stats, worker_notify).await;
|
||||
ExpiryState::worker(&mut rx, worker_store, worker_stats, worker_notify).await;
|
||||
});
|
||||
let oi = ObjectInfo {
|
||||
bucket: format!("missing-bucket-{}", Uuid::new_v4()),
|
||||
|
||||
@@ -248,10 +248,9 @@ fn validate_authoritative_object_lock_config(config: &ObjectLockConfiguration) -
|
||||
}
|
||||
|
||||
pub async fn init_bucket_metadata_sys(api: Arc<ECStore>, buckets: Vec<String>) {
|
||||
// The metadata system is inherently per-store (it holds the store handle
|
||||
// and that store's bucket cache), so it lives on the store's own instance
|
||||
// context (backlog#1052 S3) — a second instance initializes its own cell
|
||||
// instead of panicking on the process-global one.
|
||||
// The metadata system is inherently per-store, so it lives on the store's
|
||||
// own instance context (backlog#1052 S3). It resolves the store through a
|
||||
// weak handle so the context cache cannot keep the store and disks alive.
|
||||
let instance_ctx = api.ctx.clone();
|
||||
let is_dist_erasure = instance_ctx.is_dist_erasure().await;
|
||||
|
||||
@@ -317,18 +316,22 @@ fn start_refresh_buckets_metadata_loop(sys: Arc<RwLock<BucketMetadataSys>>) {
|
||||
warn!("bucket metadata refresh loop skipped because background cancellation token is not initialized");
|
||||
return;
|
||||
};
|
||||
let sys = Arc::downgrade(&sys);
|
||||
|
||||
tokio::spawn(async move {
|
||||
refresh_buckets_metadata_loop(sys, cancel_token).await;
|
||||
});
|
||||
}
|
||||
|
||||
async fn refresh_buckets_metadata_loop(sys: Arc<RwLock<BucketMetadataSys>>, cancel_token: CancellationToken) {
|
||||
async fn refresh_buckets_metadata_loop(sys: Weak<RwLock<BucketMetadataSys>>, cancel_token: CancellationToken) {
|
||||
loop {
|
||||
if !wait_refresh_interval_or_cancel(&cancel_token, BUCKET_METADATA_REFRESH_INTERVAL).await {
|
||||
break;
|
||||
}
|
||||
refresh_buckets_metadata_once(sys.clone()).await;
|
||||
let Some(sys) = sys.upgrade() else {
|
||||
break;
|
||||
};
|
||||
refresh_buckets_metadata_once(sys).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -455,7 +458,7 @@ pub(crate) async fn object_store_in(ctx: &crate::runtime::instance::InstanceCont
|
||||
|
||||
pub(crate) async fn object_store_if_initialized_in(ctx: &crate::runtime::instance::InstanceContext) -> Option<Arc<ECStore>> {
|
||||
let sys = ctx.bucket_metadata_sys().or_else(get_global_bucket_metadata_sys)?;
|
||||
Some(sys.read().await.api.clone())
|
||||
sys.read().await.object_store_if_live()
|
||||
}
|
||||
|
||||
pub(crate) async fn get_in(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result<Arc<BucketMetadata>> {
|
||||
@@ -475,7 +478,7 @@ pub(crate) async fn get_config_from_disk_with_presence_in(
|
||||
bucket: &str,
|
||||
) -> Result<(BucketMetadata, bool)> {
|
||||
let sys = bucket_metadata_sys_of(ctx)?;
|
||||
let api = sys.read().await.api.clone();
|
||||
let api = sys.read().await.object_store();
|
||||
load_bucket_metadata_parse_with_presence(api, bucket, true).await
|
||||
}
|
||||
|
||||
@@ -738,7 +741,7 @@ pub async fn acquire_scanner_bucket_incarnation_fence(
|
||||
) -> Result<BucketMetadataMutationGuard> {
|
||||
super::utils::check_valid_bucket_name(bucket)?;
|
||||
let sys = get_bucket_metadata_sys()?;
|
||||
if expected_owner_id.is_nil() || sys.read().await.api.id != expected_owner_id || expected_incarnation_id.is_nil() {
|
||||
if expected_owner_id.is_nil() || sys.read().await.object_store().id != expected_owner_id || expected_incarnation_id.is_nil() {
|
||||
return Err(Error::other("scanner bucket incarnation owner does not match"));
|
||||
}
|
||||
acquire_config_write_guard_with_migration(sys, bucket, Some(expected_incarnation_id), false).await
|
||||
@@ -751,7 +754,7 @@ async fn acquire_config_write_guard_with_migration(
|
||||
migrate: bool,
|
||||
) -> Result<BucketMetadataMutationGuard> {
|
||||
let metadata_sys = sys.read().await.clone();
|
||||
let lifecycle_guard = metadata_sys.api.acquire_bucket_lifecycle_read_lock(bucket).await?;
|
||||
let lifecycle_guard = metadata_sys.object_store().acquire_bucket_lifecycle_read_lock(bucket).await?;
|
||||
|
||||
// Legacy buckets are migrated while the lifecycle fence prevents a
|
||||
// same-name replacement. The second read under the write transaction is
|
||||
@@ -782,7 +785,7 @@ async fn acquire_config_write_guard_with_migration(
|
||||
"bucket config existence transaction validation",
|
||||
async {
|
||||
match metadata_sys
|
||||
.api
|
||||
.object_store()
|
||||
.get_bucket_info_from_sets(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
|
||||
.await
|
||||
{
|
||||
@@ -802,7 +805,7 @@ async fn acquire_config_write_guard_with_migration(
|
||||
Some(&transaction_guard),
|
||||
bucket,
|
||||
"bucket config incarnation transaction validation",
|
||||
load_bucket_incarnation(metadata_sys.api.clone(), bucket),
|
||||
load_bucket_incarnation(metadata_sys.object_store(), bucket),
|
||||
),
|
||||
)
|
||||
.await?
|
||||
@@ -1461,7 +1464,7 @@ pub struct BucketMetadataSys {
|
||||
/// Physically missing names are TTL-bounded to limit memory under bogus
|
||||
/// name floods while avoiding repeated namespace and erasure reads.
|
||||
missing_buckets: moka::future::Cache<String, ()>,
|
||||
api: Arc<ECStore>,
|
||||
api: Weak<ECStore>,
|
||||
}
|
||||
|
||||
impl BucketMetadataSys {
|
||||
@@ -1489,12 +1492,17 @@ impl BucketMetadataSys {
|
||||
.max_capacity(MISSING_BUCKET_MAX_ENTRIES)
|
||||
.time_to_live(MISSING_BUCKET_TTL)
|
||||
.build(),
|
||||
api,
|
||||
api: Arc::downgrade(&api),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn object_store(&self) -> Arc<ECStore> {
|
||||
self.api.clone()
|
||||
self.object_store_if_live()
|
||||
.expect("bucket metadata object store should still be live")
|
||||
}
|
||||
|
||||
fn object_store_if_live(&self) -> Option<Arc<ECStore>> {
|
||||
self.api.upgrade()
|
||||
}
|
||||
|
||||
fn metadata_publish_lock(&self, bucket: &str) -> Arc<Mutex<MetadataPublishLockState>> {
|
||||
@@ -1549,7 +1557,7 @@ impl BucketMetadataSys {
|
||||
) -> Result<bool> {
|
||||
await_bucket_namespace_operation(Some(namespace_guard), bucket, operation, async {
|
||||
match self
|
||||
.api
|
||||
.object_store()
|
||||
.get_bucket_info_from_sets(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
|
||||
.await
|
||||
{
|
||||
@@ -1566,7 +1574,7 @@ impl BucketMetadataSys {
|
||||
}
|
||||
async fn init_internal(&self, buckets: Vec<String>) -> Result<()> {
|
||||
let count = self
|
||||
.api
|
||||
.object_store()
|
||||
.pools
|
||||
.iter()
|
||||
.map(|pool| pool.disk_set.len())
|
||||
@@ -1598,7 +1606,7 @@ impl BucketMetadataSys {
|
||||
let mut futures = Vec::new();
|
||||
|
||||
for bucket in buckets.iter() {
|
||||
let api = self.api.clone();
|
||||
let api = self.object_store();
|
||||
let bucket = bucket.clone();
|
||||
futures.push(async move {
|
||||
sleep(Duration::from_millis(30)).await;
|
||||
@@ -1644,7 +1652,9 @@ impl BucketMetadataSys {
|
||||
let bucket = bucket.clone();
|
||||
futures.push(async move {
|
||||
sleep(Duration::from_millis(30)).await;
|
||||
let api = sys.read().await.api.clone();
|
||||
let Some(api) = sys.read().await.object_store_if_live() else {
|
||||
return Ok(());
|
||||
};
|
||||
let namespace_lock = api.new_ns_lock(&bucket, &bucket).await?;
|
||||
let namespace_guard = namespace_lock
|
||||
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
@@ -1685,7 +1695,7 @@ impl BucketMetadataSys {
|
||||
Some(namespace_guard),
|
||||
bucket,
|
||||
"bucket metadata heal existence check",
|
||||
self.api.bucket_exists_for_heal(bucket),
|
||||
self.object_store().bucket_exists_for_heal(bucket),
|
||||
)
|
||||
.await?
|
||||
{
|
||||
@@ -1709,7 +1719,7 @@ impl BucketMetadataSys {
|
||||
Some(namespace_guard),
|
||||
bucket,
|
||||
"bucket metadata heal",
|
||||
self.api.heal_bucket(
|
||||
self.object_store().heal_bucket(
|
||||
bucket,
|
||||
&HealOpts {
|
||||
recreate: true,
|
||||
@@ -1723,7 +1733,7 @@ impl BucketMetadataSys {
|
||||
Some(namespace_guard),
|
||||
bucket,
|
||||
"bucket metadata load",
|
||||
load_bucket_metadata_parse_with_presence(self.api.clone(), bucket, true),
|
||||
load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
|
||||
)
|
||||
.await?;
|
||||
match mode {
|
||||
@@ -1903,7 +1913,7 @@ impl BucketMetadataSys {
|
||||
// (backlog#1052 S7). Reading from the ambient handle instead made the
|
||||
// read and the write of a single read-modify-write able to target
|
||||
// different instances.
|
||||
let mut bm = Box::pin(Self::load_bucket_metadata_for_update(self.api.clone(), bucket, parse)).await?;
|
||||
let mut bm = Box::pin(Self::load_bucket_metadata_for_update(self.object_store(), bucket, parse)).await?;
|
||||
if !bm.bucket_incarnation_sidecar || bm.bucket_incarnation_id != expected_incarnation_id {
|
||||
return Err(Error::BucketNotFound(bucket.to_string()));
|
||||
}
|
||||
@@ -1942,7 +1952,7 @@ impl BucketMetadataSys {
|
||||
where
|
||||
F: FnOnce(&BucketMetadata) -> Result<Vec<u8>> + Send,
|
||||
{
|
||||
let mut bm = Box::pin(Self::load_bucket_metadata_for_update(self.api.clone(), bucket, true)).await?;
|
||||
let mut bm = Box::pin(Self::load_bucket_metadata_for_update(self.object_store(), bucket, true)).await?;
|
||||
if !bm.bucket_incarnation_sidecar || bm.bucket_incarnation_id != expected_incarnation_id {
|
||||
return Err(Error::BucketNotFound(bucket.to_string()));
|
||||
}
|
||||
@@ -1993,7 +2003,7 @@ impl BucketMetadataSys {
|
||||
/// server's metadata never leaks into the ambient (first) instance.
|
||||
pub(crate) async fn persist_and_set(&self, bm: BucketMetadata) -> Result<()> {
|
||||
let mut bm = bm;
|
||||
bm.save_with_store(self.api.clone()).await?;
|
||||
bm.save_with_store(self.object_store()).await?;
|
||||
|
||||
self.set(bm.name.clone(), Arc::new(bm)).await;
|
||||
|
||||
@@ -2001,8 +2011,8 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
async fn persist_new_and_set(&self, mut bm: BucketMetadata) -> Result<()> {
|
||||
bm.save_with_store(self.api.clone()).await?;
|
||||
save_bucket_incarnation(self.api.clone(), &bm.name, bm.bucket_incarnation_id).await?;
|
||||
bm.save_with_store(self.object_store()).await?;
|
||||
save_bucket_incarnation(self.object_store(), &bm.name, bm.bucket_incarnation_id).await?;
|
||||
bm.bucket_incarnation_sidecar = true;
|
||||
self.set(bm.name.clone(), Arc::new(bm)).await;
|
||||
Ok(())
|
||||
@@ -2019,7 +2029,7 @@ impl BucketMetadataSys {
|
||||
return Err(Error::other("errInvalidArgument"));
|
||||
}
|
||||
|
||||
load_bucket_metadata(self.api.clone(), bucket).await
|
||||
load_bucket_metadata(self.object_store(), bucket).await
|
||||
}
|
||||
|
||||
/// Reload persisted metadata under the bucket namespace generation fence.
|
||||
@@ -2033,7 +2043,7 @@ impl BucketMetadataSys {
|
||||
return Err(Error::other("errInvalidArgument"));
|
||||
}
|
||||
|
||||
let namespace_lock = self.api.new_ns_lock(bucket, bucket).await?;
|
||||
let namespace_lock = self.object_store().new_ns_lock(bucket, bucket).await?;
|
||||
let namespace_guard = namespace_lock
|
||||
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await?;
|
||||
@@ -2056,7 +2066,7 @@ impl BucketMetadataSys {
|
||||
Some(namespace_guard),
|
||||
bucket,
|
||||
"peer bucket metadata load",
|
||||
load_bucket_metadata_parse_with_presence(self.api.clone(), bucket, true),
|
||||
load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
|
||||
)
|
||||
.await?;
|
||||
if !persisted {
|
||||
@@ -2101,11 +2111,11 @@ impl BucketMetadataSys {
|
||||
#[cfg(test)]
|
||||
self.lazy_disk_loads.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
|
||||
let lock = self.api.new_ns_lock(bucket, bucket).await?;
|
||||
let lock = self.object_store().new_ns_lock(bucket, bucket).await?;
|
||||
let guard = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
|
||||
#[cfg(test)]
|
||||
if self.lazy_load_lock_probe.load(std::sync::atomic::Ordering::Relaxed) {
|
||||
let competing = self.api.new_ns_lock(bucket, bucket).await?;
|
||||
let competing = self.object_store().new_ns_lock(bucket, bucket).await?;
|
||||
assert!(
|
||||
competing.get_write_lock(Duration::from_millis(20)).await.is_err(),
|
||||
"lazy metadata IO must start while the bucket namespace read lock is held"
|
||||
@@ -2115,7 +2125,7 @@ impl BucketMetadataSys {
|
||||
Some(&guard),
|
||||
bucket,
|
||||
"lazy bucket metadata load",
|
||||
Box::pin(load_bucket_metadata_parse_with_presence(self.api.clone(), bucket, true)),
|
||||
Box::pin(load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true)),
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -2127,7 +2137,7 @@ impl BucketMetadataSys {
|
||||
bucket,
|
||||
"lazy bucket metadata existence check",
|
||||
Box::pin(async {
|
||||
self.api
|
||||
self.object_store()
|
||||
.get_bucket_info_from_sets(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
|
||||
.await
|
||||
.map(|_| ())
|
||||
@@ -2334,13 +2344,13 @@ impl BucketMetadataSys {
|
||||
|
||||
async fn get_bucket_incarnation_id_from_disk(&self, bucket: &str) -> Result<Uuid> {
|
||||
let transaction_lock = self
|
||||
.api
|
||||
.object_store()
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
|
||||
.await?;
|
||||
let _transaction_guard = transaction_lock
|
||||
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await?;
|
||||
let incarnation_id = load_bucket_incarnation(self.api.clone(), bucket).await?;
|
||||
let incarnation_id = load_bucket_incarnation(self.object_store(), bucket).await?;
|
||||
if _transaction_guard.is_lock_lost() {
|
||||
return Err(Error::other(format!("bucket incarnation metadata transaction lock was lost: {bucket}")));
|
||||
}
|
||||
@@ -2376,7 +2386,7 @@ impl BucketMetadataSys {
|
||||
|
||||
async fn migrate_legacy_metadata(&self, bucket: &str) -> Result<BucketMetadataAuthority> {
|
||||
let transaction_lock = self
|
||||
.api
|
||||
.object_store()
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
|
||||
.await?;
|
||||
let _transaction_guard = transaction_lock
|
||||
@@ -2401,7 +2411,7 @@ impl BucketMetadataSys {
|
||||
return Err(Error::other(format!("injected Object Lock metadata disk read failure: {bucket}")));
|
||||
}
|
||||
|
||||
let namespace_lock = self.api.new_ns_lock(bucket, bucket).await?;
|
||||
let namespace_lock = self.object_store().new_ns_lock(bucket, bucket).await?;
|
||||
let namespace_guard = namespace_lock
|
||||
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await?;
|
||||
@@ -2411,7 +2421,7 @@ impl BucketMetadataSys {
|
||||
bucket,
|
||||
"legacy bucket metadata existence check",
|
||||
async {
|
||||
self.api
|
||||
self.object_store()
|
||||
.get_bucket_info_from_sets(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
|
||||
.await
|
||||
},
|
||||
@@ -2435,7 +2445,7 @@ impl BucketMetadataSys {
|
||||
Some(&namespace_guard),
|
||||
bucket,
|
||||
"legacy bucket metadata confirmation",
|
||||
load_bucket_metadata_parse_with_presence(self.api.clone(), bucket, true),
|
||||
load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
|
||||
)
|
||||
.await?;
|
||||
if persisted && !metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() {
|
||||
@@ -2457,20 +2467,20 @@ impl BucketMetadataSys {
|
||||
}
|
||||
#[cfg(test)]
|
||||
if self.legacy_migration_lock_probe.load(std::sync::atomic::Ordering::Relaxed) {
|
||||
let competing = self.api.new_ns_lock(bucket, bucket).await?;
|
||||
let competing = self.object_store().new_ns_lock(bucket, bucket).await?;
|
||||
assert!(
|
||||
competing.get_write_lock(Duration::from_millis(20)).await.is_err(),
|
||||
"bucket delete/recreate must not cross the legacy metadata migration fence"
|
||||
);
|
||||
}
|
||||
save_bucket_incarnation(self.api.clone(), bucket, metadata.bucket_incarnation_id).await?;
|
||||
save_bucket_incarnation(self.object_store(), bucket, metadata.bucket_incarnation_id).await?;
|
||||
metadata.bucket_incarnation_sidecar = true;
|
||||
if !persisted {
|
||||
await_bucket_namespace_operation(
|
||||
Some(&namespace_guard),
|
||||
bucket,
|
||||
"legacy bucket metadata migration",
|
||||
metadata.save_with_store(self.api.clone()),
|
||||
metadata.save_with_store(self.object_store()),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -2506,7 +2516,7 @@ impl BucketMetadataSys {
|
||||
return Err(Error::other(format!("injected Object Lock metadata disk read failure: {bucket}")));
|
||||
}
|
||||
|
||||
let namespace_lock = self.api.new_ns_lock(bucket, bucket).await?;
|
||||
let namespace_lock = self.object_store().new_ns_lock(bucket, bucket).await?;
|
||||
let namespace_guard = namespace_lock
|
||||
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await?;
|
||||
@@ -2515,8 +2525,11 @@ impl BucketMetadataSys {
|
||||
bucket,
|
||||
"bucket metadata snapshot existence check",
|
||||
async {
|
||||
self.api
|
||||
.get_bucket_info_from_sets(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
|
||||
self.object_store()
|
||||
.get_bucket_info_from_sets_at_read_quorum(
|
||||
bucket,
|
||||
&crate::storage_api_contracts::bucket::BucketOptions::default(),
|
||||
)
|
||||
.await
|
||||
},
|
||||
)
|
||||
@@ -2531,7 +2544,7 @@ impl BucketMetadataSys {
|
||||
Some(&namespace_guard),
|
||||
bucket,
|
||||
"bucket metadata authoritative snapshot",
|
||||
load_bucket_metadata_parse_with_presence(self.api.clone(), bucket, true),
|
||||
load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
|
||||
)
|
||||
.await?;
|
||||
if persisted {
|
||||
@@ -3695,7 +3708,7 @@ mod tests {
|
||||
let mut stale = BucketMetadata::new("recreated-bucket");
|
||||
stale.policy_config_json = b"old-generation".to_vec();
|
||||
let namespace_lock = sys
|
||||
.api
|
||||
.object_store()
|
||||
.new_ns_lock("recreated-bucket", "recreated-bucket")
|
||||
.await
|
||||
.expect("namespace lock should be created");
|
||||
|
||||
@@ -303,8 +303,16 @@ fn scanner_scoped_dirty_usage_ack_response_matches(
|
||||
&& cleared_within_request
|
||||
}
|
||||
|
||||
fn scanner_scoped_dirty_usage_ack_reconciled(activity: &ScannerPeerActivity, expected_instance_id: &str) -> bool {
|
||||
activity.instance_id == expected_instance_id && activity.dirty_usage_pending == Some(false)
|
||||
fn scanner_scoped_dirty_usage_ack_reconciled(
|
||||
activity: &ScannerPeerActivity,
|
||||
expected_instance_id: &str,
|
||||
expected_generation: u64,
|
||||
) -> bool {
|
||||
activity.instance_id == expected_instance_id
|
||||
&& activity.dirty_usage_pending == Some(false)
|
||||
&& activity
|
||||
.dirty_usage_generation
|
||||
.is_some_and(|generation| generation >= expected_generation)
|
||||
}
|
||||
|
||||
fn scanner_instance_id_is_valid(instance_id: &str) -> bool {
|
||||
@@ -2290,6 +2298,7 @@ impl PeerRestClient {
|
||||
entries: Vec<ScannerScopedDirtyUsageAckEntry>,
|
||||
) -> Result<ScannerPeerActivity> {
|
||||
use rustfs_protos::scoped_dirty_usage::*;
|
||||
let expected_generation = entries.iter().map(|entry| entry.generation).max().unwrap_or(0);
|
||||
let payloads = scanner_scoped_dirty_usage_ack_payloads(owner_id, instance_id.clone(), false, entries)?;
|
||||
let ack_attempt = async {
|
||||
let mut client = super::client::scanner_control_time_out_client(
|
||||
@@ -2337,7 +2346,9 @@ impl PeerRestClient {
|
||||
.await;
|
||||
}
|
||||
match self.scanner_scoped_dirty_usage_activity_confirmation().await {
|
||||
Ok(activity) if scanner_scoped_dirty_usage_ack_reconciled(&activity, &instance_id) => Ok(activity),
|
||||
Ok(activity) if scanner_scoped_dirty_usage_ack_reconciled(&activity, &instance_id, expected_generation) => {
|
||||
Ok(activity)
|
||||
}
|
||||
_ => Err(err),
|
||||
}
|
||||
}
|
||||
@@ -3176,30 +3187,48 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn scanner_scoped_dirty_usage_ack_reconciliation_requires_same_clean_instance() {
|
||||
let activity = |instance_id: &str, pending| ScannerPeerActivity {
|
||||
let activity = |instance_id: &str, generation, pending| ScannerPeerActivity {
|
||||
instance_id: instance_id.to_string(),
|
||||
namespace_generation: 1,
|
||||
maintenance_generation: 1,
|
||||
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
topology_digest: Some([1; 32]),
|
||||
data_movement_active: Some(false),
|
||||
dirty_usage_generation: Some(9),
|
||||
dirty_usage_generation: generation,
|
||||
dirty_usage_pending: pending,
|
||||
movement_generation: Some(1),
|
||||
publication_blocked: Some(false),
|
||||
};
|
||||
|
||||
assert!(scanner_scoped_dirty_usage_ack_reconciled(
|
||||
&activity("0123456789abcdef0123456789abcdef", Some(false)),
|
||||
"0123456789abcdef0123456789abcdef"
|
||||
&activity("0123456789abcdef0123456789abcdef", Some(9), Some(false)),
|
||||
"0123456789abcdef0123456789abcdef",
|
||||
9
|
||||
));
|
||||
assert!(scanner_scoped_dirty_usage_ack_reconciled(
|
||||
&activity("0123456789abcdef0123456789abcdef", Some(10), Some(false)),
|
||||
"0123456789abcdef0123456789abcdef",
|
||||
9
|
||||
));
|
||||
assert!(!scanner_scoped_dirty_usage_ack_reconciled(
|
||||
&activity("0123456789abcdef0123456789abcdef", Some(true)),
|
||||
"0123456789abcdef0123456789abcdef"
|
||||
&activity("0123456789abcdef0123456789abcdef", Some(8), Some(false)),
|
||||
"0123456789abcdef0123456789abcdef",
|
||||
9
|
||||
));
|
||||
assert!(!scanner_scoped_dirty_usage_ack_reconciled(
|
||||
&activity("fedcba9876543210fedcba9876543210", Some(false)),
|
||||
"0123456789abcdef0123456789abcdef"
|
||||
&activity("0123456789abcdef0123456789abcdef", None, Some(false)),
|
||||
"0123456789abcdef0123456789abcdef",
|
||||
9
|
||||
));
|
||||
assert!(!scanner_scoped_dirty_usage_ack_reconciled(
|
||||
&activity("0123456789abcdef0123456789abcdef", Some(9), Some(true)),
|
||||
"0123456789abcdef0123456789abcdef",
|
||||
9
|
||||
));
|
||||
assert!(!scanner_scoped_dirty_usage_ack_reconciled(
|
||||
&activity("fedcba9876543210fedcba9876543210", Some(9), Some(false)),
|
||||
"0123456789abcdef0123456789abcdef",
|
||||
9
|
||||
));
|
||||
}
|
||||
|
||||
|
||||
@@ -152,6 +152,7 @@ pub(crate) const DECOMMISSION_VERSION_COPY_ATTEMPTS: usize = 3;
|
||||
const DECOMMISSION_COPY_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(50);
|
||||
const DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT: usize = 100;
|
||||
const DECOMMISSION_TERMINAL_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(1);
|
||||
const DECOMMISSION_CANCEL_TARGET_LOCK_MAX_ATTEMPTS: usize = 3;
|
||||
const DECOMMISSION_DURABLE_ILM_RECEIPT_ROOT: &str = "decommission/ilm-receipts";
|
||||
const DECOMMISSION_DURABLE_ILM_MANIFEST_ROOT: &str = "decommission/ilm-manifests";
|
||||
const DECOMMISSION_DURABLE_ILM_RECEIPT_SCHEMA: &str = "v2";
|
||||
@@ -9624,6 +9625,10 @@ struct DecommissionCapacityLockOrderBarrierState {
|
||||
cancel_before_start_entered: tokio::sync::Notify,
|
||||
cancel_before_start_release: tokio::sync::Notify,
|
||||
cancel_before_start_paused: AtomicBool,
|
||||
cancel_target_timeout_entered: tokio::sync::Notify,
|
||||
cancel_target_timeout_release: tokio::sync::Notify,
|
||||
cancel_target_timeout_paused: AtomicBool,
|
||||
cancel_target_timeouts: AtomicUsize,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -9663,6 +9668,10 @@ impl DecommissionCapacityLockOrderBarrier {
|
||||
cancel_before_start_entered: tokio::sync::Notify::new(),
|
||||
cancel_before_start_release: tokio::sync::Notify::new(),
|
||||
cancel_before_start_paused: AtomicBool::new(false),
|
||||
cancel_target_timeout_entered: tokio::sync::Notify::new(),
|
||||
cancel_target_timeout_release: tokio::sync::Notify::new(),
|
||||
cancel_target_timeout_paused: AtomicBool::new(false),
|
||||
cancel_target_timeouts: AtomicUsize::new(0),
|
||||
});
|
||||
let mut slot = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
@@ -9784,6 +9793,21 @@ impl DecommissionCapacityLockOrderBarrier {
|
||||
self.state.cancel_before_start_release.notify_one();
|
||||
}
|
||||
|
||||
fn pause_cancel_target_timeout(&self) {
|
||||
self.state.cancel_target_timeout_paused.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
async fn wait_until_cancel_target_timeout(&self) {
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30), self.state.cancel_target_timeout_entered.notified())
|
||||
.await
|
||||
.expect("cancel should observe contention on its target capacity fence");
|
||||
}
|
||||
|
||||
fn release_cancel_target_timeout(&self) {
|
||||
self.state.cancel_target_timeout_paused.store(false, Ordering::Release);
|
||||
self.state.cancel_target_timeout_release.notify_one();
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub(crate) fn release_owner(&self) {
|
||||
self.state.owner_release.notify_one();
|
||||
@@ -9826,6 +9850,7 @@ impl Drop for DecommissionCapacityLockOrderBarrier {
|
||||
self.state.external_object_capacity_probe_release.notify_one();
|
||||
self.state.external_object_commit_phase_release.notify_one();
|
||||
self.state.cancel_before_start_release.notify_one();
|
||||
self.state.cancel_target_timeout_release.notify_one();
|
||||
let mut slot = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
@@ -9884,6 +9909,24 @@ async fn pause_decommission_cancel_before_start_gate(store_id: uuid::Uuid) {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn pause_decommission_cancel_target_timeout(store_id: uuid::Uuid) {
|
||||
let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("decommission capacity lock-order barrier should not be poisoned")
|
||||
.as_ref()
|
||||
.filter(|state| state.owner_store_id == store_id)
|
||||
.cloned();
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.cancel_target_timeouts.fetch_add(1, Ordering::AcqRel);
|
||||
barrier.cancel_target_timeout_entered.notify_one();
|
||||
if barrier.cancel_target_timeout_paused.load(Ordering::Acquire) {
|
||||
barrier.cancel_target_timeout_release.notified().await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn notify_decommission_target_gate_retry(store_id: uuid::Uuid) {
|
||||
let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
|
||||
@@ -10521,6 +10564,7 @@ impl ECStore {
|
||||
|
||||
async fn acquire_decommission_capacity_terminal_guards(
|
||||
&self,
|
||||
source_pool_index: usize,
|
||||
plan: Option<&DecommissionCapacityTerminalFencePlan>,
|
||||
) -> Result<Vec<rustfs_lock::NamespaceLockGuard>> {
|
||||
let Some(plan) = plan else {
|
||||
@@ -10544,26 +10588,69 @@ impl ECStore {
|
||||
"no storage pools available".to_string(),
|
||||
)
|
||||
})?;
|
||||
let mut guards = Vec::with_capacity(plan.target_pool_indices.len());
|
||||
for &target_pool_index in &plan.target_pool_indices {
|
||||
let object = format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{target_pool_index}");
|
||||
let target_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
|
||||
let guard = target_lock
|
||||
.get_write_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(|err| match err {
|
||||
rustfs_lock::LockError::QuorumNotReached { required, achieved } => Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "write",
|
||||
bucket: RUSTFS_META_BUCKET.to_string(),
|
||||
object,
|
||||
required,
|
||||
achieved,
|
||||
},
|
||||
other => Error::Lock(other),
|
||||
})?;
|
||||
guards.push(guard);
|
||||
// Retry only target acquisition, never persistence. Keep the original
|
||||
// owner/cohort pinned so a remote Clear/start cannot retarget a cancel.
|
||||
let mut attempt = 1;
|
||||
'acquire_targets: loop {
|
||||
let mut guards = Vec::with_capacity(plan.target_pool_indices.len());
|
||||
for &target_pool_index in &plan.target_pool_indices {
|
||||
let object = format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{target_pool_index}");
|
||||
let target_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
|
||||
let started = std::time::Instant::now();
|
||||
match target_lock.get_write_lock(get_lock_acquire_timeout()).await {
|
||||
Ok(guard) => guards.push(guard),
|
||||
Err(err @ rustfs_lock::LockError::Timeout { .. }) => {
|
||||
// Release the entire partial cohort before backoff or
|
||||
// metadata reads; workers need these gates to settle I/O.
|
||||
drop(guards);
|
||||
#[cfg(test)]
|
||||
pause_decommission_cancel_target_timeout(self.id).await;
|
||||
if attempt >= DECOMMISSION_CANCEL_TARGET_LOCK_MAX_ATTEMPTS {
|
||||
return Err(Error::Lock(err));
|
||||
}
|
||||
warn!(
|
||||
event = EVENT_DECOMMISSION_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
state = "cancel_target_fence_retry",
|
||||
pool_index = source_pool_index,
|
||||
target_pool_index,
|
||||
operation_id = %plan.operation_id,
|
||||
generation = plan.generation,
|
||||
owner_nonce = %plan.owner_nonce,
|
||||
attempt,
|
||||
max_attempts = DECOMMISSION_CANCEL_TARGET_LOCK_MAX_ATTEMPTS,
|
||||
wait_ms = %started.elapsed().as_millis(),
|
||||
error = %err,
|
||||
"Decommission cancel will retry target capacity fencing"
|
||||
);
|
||||
tokio::time::sleep(DECOMMISSION_TERMINAL_RETRY_DELAY).await;
|
||||
let save_guard = self.pool_meta_save_gate.lock().await;
|
||||
let (_read_guard, snapshot) = self
|
||||
.acquire_pool_meta_read_guard(&save_guard, "decommission cancel fence retry failed")
|
||||
.await?;
|
||||
if decommission_capacity_terminal_fence_plan(&snapshot, source_pool_index)?.as_ref() != Some(plan) {
|
||||
return Err(decommission_capacity_blocked_error(
|
||||
"decommission capacity owner or target cohort changed while retrying terminal fences",
|
||||
));
|
||||
}
|
||||
attempt += 1;
|
||||
continue 'acquire_targets;
|
||||
}
|
||||
Err(rustfs_lock::LockError::QuorumNotReached { required, achieved }) => {
|
||||
return Err(Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "write",
|
||||
bucket: RUSTFS_META_BUCKET.to_string(),
|
||||
object,
|
||||
required,
|
||||
achieved,
|
||||
});
|
||||
}
|
||||
Err(err) => return Err(Error::Lock(err)),
|
||||
}
|
||||
}
|
||||
return Ok(guards);
|
||||
}
|
||||
Ok(guards)
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire_external_decommission_capacity_fence(
|
||||
@@ -12493,7 +12580,7 @@ impl ECStore {
|
||||
None
|
||||
};
|
||||
let _capacity_target_guards = if acquire_runtime_fence {
|
||||
self.acquire_decommission_capacity_terminal_guards(terminal_fence_plan.as_ref())
|
||||
self.acquire_decommission_capacity_terminal_guards(idx, terminal_fence_plan.as_ref())
|
||||
.await?
|
||||
} else {
|
||||
Vec::new()
|
||||
@@ -18736,6 +18823,429 @@ mod tests {
|
||||
assert_eq!(reservation.inflight_target_physical_bytes, 0);
|
||||
}
|
||||
|
||||
async fn start_target_fenced_cancel_test(
|
||||
store: &Arc<ECStore>,
|
||||
first_target_free: usize,
|
||||
) -> (DecommissionCapacityTerminalFencePlan, DecommissionCanceler) {
|
||||
crate::services::rebalance::promote_test_pool_meta_to_v2(store).await;
|
||||
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
|
||||
set_decommission_capacity_info_overrides_for_test(
|
||||
store.id,
|
||||
vec![vec![
|
||||
DecommissionPoolCapacityInfo::for_test(0, layout, 0, 10, 10),
|
||||
DecommissionPoolCapacityInfo::for_test(1, layout, first_target_free, 10, 10 - first_target_free),
|
||||
DecommissionPoolCapacityInfo::for_test(2, layout, 40, 40, 0),
|
||||
]],
|
||||
);
|
||||
store
|
||||
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
|
||||
.await
|
||||
.expect("persist an active target-fenced decommission");
|
||||
let plan = decommission_capacity_terminal_fence_plan(&*store.pool_meta.read().await, 0)
|
||||
.expect("active decommission should have a valid fence plan")
|
||||
.expect("active decommission should retain its reservation");
|
||||
assert_eq!(plan.model_version, DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION);
|
||||
let canceler = DecommissionCanceler::new(CancellationToken::new());
|
||||
*store.decommission_cancelers.write().await = vec![Some(canceler.clone()), None, None];
|
||||
(plan, canceler)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn decommission_cancel_waits_for_target_contention_past_one_lock_timeout() {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("5"))], async {
|
||||
let (_temp_dirs, store, _other_store) =
|
||||
crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
|
||||
let (plan, canceler) = start_target_fenced_cancel_test(&store, 0).await;
|
||||
assert_eq!(plan.target_pool_indices, vec![2]);
|
||||
let target_lock = store.pools[0]
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2"))
|
||||
.await
|
||||
.expect("create the active migration target gate");
|
||||
let target_guard = target_lock
|
||||
.get_write_lock(std::time::Duration::from_secs(30))
|
||||
.await
|
||||
.expect("hold the target gate across the first cancel acquisition timeout");
|
||||
let movement_gate = store.ctx.data_movement_operation_gate();
|
||||
let movement_guard = movement_gate.read().await;
|
||||
let cancel_store = Arc::clone(&store);
|
||||
let mut cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await });
|
||||
|
||||
tokio::time::timeout(get_lock_acquire_timeout() + std::time::Duration::from_secs(1), &mut cancel)
|
||||
.await
|
||||
.expect_err("one target-lock timeout must not end a legitimate cancel while its retry budget remains");
|
||||
assert!(!canceler.is_cancelled(), "cancel must not signal its worker before durable fencing");
|
||||
let mut durable = PoolMeta::default();
|
||||
durable
|
||||
.load_no_lock_from_replicas(store.pools.clone())
|
||||
.await
|
||||
.expect("the active reservation must remain readable while cancel waits");
|
||||
assert_eq!(
|
||||
decommission_capacity_terminal_fence_plan(&durable, 0).expect("read the durable active plan"),
|
||||
Some(plan),
|
||||
"a failed target acquisition must not release or replace the durable reservation"
|
||||
);
|
||||
assert!(!durable.pools[0].decommission.as_ref().expect("active decommission").canceled);
|
||||
|
||||
drop(target_guard);
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30), canceler.token().cancelled())
|
||||
.await
|
||||
.expect("cancel should persist and signal its worker after target contention is released");
|
||||
durable
|
||||
.load_no_lock_from_replicas(store.pools.clone())
|
||||
.await
|
||||
.expect("reload the committed cancellation from native replicas");
|
||||
let info = durable.pools[0]
|
||||
.decommission
|
||||
.as_ref()
|
||||
.expect("canceled state must remain durable");
|
||||
assert!(info.canceled);
|
||||
assert!(!info.complete && !info.failed);
|
||||
let reservation = info
|
||||
.capacity_reservation
|
||||
.as_ref()
|
||||
.expect("terminal capacity accounting must remain inspectable");
|
||||
assert!(!reservation.active());
|
||||
assert_eq!(reservation.pending_target_physical_bytes, 0);
|
||||
assert_eq!(reservation.inflight_target_physical_bytes, 0);
|
||||
tokio::time::timeout(std::time::Duration::from_millis(500), &mut cancel)
|
||||
.await
|
||||
.expect_err("the runtime-fenced cancel must wait for in-flight movement after signaling");
|
||||
drop(movement_guard);
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30), cancel)
|
||||
.await
|
||||
.expect("cancel should return after in-flight movement quiesces")
|
||||
.expect("cancel task should not panic")
|
||||
.expect("the same cancel request should finish its durable terminal transition");
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn decommission_cancel_target_contention_exhausts_bounded_attempts_without_committing() {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
|
||||
let (_temp_dirs, store, _other_store) =
|
||||
crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
|
||||
let (plan, canceler) = start_target_fenced_cancel_test(&store, 0).await;
|
||||
let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
|
||||
let target_lock = store.pools[0]
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2"))
|
||||
.await
|
||||
.expect("create the persistently contended target gate");
|
||||
let target_guard = target_lock
|
||||
.get_write_lock(std::time::Duration::from_secs(30))
|
||||
.await
|
||||
.expect("hold the target gate for every cancel attempt");
|
||||
|
||||
let err = tokio::time::timeout(std::time::Duration::from_secs(30), store.decommission_cancel(0))
|
||||
.await
|
||||
.expect("target contention must not retry indefinitely")
|
||||
.expect_err("exhausting the acquisition budget must not report cancellation success");
|
||||
match err {
|
||||
Error::Lock(rustfs_lock::LockError::Timeout { resource, timeout }) => {
|
||||
assert_eq!(resource, ".rustfs.sys/decommission/capacity-target/2@latest");
|
||||
assert_eq!(timeout, std::time::Duration::from_secs(1));
|
||||
}
|
||||
other => panic!("expected the final typed target-lock timeout, got {other:?}"),
|
||||
}
|
||||
assert_eq!(barrier.state.cancel_target_timeouts.load(Ordering::Acquire), 3);
|
||||
assert!(!canceler.is_cancelled());
|
||||
assert!(canceler.is_active());
|
||||
let mut durable = PoolMeta::default();
|
||||
durable
|
||||
.load_no_lock_from_replicas(store.pools.clone())
|
||||
.await
|
||||
.expect("reload the reservation after the canceled request exhausted its budget");
|
||||
assert_eq!(
|
||||
decommission_capacity_terminal_fence_plan(&durable, 0).expect("the active plan should remain valid"),
|
||||
Some(plan)
|
||||
);
|
||||
assert!(!durable.pools[0].decommission.as_ref().expect("active decommission").canceled);
|
||||
drop(target_guard);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn decommission_cancel_target_retry_releases_partial_cohort_and_rejects_remote_replacement() {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
|
||||
let (_temp_dirs, store, other_store) =
|
||||
crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
|
||||
let (plan, _old_canceler) = start_target_fenced_cancel_test(&store, 8).await;
|
||||
assert_eq!(plan.target_pool_indices, vec![1, 2]);
|
||||
let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
|
||||
barrier.pause_cancel_target_timeout();
|
||||
let target_lock = store.pools[0]
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2"))
|
||||
.await
|
||||
.expect("create the second target gate");
|
||||
let target_guard = target_lock
|
||||
.get_write_lock(std::time::Duration::from_secs(30))
|
||||
.await
|
||||
.expect("block cancellation after it acquires the first target");
|
||||
let cancel_store = Arc::clone(&store);
|
||||
let cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await });
|
||||
barrier.wait_until_cancel_target_timeout().await;
|
||||
|
||||
let first_target_lock = store.pools[0]
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/1"))
|
||||
.await
|
||||
.expect("create the partial-cohort release probe");
|
||||
let first_target_guard = first_target_lock
|
||||
.get_write_lock(std::time::Duration::from_secs(1))
|
||||
.await
|
||||
.expect("cancel must release its first target before retrying the blocked second target");
|
||||
drop(first_target_guard);
|
||||
drop(target_guard);
|
||||
|
||||
other_store
|
||||
.reload_pool_meta()
|
||||
.await
|
||||
.expect("load the active generation on the remote node");
|
||||
other_store
|
||||
.decommission_cancel(0)
|
||||
.await
|
||||
.expect("the remote node should cancel the old generation");
|
||||
other_store
|
||||
.clear_decommission(0)
|
||||
.await
|
||||
.expect("the remote node should clear the old generation");
|
||||
let (replacement_plan, replacement_canceler) = start_target_fenced_cancel_test(&other_store, 8).await;
|
||||
assert_ne!(replacement_plan.operation_id, plan.operation_id);
|
||||
|
||||
barrier.release_cancel_target_timeout();
|
||||
let err = tokio::time::timeout(std::time::Duration::from_secs(30), cancel)
|
||||
.await
|
||||
.expect("the stale request must finish without retrying a replacement operation")
|
||||
.expect("the stale cancel task should not panic")
|
||||
.expect_err("the original request must not cancel a remotely replaced generation");
|
||||
assert!(err.to_string().contains("changed while retrying terminal fences"));
|
||||
assert_eq!(barrier.state.cancel_target_timeouts.load(Ordering::Acquire), 1);
|
||||
assert!(!replacement_canceler.is_cancelled());
|
||||
let mut durable = PoolMeta::default();
|
||||
durable
|
||||
.load_no_lock_from_replicas(store.pools.clone())
|
||||
.await
|
||||
.expect("reload the replacement after the stale cancel returns");
|
||||
assert_eq!(
|
||||
decommission_capacity_terminal_fence_plan(&durable, 0).expect("the replacement must retain its valid plan"),
|
||||
Some(replacement_plan)
|
||||
);
|
||||
assert!(
|
||||
!durable.pools[0]
|
||||
.decommission
|
||||
.as_ref()
|
||||
.expect("replacement decommission")
|
||||
.canceled
|
||||
);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn decommission_cancel_target_retry_survives_caller_abort_and_settles_inflight_mutation() {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
|
||||
let (_temp_dirs, store, _other_store) =
|
||||
crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
|
||||
let (plan, canceler) = start_target_fenced_cancel_test(&store, 0).await;
|
||||
let owner = DecommissionCapacityOwner {
|
||||
source_pool_index: 0,
|
||||
operation_id: plan.operation_id,
|
||||
generation: plan.generation,
|
||||
owner_nonce: plan.owner_nonce,
|
||||
mutation_id: Some(uuid::Uuid::new_v4()),
|
||||
};
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
|
||||
let mutation_store = Arc::clone(&store);
|
||||
let mutation = tokio::spawn(async move {
|
||||
mutation_store
|
||||
.run_decommission_capacity_admitted_mutation(2, Some(owner), Some(1), || async {
|
||||
entered_tx
|
||||
.send(())
|
||||
.expect("the test should observe the admitted target mutation");
|
||||
release_rx.await.expect("the test should release the in-flight mutation");
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30), entered_rx)
|
||||
.await
|
||||
.expect("target mutation should reach its controlled I/O phase")
|
||||
.expect("target admission should succeed before cancel starts");
|
||||
let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
|
||||
let cancel_store = Arc::clone(&store);
|
||||
let cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await });
|
||||
barrier.wait_until_cancel_target_timeout().await;
|
||||
assert!(!canceler.is_cancelled(), "in-flight capacity settlement must precede the cancel signal");
|
||||
cancel.abort();
|
||||
assert!(cancel.await.expect_err("the RPC waiter should be aborted").is_cancelled());
|
||||
|
||||
release_tx
|
||||
.send(())
|
||||
.expect("the mutation must still be alive after the caller disconnects");
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30), mutation)
|
||||
.await
|
||||
.expect("the in-flight mutation must be able to settle without a metadata lock cycle")
|
||||
.expect("the mutation task should not panic")
|
||||
.expect("the admitted mutation must settle before the target gate is released");
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30), async {
|
||||
loop {
|
||||
if store.pool_meta.read().await.pools[0]
|
||||
.decommission
|
||||
.as_ref()
|
||||
.is_some_and(|info| info.canceled)
|
||||
{
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("the detached cancel transaction must finish after target contention clears");
|
||||
assert!(canceler.is_cancelled());
|
||||
let mut durable = PoolMeta::default();
|
||||
durable
|
||||
.load_no_lock_from_replicas(store.pools.clone())
|
||||
.await
|
||||
.expect("reload cancellation after the RPC waiter was dropped");
|
||||
let info = durable.pools[0].decommission.as_ref().expect("durable canceled decommission");
|
||||
assert!(info.canceled && !info.failed && !info.complete);
|
||||
let reservation = info
|
||||
.capacity_reservation
|
||||
.as_ref()
|
||||
.expect("inspect settled capacity accounting");
|
||||
assert!(!reservation.active());
|
||||
assert_eq!(reservation.pending_target_physical_bytes, 0);
|
||||
assert_eq!(reservation.inflight_target_physical_bytes, 0);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn decommission_cancel_target_retry_does_not_replay_a_failed_durable_save() {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
|
||||
let (_temp_dirs, store, _other_store) =
|
||||
crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
|
||||
let (plan, canceler) = start_target_fenced_cancel_test(&store, 0).await;
|
||||
let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
|
||||
barrier.pause_cancel_target_timeout();
|
||||
let target_lock = store.pools[0]
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2"))
|
||||
.await
|
||||
.expect("create the target gate preceding the failed save");
|
||||
let target_guard = target_lock
|
||||
.get_write_lock(std::time::Duration::from_secs(30))
|
||||
.await
|
||||
.expect("force one target acquisition retry before persistence");
|
||||
let save_calls = Arc::new(AtomicUsize::new(0));
|
||||
let calls = Arc::clone(&save_calls);
|
||||
let cancel_store = Arc::clone(&store);
|
||||
let owner = canceler.clone();
|
||||
let cancel = tokio::spawn(async move {
|
||||
cancel_store
|
||||
.decommission_cancel_transaction(0, Some(owner), true, move |_, _| async move {
|
||||
calls.fetch_add(1, Ordering::AcqRel);
|
||||
Err(Error::Timeout)
|
||||
})
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_cancel_target_timeout().await;
|
||||
assert_eq!(save_calls.load(Ordering::Acquire), 0, "persistence must wait for every target fence");
|
||||
drop(target_guard);
|
||||
barrier.release_cancel_target_timeout();
|
||||
let err = tokio::time::timeout(std::time::Duration::from_secs(30), cancel)
|
||||
.await
|
||||
.expect("a persistence failure must end the cancellation transaction")
|
||||
.expect("the failed-save task should not panic")
|
||||
.expect_err("the injected durable-save failure must reach the caller");
|
||||
assert!(matches!(err, Error::Timeout));
|
||||
assert_eq!(save_calls.load(Ordering::Acquire), 1);
|
||||
assert_eq!(barrier.state.cancel_target_timeouts.load(Ordering::Acquire), 1);
|
||||
assert!(!canceler.is_cancelled());
|
||||
assert!(canceler.is_active());
|
||||
store
|
||||
.ensure_pool_meta_side_effects_safe("verify ambiguous cancellation save blocks further writes")
|
||||
.await
|
||||
.expect_err("a failed durable save must retain the existing recovery gate");
|
||||
let mut durable = PoolMeta::default();
|
||||
durable
|
||||
.load_no_lock_from_replicas(store.pools.clone())
|
||||
.await
|
||||
.expect("the original reservation should remain readable after the injected save failure");
|
||||
assert_eq!(
|
||||
decommission_capacity_terminal_fence_plan(&durable, 0).expect("the old reservation should remain valid"),
|
||||
Some(plan)
|
||||
);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn decommission_cancel_target_retry_converges_after_successive_contenders() {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
|
||||
let (_temp_dirs, store, _other_store) =
|
||||
crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
|
||||
let (_plan, canceler) = start_target_fenced_cancel_test(&store, 0).await;
|
||||
let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
|
||||
barrier.pause_cancel_target_timeout();
|
||||
let target_lock = store.pools[0]
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2"))
|
||||
.await
|
||||
.expect("create the repeatedly contended target gate");
|
||||
let first_guard = target_lock
|
||||
.get_write_lock(std::time::Duration::from_secs(30))
|
||||
.await
|
||||
.expect("the first contender should own the target gate");
|
||||
let cancel_store = Arc::clone(&store);
|
||||
let cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await });
|
||||
barrier.wait_until_cancel_target_timeout().await;
|
||||
assert_eq!(barrier.state.cancel_target_timeouts.load(Ordering::Acquire), 1);
|
||||
drop(first_guard);
|
||||
let second_guard = target_lock
|
||||
.get_write_lock(std::time::Duration::from_secs(30))
|
||||
.await
|
||||
.expect("a second contender should be able to acquire between cancel attempts");
|
||||
barrier.release_cancel_target_timeout();
|
||||
barrier.pause_cancel_target_timeout();
|
||||
barrier.wait_until_cancel_target_timeout().await;
|
||||
assert_eq!(barrier.state.cancel_target_timeouts.load(Ordering::Acquire), 2);
|
||||
assert!(!canceler.is_cancelled());
|
||||
drop(second_guard);
|
||||
barrier.release_cancel_target_timeout();
|
||||
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30), cancel)
|
||||
.await
|
||||
.expect("cancel should converge after the repeated contention ends")
|
||||
.expect("the cancel task should not panic")
|
||||
.expect("the third target acquisition should permit a durable cancellation");
|
||||
assert!(canceler.is_cancelled());
|
||||
let mut durable = PoolMeta::default();
|
||||
durable
|
||||
.load_no_lock_from_replicas(store.pools.clone())
|
||||
.await
|
||||
.expect("the successful retried cancellation must survive a native metadata reload");
|
||||
assert!(
|
||||
durable.pools[0]
|
||||
.decommission
|
||||
.as_ref()
|
||||
.expect("canceled decommission")
|
||||
.canceled
|
||||
);
|
||||
assert!(
|
||||
decommission_capacity_terminal_fence_plan(&durable, 0)
|
||||
.expect("valid terminal metadata")
|
||||
.is_none()
|
||||
);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn stale_node_cancel_cannot_replace_a_new_durable_v2_operation() {
|
||||
|
||||
@@ -55,7 +55,7 @@ use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use rustfs_utils::{crc_hash, path::path_join_buf, sip_hash};
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
sync::Arc,
|
||||
sync::{Arc, Weak},
|
||||
};
|
||||
use tokio::sync::RwLock;
|
||||
use tokio::sync::broadcast::{Receiver, Sender};
|
||||
@@ -219,7 +219,10 @@ impl Sets {
|
||||
|
||||
let mut disk_set = Vec::with_capacity(set_count);
|
||||
|
||||
let lock_registry = runtime_sources::lock_registry();
|
||||
let pool_lockers = runtime_sources::lock_registry()
|
||||
.as_ref()
|
||||
.map(|registry| registry.clients_for_endpoints(endpoints.endpoints.as_ref()))
|
||||
.unwrap_or_default();
|
||||
|
||||
for i in 0..set_count {
|
||||
let mut set_drive = Vec::with_capacity(set_drive_count);
|
||||
@@ -270,10 +273,6 @@ impl Sets {
|
||||
}
|
||||
}
|
||||
|
||||
let lockers = lock_registry
|
||||
.as_ref()
|
||||
.map(|registry| registry.clients_for_endpoints(&set_endpoints))
|
||||
.unwrap_or_default();
|
||||
let set_disks = SetDisks::new_with_instance_ctx(
|
||||
runtime_sources::local_node_name().await,
|
||||
Arc::new(RwLock::new(set_drive)),
|
||||
@@ -283,7 +282,7 @@ impl Sets {
|
||||
pool_idx,
|
||||
set_endpoints,
|
||||
fm.clone(),
|
||||
lockers,
|
||||
pool_lockers.clone(),
|
||||
instance_ctx.clone(),
|
||||
)
|
||||
.await;
|
||||
@@ -308,10 +307,9 @@ impl Sets {
|
||||
ctx: instance_ctx,
|
||||
});
|
||||
|
||||
let asets = sets.clone();
|
||||
|
||||
let rx1 = rx.resubscribe();
|
||||
tokio::spawn(async move { asets.monitor_and_connect_endpoints(rx1).await });
|
||||
let weak_sets = Arc::downgrade(&sets);
|
||||
tokio::spawn(async move { Self::monitor_and_connect_endpoints_task(weak_sets, rx1).await });
|
||||
|
||||
Ok(sets)
|
||||
}
|
||||
@@ -326,12 +324,26 @@ impl Sets {
|
||||
&self.ctx
|
||||
}
|
||||
|
||||
pub async fn monitor_and_connect_endpoints(&self, mut rx: Receiver<()>) {
|
||||
tokio::time::sleep(Duration::from_secs(5)).await;
|
||||
async fn monitor_and_connect_endpoints_task(sets: Weak<Sets>, mut rx: Receiver<()>) {
|
||||
let startup_delay = tokio::time::sleep(Duration::from_secs(5));
|
||||
tokio::pin!(startup_delay);
|
||||
|
||||
tokio::select! {
|
||||
_ = &mut startup_delay => {}
|
||||
_ = rx.recv() => {
|
||||
warn!("monitor_and_connect_endpoints ctx cancelled");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
info!("start monitor_and_connect_endpoints");
|
||||
|
||||
self.connect_disks().await;
|
||||
let Some(current) = sets.upgrade() else {
|
||||
warn!("monitor_and_connect_endpoints exit");
|
||||
return;
|
||||
};
|
||||
current.connect_disks().await;
|
||||
drop(current);
|
||||
|
||||
// TODO(backlog): make monitor_and_connect interval configurable instead of hardcoded 15s
|
||||
let mut interval = tokio::time::interval(Duration::from_secs(15));
|
||||
@@ -339,7 +351,10 @@ impl Sets {
|
||||
tokio::select! {
|
||||
_= interval.tick()=>{
|
||||
// debug!("tick...");
|
||||
self.connect_disks().await;
|
||||
let Some(current) = sets.upgrade() else {
|
||||
break;
|
||||
};
|
||||
current.connect_disks().await;
|
||||
|
||||
interval.reset();
|
||||
},
|
||||
|
||||
@@ -6433,9 +6433,48 @@ impl TierConfigMgr {
|
||||
}
|
||||
|
||||
pub(crate) async fn refresh_tier_config_handle(handle: Arc<RwLock<Self>>, api: Arc<ECStore>) {
|
||||
Self::refresh_tier_config_handle_with(handle, api).await;
|
||||
Self::refresh_tier_config_handle_with_weak(handle, Arc::downgrade(&api)).await;
|
||||
}
|
||||
|
||||
async fn refresh_tier_config_handle_with_weak(handle: Arc<RwLock<Self>>, api: Weak<ECStore>) {
|
||||
// The periodic refresh remains the recovery fallback; committed mutations
|
||||
// notify this worker so a successful peer commit converges immediately.
|
||||
let mutation_refresh = Self::mutation_refresh_notifier(&handle).await;
|
||||
let r = rand::rng().random_range(0.0..1.0);
|
||||
let rand_interval = || Duration::from_secs((r * 60_f64).round() as u64);
|
||||
|
||||
let refresh_interval = TIER_CFG_REFRESH + rand_interval();
|
||||
let mut t = delayed_tier_refresh_interval(refresh_interval);
|
||||
loop {
|
||||
select! {
|
||||
_ = t.tick() => {
|
||||
let Some(api) = Weak::upgrade(&api) else {
|
||||
return;
|
||||
};
|
||||
if let Err(err) = Self::reload_handle_with(&handle, api).await {
|
||||
warn!(
|
||||
event = EVENT_TIER_CONFIG_REFRESH,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_TIER,
|
||||
trigger = "periodic",
|
||||
result = "failed",
|
||||
error = ?err,
|
||||
"tier configuration refresh"
|
||||
);
|
||||
}
|
||||
}
|
||||
_ = mutation_refresh.notified() => {
|
||||
let Some(api) = Weak::upgrade(&api) else {
|
||||
return;
|
||||
};
|
||||
Self::reload_after_committed_mutation(&handle, api).await;
|
||||
}
|
||||
}
|
||||
t.reset();
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "used by focused tier refresh tests and non-ECStore generic harnesses")]
|
||||
pub(crate) async fn refresh_tier_config_handle_with<S>(handle: Arc<RwLock<Self>>, api: Arc<S>)
|
||||
where
|
||||
S: EcstoreObjectIO
|
||||
@@ -17540,6 +17579,66 @@ mod tests {
|
||||
assert!(current.tiers.contains_key("COLD-B"));
|
||||
}
|
||||
|
||||
async fn wait_for_reference_proof_barrier(
|
||||
barrier: &TierDriverBuildBarrier,
|
||||
update: &mut tokio::task::JoinHandle<std::result::Result<(), TierConfigUpdateError>>,
|
||||
) -> std::result::Result<(), String> {
|
||||
tokio::select! {
|
||||
biased;
|
||||
result = &mut *update => Err(format!("tier update exited before the reference proof barrier: {result:?}")),
|
||||
() = barrier.arrived.notified() => Ok(()),
|
||||
() = tokio::time::sleep(Duration::from_secs(30)) => {
|
||||
// Aborting the caller does not stop its owned mutation task.
|
||||
// Let a late arrival pass the test-only barrier.
|
||||
barrier.release.add_permits(1);
|
||||
update.abort();
|
||||
Err("timed out waiting for the reference proof barrier".to_string())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn reference_proof_barrier_reports_update_failure_before_arrival() {
|
||||
let manager = TierConfigMgr::new();
|
||||
let store = Arc::new(CasConfigStore::default());
|
||||
let mut persisted = empty_mgr();
|
||||
persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
|
||||
persisted
|
||||
.save_tiering_config_if_current(store.clone(), None)
|
||||
.await
|
||||
.expect("early update failure fixture should persist");
|
||||
let barrier = tier_reference_proof_test_barrier();
|
||||
let scoped_barrier = barrier.clone();
|
||||
let factory: TierDriverTestFactory =
|
||||
Arc::new(|_| Err(AdminError::msg("injected driver initialization failure before reference proof")));
|
||||
let mut update = tokio::spawn(async move {
|
||||
TIER_REFERENCE_PROOF_TEST_BARRIER
|
||||
.scope(
|
||||
scoped_barrier,
|
||||
TIER_DRIVER_TEST_FACTORY.scope(
|
||||
factory,
|
||||
TIER_MUTATION_TEST_PEERS.scope(
|
||||
Vec::new(),
|
||||
TierConfigMgr::update_candidate_with_config_lock(
|
||||
&manager,
|
||||
store,
|
||||
TierCandidateMutation::Remove("COLD-A".to_string(), true),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
.await
|
||||
});
|
||||
|
||||
let err = tokio::time::timeout(Duration::from_secs(5), wait_for_reference_proof_barrier(&barrier, &mut update))
|
||||
.await
|
||||
.expect("an early update failure should be observed without waiting for the barrier deadline")
|
||||
.expect_err("a failed update cannot reach the reference proof barrier");
|
||||
assert!(err.contains("Mutation"), "{err}");
|
||||
assert!(err.contains("injected driver initialization failure before reference proof"), "{err}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn reference_proof_rejects_a_changed_prepared_fence_revision_before_publish() {
|
||||
@@ -17560,7 +17659,7 @@ mod tests {
|
||||
let scoped_barrier = barrier.clone();
|
||||
let update_manager = manager.clone();
|
||||
let update_store = store.clone();
|
||||
let update = tokio::spawn(async move {
|
||||
let mut update = tokio::spawn(async move {
|
||||
TIER_REFERENCE_PROOF_TEST_BARRIER
|
||||
.scope(
|
||||
scoped_barrier,
|
||||
@@ -17575,7 +17674,9 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
});
|
||||
barrier.arrived.notified().await;
|
||||
wait_for_reference_proof_barrier(&barrier, &mut update)
|
||||
.await
|
||||
.expect("tier update should reach the reference proof barrier");
|
||||
|
||||
let unrelated = prepared_remove_intent("COLD-B", uuid::Uuid::from_u128(0x2237));
|
||||
TierConfigMgr::apply_prepared_mutation_intent_block(&manager, &unrelated)
|
||||
@@ -17583,8 +17684,9 @@ mod tests {
|
||||
.expect("an unrelated prepared fence should advance the runtime revision");
|
||||
barrier.release.add_permits(1);
|
||||
|
||||
let err = update
|
||||
let err = tokio::time::timeout(Duration::from_secs(30), update)
|
||||
.await
|
||||
.expect("tier update should finish after the reference proof barrier releases")
|
||||
.expect("tier update task should join")
|
||||
.expect_err("a reference proof cannot authorize publication across a fence revision change");
|
||||
let TierConfigUpdateError::Publish(err) = err else {
|
||||
|
||||
@@ -3076,6 +3076,23 @@ impl SetDisks {
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
) -> Result<Option<rustfs_filemeta::FileInfoVersions>> {
|
||||
self.load_file_info_versions_for_cleanup(bucket, object, false).await
|
||||
}
|
||||
|
||||
pub(crate) async fn load_file_info_versions_for_tier_cleanup(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
) -> Result<Option<rustfs_filemeta::FileInfoVersions>> {
|
||||
self.load_file_info_versions_for_cleanup(bucket, object, true).await
|
||||
}
|
||||
|
||||
async fn load_file_info_versions_for_cleanup(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
retain_unconfirmed_tier_references: bool,
|
||||
) -> Result<Option<rustfs_filemeta::FileInfoVersions>> {
|
||||
let disk_object = rustfs_utils::path::encode_dir_object(object);
|
||||
let disks = self.get_disks_internal().await;
|
||||
@@ -3152,12 +3169,24 @@ impl SetDisks {
|
||||
)));
|
||||
}
|
||||
|
||||
let file_info_versions = FileMeta {
|
||||
let mut file_info_versions = FileMeta {
|
||||
versions,
|
||||
..Default::default()
|
||||
}
|
||||
.get_all_file_info_versions(bucket, object, true)
|
||||
.map_err(decode_error)?;
|
||||
if retain_unconfirmed_tier_references {
|
||||
// A failed overwrite may leave its live source on a minority
|
||||
// of disks. Preserve that reference even if quorum merging
|
||||
// selects only the replacement and its cleanup owner.
|
||||
file_info_versions.versions.extend(
|
||||
transition_copies
|
||||
.into_values()
|
||||
.flatten()
|
||||
.map(|(version, _)| version)
|
||||
.filter(|version| !version.tier_free_version()),
|
||||
);
|
||||
}
|
||||
|
||||
for file_info in file_info_versions
|
||||
.versions
|
||||
@@ -12214,6 +12243,64 @@ mod tests {
|
||||
assert!(result.is_err(), "missing disks must prevent metadata write quorum");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tier_overwrite_cleanup_rejects_unreadable_disk_despite_metadata_quorum() {
|
||||
let bucket = "tier-unreadable-disk";
|
||||
let object = "object";
|
||||
let mut dirs = Vec::new();
|
||||
let mut disks = Vec::new();
|
||||
let mut fi = metadata_test_fileinfo(object);
|
||||
fi.mod_time = Some(OffsetDateTime::now_utc());
|
||||
for index in 1..=3 {
|
||||
let (dir, disk) = read_multiple_test_disk(bucket, &[]).await;
|
||||
fi.erasure.index = index;
|
||||
disk.write_metadata(bucket, bucket, object, fi.clone())
|
||||
.await
|
||||
.expect("seed metadata quorum");
|
||||
dirs.push(dir);
|
||||
disks.push(Some(disk));
|
||||
}
|
||||
disks.push(None);
|
||||
let set = io_primitives_test_set(disks, 2).await;
|
||||
assert!(
|
||||
set.load_file_info_versions_exact(bucket, object).await.is_err(),
|
||||
"exact reads must preserve release's unreadable-replica fence"
|
||||
);
|
||||
assert!(
|
||||
set.load_file_info_versions_for_tier_cleanup(bucket, object).await.is_err(),
|
||||
"unreadable replica may still reference the old remote object"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tier_overwrite_cleanup_rejects_minority_metadata_in_an_absent_set() {
|
||||
let bucket = "tier-minority-metadata";
|
||||
let object = "object";
|
||||
let mut dirs = Vec::new();
|
||||
let mut disks = Vec::new();
|
||||
for index in 1..=4 {
|
||||
let (dir, disk) = read_multiple_test_disk(bucket, &[]).await;
|
||||
if index == 1 {
|
||||
let mut fi = metadata_test_fileinfo(object);
|
||||
fi.mod_time = Some(OffsetDateTime::now_utc());
|
||||
disk.write_metadata(bucket, bucket, object, fi)
|
||||
.await
|
||||
.expect("seed minority metadata");
|
||||
}
|
||||
dirs.push(dir);
|
||||
disks.push(Some(disk));
|
||||
}
|
||||
let set = io_primitives_test_set(disks, 2).await;
|
||||
assert!(
|
||||
set.load_file_info_versions_exact(bucket, object).await.is_err(),
|
||||
"exact reads must preserve release's minority-ownership fence"
|
||||
);
|
||||
assert!(
|
||||
set.load_file_info_versions_for_tier_cleanup(bucket, object).await.is_err(),
|
||||
"absence on a majority cannot prove this physical set has no remote reference"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn load_file_info_versions_exact_returns_versions_from_read_quorum() {
|
||||
let bucket = "exact-versions-bucket";
|
||||
|
||||
@@ -3852,7 +3852,7 @@ pub struct SetDisks {
|
||||
pub default_parity_count: usize,
|
||||
pub set_index: usize,
|
||||
pub pool_index: usize,
|
||||
/// Stable namespace shared by every object lock created for this set.
|
||||
/// Stable namespace shared by every object lock created for this pool.
|
||||
set_lock_namespace: Arc<str>,
|
||||
pub format: FormatV3,
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
@@ -4491,7 +4491,7 @@ impl SetDisks {
|
||||
instance_ctx: Arc<InstanceContext>,
|
||||
) -> Arc<Self> {
|
||||
let ctx = instance_ctx;
|
||||
let set_lock_namespace: Arc<str> = format!("set-{pool_index}-{set_index}").into();
|
||||
let set_lock_namespace: Arc<str> = format!("pool-{pool_index}").into();
|
||||
let shared_lockers = Arc::from(lockers.to_vec());
|
||||
Arc::new(SetDisks {
|
||||
locker_owner,
|
||||
@@ -4605,7 +4605,9 @@ impl SetDisks {
|
||||
pub(crate) async fn shares_namespace_lock_domain(&self, other: &Self) -> bool {
|
||||
match (self.ctx.is_dist_erasure().await, other.ctx.is_dist_erasure().await) {
|
||||
(false, false) => Arc::ptr_eq(&self.local_lock_manager, &other.local_lock_manager),
|
||||
(true, true) => same_distributed_lock_domain(&self.lockers, &other.lockers),
|
||||
(true, true) => {
|
||||
self.set_lock_namespace == other.set_lock_namespace && same_distributed_lock_domain(&self.lockers, &other.lockers)
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
@@ -7123,7 +7125,7 @@ mod tests {
|
||||
ctx.update_erasure_type(SetupType::Erasure).await;
|
||||
let set = make_test_set_disks_with_ctx(Vec::new(), ctx).await;
|
||||
|
||||
assert_eq!(&*set.set_lock_namespace, "set-0-0");
|
||||
assert_eq!(&*set.set_lock_namespace, "pool-0");
|
||||
let before = Arc::strong_count(&set.set_lock_namespace);
|
||||
let lock = set
|
||||
.new_ns_lock("bucket", "object")
|
||||
@@ -8348,6 +8350,78 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn test_new_ns_lock_distributed_write_succeeds_with_three_lockers_one_offline() {
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
|
||||
let manager_a = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let manager_b = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let healthy_a: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager_a));
|
||||
let healthy_b: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager_b));
|
||||
let failing_client: Arc<dyn LockClient> = Arc::new(FailingClient);
|
||||
let set_disks = make_test_set_disks(vec![healthy_a, failing_client, healthy_b]).await;
|
||||
|
||||
let guard = set_disks
|
||||
.new_ns_lock("bucket", "object")
|
||||
.await
|
||||
.expect("namespace lock should be created")
|
||||
.get_write_lock(Duration::from_millis(500))
|
||||
.await
|
||||
.expect("two healthy lockers should satisfy the three-locker write quorum");
|
||||
|
||||
match guard {
|
||||
NamespaceLockGuard::Standard(_) => {}
|
||||
NamespaceLockGuard::Fast(_) => panic!("Expected distributed guard for dist-erasure"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn namespace_lock_domain_includes_pool_namespace() {
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
|
||||
let first: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
let second: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
let lockers = vec![first, second];
|
||||
let same_pool_first_set = make_test_set_disks_with_ctx(lockers.clone(), bootstrap_ctx()).await;
|
||||
let same_pool_second_set = SetDisks::new_with_instance_ctx(
|
||||
"test-owner".to_string(),
|
||||
Arc::new(RwLock::new(vec![None, None])),
|
||||
2,
|
||||
1,
|
||||
1,
|
||||
0,
|
||||
same_pool_first_set.set_endpoints.clone(),
|
||||
FormatV3::new(2, 2),
|
||||
lockers.clone(),
|
||||
bootstrap_ctx(),
|
||||
)
|
||||
.await;
|
||||
let other_pool_set = SetDisks::new_with_instance_ctx(
|
||||
"test-owner".to_string(),
|
||||
Arc::new(RwLock::new(vec![None, None])),
|
||||
2,
|
||||
1,
|
||||
0,
|
||||
1,
|
||||
same_pool_first_set.set_endpoints.clone(),
|
||||
FormatV3::new(1, 2),
|
||||
lockers,
|
||||
bootstrap_ctx(),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
same_pool_first_set.shares_namespace_lock_domain(&same_pool_second_set).await,
|
||||
"sets in the same pool share the object namespace lock domain"
|
||||
);
|
||||
assert!(
|
||||
!same_pool_first_set.shares_namespace_lock_domain(&other_pool_set).await,
|
||||
"different pool namespaces must not be deduplicated solely by identical clients"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn streaming_reader_holds_read_lock_until_eof() {
|
||||
|
||||
@@ -3821,6 +3821,13 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
fi.metadata = user_defined;
|
||||
if fi.version_id.is_none_or(|id| id.is_nil()) && !opts.data_movement && expected_restore_operation_id.is_none() {
|
||||
// Every disk must publish the same cleanup owner alongside a
|
||||
// replaced null version. This transient key is not persisted
|
||||
// on the new object; recovery discovers the free-version in
|
||||
// the committed xl.meta even if this request is cancelled.
|
||||
fi.set_tier_free_version_id(&Uuid::new_v4().to_string());
|
||||
}
|
||||
fi.mod_time = mod_time;
|
||||
fi.size = w_size as i64;
|
||||
fi.versioned = opts.versioned || opts.version_suspended;
|
||||
@@ -13807,7 +13814,17 @@ mod transition_commit_failure_tests {
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn restore_failure_after_snapshot_cleans_exact_generation_and_returns_primary_error() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
assert_restore_failure_cleanup_boundary(true).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn restore_failure_after_snapshot_preserves_corrupt_known_transition_metadata() {
|
||||
assert_restore_failure_cleanup_boundary(false).await;
|
||||
}
|
||||
|
||||
async fn assert_restore_failure_cleanup_boundary(legacy_unknown: bool) {
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "restore-post-snapshot-cleanup-bucket";
|
||||
let object = "object.bin";
|
||||
for disk in &disk_stores {
|
||||
@@ -13816,7 +13833,15 @@ mod transition_commit_failure_tests {
|
||||
|
||||
let mut reader = PutObjReader::from_vec(b"post-snapshot cleanup source".repeat(1024));
|
||||
let original = set_disks
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("source object should be written");
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
@@ -13857,16 +13882,70 @@ mod transition_commit_failure_tests {
|
||||
.await
|
||||
.expect("transitioned metadata should be readable")
|
||||
.into_owned();
|
||||
let known_state = source_fi.transition_version_state;
|
||||
assert_ne!(known_state, rustfs_filemeta::TransitionVersionState::Unknown);
|
||||
source_fi.metadata.extend(restore_metadata(operation_id, true));
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut source_fi.metadata,
|
||||
rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
"invalid".to_string(),
|
||||
);
|
||||
set_disks
|
||||
.update_object_meta(bucket, object, source_fi, &online_disks)
|
||||
.await
|
||||
.expect("invalid backend identity fixture should be persisted");
|
||||
.expect("restore markers should be persisted");
|
||||
|
||||
// Normal writes reject damage to a reconciled binding. Model on-disk
|
||||
// corruption directly, with and without the legacy missing-state field.
|
||||
let mut corrupted_metadata = Vec::new();
|
||||
for temp_dir in &temp_dirs {
|
||||
let metadata_path = temp_dir.path().join(bucket).join(object).join(STORAGE_FORMAT_FILE);
|
||||
let encoded = tokio::fs::read(&metadata_path)
|
||||
.await
|
||||
.expect("transition metadata should be readable");
|
||||
let mut metadata = FileMeta::load(&encoded).expect("transition metadata should decode");
|
||||
let (version_index, mut version) = metadata
|
||||
.find_version(original.version_id)
|
||||
.expect("transitioned version should exist");
|
||||
let object_meta = version.object.as_mut().expect("transitioned version should be an object");
|
||||
rustfs_utils::http::insert_bytes(
|
||||
&mut object_meta.meta_sys,
|
||||
rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
b"invalid".to_vec(),
|
||||
);
|
||||
if legacy_unknown {
|
||||
rustfs_utils::http::remove_bytes(
|
||||
&mut object_meta.meta_sys,
|
||||
rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
);
|
||||
}
|
||||
metadata.versions[version_index] =
|
||||
rustfs_filemeta::FileMetaShallowVersion::try_from(version).expect("corrupt fixture should re-encode");
|
||||
tokio::fs::write(&metadata_path, metadata.marshal_msg().expect("corrupt fixture should encode"))
|
||||
.await
|
||||
.expect("corrupt fixture should be written");
|
||||
let persisted = tokio::fs::read(&metadata_path)
|
||||
.await
|
||||
.expect("corrupt fixture should be readable");
|
||||
let fixture = FileMeta::load(&persisted)
|
||||
.expect("corrupt fixture should decode")
|
||||
.find_version(original.version_id)
|
||||
.expect("corrupt version should exist")
|
||||
.1
|
||||
.into_fileinfo(bucket, object, true)
|
||||
.expect("corrupt version should decode");
|
||||
assert_eq!(
|
||||
fixture.transition_version_state,
|
||||
if legacy_unknown {
|
||||
rustfs_filemeta::TransitionVersionState::Unknown
|
||||
} else {
|
||||
known_state
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
rustfs_utils::http::get_str(&fixture.metadata, rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID),
|
||||
Some("invalid".to_string())
|
||||
);
|
||||
for (key, value) in restore_metadata(operation_id, true) {
|
||||
assert_eq!(fixture.metadata.get(&key), Some(&value), "fixture must retain restore marker {key}");
|
||||
}
|
||||
corrupted_metadata.push((metadata_path, persisted));
|
||||
}
|
||||
set_disks.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||
|
||||
let mut opts = ObjectOptions::default();
|
||||
@@ -13889,6 +13968,23 @@ mod transition_commit_failure_tests {
|
||||
.await
|
||||
.expect("cleanup should leave the transitioned object readable");
|
||||
assert_eq!(cleaned.transitioned_object.status, TRANSITION_COMPLETE);
|
||||
if !legacy_unknown {
|
||||
// Known bindings with corrupt identities must be repaired before
|
||||
// cleanup; rejection must preserve both the binding and markers.
|
||||
for (key, value) in restore_metadata(operation_id, true) {
|
||||
assert_eq!(cleaned.user_defined.get(&key), Some(&value), "cleanup must preserve restore marker {key}");
|
||||
}
|
||||
for (metadata_path, before) in corrupted_metadata {
|
||||
assert_eq!(
|
||||
tokio::fs::read(metadata_path)
|
||||
.await
|
||||
.expect("rejected cleanup metadata should remain readable"),
|
||||
before,
|
||||
"rejected cleanup must leave corrupt known metadata unchanged"
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
assert!(!cleaned.user_defined.contains_key(s3s::header::X_AMZ_RESTORE.as_str()));
|
||||
assert!(
|
||||
rustfs_utils::http::get_str(cleaned.user_defined.as_ref(), rustfs_utils::http::SUFFIX_RESTORE_OPERATION_ID,)
|
||||
@@ -18085,6 +18181,102 @@ mod put_object_tmp_cleanup_tests {
|
||||
drop(temp_dirs);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn tier_overwrite_failed_quorum_and_cancellation_preserve_live_source() {
|
||||
for cancel_before_rename in [false, true] {
|
||||
let (dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "tier-overwrite-failure";
|
||||
let object = "still-live";
|
||||
make_completion_test_bucket(&disks, bucket).await;
|
||||
let old_body = vec![0x31; TEST_OBJECT_SIZE];
|
||||
let mut metadata = HashMap::from([(
|
||||
"x-amz-restore".to_string(),
|
||||
"ongoing-request=\"false\", expiry-date=\"2099-01-01T00:00:00Z\"".to_string(),
|
||||
)]);
|
||||
for (suffix, value) in [
|
||||
(rustfs_utils::http::SUFFIX_TRANSITION_STATUS, "complete".to_string()),
|
||||
(rustfs_utils::http::SUFFIX_TRANSITION_TIER, "WARM".to_string()),
|
||||
(rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME, "remote/still-live".to_string()),
|
||||
(rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID, "exact-live-version".to_string()),
|
||||
(rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE, "exact".to_string()),
|
||||
(rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID, "ab".repeat(32)),
|
||||
] {
|
||||
rustfs_utils::http::insert_str(&mut metadata, suffix, value);
|
||||
}
|
||||
set.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(old_body.clone()),
|
||||
&ObjectOptions {
|
||||
user_defined: metadata,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed live transitioned source");
|
||||
wait_for_tmp_workspace_to_drain(&dirs, "seed write must drain").await;
|
||||
let before = set
|
||||
.load_file_info_versions_exact(bucket, object)
|
||||
.await
|
||||
.expect("read original metadata")
|
||||
.expect("original exists");
|
||||
|
||||
if cancel_before_rename {
|
||||
let barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::AfterQuotaReservation);
|
||||
let writer = Arc::clone(&set);
|
||||
let put = tokio::spawn(async move {
|
||||
writer
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![0x32; TEST_OBJECT_SIZE]),
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
put.abort();
|
||||
assert!(put.await.expect_err("cancel paused replacement").is_cancelled());
|
||||
wait_for_tmp_workspace_to_drain(&dirs, "cancelled replacement must roll back").await;
|
||||
drop(barrier);
|
||||
} else {
|
||||
let _fault = rename_fault_injection::fail_rename_on(object, &[2, 3]);
|
||||
let err = set
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![0x32; TEST_OBJECT_SIZE]),
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("two disk commits cannot satisfy write quorum three");
|
||||
assert!(matches!(err, Error::ErasureWriteQuorum | Error::InsufficientWriteQuorum(_, _)), "{err}");
|
||||
}
|
||||
let after = set
|
||||
.load_file_info_versions_exact(bucket, object)
|
||||
.await
|
||||
.expect("read rolled-back metadata")
|
||||
.expect("live source must survive");
|
||||
assert_eq!(after.versions, before.versions, "failed replacement must preserve the live version");
|
||||
assert_eq!(
|
||||
after.free_versions, before.free_versions,
|
||||
"failed replacement must not publish a cleanup owner"
|
||||
);
|
||||
let mut reader = set
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("live source remains readable");
|
||||
let mut actual = Vec::new();
|
||||
reader.stream.read_to_end(&mut actual).await.expect("read original bytes");
|
||||
assert_eq!(actual, old_body);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cooperative_cancellation_while_waiting_for_namespace_lock_cleans_tmp_workspace() {
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
|
||||
@@ -801,6 +801,15 @@ impl ECStore {
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn get_bucket_info_from_sets_at_read_quorum(
|
||||
&self,
|
||||
bucket: &str,
|
||||
opts: &BucketOptions,
|
||||
) -> Result<BucketInfo> {
|
||||
self.get_bucket_info_from_sets_with_quorum(bucket, opts, BucketInfoQuorum::Read)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get_bucket_info_from_sets_with_quorum(
|
||||
&self,
|
||||
bucket: &str,
|
||||
|
||||
@@ -2832,6 +2832,224 @@ mod tests {
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn tier_overwrite_put_and_self_copy_recover_persisted_cleanup_owners() {
|
||||
use crate::bucket::lifecycle::bucket_lifecycle_ops::ExpiryState;
|
||||
use crate::bucket::lifecycle::tier_free_version_recovery::recover_tier_free_versions;
|
||||
use rustfs_filemeta::TransitionVersionState::{Exact, KnownDisabled, SuspendedNull};
|
||||
use rustfs_s3_client::transition_api::ReaderImpl;
|
||||
use rustfs_utils::http::{
|
||||
SUFFIX_TRANSITION_STATUS, SUFFIX_TRANSITION_TIER, SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
SUFFIX_TRANSITIONED_OBJECTNAME, SUFFIX_TRANSITIONED_VERSION_ID, SUFFIX_TRANSITIONED_VERSION_STATE, insert_str,
|
||||
};
|
||||
|
||||
let temp_dir = tempfile::tempdir().expect("create tier overwrite store");
|
||||
let (mut ctx, mut store, mut shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-overwrite", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let tier = "OVERWRITE-TIER";
|
||||
let backend = register_mock_tier(&ctx.tier_config_mgr(), tier).await;
|
||||
let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier)
|
||||
.await
|
||||
.expect("tier identity");
|
||||
let identity = rustfs_utils::crypto::hex(lease.backend_identity());
|
||||
drop(lease);
|
||||
|
||||
for state in [Exact, KnownDisabled, SuspendedNull] {
|
||||
for suspended in [false, true] {
|
||||
for self_copy in [false, true] {
|
||||
let bucket = format!("tier-overwrite-{}", Uuid::new_v4());
|
||||
let object = "object";
|
||||
let remote = format!("remote/{bucket}");
|
||||
let version = match state {
|
||||
Exact => "opaque-overwrite-version",
|
||||
SuspendedNull => "null",
|
||||
_ => "",
|
||||
};
|
||||
let payload = vec![0x5b; if suspended { 512 * 1024 } else { 257 }];
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
backend.set_put_remote_version(Some(version.to_string())).await;
|
||||
let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier)
|
||||
.await
|
||||
.expect("seed tier lease");
|
||||
lease
|
||||
.put(
|
||||
&remote,
|
||||
ReaderImpl::Body(bytes::Bytes::from(payload.clone())),
|
||||
payload.len().try_into().expect("payload size"),
|
||||
)
|
||||
.await
|
||||
.expect("seed remote bytes");
|
||||
drop(lease);
|
||||
let mut metadata = HashMap::from([
|
||||
("content-type".to_string(), "application/octet-stream".to_string()),
|
||||
(
|
||||
"x-amz-restore".to_string(),
|
||||
"ongoing-request=\"false\", expiry-date=\"2099-01-01T00:00:00Z\"".to_string(),
|
||||
),
|
||||
]);
|
||||
for (suffix, value) in [
|
||||
(SUFFIX_TRANSITION_STATUS, "complete"),
|
||||
(SUFFIX_TRANSITION_TIER, tier),
|
||||
(SUFFIX_TRANSITION_TIER_DESTINATION_ID, identity.as_str()),
|
||||
(SUFFIX_TRANSITIONED_OBJECTNAME, remote.as_str()),
|
||||
(SUFFIX_TRANSITIONED_VERSION_STATE, state.as_str()),
|
||||
] {
|
||||
insert_str(&mut metadata, suffix, value.to_string());
|
||||
}
|
||||
if !version.is_empty() {
|
||||
insert_str(&mut metadata, SUFFIX_TRANSITIONED_VERSION_ID, version.to_string());
|
||||
}
|
||||
let options = ObjectOptions {
|
||||
version_suspended: suspended,
|
||||
..Default::default()
|
||||
};
|
||||
store
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(payload.clone()),
|
||||
&ObjectOptions {
|
||||
user_defined: metadata,
|
||||
..options.clone()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed transitioned source with locally restored bytes");
|
||||
let expected = if self_copy {
|
||||
payload.clone()
|
||||
} else {
|
||||
vec![0x73; payload.len()]
|
||||
};
|
||||
let new_metadata = HashMap::from([
|
||||
("content-type".to_string(), "text/plain".to_string()),
|
||||
("x-amz-meta-replacement".to_string(), "kept".to_string()),
|
||||
]);
|
||||
if self_copy {
|
||||
let mut source = store
|
||||
.get_object_info(&bucket, object, &options)
|
||||
.await
|
||||
.expect("self-copy source");
|
||||
source.metadata_only = false;
|
||||
source.user_defined = Arc::new(new_metadata);
|
||||
source.put_object_reader = Some(PutObjReader::from_vec(expected.clone()));
|
||||
store
|
||||
.copy_object(&bucket, object, &bucket, object, &mut source, &options, &options)
|
||||
.await
|
||||
.expect("materialized self-copy");
|
||||
} else {
|
||||
store
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(expected.clone()),
|
||||
&ObjectOptions {
|
||||
user_defined: new_metadata,
|
||||
..options.clone()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("overwrite transitioned null version");
|
||||
}
|
||||
|
||||
let set = store.pools[0].get_disks_by_key(object);
|
||||
let versions = set
|
||||
.load_file_info_versions_exact(&bucket, object)
|
||||
.await
|
||||
.expect("read committed disk metadata")
|
||||
.expect("replacement metadata exists");
|
||||
let free: Vec<_> = versions
|
||||
.versions
|
||||
.iter()
|
||||
.chain(versions.free_versions.iter())
|
||||
.filter(|fi| fi.tier_free_version())
|
||||
.collect();
|
||||
assert_eq!(free.len(), 1, "{state:?}, suspended={suspended}, copy={self_copy}");
|
||||
assert_eq!(free[0].transitioned_objname, remote);
|
||||
assert_eq!(free[0].transition_version_state, state);
|
||||
assert!(backend.contains(&remote).await, "commit must not delete remote bytes before cleanup");
|
||||
let removed_before = backend.remove_count().await;
|
||||
|
||||
// Restart before queue delivery. The new runtime must
|
||||
// reconstruct ownership solely from the committed xl.meta.
|
||||
let tier_config = ctx
|
||||
.tier_config_mgr()
|
||||
.read()
|
||||
.await
|
||||
.tiers
|
||||
.get(tier)
|
||||
.expect("tier configuration survives restart")
|
||||
.clone_with_credentials();
|
||||
drop(set);
|
||||
shutdown.cancel();
|
||||
drop(store);
|
||||
drop(ctx);
|
||||
(ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-overwrite-restart", &[4]))
|
||||
.await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
{
|
||||
let manager = ctx.tier_config_mgr();
|
||||
let mut manager = manager.write().await;
|
||||
manager.tiers.insert(tier.to_string(), tier_config);
|
||||
manager
|
||||
.install_test_driver(tier, Box::new(backend.clone()))
|
||||
.expect("rebind the same remote destination after restart");
|
||||
}
|
||||
let set = store.pools[0].get_disks_by_key(object);
|
||||
ExpiryState::resize_workers(1, Arc::clone(&store)).await;
|
||||
let recovered = recover_tier_free_versions(Arc::clone(&store), 100, None, None)
|
||||
.await
|
||||
.expect("recover persisted cleanup owner");
|
||||
assert!(recovered.enqueued >= 1);
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
let versions = set
|
||||
.load_file_info_versions_exact(&bucket, object)
|
||||
.await
|
||||
.expect("read cleanup progress")
|
||||
.expect("new object must survive cleanup");
|
||||
if versions
|
||||
.versions
|
||||
.iter()
|
||||
.chain(versions.free_versions.iter())
|
||||
.all(|fi| !fi.tier_free_version())
|
||||
{
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("cleanup must converge");
|
||||
assert!(!backend.contains(&remote).await);
|
||||
assert_eq!(backend.remove_count().await, removed_before + 1, "one remote DELETE per owner");
|
||||
assert_eq!(backend.remove_versions().await.last(), Some(&(remote.clone(), version.to_string())));
|
||||
let mut reader = store
|
||||
.get_object_reader(&bucket, object, None, HeaderMap::new(), &options)
|
||||
.await
|
||||
.expect("replacement remains readable");
|
||||
let mut actual = Vec::new();
|
||||
reader.stream.read_to_end(&mut actual).await.expect("read replacement bytes");
|
||||
assert_eq!(actual, expected);
|
||||
let current = store
|
||||
.get_object_info(&bucket, object, &options)
|
||||
.await
|
||||
.expect("replacement metadata");
|
||||
assert_eq!(current.user_defined.get("content-type").map(String::as_str), Some("text/plain"));
|
||||
assert_eq!(current.user_defined.get("x-amz-meta-replacement").map(String::as_str), Some("kept"));
|
||||
assert!(current.transitioned_object.status.is_empty());
|
||||
}
|
||||
}
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
@@ -3707,6 +3925,31 @@ mod tests {
|
||||
OfflineTestDisks { disks }
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn force_set_disk_range_offline_for_test(
|
||||
set: &Arc<crate::set_disk::SetDisks>,
|
||||
range: std::ops::Range<usize>,
|
||||
) -> OfflineTestDisks {
|
||||
let disks = set
|
||||
.disks
|
||||
.read()
|
||||
.await
|
||||
.get(range)
|
||||
.expect("offline test range must fit the set")
|
||||
.iter()
|
||||
.map(|disk| disk.clone().expect("fault-injection disk should start online"))
|
||||
.collect::<Vec<_>>();
|
||||
for disk in &disks {
|
||||
disk.close().await.expect("fault injection should stop per-disk monitoring");
|
||||
disk.force_runtime_state_for_test(crate::disk::health_state::RuntimeDriveHealthState::Offline);
|
||||
}
|
||||
set.connect_disks().await;
|
||||
for disk in &disks {
|
||||
assert_eq!(disk.runtime_state(), crate::disk::health_state::RuntimeDriveHealthState::Offline);
|
||||
}
|
||||
OfflineTestDisks { disks }
|
||||
}
|
||||
|
||||
fn active_rebalance_meta_for_pool(pool_count: usize, active_pool_idx: usize) -> RebalanceMeta {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let mut pool_stats = vec![RebalanceStats::default(); pool_count];
|
||||
@@ -15661,25 +15904,24 @@ mod tests {
|
||||
assert!(deleted[0].found, "the aggregate error must retain the committed pool result");
|
||||
drop(injection);
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
let mut metadata_absent = true;
|
||||
for pool in &store.pools {
|
||||
metadata_absent &= pool
|
||||
.get_disks_by_key(object)
|
||||
.load_file_info_versions_exact(bucket, object)
|
||||
.await
|
||||
.expect("aggregate-error cleanup metadata should remain readable")
|
||||
.is_none();
|
||||
}
|
||||
if metadata_absent && backend.remove_count().await == 1 {
|
||||
return;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("aggregate failure must not suppress committed receipt dispatch");
|
||||
// Exact reads can see subquorum metadata while workers remove each
|
||||
// disk's free version. Inspect the final state after cleanup drains.
|
||||
wait_for_expiry_workers_idle(&store).await;
|
||||
for pool in &store.pools {
|
||||
assert!(
|
||||
pool.get_disks_by_key(object)
|
||||
.load_file_info_versions_exact(bucket, object)
|
||||
.await
|
||||
.expect("aggregate-error cleanup metadata should remain readable")
|
||||
.is_none(),
|
||||
"aggregate failure must not suppress committed receipt cleanup"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
backend.remove_count().await,
|
||||
1,
|
||||
"committed receipts must remove the shared remote object once"
|
||||
);
|
||||
assert_eq!(backend.object_count().await, 0, "the shared remote object should be removed exactly once");
|
||||
store
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
@@ -17323,6 +17565,55 @@ mod tests {
|
||||
assert_eq!(body, original_body);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn object_lock_snapshot_uses_read_quorum_bucket_existence_probe() {
|
||||
let temp = tempfile::tempdir().expect("create degraded snapshot store dir");
|
||||
let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
|
||||
temp.path(),
|
||||
"degraded-object-lock-snapshot",
|
||||
&[(2, 12)],
|
||||
CancellationToken::new(),
|
||||
None,
|
||||
))
|
||||
.await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
|
||||
let bucket = format!("degraded-ol-{}", uuid::Uuid::new_v4());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create snapshot bucket");
|
||||
let expected_incarnation = store
|
||||
.bucket_incarnation_id(&bucket)
|
||||
.await
|
||||
.expect("read bucket incarnation before degrading sets");
|
||||
|
||||
let mut offline_disks = Vec::new();
|
||||
for set in store.all_set_disks() {
|
||||
offline_disks.push(force_set_disk_range_offline_for_test(&set, 6..12).await);
|
||||
}
|
||||
|
||||
let snapshot = store
|
||||
.object_lock_config_snapshot(&bucket)
|
||||
.await
|
||||
.expect("read-quorum bucket existence should admit guarded Object Lock snapshot");
|
||||
assert!(matches!(
|
||||
snapshot.state(),
|
||||
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent
|
||||
));
|
||||
assert!(snapshot.is_valid_for_destructive_put(store.id, &bucket, expected_incarnation));
|
||||
|
||||
let current_incarnation = crate::bucket::metadata_sys::get_object_lock_config_and_incarnation_from_disk_in(&ctx, &bucket)
|
||||
.await
|
||||
.expect("authoritative metadata read should also survive at read quorum")
|
||||
.1;
|
||||
assert_eq!(current_incarnation, expected_incarnation);
|
||||
|
||||
drop(offline_disks);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn force_create_existing_bucket_preserves_incarnation_and_inflight_request() {
|
||||
@@ -19057,27 +19348,34 @@ mod tests {
|
||||
.await
|
||||
.expect("transition metadata should be readable");
|
||||
let mut metadata = FileMeta::load(&encoded).expect("transition metadata should decode");
|
||||
let mut transitioned = metadata
|
||||
.get_all_file_info_versions(bucket, object, true)
|
||||
.expect("transitioned versions should decode")
|
||||
.versions
|
||||
.into_iter()
|
||||
.find(|version| version.version_id == history.version_id)
|
||||
let (version_index, mut transitioned) = metadata
|
||||
.find_version(history.version_id)
|
||||
.expect("transitioned history should exist");
|
||||
transitioned.transition_version_state = rustfs_filemeta::TransitionVersionState::Unknown;
|
||||
rustfs_utils::http::metadata_compat::remove_str(
|
||||
&mut transitioned.metadata,
|
||||
// Rewrite the serialized record to model legacy metadata;
|
||||
// ordinary writes preserve an already reconciled state.
|
||||
rustfs_utils::http::metadata_compat::remove_bytes(
|
||||
&mut transitioned.object.as_mut().expect("history should be an object").meta_sys,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
);
|
||||
metadata
|
||||
.add_version(transitioned)
|
||||
.expect("unknown state should replace the transitioned version");
|
||||
metadata.versions[version_index] = rustfs_filemeta::FileMetaShallowVersion::try_from(transitioned)
|
||||
.expect("legacy history should re-encode");
|
||||
tokio::fs::write(
|
||||
&metadata_path,
|
||||
metadata.marshal_msg().expect("unknown transition metadata should encode"),
|
||||
)
|
||||
.await
|
||||
.expect("unknown transition metadata should be written");
|
||||
let encoded = tokio::fs::read(&metadata_path)
|
||||
.await
|
||||
.expect("legacy transition metadata should be readable");
|
||||
let legacy = FileMeta::load(&encoded)
|
||||
.expect("legacy transition metadata should decode")
|
||||
.find_version(history.version_id)
|
||||
.expect("legacy history should exist")
|
||||
.1
|
||||
.into_fileinfo(bucket, object, true)
|
||||
.expect("legacy history should decode");
|
||||
assert_eq!(legacy.transition_version_state, rustfs_filemeta::TransitionVersionState::Unknown);
|
||||
}
|
||||
let lifecycle_event = crate::bucket::lifecycle::lifecycle::Event {
|
||||
action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction,
|
||||
|
||||
@@ -50,7 +50,7 @@ use crate::services::notification_sys::{
|
||||
use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease, tier_destination_id_from_metadata};
|
||||
use crate::set_disk::{
|
||||
SetDisks, get_lock_acquire_timeout, get_object_lock_diag_slow_acquire_threshold, get_object_lock_diag_slow_hold_threshold,
|
||||
is_lock_optimization_enabled, is_object_lock_diag_enabled, same_distributed_lock_domain,
|
||||
is_lock_optimization_enabled, is_object_lock_diag_enabled,
|
||||
};
|
||||
use crate::storage_api_contracts::{
|
||||
list::ListOperations as _,
|
||||
@@ -3440,10 +3440,15 @@ impl ECStore {
|
||||
|
||||
for pool in &self.pools {
|
||||
let hashed_set = pool.get_disks_by_key(object);
|
||||
let lock_domain_already_held = !distributed
|
||||
|| locked_sets
|
||||
.iter()
|
||||
.any(|locked_set| same_distributed_lock_domain(&locked_set.lockers, &hashed_set.lockers));
|
||||
let mut lock_domain_already_held = !distributed;
|
||||
if !lock_domain_already_held {
|
||||
for locked_set in &locked_sets {
|
||||
if locked_set.shares_namespace_lock_domain(&hashed_set).await {
|
||||
lock_domain_already_held = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if lock_domain_already_held {
|
||||
continue;
|
||||
}
|
||||
@@ -3500,10 +3505,15 @@ impl ECStore {
|
||||
let mut locked_sets = vec![fixed_set];
|
||||
for pool in &self.pools {
|
||||
for set in &pool.disk_set {
|
||||
let lock_domain_already_held = !distributed
|
||||
|| locked_sets
|
||||
.iter()
|
||||
.any(|locked_set| same_distributed_lock_domain(&locked_set.lockers, &set.lockers));
|
||||
let mut lock_domain_already_held = !distributed;
|
||||
if !lock_domain_already_held {
|
||||
for locked_set in &locked_sets {
|
||||
if locked_set.shares_namespace_lock_domain(set).await {
|
||||
lock_domain_already_held = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if lock_domain_already_held {
|
||||
continue;
|
||||
}
|
||||
@@ -3581,10 +3591,15 @@ impl ECStore {
|
||||
let mut locked_sets = vec![fixed_set];
|
||||
for pool in &self.pools {
|
||||
for set in &pool.disk_set {
|
||||
let lock_domain_already_held = !distributed
|
||||
|| locked_sets
|
||||
.iter()
|
||||
.any(|locked_set| same_distributed_lock_domain(&locked_set.lockers, &set.lockers));
|
||||
let mut lock_domain_already_held = !distributed;
|
||||
if !lock_domain_already_held {
|
||||
for locked_set in &locked_sets {
|
||||
if locked_set.shares_namespace_lock_domain(set).await {
|
||||
lock_domain_already_held = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if lock_domain_already_held {
|
||||
continue;
|
||||
}
|
||||
@@ -3667,11 +3682,15 @@ impl ECStore {
|
||||
.get(pool_idx)
|
||||
.ok_or_else(|| Error::other(format!("invalid data movement publication pool {pool_idx}")))?;
|
||||
let set = pool.get_disks_by_key(object);
|
||||
let lock_domain_already_held = !locked_sets.is_empty()
|
||||
&& (!distributed
|
||||
|| locked_sets.iter().any(|locked_set: &Arc<crate::set_disk::SetDisks>| {
|
||||
same_distributed_lock_domain(&locked_set.lockers, &set.lockers)
|
||||
}));
|
||||
let mut lock_domain_already_held = !locked_sets.is_empty() && !distributed;
|
||||
if !lock_domain_already_held {
|
||||
for locked_set in &locked_sets {
|
||||
if locked_set.shares_namespace_lock_domain(&set).await {
|
||||
lock_domain_already_held = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if lock_domain_already_held {
|
||||
continue;
|
||||
}
|
||||
@@ -5717,6 +5736,7 @@ mod tests {
|
||||
GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, GetObjectBodySource, clear_get_object_body_cache_hook,
|
||||
lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
|
||||
};
|
||||
use crate::set_disk::same_distributed_lock_domain;
|
||||
use crate::set_disk::{SetDisks, disk_call_counters};
|
||||
use crate::storage_api_contracts::bucket::MakeBucketOptions;
|
||||
use crate::storage_api_contracts::lifecycle::TransitionedObject;
|
||||
|
||||
@@ -480,7 +480,110 @@ impl FileMeta {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn add_version(&mut self, mut fi: FileInfo) -> Result<()> {
|
||||
pub fn add_version(&mut self, fi: FileInfo) -> Result<()> {
|
||||
if let Some(free_version) = self.overwritten_tier_free_version(&fi)? {
|
||||
// The replacement and its cleanup owner must share one xl.meta
|
||||
// commit. Keep the original intact if either insertion fails.
|
||||
let mut next = self.clone();
|
||||
next.add_version_inner(fi)?;
|
||||
next.add_version_filemata(free_version)?;
|
||||
*self = next;
|
||||
return Ok(());
|
||||
}
|
||||
self.add_version_inner(fi)
|
||||
}
|
||||
|
||||
fn overwritten_tier_free_version(&self, fi: &FileInfo) -> Result<Option<FileMetaVersion>> {
|
||||
use rustfs_utils::http::{
|
||||
SUFFIX_TIER_FV_ID, SUFFIX_TRANSITION_STATUS, SUFFIX_TRANSITION_TIER, SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
SUFFIX_TRANSITIONED_OBJECTNAME, SUFFIX_TRANSITIONED_VERSION_ID, SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
get_consistent_bytes, get_consistent_str, has_internal_suffix, strip_internal_prefix_preserving_case,
|
||||
};
|
||||
|
||||
if fi.version_id.is_some_and(|id| !id.is_nil()) || !contains_key_str(&fi.metadata, SUFFIX_TIER_FV_ID) {
|
||||
return Ok(None);
|
||||
}
|
||||
let Some(existing) = self
|
||||
.versions
|
||||
.iter()
|
||||
.find(|v| v.header.version_id.is_none_or(|id| id.is_nil()))
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let old = existing.parse_version_meta()?;
|
||||
let Some(mut object) = old.object else {
|
||||
return Ok(None);
|
||||
};
|
||||
let status = get_consistent_bytes(&object.meta_sys, SUFFIX_TRANSITION_STATUS);
|
||||
if status.is_none()
|
||||
&& object
|
||||
.meta_sys
|
||||
.keys()
|
||||
.any(|key| has_internal_suffix(key, SUFFIX_TRANSITION_STATUS))
|
||||
{
|
||||
// Empty status is a valid local object. The reader distinguishes
|
||||
// it from conflicting aliases before the ordinary overwrite.
|
||||
object.into_fileinfo(&fi.volume, &fi.name, false)?;
|
||||
return Ok(None);
|
||||
}
|
||||
if status != Some(TRANSITION_COMPLETE.as_bytes()) {
|
||||
return Ok(None);
|
||||
}
|
||||
// Reuse the reader's alias/state validation. A legacy empty remote
|
||||
// version is valid and must not be mistaken for conflicting aliases.
|
||||
object.into_fileinfo(&fi.volume, &fi.name, false)?;
|
||||
if object
|
||||
.meta_sys
|
||||
.keys()
|
||||
.any(|key| has_internal_suffix(key, SUFFIX_TRANSITION_TIER_DESTINATION_ID))
|
||||
&& get_consistent_bytes(&object.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID).is_none()
|
||||
{
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
let transition_suffixes = [
|
||||
SUFFIX_TRANSITION_STATUS,
|
||||
SUFFIX_TRANSITION_TIER,
|
||||
SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
SUFFIX_TRANSITIONED_OBJECTNAME,
|
||||
SUFFIX_TRANSITIONED_VERSION_ID,
|
||||
SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
];
|
||||
let replacement = MetaObject::from(fi.clone());
|
||||
if transition_suffixes
|
||||
.iter()
|
||||
.all(|suffix| get_consistent_bytes(&object.meta_sys, suffix) == get_consistent_bytes(&replacement.meta_sys, suffix))
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
let id = get_consistent_str(&fi.metadata, SUFFIX_TIER_FV_ID).ok_or(Error::FileCorrupt)?;
|
||||
let id = Uuid::parse_str(id)?;
|
||||
if id.is_nil() || self.versions.iter().any(|version| version.header.version_id == Some(id)) {
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
// The reader also accepts legacy key casing. Canonicalize only this
|
||||
// cleanup source so init_free_version preserves every accepted field,
|
||||
// including an explicitly empty unversioned remote version.
|
||||
for suffix in transition_suffixes {
|
||||
let value = object
|
||||
.meta_sys
|
||||
.iter()
|
||||
.find(|(key, _)| {
|
||||
strip_internal_prefix_preserving_case(key).is_some_and(|found| found.eq_ignore_ascii_case(suffix))
|
||||
})
|
||||
.map(|(_, value)| value.clone());
|
||||
if let Some(value) = value {
|
||||
rustfs_utils::http::insert_bytes(&mut object.meta_sys, suffix, value);
|
||||
}
|
||||
}
|
||||
let (free_version, created) = object.init_free_version(fi)?;
|
||||
if !created {
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
Ok(Some(free_version))
|
||||
}
|
||||
|
||||
fn add_version_inner(&mut self, mut fi: FileInfo) -> Result<()> {
|
||||
rustfs_utils::http::remove_str(&mut fi.metadata, rustfs_utils::http::SUFFIX_TIER_FV_ID);
|
||||
// empty version_id means "null" (versioning disabled/suspended)
|
||||
if fi.version_id.is_none() {
|
||||
fi.version_id = Some(Uuid::nil());
|
||||
@@ -1463,6 +1566,235 @@ mod test {
|
||||
});
|
||||
}
|
||||
|
||||
fn tier_overwrite_fixture(state: crate::TransitionVersionState) -> (FileMeta, FileInfo) {
|
||||
let mut source = FileInfo::new("object", 2, 2);
|
||||
source.erasure.index = 1;
|
||||
source.mod_time = Some(OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixture timestamp"));
|
||||
source.data_dir = Some(Uuid::new_v4());
|
||||
source.transition_status = TRANSITION_COMPLETE.to_string();
|
||||
source.transition_tier = "WARM".to_string();
|
||||
source.transitioned_objname = "remote/old-object".to_string();
|
||||
source.transition_version_state = state;
|
||||
source.transition_version = match state {
|
||||
crate::TransitionVersionState::Exact => Some("opaque-provider-version".to_string()),
|
||||
crate::TransitionVersionState::SuspendedNull => Some("null".to_string()),
|
||||
_ => None,
|
||||
};
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut source.metadata,
|
||||
rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
"ab".repeat(32),
|
||||
);
|
||||
if state == crate::TransitionVersionState::KnownDisabled {
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut source.metadata,
|
||||
rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID,
|
||||
String::new(),
|
||||
);
|
||||
}
|
||||
let mut meta = FileMeta::new();
|
||||
meta.add_version(source.clone()).expect("seed transitioned null version");
|
||||
(meta, source)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_overwrite_preserves_exact_cleanup_owner_across_reload() {
|
||||
use crate::TransitionVersionState::{Exact, KnownDisabled, SuspendedNull, Unknown};
|
||||
use rustfs_utils::http::{MINIO_INTERNAL_PREFIX, RUSTFS_INTERNAL_PREFIX, SUFFIX_TIER_FV_ID};
|
||||
|
||||
for state in [Exact, KnownDisabled, SuspendedNull, Unknown] {
|
||||
for inline in [false, true] {
|
||||
let (mut meta, _) = tier_overwrite_fixture(state);
|
||||
let old = meta.versions[0].parse_version_meta().expect("old metadata");
|
||||
let old = old.object.expect("old object");
|
||||
let id = Uuid::new_v4();
|
||||
let mut replacement = FileInfo::new("object", 2, 2);
|
||||
replacement.version_id = inline.then_some(Uuid::nil());
|
||||
replacement.mod_time = Some(OffsetDateTime::from_unix_timestamp(1_700_000_001).expect("fixture timestamp"));
|
||||
replacement.data_dir = Some(Uuid::new_v4());
|
||||
replacement.size = 3;
|
||||
if inline {
|
||||
replacement.data = Some(Bytes::from_static(b"new"));
|
||||
}
|
||||
replacement.set_tier_free_version_id(&id.to_string());
|
||||
|
||||
meta.add_version(replacement.clone())
|
||||
.expect("replace transitioned null version");
|
||||
let bytes = meta.marshal_msg().expect("persist replacement and cleanup owner");
|
||||
let mut reopened = FileMeta::load(&bytes).expect("reopen committed metadata");
|
||||
assert_eq!(reopened.versions.len(), 2);
|
||||
let (_, free) = reopened.find_version(Some(id)).expect("durable cleanup owner");
|
||||
assert!(free.free_version());
|
||||
let marker = free.delete_marker.expect("cleanup marker");
|
||||
for suffix in [
|
||||
rustfs_utils::http::SUFFIX_TRANSITION_TIER,
|
||||
rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME,
|
||||
rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID,
|
||||
rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
] {
|
||||
for prefix in [RUSTFS_INTERNAL_PREFIX, MINIO_INTERNAL_PREFIX] {
|
||||
let key = format!("{prefix}{suffix}");
|
||||
assert_eq!(marker.meta_sys.get(&key), old.meta_sys.get(&key), "{state:?}: {key}");
|
||||
}
|
||||
}
|
||||
let (_, current) = reopened.find_version(None).expect("replacement survives restart");
|
||||
let current = current.object.expect("replacement object");
|
||||
assert_eq!(current.size, 3);
|
||||
assert!(!rustfs_utils::http::contains_key_bytes(¤t.meta_sys, SUFFIX_TIER_FV_ID));
|
||||
reopened
|
||||
.add_version(replacement)
|
||||
.expect("replaying replacement is idempotent");
|
||||
assert_eq!(reopened.versions.len(), 2);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_overwrite_rejects_cleanup_failure_without_mutating_source() {
|
||||
for id in ["not-a-uuid".to_string(), Uuid::nil().to_string()] {
|
||||
let (mut meta, _) = tier_overwrite_fixture(crate::TransitionVersionState::Exact);
|
||||
let before = meta.clone();
|
||||
let mut replacement = FileInfo::new("object", 2, 2);
|
||||
replacement.mod_time = Some(OffsetDateTime::now_utc());
|
||||
replacement.set_tier_free_version_id(&id);
|
||||
assert!(meta.add_version(replacement).is_err());
|
||||
assert_eq!(meta, before, "invalid cleanup identity must preserve source");
|
||||
}
|
||||
with_object_max_versions_for_test(1, || {
|
||||
let (mut meta, _) = tier_overwrite_fixture(crate::TransitionVersionState::KnownDisabled);
|
||||
let before = meta.clone();
|
||||
let mut replacement = FileInfo::new("object", 2, 2);
|
||||
replacement.mod_time = Some(OffsetDateTime::now_utc());
|
||||
replacement.data = Some(Bytes::from_static(b"new"));
|
||||
replacement.set_tier_free_version_id(&Uuid::new_v4().to_string());
|
||||
assert_eq!(
|
||||
meta.add_version(replacement)
|
||||
.expect_err("cleanup owner exceeds version limit"),
|
||||
Error::MaxVersionsExceeded
|
||||
);
|
||||
assert_eq!(meta, before, "failed cleanup insertion must also preserve inline bytes");
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_overwrite_allows_empty_transition_status_on_local_source() {
|
||||
let mut source = FileInfo::new("object", 2, 2);
|
||||
source.mod_time = Some(OffsetDateTime::now_utc());
|
||||
source.data = Some(Bytes::from_static(b"old"));
|
||||
rustfs_utils::http::insert_str(&mut source.metadata, rustfs_utils::http::SUFFIX_TRANSITION_STATUS, String::new());
|
||||
let mut meta = FileMeta::new();
|
||||
meta.add_version(source)
|
||||
.expect("seed readable local metadata with empty status");
|
||||
let mut replacement = FileInfo::new("object", 2, 2);
|
||||
replacement.mod_time = Some(OffsetDateTime::now_utc());
|
||||
replacement.size = 3;
|
||||
replacement.data = Some(Bytes::from_static(b"new"));
|
||||
replacement.set_tier_free_version_id(&Uuid::new_v4().to_string());
|
||||
meta.add_version(replacement)
|
||||
.expect("an ordinary overwrite must still succeed");
|
||||
assert_eq!(meta.versions.len(), 1);
|
||||
let (_, current) = meta.find_version(None).expect("replacement remains visible");
|
||||
assert_eq!(current.object.expect("ordinary object").size, 3);
|
||||
assert!(!meta.versions[0].header.free_version());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_overwrite_preserves_legacy_metadata_casing() {
|
||||
use rustfs_utils::http::{MINIO_INTERNAL_PREFIX, RUSTFS_INTERNAL_PREFIX};
|
||||
|
||||
for state in [
|
||||
crate::TransitionVersionState::Exact,
|
||||
crate::TransitionVersionState::KnownDisabled,
|
||||
] {
|
||||
let (mut meta, _) = tier_overwrite_fixture(state);
|
||||
let mut source = meta.versions[0].parse_version_meta().expect("seeded source");
|
||||
let object = source.object.as_mut().expect("transitioned source");
|
||||
let expected = object.meta_sys.clone();
|
||||
object.meta_sys = object
|
||||
.meta_sys
|
||||
.drain()
|
||||
.map(|(key, value)| (key.to_ascii_uppercase(), value))
|
||||
.collect();
|
||||
meta.versions[0] = FileMetaShallowVersion::try_from(source).expect("legacy key casing");
|
||||
let id = Uuid::new_v4();
|
||||
let mut replacement = FileInfo::new("object", 2, 2);
|
||||
replacement.mod_time = Some(OffsetDateTime::now_utc());
|
||||
replacement.set_tier_free_version_id(&id.to_string());
|
||||
meta.add_version(replacement).expect("overwrite readable legacy source");
|
||||
let reopened = FileMeta::load(&meta.marshal_msg().expect("persist overwrite")).expect("reopen overwrite");
|
||||
let (_, owner) = reopened
|
||||
.find_version(Some(id))
|
||||
.expect("legacy source must retain cleanup ownership");
|
||||
let marker = owner.delete_marker.expect("cleanup marker");
|
||||
for suffix in [
|
||||
rustfs_utils::http::SUFFIX_TRANSITION_TIER,
|
||||
rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME,
|
||||
rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID,
|
||||
rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
] {
|
||||
for prefix in [RUSTFS_INTERNAL_PREFIX, MINIO_INTERNAL_PREFIX] {
|
||||
let key = format!("{prefix}{suffix}");
|
||||
assert_eq!(marker.meta_sys.get(&key), expected.get(&key), "legacy {state:?}: {key}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_overwrite_rejects_conflicting_remote_metadata_aliases() {
|
||||
use rustfs_utils::http::{
|
||||
MINIO_INTERNAL_PREFIX, SUFFIX_TRANSITION_STATUS, SUFFIX_TRANSITION_TIER, SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
SUFFIX_TRANSITIONED_OBJECTNAME, SUFFIX_TRANSITIONED_VERSION_ID, SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
};
|
||||
for suffix in [
|
||||
SUFFIX_TRANSITION_STATUS,
|
||||
SUFFIX_TRANSITION_TIER,
|
||||
SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
SUFFIX_TRANSITIONED_OBJECTNAME,
|
||||
SUFFIX_TRANSITIONED_VERSION_ID,
|
||||
SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
] {
|
||||
let (mut meta, _) = tier_overwrite_fixture(crate::TransitionVersionState::Exact);
|
||||
let mut old = meta.versions[0].parse_version_meta().expect("seeded source metadata");
|
||||
old.object
|
||||
.as_mut()
|
||||
.expect("transitioned source")
|
||||
.meta_sys
|
||||
.insert(format!("{MINIO_INTERNAL_PREFIX}{suffix}"), b"conflicting-value".to_vec());
|
||||
meta.versions[0] = FileMetaShallowVersion::try_from(old).expect("encode conflicting aliases");
|
||||
let before = meta.clone();
|
||||
let mut replacement = FileInfo::new("object", 2, 2);
|
||||
replacement.mod_time = Some(OffsetDateTime::now_utc());
|
||||
replacement.set_tier_free_version_id(&Uuid::new_v4().to_string());
|
||||
assert_eq!(
|
||||
meta.add_version(replacement)
|
||||
.expect_err("ambiguous ownership must fail closed"),
|
||||
Error::FileCorrupt
|
||||
);
|
||||
assert_eq!(meta, before, "conflicting {suffix} must not erase the old remote tuple");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_overwrite_keeps_retained_remote_and_versioned_copy_ownership() {
|
||||
let (mut meta, mut source) = tier_overwrite_fixture(crate::TransitionVersionState::Exact);
|
||||
source.set_tier_free_version_id(&Uuid::new_v4().to_string());
|
||||
meta.add_version(source.clone())
|
||||
.expect("restore retains the same remote owner");
|
||||
assert_eq!(meta.versions.len(), 1);
|
||||
source.version_id = Some(Uuid::new_v4());
|
||||
source.transition_status.clear();
|
||||
source.transition_tier.clear();
|
||||
source.transitioned_objname.clear();
|
||||
source.transition_version = None;
|
||||
source.transition_version_state = crate::TransitionVersionState::Unknown;
|
||||
meta.add_version(source).expect("versioned write retains historical source");
|
||||
assert_eq!(meta.versions.len(), 2);
|
||||
assert!(meta.versions.iter().all(|version| !version.header.free_version()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn add_version_filemata_uses_canonical_equal_time_order() {
|
||||
let mod_time = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid test timestamp");
|
||||
|
||||
@@ -1321,6 +1321,38 @@ mod tests {
|
||||
.expect("token auth must map to a source");
|
||||
}
|
||||
|
||||
/// A pending acquisition cannot take the returned-error fallback: the
|
||||
/// outer login policy must cut it off before publishing a client.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_stalled_initial_login_is_bounded_by_the_attempt_timeout() {
|
||||
let state = Arc::new(ScriptedState::default());
|
||||
let source = ScriptedSource {
|
||||
state: state.clone(),
|
||||
ttl: Duration::ZERO,
|
||||
renewable: false,
|
||||
login_delay: Duration::from_secs(60),
|
||||
};
|
||||
let policy = test_policy(Duration::from_secs(10), Duration::from_secs(5));
|
||||
let attempt_timeout = policy.retry.attempt_timeout;
|
||||
let started = Instant::now();
|
||||
let result = VaultCredentialProvider::new(test_settings(), Box::new(source), policy).await;
|
||||
assert!(
|
||||
matches!(&result, Err(KmsError::OperationTimedOut { message }) if message.starts_with("vault_login attempt 1 timed out")),
|
||||
"a stalled login must return its typed timeout without publishing a client"
|
||||
);
|
||||
assert_eq!(
|
||||
started.elapsed(),
|
||||
attempt_timeout,
|
||||
"login must consume exactly one virtual attempt budget"
|
||||
);
|
||||
assert_eq!(state.login_calls.load(Ordering::SeqCst), 0, "the acquisition must not complete");
|
||||
assert_eq!(
|
||||
state.renew_calls.load(Ordering::SeqCst),
|
||||
0,
|
||||
"failed initialization must not start renewal"
|
||||
);
|
||||
}
|
||||
|
||||
/// backlog#2369 P3: `vault token create` defaults to a 768-hour TTL, so
|
||||
/// hard-coding "no lease" for token auth left the renewal task unstarted
|
||||
/// and turned a healthy cluster into one that answers 403 a month later.
|
||||
|
||||
@@ -14,8 +14,7 @@
|
||||
|
||||
//! Fault-injection matrix for the Vault backend operation policy.
|
||||
//!
|
||||
//! Offline cases run against locally injected transport faults (a listener
|
||||
//! that never responds) — deterministic, no external
|
||||
//! Offline cases run against locally injected HTTP and transport faults — no external
|
||||
//! dependencies. Real-Vault cases are `#[ignore]`d and need a dev Vault
|
||||
//! (default `http://127.0.0.1:8200`, override with `RUSTFS_KMS_VAULT_ADDR`).
|
||||
//!
|
||||
@@ -38,9 +37,100 @@ use rustfs_kms::backends::vault::VaultKmsBackend;
|
||||
use rustfs_kms::{
|
||||
BackendConfig, DescribeKeyRequest, KmsBackend as KmsBackendKind, KmsConfig, KmsError, VaultAuthMethod, VaultConfig,
|
||||
};
|
||||
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::task::JoinHandle;
|
||||
|
||||
const OPERATIONS_TOTAL: &str = "rustfs_kms_backend_operations_total";
|
||||
const ATTEMPT_FAILURES_TOTAL: &str = "rustfs_kms_backend_attempt_failures_total";
|
||||
const LOGIN: &str = "vault_login";
|
||||
const READ_KEY: &str = "vault_kv2_read_key";
|
||||
const LOOKUP_REQUEST: &str = "GET /v1/auth/token/lookup-self HTTP/1.1";
|
||||
|
||||
/// Unlike the unit-test scripted Vault, this fixture records the credential
|
||||
/// probe too and can fail it independently of the subsequent key request.
|
||||
/// `None` parks a connection without responding; extra requests receive 599.
|
||||
struct FaultVault {
|
||||
address: String,
|
||||
requests: mpsc::UnboundedReceiver<String>,
|
||||
task: JoinHandle<()>,
|
||||
}
|
||||
|
||||
impl FaultVault {
|
||||
async fn serve(responses: Vec<Option<(u16, serde_json::Value)>>) -> Self {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind fault-injection Vault");
|
||||
let address = format!("http://{}", listener.local_addr().expect("fault-injection Vault address"));
|
||||
let (recorded, requests) = mpsc::unbounded_channel();
|
||||
let task = tokio::spawn(async move {
|
||||
let mut responses = responses.into_iter();
|
||||
let mut parked = Vec::new();
|
||||
loop {
|
||||
let (stream, _) = listener.accept().await.expect("accept Vault request");
|
||||
let mut stream = BufReader::new(stream);
|
||||
let mut line = String::new();
|
||||
assert_ne!(stream.read_line(&mut line).await.expect("read request line"), 0);
|
||||
recorded.send(line.trim_end().to_string()).expect("record Vault request");
|
||||
loop {
|
||||
line.clear();
|
||||
assert_ne!(stream.read_line(&mut line).await.expect("read request header"), 0);
|
||||
if line == "\r\n" {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let mut stream = stream.into_inner();
|
||||
let response = responses
|
||||
.next()
|
||||
.unwrap_or_else(|| Some((599, serde_json::json!({"errors": ["unexpected Vault request"]}))));
|
||||
if let Some((status, body)) = response {
|
||||
let body = body.to_string();
|
||||
let response = format!(
|
||||
"HTTP/1.1 {status} Scripted\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
|
||||
body.len()
|
||||
);
|
||||
stream.write_all(response.as_bytes()).await.expect("write Vault response");
|
||||
stream.shutdown().await.expect("close Vault response");
|
||||
} else {
|
||||
parked.push(stream);
|
||||
}
|
||||
}
|
||||
});
|
||||
Self { address, requests, task }
|
||||
}
|
||||
|
||||
async fn finish(&mut self) {
|
||||
assert!(self.requests.try_recv().is_err(), "no unexpected requests may remain");
|
||||
self.task.abort();
|
||||
let error = (&mut self.task).await.expect_err("fault server runs until aborted");
|
||||
assert!(error.is_cancelled(), "fault server must not panic: {error}");
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for FaultVault {
|
||||
fn drop(&mut self) {
|
||||
self.task.abort();
|
||||
}
|
||||
}
|
||||
|
||||
fn healthy_token_lookup() -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"data": {
|
||||
"accessor": "fault-injection-accessor",
|
||||
"creation_time": 1_700_000_000u64,
|
||||
"creation_ttl": 0,
|
||||
"display_name": "token",
|
||||
"entity_id": "",
|
||||
"explicit_max_ttl": 0,
|
||||
"id": "unused",
|
||||
"num_uses": 0,
|
||||
"orphan": true,
|
||||
"path": "auth/token/create",
|
||||
"policies": ["default"],
|
||||
"renewable": false,
|
||||
"ttl": 0
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn vault_config(address: &str, token: &str) -> VaultConfig {
|
||||
VaultConfig {
|
||||
@@ -125,39 +215,112 @@ fn counter_value(snapshot: &[MetricEntry], name: &str, labels: &[(&str, &str)])
|
||||
fn stalled_connection_is_cut_off_by_the_attempt_timeout() {
|
||||
let snapshot = record_metrics(|| {
|
||||
Box::pin(async move {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
let mut vault = FaultVault::serve(vec![Some((200, healthy_token_lookup())), None]).await;
|
||||
let attempt_timeout = Duration::from_millis(250);
|
||||
let client = VaultKmsBackend::new(kms_config(vault_config(&vault.address, "unused"), attempt_timeout, 1))
|
||||
.await
|
||||
.expect("bind stall listener");
|
||||
let address = format!("http://{}", listener.local_addr().expect("stall listener addr"));
|
||||
// Accept and park every connection without ever responding.
|
||||
tokio::spawn(async move {
|
||||
let mut parked = Vec::new();
|
||||
loop {
|
||||
let Ok((socket, _)) = listener.accept().await else { return };
|
||||
parked.push(socket);
|
||||
}
|
||||
});
|
||||
.expect("the token lookup must succeed before injecting the stalled key read");
|
||||
assert_eq!(vault.requests.try_recv().as_deref(), Ok(LOOKUP_REQUEST));
|
||||
|
||||
let client = VaultKmsBackend::new(kms_config(vault_config(&address, "unused"), Duration::from_millis(250), 1))
|
||||
.await
|
||||
.expect("client construction performs no network calls");
|
||||
let error = KmsBackendTrait::describe_key(&client, describe_key_request("fault-injection-stalled"))
|
||||
let read = KmsBackendTrait::describe_key(&client, describe_key_request("fault-injection-stalled"));
|
||||
tokio::pin!(read);
|
||||
tokio::select! {
|
||||
request = vault.requests.recv() => assert_eq!(
|
||||
request.as_deref(),
|
||||
Some("GET /v1/secret/data/rustfs/kms/fault-injection/fault-injection-stalled? HTTP/1.1")
|
||||
),
|
||||
result = &mut read => panic!("the key request must reach the stall listener: {result:?}"),
|
||||
}
|
||||
// Pause only after real loopback I/O reaches the intended request;
|
||||
// otherwise auto-advancing time could expire the login instead.
|
||||
tokio::time::pause();
|
||||
let stalled_at = tokio::time::Instant::now();
|
||||
// Tokio rounds timer deadlines up to the next millisecond.
|
||||
let virtual_step = attempt_timeout + Duration::from_millis(1);
|
||||
tokio::time::advance(virtual_step).await;
|
||||
let error = tokio::time::timeout(Duration::from_secs(1), read)
|
||||
.await
|
||||
.expect("the attempt timer must resolve without further network activity")
|
||||
.expect_err("a stalled request must be cut off by the attempt timeout");
|
||||
assert_eq!(
|
||||
stalled_at.elapsed(),
|
||||
virtual_step,
|
||||
"the read must resolve within the attempt budget plus one timer tick"
|
||||
);
|
||||
assert!(
|
||||
matches!(error, KmsError::OperationTimedOut { .. } | KmsError::BackendError { .. }),
|
||||
"got {error:?}"
|
||||
);
|
||||
vault.finish().await;
|
||||
})
|
||||
});
|
||||
|
||||
// The policy timer reports attempt_timeout; the client-level HTTP timeout
|
||||
// surfaces as a connection-class failure. Either way it is exactly one
|
||||
// attempt that was cut off.
|
||||
let cut_off = counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("error_class", "attempt_timeout")])
|
||||
+ counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("error_class", "retryable_conn")]);
|
||||
let cut_off = counter_value(
|
||||
&snapshot,
|
||||
ATTEMPT_FAILURES_TOTAL,
|
||||
&[("operation", READ_KEY), ("error_class", "attempt_timeout")],
|
||||
) + counter_value(
|
||||
&snapshot,
|
||||
ATTEMPT_FAILURES_TOTAL,
|
||||
&[("operation", READ_KEY), ("error_class", "retryable_conn")],
|
||||
);
|
||||
assert_eq!(cut_off, 1, "the single budgeted attempt must be cut off by a timeout");
|
||||
assert_eq!(counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "budget_exhausted")]), 1);
|
||||
assert_eq!(counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("operation", LOGIN)]), 0);
|
||||
assert_eq!(
|
||||
counter_value(&snapshot, OPERATIONS_TOTAL, &[("operation", LOGIN), ("outcome", "success")]),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
counter_value(&snapshot, OPERATIONS_TOTAL, &[("operation", READ_KEY), ("outcome", "budget_exhausted")]),
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
/// A returned lookup error degrades lease discovery, not Vault authorization:
|
||||
/// the subsequent forbidden key read must still fail once, without retrying.
|
||||
#[test]
|
||||
fn token_lookup_errors_do_not_bypass_key_authorization() {
|
||||
for lookup_status in [403, 503] {
|
||||
let snapshot = record_metrics(|| {
|
||||
Box::pin(async move {
|
||||
let mut vault = FaultVault::serve(vec![
|
||||
Some((lookup_status, serde_json::json!({"errors": ["token lookup unavailable"]}))),
|
||||
Some((403, serde_json::json!({"errors": ["permission denied"]}))),
|
||||
])
|
||||
.await;
|
||||
let client = VaultKmsBackend::new(kms_config(vault_config(&vault.address, "unused"), Duration::from_secs(5), 3))
|
||||
.await
|
||||
.expect("a returned token lookup error must preserve static-token fallback");
|
||||
assert_eq!(vault.requests.try_recv().as_deref(), Ok(LOOKUP_REQUEST));
|
||||
let error = KmsBackendTrait::describe_key(&client, describe_key_request("fault-injection-forbidden"))
|
||||
.await
|
||||
.expect_err("lease discovery fallback must not authorize a forbidden key read");
|
||||
assert!(matches!(error, KmsError::BackendError { .. }), "got {error:?}");
|
||||
assert_eq!(
|
||||
vault.requests.try_recv().as_deref(),
|
||||
Ok("GET /v1/secret/data/rustfs/kms/fault-injection/fault-injection-forbidden? HTTP/1.1")
|
||||
);
|
||||
vault.finish().await;
|
||||
})
|
||||
});
|
||||
assert_eq!(counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("operation", LOGIN)]), 0);
|
||||
assert_eq!(
|
||||
counter_value(&snapshot, OPERATIONS_TOTAL, &[("operation", LOGIN), ("outcome", "success")]),
|
||||
1
|
||||
);
|
||||
assert_eq!(counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("operation", READ_KEY)]), 1);
|
||||
assert_eq!(
|
||||
counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("operation", READ_KEY), ("error_class", "fatal")]),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
counter_value(&snapshot, OPERATIONS_TOTAL, &[("operation", READ_KEY), ("outcome", "fatal")]),
|
||||
1
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn real_vault_address() -> String {
|
||||
@@ -174,7 +337,7 @@ fn real_vault_invalid_token_is_fatal_and_never_retried() {
|
||||
let config = vault_config(&real_vault_address(), "fault-injection-invalid-token");
|
||||
let client = VaultKmsBackend::new(kms_config(config, Duration::from_secs(5), 3))
|
||||
.await
|
||||
.expect("client construction performs no network calls");
|
||||
.expect("a returned token lookup error must preserve static-token fallback");
|
||||
let error = KmsBackendTrait::describe_key(&client, describe_key_request("fault-injection-forbidden"))
|
||||
.await
|
||||
.expect_err("an invalid token must be rejected");
|
||||
@@ -183,13 +346,20 @@ fn real_vault_invalid_token_is_fatal_and_never_retried() {
|
||||
});
|
||||
|
||||
assert_eq!(
|
||||
counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("error_class", "fatal")]),
|
||||
counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("operation", READ_KEY), ("error_class", "fatal")]),
|
||||
1,
|
||||
"a 403 must be observed by exactly one attempt"
|
||||
);
|
||||
assert_eq!(counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "fatal")]), 1);
|
||||
assert_eq!(
|
||||
counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("error_class", "retryable_status")]),
|
||||
counter_value(&snapshot, OPERATIONS_TOTAL, &[("operation", READ_KEY), ("outcome", "fatal")]),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
counter_value(
|
||||
&snapshot,
|
||||
ATTEMPT_FAILURES_TOTAL,
|
||||
&[("operation", READ_KEY), ("error_class", "retryable_status")]
|
||||
),
|
||||
0,
|
||||
"an auth failure must never be classified as retryable"
|
||||
);
|
||||
@@ -207,7 +377,7 @@ fn real_vault_missing_key_is_resolved_in_one_attempt() {
|
||||
let config = vault_config(&real_vault_address(), &token);
|
||||
let client = VaultKmsBackend::new(kms_config(config, Duration::from_secs(5), 3))
|
||||
.await
|
||||
.expect("client construction performs no network calls");
|
||||
.expect("static-token initialization must complete against the running Vault");
|
||||
let error = KmsBackendTrait::describe_key(&client, describe_key_request("fault-injection-definitely-missing"))
|
||||
.await
|
||||
.expect_err("a missing key must resolve to key-not-found");
|
||||
@@ -216,9 +386,12 @@ fn real_vault_missing_key_is_resolved_in_one_attempt() {
|
||||
});
|
||||
|
||||
assert_eq!(
|
||||
counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("error_class", "fatal")]),
|
||||
counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("operation", READ_KEY), ("error_class", "fatal")]),
|
||||
1,
|
||||
"a 404 must be observed by exactly one attempt"
|
||||
);
|
||||
assert_eq!(counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "fatal")]), 1);
|
||||
assert_eq!(
|
||||
counter_value(&snapshot, OPERATIONS_TOTAL, &[("operation", READ_KEY), ("outcome", "fatal")]),
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
@@ -96,14 +96,14 @@ pub use scanner::{
|
||||
scanner_topology_digest,
|
||||
};
|
||||
pub use scanner_io::{
|
||||
ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageClearObserver, ScannerDirtyUsageSnapshot,
|
||||
ScannerDirtyUsageState, ScannerDurableDirtyUsageReplayEntry, ScannerDurableDirtyUsageReplayError,
|
||||
ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageClearObserver, ScannerDirtyUsageMutationObserver,
|
||||
ScannerDirtyUsageSnapshot, ScannerDirtyUsageState, ScannerDurableDirtyUsageReplayEntry, ScannerDurableDirtyUsageReplayError,
|
||||
ScannerDurableDirtyUsageReplayRecord, ScannerDurableDirtyUsageReplayScope, acknowledge_dirty_usage_generation,
|
||||
acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, encode_durable_dirty_usage_producer_replay_record,
|
||||
record_dirty_usage_bucket, record_dirty_usage_bucket_from_producer, record_dirty_usage_bucket_from_producers,
|
||||
record_dirty_usage_object, record_dirty_usage_object_from_producer, record_scanner_maintenance_change,
|
||||
replay_durable_dirty_usage_producer_record, scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
|
||||
scanner_maintenance_generation, set_scanner_dirty_usage_clear_observer,
|
||||
scanner_maintenance_generation, set_scanner_dirty_usage_clear_observer, set_scanner_dirty_usage_mutation_observer,
|
||||
};
|
||||
pub use segment_invalidation::SegmentInvalidationProducerIdentity;
|
||||
pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER};
|
||||
|
||||
@@ -105,8 +105,14 @@ pub(super) fn remote_dirty_usage_acknowledgement_loss_reconciled(
|
||||
if !acknowledged_hosts.insert(acknowledgement.host.as_str()) {
|
||||
return false;
|
||||
}
|
||||
scanner_activity_dirty_usage_state_for_host(&activity_after_error, &acknowledgement.host)
|
||||
.is_some_and(|(instance_id, _generation, pending)| instance_id == acknowledgement.instance_id && !pending)
|
||||
let Some(expected_generation) = acknowledgement.expected_dirty_usage_generation() else {
|
||||
return false;
|
||||
};
|
||||
scanner_activity_dirty_usage_state_for_host(&activity_after_error, &acknowledgement.host).is_some_and(
|
||||
|(instance_id, generation, pending)| {
|
||||
instance_id == acknowledgement.instance_id && generation >= expected_generation && !pending
|
||||
},
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -509,6 +515,15 @@ pub(crate) enum ScannerDirtyUsageAcknowledgementKind {
|
||||
},
|
||||
}
|
||||
|
||||
impl ScannerDirtyUsageAcknowledgement {
|
||||
fn expected_dirty_usage_generation(&self) -> Option<u64> {
|
||||
match &self.kind {
|
||||
ScannerDirtyUsageAcknowledgementKind::Generation(generation) => Some(*generation),
|
||||
ScannerDirtyUsageAcknowledgementKind::Scoped { entries, .. } => entries.iter().map(|entry| entry.generation).max(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<ScannerDirtyUsageAcknowledgement> for crate::storage_api::EcstoreScannerDirtyUsageAcknowledgement {
|
||||
fn from(acknowledgement: ScannerDirtyUsageAcknowledgement) -> Self {
|
||||
match acknowledgement.kind {
|
||||
|
||||
@@ -1798,6 +1798,8 @@ pub(super) fn scanner_pause_backlog_now() -> u64 {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const NATIVE_RETIREMENT_DRIVES_PER_SET: usize = 2;
|
||||
|
||||
fn run_native_retirement_test<C, F>(case: C)
|
||||
where
|
||||
C: FnOnce() -> F + Send + 'static,
|
||||
@@ -1825,10 +1827,29 @@ mod tests {
|
||||
async fn native_retirement_store() -> (tempfile::TempDir, Arc<ECStore>) {
|
||||
register_scanner_pause_backlog_retirement();
|
||||
let root = tempfile::tempdir().expect("native retirement fixture directory");
|
||||
let store = super::super::tests::setup_scanner_cycle_store_at_path_with_sets(root.path(), false, 3, 2).await;
|
||||
let store = super::super::tests::setup_scanner_cycle_store_at_path_with_layout_and_disk_preinit(
|
||||
root.path(),
|
||||
false,
|
||||
3,
|
||||
2,
|
||||
NATIVE_RETIREMENT_DRIVES_PER_SET,
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
(root, store)
|
||||
}
|
||||
|
||||
async fn shutdown_native_retirement_store(store: Arc<ECStore>) {
|
||||
if let Some(token) = store.background_cancel_token() {
|
||||
token.cancel();
|
||||
}
|
||||
drop(store);
|
||||
for _ in 0..8 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
|
||||
async fn native_replica_bytes(set: &SetDisks) -> (Vec<u8>, String) {
|
||||
let mut reader = set
|
||||
.get_object_reader(
|
||||
@@ -1871,7 +1892,15 @@ mod tests {
|
||||
|
||||
register_scanner_pause_backlog_retirement();
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let old = super::super::tests::setup_scanner_cycle_store_at_path_with_sets(root.path(), false, old_pool_count, 2).await;
|
||||
let old = super::super::tests::setup_scanner_cycle_store_at_path_with_layout_and_disk_preinit(
|
||||
root.path(),
|
||||
false,
|
||||
old_pool_count,
|
||||
2,
|
||||
NATIVE_RETIREMENT_DRIVES_PER_SET,
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
let fault = unstable_source
|
||||
.then(|| NativeScannerPauseBacklogWriteFault::fail_before_write(Arc::clone(&old.pools[0].disk_set[0]), "publish", 2));
|
||||
let now = unix_now();
|
||||
@@ -1889,10 +1918,14 @@ mod tests {
|
||||
} else {
|
||||
controller.observe(observation(now + 1, true, 4)).await;
|
||||
controller.observe(observation(now + 2, false, 4)).await;
|
||||
assert!(matches!(
|
||||
controller.begin_attempt(now + 2).await,
|
||||
ScannerPauseBacklogAttemptDecision::Tracked(_)
|
||||
));
|
||||
let decision = controller.begin_attempt(now + 2).await;
|
||||
assert!(
|
||||
matches!(decision, ScannerPauseBacklogAttemptDecision::Tracked(_)),
|
||||
"expected tracked native expansion setup attempt, got {decision:?}; ledger={:?}; persistence_disabled={}; runtime_error={:?}",
|
||||
controller.loaded.ledger,
|
||||
controller.persistence_disabled,
|
||||
runtime_error()
|
||||
);
|
||||
assert!(controller.loaded.ledger.has_unfinished_attempt());
|
||||
}
|
||||
let original = controller.loaded.ledger.clone();
|
||||
@@ -1902,9 +1935,16 @@ mod tests {
|
||||
old.pool_meta_write_status()
|
||||
.await
|
||||
.expect("healthy pool metadata keeps the background recovery loop read-only");
|
||||
old.background_cancel_token().expect("old store shutdown token").cancel();
|
||||
drop(old);
|
||||
let expanded = super::super::tests::setup_scanner_cycle_store_at_path_with_sets(root.path(), false, 3, 2).await;
|
||||
shutdown_native_retirement_store(old).await;
|
||||
let expanded = super::super::tests::setup_scanner_cycle_store_at_path_with_layout_and_disk_preinit(
|
||||
root.path(),
|
||||
false,
|
||||
3,
|
||||
2,
|
||||
NATIVE_RETIREMENT_DRIVES_PER_SET,
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
for pool in expanded.pools.iter().skip(old_pool_count) {
|
||||
for set in &pool.disk_set {
|
||||
let replica = read_scanner_pause_backlog_replica(Arc::clone(set)).await;
|
||||
@@ -1999,12 +2039,16 @@ mod tests {
|
||||
}
|
||||
|
||||
store.pool_meta_write_status().await.expect("healthy metadata before restart");
|
||||
store
|
||||
.background_cancel_token()
|
||||
.expect("expanded store shutdown token")
|
||||
.cancel();
|
||||
drop(store);
|
||||
let restarted = super::super::tests::setup_scanner_cycle_store_at_path_with_sets(root.path(), false, 3, 2).await;
|
||||
shutdown_native_retirement_store(store).await;
|
||||
let restarted = super::super::tests::setup_scanner_cycle_store_at_path_with_layout_and_disk_preinit(
|
||||
root.path(),
|
||||
false,
|
||||
3,
|
||||
2,
|
||||
NATIVE_RETIREMENT_DRIVES_PER_SET,
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
let reloaded = load_scanner_pause_backlog(Arc::clone(&restarted))
|
||||
.await
|
||||
.expect("a new store must recover from disk without the failed controller");
|
||||
@@ -2021,6 +2065,8 @@ mod tests {
|
||||
assert_eq!(retry_ledger.current_attempt_serial, original.current_attempt_serial);
|
||||
assert_eq!(retry_ledger.last_finished_attempt_serial, original.current_attempt_serial);
|
||||
assert_eq!(retry_ledger.consecutive_failures, original.consecutive_failures + 1);
|
||||
drop(retried);
|
||||
shutdown_native_retirement_store(restarted).await;
|
||||
}
|
||||
|
||||
async fn assert_current_native_writer_ledger(store: &Arc<ECStore>, expected: &ScannerPauseBacklogLedger) {
|
||||
@@ -2124,6 +2170,8 @@ mod tests {
|
||||
.await
|
||||
.expect("retry must seed, commit and stabilize the stale member");
|
||||
assert_current_native_writer_ledger(&store, &expected).await;
|
||||
drop(target);
|
||||
shutdown_native_retirement_store(store).await;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -2175,6 +2223,8 @@ mod tests {
|
||||
.await
|
||||
.expect("a fresh caller may finish the seeded membership transition");
|
||||
assert_current_native_writer_ledger(&store, &expected).await;
|
||||
drop(pool_meta_lock);
|
||||
shutdown_native_retirement_store(store).await;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -2235,6 +2285,7 @@ mod tests {
|
||||
.await
|
||||
.expect("retry must seed the newly visible members before committing");
|
||||
assert_current_native_writer_ledger(&store, &expected).await;
|
||||
shutdown_native_retirement_store(store).await;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -2292,6 +2343,7 @@ mod tests {
|
||||
assert_current_native_ledger(&store, &original).await;
|
||||
}
|
||||
assert!(original.has_unfinished_attempt());
|
||||
shutdown_native_retirement_store(store).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -2337,14 +2389,22 @@ mod tests {
|
||||
.pool_meta_write_status()
|
||||
.await
|
||||
.expect("healthy pool metadata keeps the background recovery loop read-only");
|
||||
store.background_cancel_token().expect("old store shutdown token").cancel();
|
||||
drop(store);
|
||||
let restarted = super::super::tests::setup_scanner_cycle_store_at_path_with_sets(root.path(), false, 3, 2).await;
|
||||
shutdown_native_retirement_store(store).await;
|
||||
let restarted = super::super::tests::setup_scanner_cycle_store_at_path_with_layout_and_disk_preinit(
|
||||
root.path(),
|
||||
false,
|
||||
3,
|
||||
2,
|
||||
NATIVE_RETIREMENT_DRIVES_PER_SET,
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
for set_index in 0..2 {
|
||||
restarted.retire_scanner_pause_backlog_for_test(0, set_index).await.unwrap();
|
||||
assert_native_source_missing(&restarted, set_index).await;
|
||||
}
|
||||
assert_current_native_ledger(&restarted, &original).await;
|
||||
shutdown_native_retirement_store(restarted).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -2384,9 +2444,16 @@ mod tests {
|
||||
.pool_meta_write_status()
|
||||
.await
|
||||
.expect("healthy pool metadata keeps the background recovery loop read-only");
|
||||
store.background_cancel_token().expect("old store shutdown token").cancel();
|
||||
drop(store);
|
||||
let restarted = super::super::tests::setup_scanner_cycle_store_at_path_with_sets(root.path(), false, 3, 2).await;
|
||||
shutdown_native_retirement_store(store).await;
|
||||
let restarted = super::super::tests::setup_scanner_cycle_store_at_path_with_layout_and_disk_preinit(
|
||||
root.path(),
|
||||
false,
|
||||
3,
|
||||
2,
|
||||
NATIVE_RETIREMENT_DRIVES_PER_SET,
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
for set_index in 0..2 {
|
||||
restarted.retire_scanner_pause_backlog_for_test(0, set_index).await.unwrap();
|
||||
assert_native_source_missing(&restarted, set_index).await;
|
||||
@@ -2396,6 +2463,7 @@ mod tests {
|
||||
assert_eq!(bootstrapped.ledger.generation, 1);
|
||||
assert!(bootstrapped.ledger.last_updated_at_unix_secs < future_now);
|
||||
assert_current_native_ledger(&restarted, &bootstrapped.ledger).await;
|
||||
shutdown_native_retirement_store(restarted).await;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -2441,6 +2509,8 @@ mod tests {
|
||||
assert_current_native_ledger(&store, &original).await;
|
||||
store.retire_scanner_pause_backlog_for_test(0, 1).await.unwrap();
|
||||
assert_native_source_missing(&store, 1).await;
|
||||
drop(pool_meta_lock);
|
||||
shutdown_native_retirement_store(store).await;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -2536,6 +2606,9 @@ mod tests {
|
||||
"retirement never copied a stale source record"
|
||||
);
|
||||
}
|
||||
drop(pool_meta_lock);
|
||||
drop(barrier);
|
||||
shutdown_native_retirement_store(store).await;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -2669,11 +2742,12 @@ mod tests {
|
||||
.retire_scanner_pause_backlog_for_test(0, 1)
|
||||
.await
|
||||
.expect("the remaining source set follows the same native proof");
|
||||
let after = load_scanner_pause_backlog(store)
|
||||
let after = load_scanner_pause_backlog(Arc::clone(&store))
|
||||
.await
|
||||
.expect("native restart selection after handoff");
|
||||
assert_eq!(after.ledger, new_ledger);
|
||||
assert!(after.durable && after.stable_matches_ledger);
|
||||
shutdown_native_retirement_store(store).await;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -2745,6 +2819,7 @@ mod tests {
|
||||
.expect("retained target intent snapshot")
|
||||
};
|
||||
assert_eq!(after, before, "native cleanup never clears or estimates a target mutation intent");
|
||||
shutdown_native_retirement_store(store).await;
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -69,13 +69,42 @@ pub(super) async fn setup_scanner_cycle_store_at_path_with_sets(
|
||||
seed_usage_baseline: bool,
|
||||
pool_count: usize,
|
||||
sets_per_pool: usize,
|
||||
) -> Arc<ECStore> {
|
||||
setup_scanner_cycle_store_at_path_with_layout(root, seed_usage_baseline, pool_count, sets_per_pool, 4).await
|
||||
}
|
||||
|
||||
pub(super) async fn setup_scanner_cycle_store_at_path_with_layout(
|
||||
root: &Path,
|
||||
seed_usage_baseline: bool,
|
||||
pool_count: usize,
|
||||
sets_per_pool: usize,
|
||||
drives_per_set: usize,
|
||||
) -> Arc<ECStore> {
|
||||
setup_scanner_cycle_store_at_path_with_layout_and_disk_preinit(
|
||||
root,
|
||||
seed_usage_baseline,
|
||||
pool_count,
|
||||
sets_per_pool,
|
||||
drives_per_set,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn setup_scanner_cycle_store_at_path_with_layout_and_disk_preinit(
|
||||
root: &Path,
|
||||
seed_usage_baseline: bool,
|
||||
pool_count: usize,
|
||||
sets_per_pool: usize,
|
||||
drives_per_set: usize,
|
||||
preinitialize_disks: bool,
|
||||
) -> Arc<ECStore> {
|
||||
init_ecstore_config_for_scanner_tests();
|
||||
let mut pools = Vec::with_capacity(pool_count);
|
||||
for pool_index in 0..pool_count {
|
||||
let mut endpoints = Vec::new();
|
||||
for set_index in 0..sets_per_pool {
|
||||
for disk_index in 0..4 {
|
||||
for disk_index in 0..drives_per_set {
|
||||
let disk_path = if sets_per_pool == 1 {
|
||||
root.join(format!("pool{pool_index}/disk{disk_index}"))
|
||||
} else {
|
||||
@@ -95,7 +124,7 @@ pub(super) async fn setup_scanner_cycle_store_at_path_with_sets(
|
||||
pools.push(PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: sets_per_pool,
|
||||
drives_per_set: 4,
|
||||
drives_per_set,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: if pool_count == 1 && sets_per_pool == 1 {
|
||||
"scanner-cycle-metrics".to_string()
|
||||
@@ -108,9 +137,11 @@ pub(super) async fn setup_scanner_cycle_store_at_path_with_sets(
|
||||
let endpoint_pools = EndpointServerPools::from(pools);
|
||||
let instance_ctx = Arc::new(InstanceContext::new());
|
||||
instance_ctx.set_endpoints(endpoint_pools.clone());
|
||||
init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
|
||||
.await
|
||||
.expect("scanner cycle test disks should initialize");
|
||||
if preinitialize_disks {
|
||||
init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
|
||||
.await
|
||||
.expect("scanner cycle test disks should initialize");
|
||||
}
|
||||
let store = ECStore::new_with_instance_ctx(
|
||||
"127.0.0.1:0".parse().expect("test address should parse"),
|
||||
endpoint_pools,
|
||||
@@ -7649,6 +7680,25 @@ async fn scanner_cycle_confirms_lost_remote_ack_from_activity_snapshot() {
|
||||
"a new peer instance cannot confirm whether the old ACK reached durable dirty state"
|
||||
);
|
||||
|
||||
let mut stale_activity = scanner_node_activity("epoch-a", 7, 3);
|
||||
stale_activity.dirty_usage_generation = 4;
|
||||
let stale_clean_activity = BTreeMap::from([("node-2".to_string(), stale_activity)]);
|
||||
let stale_clean = remote_dirty_usage_acknowledgement_pending(
|
||||
8,
|
||||
1,
|
||||
std::slice::from_ref(&acknowledgement),
|
||||
std::future::ready(Err::<bool, _>(std::io::Error::other(
|
||||
"response lost before newer generation was observed",
|
||||
))),
|
||||
|| async { Ok(stale_clean_activity) },
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, stale_clean),
|
||||
ScannerCycleOutcome::CompletedWithPendingMaintenance,
|
||||
"a clean peer snapshot from before the acknowledged generation cannot prove the ACK reached durable dirty state"
|
||||
);
|
||||
|
||||
let mut written_activity = scanner_node_activity("epoch-a", 7, 3);
|
||||
written_activity.dirty_usage_generation = 6;
|
||||
written_activity.dirty_usage_pending = true;
|
||||
@@ -7722,6 +7772,47 @@ async fn scanner_cycle_confirms_lost_scoped_ack_only_after_same_instance_clean_a
|
||||
"a restarted peer cannot prove the scoped ACK reached the old scanner instance"
|
||||
);
|
||||
|
||||
let mut stale_activity = scanner_node_activity("epoch-a", 7, 3);
|
||||
stale_activity.dirty_usage_generation = 4;
|
||||
let stale_clean_activity = BTreeMap::from([("node-2".to_string(), stale_activity)]);
|
||||
let stale_clean = remote_dirty_usage_acknowledgement_pending(
|
||||
8,
|
||||
1,
|
||||
std::slice::from_ref(&acknowledgement),
|
||||
std::future::ready(Err::<bool, _>(std::io::Error::other(
|
||||
"scoped ACK transport failed before the requested generation was observed",
|
||||
))),
|
||||
|| async { Ok(stale_clean_activity) },
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, stale_clean),
|
||||
ScannerCycleOutcome::CompletedWithPendingMaintenance,
|
||||
"a clean peer snapshot from before the scoped ACK generation cannot prove the ACK reached durable dirty state"
|
||||
);
|
||||
|
||||
let empty_scoped_ack = ScannerDirtyUsageAcknowledgement {
|
||||
host: "node-2".to_string(),
|
||||
instance_id: "epoch-a".to_string(),
|
||||
kind: ScannerDirtyUsageAcknowledgementKind::Scoped {
|
||||
owner_id: Uuid::from_u128(0x11111111111111111111111111111111).to_string(),
|
||||
entries: Vec::new(),
|
||||
},
|
||||
};
|
||||
let empty_scoped_clean = remote_dirty_usage_acknowledgement_pending(
|
||||
8,
|
||||
1,
|
||||
&[empty_scoped_ack],
|
||||
std::future::ready(Err::<bool, _>(std::io::Error::other("empty scoped ACK failed before peer delivery"))),
|
||||
|| async { Ok(BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))])) },
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, empty_scoped_clean),
|
||||
ScannerCycleOutcome::CompletedWithPendingMaintenance,
|
||||
"an empty scoped ACK has no durable generation to reconcile after response loss"
|
||||
);
|
||||
|
||||
let mut written_activity = scanner_node_activity("epoch-a", 7, 3);
|
||||
written_activity.dirty_usage_generation = 6;
|
||||
written_activity.dirty_usage_pending = true;
|
||||
|
||||
@@ -432,7 +432,12 @@ async fn scoped_ack_publication_stale_baseline_cannot_prove_a_replaced_root() {
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scoped_ack_publication_rejects_builder_mutation_after_real_root_publish() {
|
||||
for mutation in ["remote_ack_target", "publication_epoch", "remote_lease_targets"] {
|
||||
for mutation in [
|
||||
"remote_ack_target",
|
||||
"remote_scoped_ack_target",
|
||||
"publication_epoch",
|
||||
"remote_lease_targets",
|
||||
] {
|
||||
let (_directory, store) = candidate_store().await;
|
||||
let (scan, candidate) = complete_candidate(&store, PROOF_CYCLE).await;
|
||||
let baseline = read_data_usage_persist_baseline(store.clone())
|
||||
@@ -465,6 +470,18 @@ async fn scoped_ack_publication_rejects_builder_mutation_after_real_root_publish
|
||||
instance_id: crate::scanner_activity_epoch().to_string(),
|
||||
kind: crate::scanner::ScannerDirtyUsageAcknowledgementKind::Generation(changed_generation),
|
||||
}]),
|
||||
"remote_scoped_ack_target" => scan.with_remote_dirty_usage_acknowledgements(vec![ScannerDirtyUsageAcknowledgement {
|
||||
host: "proof-peer:9000".to_string(),
|
||||
instance_id: crate::scanner_activity_epoch().to_string(),
|
||||
kind: crate::scanner::ScannerDirtyUsageAcknowledgementKind::Scoped {
|
||||
owner_id: crate::scanner_activity_epoch().to_string(),
|
||||
entries: vec![crate::storage_api::EcstoreScannerScopedDirtyUsageAckEntry {
|
||||
bucket: PROOF_BUCKET.to_string(),
|
||||
bucket_incarnation: uuid::Uuid::from_u128(0x11111111111111111111111111111111),
|
||||
generation: changed_generation,
|
||||
}],
|
||||
},
|
||||
}]),
|
||||
"publication_epoch" => scan.with_publication_epoch(Some(changed_epoch)),
|
||||
"remote_lease_targets" => scan.with_remote_publication_lease_targets(vec![(
|
||||
"proof-peer:9000".to_string(),
|
||||
|
||||
@@ -306,7 +306,7 @@ fn scanner_scoped_dirty_usage_ack_exceeds_cost_threshold(
|
||||
|
||||
fn resolve_remote_dirty_usage_scope(
|
||||
requested_scope: ScannerBucketScanScope,
|
||||
mut dirty_buckets: HashSet<String>,
|
||||
dirty_usage_snapshot: &DirtyUsageSnapshot,
|
||||
remote_dirty_usage: VerifiedRemoteDirtyUsage,
|
||||
all_buckets: &[BucketInfo],
|
||||
baseline_proof: ScannerCacheBaselineProof<'_>,
|
||||
@@ -319,12 +319,21 @@ fn resolve_remote_dirty_usage_scope(
|
||||
|
||||
let peer_count = remote_dirty_usage.peer_count;
|
||||
let dirty_peer_count = remote_dirty_usage.dirty_peer_count;
|
||||
let remote_dirty_buckets = remote_dirty_usage.dirty_buckets.clone();
|
||||
let mut dirty_buckets = dirty_usage_snapshot.buckets.keys().cloned().collect::<HashSet<_>>();
|
||||
dirty_buckets.extend(remote_dirty_usage.dirty_buckets);
|
||||
// Peer snapshots contribute bucket names only; the local prefix scopes
|
||||
// would narrow a bucket a peer dirtied elsewhere, so the merged scope
|
||||
// stays at bucket granularity (same rule as the local fallthrough).
|
||||
let scope =
|
||||
scoped_scan_scope_from_dirty_buckets(requested_scope, dirty_buckets, None, true, false, all_buckets, baseline_proof);
|
||||
let scope = scoped_scan_scope_from_dirty_buckets(
|
||||
requested_scope.clone(),
|
||||
dirty_buckets.clone(),
|
||||
None,
|
||||
true,
|
||||
false,
|
||||
all_buckets,
|
||||
baseline_proof,
|
||||
);
|
||||
if scope.is_default() {
|
||||
return default_result(scope);
|
||||
}
|
||||
@@ -358,19 +367,45 @@ fn resolve_remote_dirty_usage_scope(
|
||||
}
|
||||
let has_scoped_acknowledgements = !scoped_acknowledgements.is_empty();
|
||||
|
||||
let distributed_segment_invalidation_evidence =
|
||||
(dirty_peer_count > 0 && has_scoped_acknowledgements).then_some(DistributedSegmentInvalidationEvidence {
|
||||
invalidation_domain: crate::segment_invalidation::SegmentInvalidationDomain::DistributedEc,
|
||||
distributed_ec_invalidation: true,
|
||||
peer_count,
|
||||
dirty_peer_count,
|
||||
same_window_remote_proof: true,
|
||||
all_peers_bound_to_generation_window: true,
|
||||
});
|
||||
let segment_reuse_activation_preflight = scanner_segment_reuse_activation_preflight_for_baseline_with_evidence(
|
||||
dirty_usage_snapshot,
|
||||
true,
|
||||
baseline_proof,
|
||||
distributed_segment_invalidation_evidence,
|
||||
);
|
||||
let scope = if segment_reuse_activation_preflight.scanner_segment_reuse_activated {
|
||||
let local_only_scopes = dirty_usage_snapshot
|
||||
.scopes
|
||||
.iter()
|
||||
.filter(|(bucket, _)| !remote_dirty_buckets.contains(bucket.as_str()))
|
||||
.map(|(bucket, scope)| (bucket.clone(), scope.clone()))
|
||||
.collect::<DirtyUsageBucketScopes>();
|
||||
scoped_scan_scope_from_dirty_buckets(
|
||||
requested_scope,
|
||||
dirty_buckets,
|
||||
(!local_only_scopes.is_empty()).then_some(&local_only_scopes),
|
||||
true,
|
||||
true,
|
||||
all_buckets,
|
||||
baseline_proof,
|
||||
)
|
||||
} else {
|
||||
scope
|
||||
};
|
||||
|
||||
ScannerBucketScopeResolutionResult {
|
||||
scope,
|
||||
remote_dirty_usage_acknowledgements: scoped_acknowledgements,
|
||||
distributed_segment_invalidation_evidence: (dirty_peer_count > 0 && has_scoped_acknowledgements).then_some(
|
||||
DistributedSegmentInvalidationEvidence {
|
||||
invalidation_domain: crate::segment_invalidation::SegmentInvalidationDomain::DistributedEc,
|
||||
distributed_ec_invalidation: true,
|
||||
peer_count,
|
||||
dirty_peer_count,
|
||||
same_window_remote_proof: true,
|
||||
all_peers_bound_to_generation_window: true,
|
||||
},
|
||||
),
|
||||
distributed_segment_invalidation_evidence,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -596,7 +631,7 @@ fn scanner_segment_reuse_activation_preflight_for_cycle(
|
||||
production_activation: true,
|
||||
producer_identity_coverage_complete: dirty_usage_producer_evidence.producer_identity_coverage_complete,
|
||||
durable_producer_identity: dirty_usage_producer_evidence.durable_producer_identity,
|
||||
durable_dirty_producer_journal: dirty_usage_producer_evidence.durable_producer_identity,
|
||||
durable_dirty_producer_journal: dirty_usage_producer_evidence.durable_dirty_producer_journal,
|
||||
restart_gap_absent: dirty_usage_producer_evidence.restart_gap_absent,
|
||||
generation_window_bound: dirty_usage_snapshot.covers_all_pending
|
||||
&& dirty_usage_snapshot.generation != 0
|
||||
@@ -616,6 +651,15 @@ fn scanner_segment_reuse_activation_preflight_for_baseline(
|
||||
dirty_usage_snapshot: &DirtyUsageSnapshot,
|
||||
distributed: bool,
|
||||
baseline_proof: ScannerCacheBaselineProof<'_>,
|
||||
) -> ScannerSegmentReuseActivationPreflight {
|
||||
scanner_segment_reuse_activation_preflight_for_baseline_with_evidence(dirty_usage_snapshot, distributed, baseline_proof, None)
|
||||
}
|
||||
|
||||
fn scanner_segment_reuse_activation_preflight_for_baseline_with_evidence(
|
||||
dirty_usage_snapshot: &DirtyUsageSnapshot,
|
||||
distributed: bool,
|
||||
baseline_proof: ScannerCacheBaselineProof<'_>,
|
||||
distributed_segment_invalidation_evidence: Option<DistributedSegmentInvalidationEvidence>,
|
||||
) -> ScannerSegmentReuseActivationPreflight {
|
||||
let (dirty_usage_producer_evidence, cold_zero_walk_oracle) =
|
||||
scanner_segment_reuse_baseline_producer_evidence(dirty_usage_snapshot, baseline_proof);
|
||||
@@ -623,7 +667,7 @@ fn scanner_segment_reuse_activation_preflight_for_baseline(
|
||||
dirty_usage_snapshot,
|
||||
dirty_usage_producer_evidence,
|
||||
distributed,
|
||||
None,
|
||||
distributed_segment_invalidation_evidence,
|
||||
cold_zero_walk_oracle,
|
||||
)
|
||||
}
|
||||
@@ -1568,14 +1612,14 @@ pub(crate) use cache::{
|
||||
current_cache_root_or_prepare_with_generation,
|
||||
};
|
||||
pub use dirty_usage::{
|
||||
ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageClearObserver, ScannerDirtyUsageSnapshot,
|
||||
ScannerDirtyUsageState, ScannerDurableDirtyUsageReplayEntry, ScannerDurableDirtyUsageReplayError,
|
||||
ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageClearObserver, ScannerDirtyUsageMutationObserver,
|
||||
ScannerDirtyUsageSnapshot, ScannerDirtyUsageState, ScannerDurableDirtyUsageReplayEntry, ScannerDurableDirtyUsageReplayError,
|
||||
ScannerDurableDirtyUsageReplayRecord, ScannerDurableDirtyUsageReplayScope, acknowledge_dirty_usage_generation,
|
||||
acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, encode_durable_dirty_usage_producer_replay_record,
|
||||
record_dirty_usage_bucket, record_dirty_usage_bucket_from_producer, record_dirty_usage_bucket_from_producers,
|
||||
record_dirty_usage_object, record_dirty_usage_object_from_producer, record_scanner_maintenance_change,
|
||||
replay_durable_dirty_usage_producer_record, scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
|
||||
scanner_maintenance_generation, set_scanner_dirty_usage_clear_observer,
|
||||
scanner_maintenance_generation, set_scanner_dirty_usage_clear_observer, set_scanner_dirty_usage_mutation_observer,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use dirty_usage::{clear_dirty_usage_buckets_for_tests, dirty_usage_buckets_for_tests};
|
||||
|
||||
@@ -30,6 +30,8 @@ pub(super) static DIRTY_USAGE_PRODUCER_IDENTITIES: LazyLock<StdMutex<DirtyUsageP
|
||||
LazyLock::new(|| StdMutex::new(BTreeMap::new()));
|
||||
static DIRTY_USAGE_CLEAR_OBSERVER: LazyLock<StdRwLock<Option<ScannerDirtyUsageClearObserver>>> =
|
||||
LazyLock::new(|| StdRwLock::new(None));
|
||||
static DIRTY_USAGE_MUTATION_OBSERVER: LazyLock<StdRwLock<Option<ScannerDirtyUsageMutationObserver>>> =
|
||||
LazyLock::new(|| StdRwLock::new(None));
|
||||
pub(super) static DIRTY_USAGE_PRODUCER_COVERAGE: AtomicU64 = AtomicU64::new(0);
|
||||
pub(super) static DIRTY_USAGE_BUCKET_NOTIFY: LazyLock<Notify> = LazyLock::new(Notify::new);
|
||||
pub(super) static SCANNER_ACTIVITY_EPOCH: LazyLock<String> = LazyLock::new(|| format!("{:032x}", rand::random::<u128>()));
|
||||
@@ -53,6 +55,8 @@ pub struct ScannerDirtyUsageBucket {
|
||||
}
|
||||
|
||||
pub type ScannerDirtyUsageClearObserver = Arc<dyn Fn(Vec<ScannerDirtyUsageBucket>) + Send + Sync + 'static>;
|
||||
pub type ScannerDirtyUsageMutationObserver =
|
||||
Arc<dyn Fn(&str, &str, crate::segment_invalidation::SegmentInvalidationProducerIdentity) + Send + Sync + 'static>;
|
||||
|
||||
/// A non-durable optimization hint for a dirty bucket.
|
||||
///
|
||||
@@ -83,6 +87,7 @@ pub(super) type DirtyUsageProducerIdentities = BTreeMap<String, DirtyUsageProduc
|
||||
pub(super) struct DirtyUsageProducerEvidence {
|
||||
pub(super) producer_identity_coverage_complete: bool,
|
||||
pub(super) durable_producer_identity: bool,
|
||||
pub(super) durable_dirty_producer_journal: bool,
|
||||
pub(super) restart_gap_absent: bool,
|
||||
pub(super) generation_window_bound: bool,
|
||||
pub(super) generation_start: u64,
|
||||
@@ -112,6 +117,15 @@ pub fn set_scanner_dirty_usage_clear_observer(
|
||||
std::mem::replace(&mut *slot, observer)
|
||||
}
|
||||
|
||||
pub fn set_scanner_dirty_usage_mutation_observer(
|
||||
observer: Option<ScannerDirtyUsageMutationObserver>,
|
||||
) -> Option<ScannerDirtyUsageMutationObserver> {
|
||||
let mut slot = DIRTY_USAGE_MUTATION_OBSERVER
|
||||
.write()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
std::mem::replace(&mut *slot, observer)
|
||||
}
|
||||
|
||||
fn notify_dirty_usage_clear(cleared: Vec<ScannerDirtyUsageBucket>) {
|
||||
if cleared.is_empty() {
|
||||
return;
|
||||
@@ -125,6 +139,30 @@ fn notify_dirty_usage_clear(cleared: Vec<ScannerDirtyUsageBucket>) {
|
||||
}
|
||||
}
|
||||
|
||||
fn notify_dirty_usage_mutation(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
producers: &[crate::segment_invalidation::SegmentInvalidationProducerIdentity],
|
||||
) {
|
||||
if bucket.is_empty() {
|
||||
return;
|
||||
}
|
||||
let observer = DIRTY_USAGE_MUTATION_OBSERVER
|
||||
.read()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.clone();
|
||||
let Some(observer) = observer else {
|
||||
return;
|
||||
};
|
||||
if producers.is_empty() {
|
||||
observer(bucket, object, crate::segment_invalidation::SegmentInvalidationProducerIdentity::Unknown);
|
||||
} else {
|
||||
for producer in producers {
|
||||
observer(bucket, object, *producer);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A point-in-time view of the local dirty bucket generations.
|
||||
///
|
||||
/// `complete == false` is an all-or-nothing overflow signal: `buckets` is
|
||||
@@ -566,6 +604,7 @@ mod scoped_dirty_usage_tests {
|
||||
let evidence = dirty_usage_producer_evidence(&snapshot);
|
||||
assert!(evidence.producer_identity_coverage_complete);
|
||||
assert!(evidence.durable_producer_identity);
|
||||
assert!(evidence.durable_dirty_producer_journal);
|
||||
assert!(evidence.restart_gap_absent);
|
||||
assert_eq!(evidence.generation_start, 7);
|
||||
assert_eq!(evidence.generation_end, 7);
|
||||
@@ -583,10 +622,45 @@ mod scoped_dirty_usage_tests {
|
||||
);
|
||||
let changed_evidence = dirty_usage_producer_evidence(&changed);
|
||||
assert!(!changed_evidence.durable_producer_identity);
|
||||
assert!(!changed_evidence.durable_dirty_producer_journal);
|
||||
assert!(!changed_evidence.restart_gap_absent);
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn dirty_usage_mutation_observer_tracks_typed_and_conservative_events() {
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
let observed = Arc::new(Mutex::new(Vec::new()));
|
||||
let observed_clone = observed.clone();
|
||||
let previous = set_scanner_dirty_usage_mutation_observer(Some(Arc::new(move |bucket, object, producer| {
|
||||
observed_clone.lock().expect("observer lock should not be poisoned").push((
|
||||
bucket.to_string(),
|
||||
object.to_string(),
|
||||
producer,
|
||||
));
|
||||
})));
|
||||
|
||||
record_dirty_usage_object_from_producer("photos", "2026/image.jpg", SegmentInvalidationProducerIdentity::PutObject);
|
||||
record_dirty_usage_bucket("archive");
|
||||
set_scanner_dirty_usage_mutation_observer(previous);
|
||||
|
||||
assert_eq!(
|
||||
*observed.lock().expect("observer lock should not be poisoned"),
|
||||
vec![
|
||||
(
|
||||
"photos".to_string(),
|
||||
"2026/image.jpg".to_string(),
|
||||
SegmentInvalidationProducerIdentity::PutObject,
|
||||
),
|
||||
("archive".to_string(), String::new(), SegmentInvalidationProducerIdentity::Unknown,),
|
||||
]
|
||||
);
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn durable_dirty_usage_replay_rejects_invalid_records_without_partial_state() {
|
||||
@@ -745,6 +819,7 @@ fn record_dirty_usage_bucket_inner<I>(bucket: &str, producers: I)
|
||||
where
|
||||
I: IntoIterator<Item = crate::segment_invalidation::SegmentInvalidationProducerIdentity>,
|
||||
{
|
||||
let producers = producers.into_iter().collect::<Vec<_>>();
|
||||
let pending_buckets = {
|
||||
let mut dirty_buckets = dirty_usage_buckets();
|
||||
let mut dirty_scopes = dirty_usage_bucket_scopes();
|
||||
@@ -752,7 +827,12 @@ where
|
||||
let generation = advance_generation(&DIRTY_USAGE_BUCKET_GENERATION);
|
||||
dirty_buckets.insert(bucket.to_string(), generation);
|
||||
dirty_scopes.insert(bucket.to_string(), DirtyUsageBucketScope::WholeBucket);
|
||||
record_segment_invalidation_producer_identities_for_generation(&mut producer_identities, bucket, generation, producers);
|
||||
record_segment_invalidation_producer_identities_for_generation(
|
||||
&mut producer_identities,
|
||||
bucket,
|
||||
generation,
|
||||
producers.iter().copied(),
|
||||
);
|
||||
dirty_buckets.len()
|
||||
};
|
||||
global_metrics().record_scanner_dirty_usage_pending(usize_to_u64_saturated(pending_buckets));
|
||||
@@ -760,6 +840,7 @@ where
|
||||
// admin/console consumers never ride the full TTL after a change
|
||||
// (rustfs/backlog#1872).
|
||||
crate::prefix_usage::invalidate_prefix_usage_cache(bucket);
|
||||
notify_dirty_usage_mutation(bucket, "", &producers);
|
||||
DIRTY_USAGE_BUCKET_NOTIFY.notify_one();
|
||||
}
|
||||
|
||||
@@ -817,11 +898,17 @@ where
|
||||
if overflowed {
|
||||
*scope = DirtyUsageBucketScope::WholeBucket;
|
||||
}
|
||||
record_segment_invalidation_producer_identities_for_generation(&mut producer_identities, bucket, generation, producers);
|
||||
record_segment_invalidation_producer_identities_for_generation(
|
||||
&mut producer_identities,
|
||||
bucket,
|
||||
generation,
|
||||
producers.iter().copied(),
|
||||
);
|
||||
dirty_buckets.len()
|
||||
};
|
||||
global_metrics().record_scanner_dirty_usage_pending(usize_to_u64_saturated(pending_buckets));
|
||||
crate::prefix_usage::invalidate_prefix_usage_cache(bucket);
|
||||
notify_dirty_usage_mutation(bucket, object, &producers);
|
||||
DIRTY_USAGE_BUCKET_NOTIFY.notify_one();
|
||||
}
|
||||
|
||||
@@ -1193,7 +1280,7 @@ pub(super) fn dirty_usage_producer_evidence(snapshot: &DirtyUsageSnapshot) -> Di
|
||||
&& DIRTY_USAGE_PRODUCER_COVERAGE.load(Ordering::Acquire)
|
||||
& crate::segment_invalidation::SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION_COVERAGE_MASK
|
||||
== crate::segment_invalidation::SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION_COVERAGE_MASK;
|
||||
let durable_producer_identity = producer_identity_coverage_complete
|
||||
let durable_dirty_producer_journal = producer_identity_coverage_complete
|
||||
&& snapshot
|
||||
.buckets
|
||||
.keys()
|
||||
@@ -1205,8 +1292,9 @@ pub(super) fn dirty_usage_producer_evidence(snapshot: &DirtyUsageSnapshot) -> Di
|
||||
|
||||
DirtyUsageProducerEvidence {
|
||||
producer_identity_coverage_complete,
|
||||
durable_producer_identity,
|
||||
restart_gap_absent: durable_producer_identity,
|
||||
durable_producer_identity: durable_dirty_producer_journal,
|
||||
durable_dirty_producer_journal,
|
||||
restart_gap_absent: durable_dirty_producer_journal,
|
||||
generation_window_bound,
|
||||
generation_start: if generation_window_bound { generation_start } else { 0 },
|
||||
generation_end: if generation_window_bound { generation_end } else { 0 },
|
||||
|
||||
@@ -190,7 +190,7 @@ where
|
||||
};
|
||||
let remote_resolution = resolve_remote_dirty_usage_scope(
|
||||
resolution.requested_scope,
|
||||
dirty_buckets,
|
||||
resolution.dirty_usage_snapshot,
|
||||
remote_dirty_usage,
|
||||
resolution.all_buckets,
|
||||
resolution.baseline_proof,
|
||||
|
||||
@@ -37,7 +37,7 @@ use rustfs_concurrency::{
|
||||
};
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use serial_test::serial;
|
||||
use std::collections::BTreeMap;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::sync::Arc;
|
||||
use temp_env::with_var;
|
||||
use time::OffsetDateTime;
|
||||
@@ -285,6 +285,7 @@ fn scanner_durable_segment_invalidation_evidence_requires_matching_complete_set_
|
||||
|
||||
assert!(durable_evidence.producer_identity_coverage_complete);
|
||||
assert!(durable_evidence.durable_producer_identity);
|
||||
assert!(!durable_evidence.durable_dirty_producer_journal);
|
||||
assert!(durable_evidence.restart_gap_absent);
|
||||
|
||||
let mut stale_epoch = results.clone();
|
||||
@@ -297,12 +298,14 @@ fn scanner_durable_segment_invalidation_evidence_requires_matching_complete_set_
|
||||
let stale_evidence = scanner_durable_segment_invalidation_evidence(&dirty_usage_snapshot, &stale_epoch, &expected_sources);
|
||||
assert!(stale_evidence.producer_identity_coverage_complete);
|
||||
assert!(!stale_evidence.durable_producer_identity);
|
||||
assert!(!stale_evidence.durable_dirty_producer_journal);
|
||||
assert!(!stale_evidence.restart_gap_absent);
|
||||
|
||||
record_dirty_usage_bucket("videos");
|
||||
let changed_evidence = scanner_durable_segment_invalidation_evidence(&dirty_usage_snapshot, &results, &expected_sources);
|
||||
assert!(!changed_evidence.producer_identity_coverage_complete);
|
||||
assert!(!changed_evidence.durable_producer_identity);
|
||||
assert!(!changed_evidence.durable_dirty_producer_journal);
|
||||
assert!(!changed_evidence.restart_gap_absent);
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
@@ -316,6 +319,19 @@ fn scanner_segment_reuse_activation_replays_cold_durable_baseline() {
|
||||
for producer in SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION {
|
||||
record_dirty_usage_object_from_producer("photos", "2026/object", producer);
|
||||
}
|
||||
let replay_generation = dirty_usage_generation();
|
||||
replay_durable_dirty_usage_producer_record(
|
||||
&encode_durable_dirty_usage_producer_replay_record(vec![ScannerDurableDirtyUsageReplayEntry {
|
||||
bucket: "photos".to_string(),
|
||||
generation: replay_generation,
|
||||
scope: ScannerDurableDirtyUsageReplayScope::TopLevelEntries {
|
||||
entries: BTreeSet::from(["2026".to_string()]),
|
||||
},
|
||||
producers: SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION.into_iter().collect(),
|
||||
}])
|
||||
.expect("durable producer replay should encode"),
|
||||
)
|
||||
.expect("durable producer replay should restore restart authority");
|
||||
let dirty_usage_snapshot =
|
||||
snapshot_dirty_usage_buckets(&[bucket_info("photos"), bucket_info("archive")], dirty_usage_generation());
|
||||
let mut segment_proof = dirty_usage_producer_evidence(&dirty_usage_snapshot)
|
||||
@@ -413,8 +429,11 @@ fn scanner_segment_reuse_activation_replays_cold_durable_baseline() {
|
||||
scan_plan_digest,
|
||||
},
|
||||
);
|
||||
assert!(preflight.scanner_segment_reuse_activated);
|
||||
assert_eq!(preflight.fail_closed_blockers().collect::<Vec<_>>(), Vec::<&str>::new());
|
||||
assert!(!preflight.scanner_segment_reuse_activated);
|
||||
assert_eq!(
|
||||
preflight.fail_closed_blockers().collect::<Vec<_>>(),
|
||||
vec!["missing_durable_journal_replay"]
|
||||
);
|
||||
|
||||
record_dirty_usage_bucket("photos");
|
||||
let unidentified_snapshot =
|
||||
@@ -473,6 +492,7 @@ fn complete_process_local_producer_evidence() -> DirtyUsageProducerEvidence {
|
||||
DirtyUsageProducerEvidence {
|
||||
producer_identity_coverage_complete: true,
|
||||
durable_producer_identity: false,
|
||||
durable_dirty_producer_journal: false,
|
||||
restart_gap_absent: false,
|
||||
generation_window_bound: true,
|
||||
generation_start: 7,
|
||||
@@ -1468,6 +1488,7 @@ fn dirty_usage_producer_evidence_tracks_process_local_coverage_without_durable_r
|
||||
assert!(evidence.generation_window_bound);
|
||||
assert!(evidence.producer_identity_coverage_complete);
|
||||
assert!(!evidence.durable_producer_identity);
|
||||
assert!(!evidence.durable_dirty_producer_journal);
|
||||
assert!(!evidence.restart_gap_absent);
|
||||
assert_eq!(evidence.generation_start, snapshot.buckets["photos"]);
|
||||
assert_eq!(evidence.generation_end, snapshot.buckets["photos"]);
|
||||
@@ -1858,6 +1879,44 @@ fn complete_usage_baseline(
|
||||
bytes::Bytes::from(serde_json::to_vec(&baseline).expect("test baseline should encode"))
|
||||
}
|
||||
|
||||
fn complete_segment_reuse_baseline(
|
||||
source: DataUsageCacheSource,
|
||||
scan_plan_digest: DataUsageScanPlanDigest,
|
||||
scanner_cycle: u64,
|
||||
scanner_epoch: u64,
|
||||
evidence: DirtyUsageProducerEvidence,
|
||||
buckets: &[&str],
|
||||
) -> bytes::Bytes {
|
||||
let mut proof = evidence
|
||||
.segment_invalidation_proof()
|
||||
.expect("durable producer evidence should produce segment proof");
|
||||
proof.cold_zero_walk_oracle = true;
|
||||
let baseline = DataUsageInfo {
|
||||
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(10)),
|
||||
scanner_cycle: Some(scanner_cycle),
|
||||
scanner_epoch: Some(scanner_epoch),
|
||||
buckets_count: u64::try_from(buckets.len()).expect("test bucket count should fit"),
|
||||
buckets_usage: buckets
|
||||
.iter()
|
||||
.map(|bucket| ((*bucket).to_string(), Default::default()))
|
||||
.collect(),
|
||||
usage_snapshot_complete: true,
|
||||
usage_snapshot_converged: Some(true),
|
||||
usage_snapshot_set_states: vec![DataUsageSnapshotSetState {
|
||||
pool_index: u64::try_from(source.pool_index).expect("test pool index should fit"),
|
||||
set_index: u64::try_from(source.set_index).expect("test set index should fit"),
|
||||
scanner_cycle: Some(scanner_cycle),
|
||||
scanner_epoch: Some(scanner_epoch),
|
||||
scan_plan_digest: Some(scan_plan_digest.0),
|
||||
complete: true,
|
||||
tombstone: false,
|
||||
segment_invalidation_proof: Some(proof),
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
bytes::Bytes::from(serde_json::to_vec(&baseline).expect("test baseline should encode"))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scoped_scan_requires_a_converged_complete_baseline_with_exact_set_provenance() {
|
||||
let source = DataUsageCacheSource::new(1, 2);
|
||||
@@ -2183,6 +2242,12 @@ fn remote_dirty_usage_invalidates_local_prefix_hints_until_distributed_proof_exi
|
||||
"photos".to_string(),
|
||||
DirtyUsageBucketScope::TopLevelEntries(HashSet::from(["2026".to_string()])),
|
||||
)]);
|
||||
let dirty_usage_snapshot = DirtyUsageSnapshot {
|
||||
buckets: Arc::new(HashMap::from([("photos".to_string(), 7)])),
|
||||
scopes: Arc::new(dirty_scopes.clone()),
|
||||
generation: 7,
|
||||
covers_all_pending: true,
|
||||
};
|
||||
let locally_scoped = scoped_scan_scope_from_dirty_buckets(
|
||||
ScannerBucketScanScope::default(),
|
||||
HashSet::from(["photos".to_string()]),
|
||||
@@ -2206,7 +2271,7 @@ fn remote_dirty_usage_invalidates_local_prefix_hints_until_distributed_proof_exi
|
||||
|
||||
let distributed = resolve_remote_dirty_usage_scope(
|
||||
ScannerBucketScanScope::default(),
|
||||
HashSet::from(["photos".to_string()]),
|
||||
&dirty_usage_snapshot,
|
||||
remote_dirty_usage,
|
||||
&[bucket_info("photos")],
|
||||
ScannerCacheBaselineProof {
|
||||
@@ -2246,6 +2311,82 @@ fn remote_dirty_usage_invalidates_local_prefix_hints_until_distributed_proof_exi
|
||||
assert!(evidence.all_peers_bound_to_generation_window);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn distributed_segment_reuse_activation_keeps_remote_dirty_buckets_at_bucket_scope() {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
let source = DataUsageCacheSource::new(1, 2);
|
||||
let expected_sources = HashSet::from([source]);
|
||||
let scan_plan_digest = DataUsageScanPlanDigest([9; 32]);
|
||||
let entries = BTreeSet::from(["2026".to_string()]);
|
||||
let bytes = encode_durable_dirty_usage_producer_replay_record(vec![ScannerDurableDirtyUsageReplayEntry {
|
||||
bucket: "photos".to_string(),
|
||||
generation: 7,
|
||||
scope: ScannerDurableDirtyUsageReplayScope::TopLevelEntries { entries },
|
||||
producers: crate::segment_invalidation::SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION
|
||||
.iter()
|
||||
.copied()
|
||||
.collect(),
|
||||
}])
|
||||
.expect("durable dirty usage replay record should encode");
|
||||
replay_durable_dirty_usage_producer_record(&bytes).expect("durable dirty usage replay should restore producer proof");
|
||||
let dirty_usage_snapshot =
|
||||
snapshot_dirty_usage_buckets(&[bucket_info("photos"), bucket_info("archive")], dirty_usage_generation());
|
||||
let evidence = dirty_usage_producer_evidence(&dirty_usage_snapshot);
|
||||
let baseline = complete_segment_reuse_baseline(source, scan_plan_digest, 7, 11, evidence, &["photos", "archive"]);
|
||||
let expected_peers = HashMap::from([(
|
||||
"node-a:9000".to_string(),
|
||||
ScannerPeerDirtyUsageExpectation {
|
||||
instance_id: "instance-a".to_string(),
|
||||
generation: 3,
|
||||
pending: true,
|
||||
},
|
||||
)]);
|
||||
let remote_dirty_usage = verified_remote_dirty_usage(
|
||||
&expected_peers,
|
||||
vec![(
|
||||
"node-a:9000".to_string(),
|
||||
peer_dirty_usage_snapshot("instance-a", 3, true, &[("archive", 3)]),
|
||||
)],
|
||||
)
|
||||
.expect("fixture remote dirty usage should verify at bucket granularity");
|
||||
|
||||
let result = resolve_remote_dirty_usage_scope(
|
||||
ScannerBucketScanScope::default(),
|
||||
&dirty_usage_snapshot,
|
||||
remote_dirty_usage,
|
||||
&[bucket_info("photos"), bucket_info("archive")],
|
||||
ScannerCacheBaselineProof {
|
||||
authoritative_data: Some(&baseline),
|
||||
observed_candidate_data: None,
|
||||
expected_sources: &expected_sources,
|
||||
leader_epoch: 11,
|
||||
want_cycle: 8,
|
||||
scan_plan_digest,
|
||||
},
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
result
|
||||
.scope
|
||||
.selected_buckets
|
||||
.as_deref()
|
||||
.expect("distributed reuse still selects both dirty buckets"),
|
||||
&HashSet::from(["photos".to_string(), "archive".to_string()])
|
||||
);
|
||||
assert!(
|
||||
result.scope.prefix_scope_for("photos").is_some(),
|
||||
"durable local producer proof may activate local segment reuse after distributed ACK capability evidence"
|
||||
);
|
||||
assert!(
|
||||
result.scope.prefix_scope_for("archive").is_none(),
|
||||
"remote dirty usage has only bucket-granularity evidence and must not be narrowed to a local prefix"
|
||||
);
|
||||
assert_eq!(result.remote_dirty_usage_acknowledgements.len(), 1);
|
||||
assert!(result.distributed_segment_invalidation_evidence.is_some());
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
fn peer_dirty_usage_snapshot(
|
||||
instance_id: &str,
|
||||
generation: u64,
|
||||
@@ -2419,7 +2560,12 @@ fn remote_dirty_usage_scope_resolution_falls_back_when_ack_batch_exceeds_thresho
|
||||
|
||||
let result = resolve_remote_dirty_usage_scope(
|
||||
ScannerBucketScanScope::default(),
|
||||
HashSet::new(),
|
||||
&DirtyUsageSnapshot {
|
||||
buckets: Arc::new(HashMap::new()),
|
||||
scopes: Arc::new(HashMap::new()),
|
||||
generation: 7,
|
||||
covers_all_pending: true,
|
||||
},
|
||||
remote_dirty_usage,
|
||||
&all_buckets,
|
||||
ScannerCacheBaselineProof {
|
||||
|
||||
@@ -226,6 +226,34 @@ fn record_segment_dirty_usage(bucket: &str) {
|
||||
}
|
||||
}
|
||||
|
||||
fn replay_segment_dirty_usage(bucket: &str) {
|
||||
replay_dirty_usage(
|
||||
bucket,
|
||||
ScannerDurableDirtyUsageReplayScope::TopLevelEntries {
|
||||
entries: BTreeSet::from(["hot-segment".to_string()]),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
fn replay_whole_bucket_dirty_usage(bucket: &str) {
|
||||
replay_dirty_usage(bucket, ScannerDurableDirtyUsageReplayScope::WholeBucket);
|
||||
}
|
||||
|
||||
fn replay_dirty_usage(bucket: &str, scope: ScannerDurableDirtyUsageReplayScope) {
|
||||
replay_durable_dirty_usage_producer_record(
|
||||
&encode_durable_dirty_usage_producer_replay_record(vec![ScannerDurableDirtyUsageReplayEntry {
|
||||
bucket: bucket.to_string(),
|
||||
generation: dirty_usage_generation(),
|
||||
scope,
|
||||
producers: crate::segment_invalidation::SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION
|
||||
.into_iter()
|
||||
.collect(),
|
||||
}])
|
||||
.expect("durable segment replay should encode"),
|
||||
)
|
||||
.expect("durable segment replay should restore producer authority");
|
||||
}
|
||||
|
||||
// The scoped fallback fixture keeps two EC pools and several scan futures live
|
||||
// at once. Run the async cases on a dedicated stack so Linux libtest defaults
|
||||
// exercise the assertions instead of aborting before the oracle finishes.
|
||||
@@ -267,6 +295,7 @@ async fn scoped_entry_fallback_distinguishes_planned_scope_from_real_cold_walks_
|
||||
create_bucket(&store, &hot).await;
|
||||
create_bucket(&store, &cold).await;
|
||||
record_segment_dirty_usage(&hot);
|
||||
replay_segment_dirty_usage(&hot);
|
||||
let baseline = run_entry(&store, 1, None, true, false, false).await;
|
||||
persist_baseline(&store, &baseline).await;
|
||||
|
||||
@@ -283,6 +312,7 @@ async fn scoped_entry_fallback_distinguishes_planned_scope_from_real_cold_walks_
|
||||
"hot-segment/object",
|
||||
crate::segment_invalidation::SegmentInvalidationProducerIdentity::PutObject,
|
||||
);
|
||||
replay_segment_dirty_usage(&hot);
|
||||
let usage = run_entry(&store, 3, Some(&hot), true, true, true).await;
|
||||
assert_eq!(usage.buckets_usage[&hot].objects_count, 2);
|
||||
assert_eq!(usage.buckets_usage[&cold].objects_count, 1);
|
||||
@@ -298,6 +328,7 @@ async fn scoped_entry_fallback_distinguishes_planned_scope_from_real_cold_walks_
|
||||
crate::segment_invalidation::SegmentInvalidationProducerIdentity::PutObject,
|
||||
);
|
||||
}
|
||||
replay_whole_bucket_dirty_usage(&hot);
|
||||
let usage = run_entry(&store, 4, Some(&hot), true, true, false).await;
|
||||
assert_eq!(usage.objects_total_count, 3);
|
||||
|
||||
|
||||
@@ -1691,7 +1691,7 @@ mod serial_tests {
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "FAILING on main: excluded from the serial ILM lane pending a fix, see rustfs/backlog#1148 (ilm-1 partial)"]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn test_noncurrent_expiry_still_works_after_immediate_compensation_transition() {
|
||||
let (disk_paths, ecstore) = setup_isolated_test_env(true).await;
|
||||
|
||||
@@ -1775,7 +1775,7 @@ mod serial_tests {
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "FAILING on main: excluded from the serial ILM lane pending a fix, see rustfs/backlog#1148 (ilm-1 partial)"]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn test_noncurrent_transition_still_works_after_immediate_compensation_transition() {
|
||||
let (disk_paths, ecstore) = setup_isolated_test_env(true).await;
|
||||
|
||||
|
||||
@@ -30,11 +30,23 @@ These are approved-target invariants. A protocol's explicitly labeled current ex
|
||||
| Remote PUT is in flight or its response is unknown | Transition transaction | Only cleanup of its own canonical candidate, subject to the transaction recovery predicate | Durable transaction identity plus a known remote-version state; the approved target also requires expiry and durable takeover of the creator fence |
|
||||
| Local transition commit is complete | Exact transitioned version in `xl.meta` | No | Recovery finds the transaction's logical bucket/object/version and requires the complete recorded source identity (version ID, data directory, modification time, size, and ETag), `TRANSITION_COMPLETE`, and the same remote object, tier, and remote version before removing only the transaction record |
|
||||
| An ordinary delete removes that transitioned version | Hidden `xl.meta` free-version | Yes | Metadata quorum atomically removes the visible version and preserves its exact tier tuple in the free-version |
|
||||
| PUT or materialized self-copy replaces a transitioned null version | Hidden `xl.meta` free-version | Yes, after replacement commit and complete physical reference checks | The coordinator supplies one cleanup UUID to every disk; replacement metadata and the old remote tuple are written in the same `xl.meta` commit |
|
||||
| A recursive prefix/delete-all operation cannot preserve per-object markers | v6 journal bound to an immutable single dispatch manifest or a chunk-parent-bound child manifest | Yes, but only after child/manifest completion and all-pool absence proof | `DispatchAuthorized`, exact local destructive mutation, every journal `Committed`, then child/manifest `Completed`; a chunk parent advances only after that child completion |
|
||||
| Tier configuration mutation, manual job, or decommission receipt | Intent/admission/copy proof only | No | These records gate configuration, scheduling, or migration; they never become remote-object cleanup owners |
|
||||
|
||||
An old journal and a free-version can coexist during compatibility recovery. That coexistence is evidence of multiple possible owners, not permission to choose one: the journal path must retain its record until the version-specific recovery rule proves which owner is authoritative.
|
||||
|
||||
Null-version replacement uses the existing free-version format and recovery
|
||||
worker. Failed metadata preparation preserves both the old version and its
|
||||
inline bytes; rename rollback restores the complete previous metadata. Recovery
|
||||
must retain cleanup while any physical replica still references the remote tuple,
|
||||
including a minority version omitted by quorum merging, or any disk cannot be
|
||||
checked. A successful replacement needs no in-memory queue receipt to survive
|
||||
restart: the normal free-version sweep discovers its committed owner. Restores
|
||||
that retain the same remote tuple and ordinary versioned writes retain their
|
||||
existing ownership. Older binaries can read this format, but all writers and
|
||||
cleanup workers need the overwrite fix before these guarantees cover the fleet.
|
||||
|
||||
## Persisted record inventory
|
||||
|
||||
All keys below are objects in the internal metadata bucket. The table gives the canonical target form. Transition-transaction runtime recovery extracts the final 32 hexadecimal characters and UUID while ignoring shard directories and accepting uppercase hex. Manual-job runtime recovery requires exactly two shards matching the filename prefix, but accepts an uppercase UUID when the shards use the same uppercase text; it then loads the lowercase canonical job by UUID. The decommission validator recomputes and rejects a noncanonical manual-job path, but currently inherits the weaker transition parser. Exact runtime canonical-path validation for both protocols is an approved target.
|
||||
|
||||
@@ -364,6 +364,29 @@ scripts/python_bin.sh scripts/check_test_wiring.py \
|
||||
--check-scanner-heal-release-bundle /path/to/release-evidence.json
|
||||
```
|
||||
|
||||
For a single Linux handoff checklist that keeps the measured runners in a
|
||||
stable order, generate the Scanner/Heal Linux evidence plan:
|
||||
|
||||
```bash
|
||||
scripts/python_bin.sh scripts/run_scanner_heal_linux_evidence_plan.py \
|
||||
--write-plan --out-dir /path/to/plan-dir
|
||||
```
|
||||
|
||||
The plan is only an execution manifest. Its `evidence_type` is `plan_only`, and
|
||||
it cannot satisfy any Gxx/Wxx/Rxx gate. Use `--run-preflight` only for the
|
||||
lightweight registry and runner self-tests before starting a long Linux run.
|
||||
After or during a Linux run, check which planned artifacts are still missing
|
||||
without approving the release bundle:
|
||||
|
||||
```bash
|
||||
scripts/python_bin.sh scripts/run_scanner_heal_linux_evidence_plan.py \
|
||||
--status-root /path/to/run-root --format json
|
||||
```
|
||||
|
||||
The status command exits nonzero while evidence is missing or malformed and
|
||||
keeps `release_approved: false`; use its `pending_gates` and `next_step` fields
|
||||
for issue writeback and failure triage.
|
||||
|
||||
Lane descriptors can be assembled into that bundle with:
|
||||
|
||||
```bash
|
||||
@@ -416,12 +439,14 @@ sample, or save-frequency cost counters. Missing, synthetic, stale, tampered,
|
||||
undersized, cross-run, or topology-mismatched evidence returns a compact blocked
|
||||
or invalid JSON result and a nonzero exit.
|
||||
|
||||
The status-and-outcome descriptor producer consumes three measured raw JSON
|
||||
artifacts for G05, G06, and R-D. Those inputs must all carry schema 1, measured
|
||||
evidence, matching `source_revision`, a shared `run_id`, a shared
|
||||
The status-and-outcome raw collector normalizes live observations into the three
|
||||
measured raw JSON artifacts for G05, G06, and R-D. The descriptor producer then
|
||||
consumes those artifacts. The inputs must all carry schema 1, measured evidence,
|
||||
matching `source_revision`, a shared `run_id`, a shared
|
||||
`measurement_window_id`, matching `started_at`/`finished_at` timestamps, and
|
||||
non-empty command provenance. The producer rejects command-line run/window/time
|
||||
overrides that would relabel raw artifacts from another status-and-outcome run.
|
||||
non-empty command provenance. The collector and producer reject synthetic input,
|
||||
missing required cases, and command-line run/window/time overrides that would
|
||||
relabel raw artifacts from another status-and-outcome run.
|
||||
|
||||
The scheduler-pressure lane must also carry the numbers needed to close W09,
|
||||
W10, and W11: bounded deferred item/byte/age limits, zero duplicate tasks,
|
||||
|
||||
@@ -28,6 +28,7 @@ use futures_util::future::join_all;
|
||||
use http::{HeaderMap, HeaderValue, Uri};
|
||||
use hyper::{Method, StatusCode};
|
||||
use matchit::Params;
|
||||
use percent_encoding::percent_decode_str;
|
||||
use rustfs_config::MAX_HEAL_REQUEST_SIZE;
|
||||
use rustfs_heal::heal::utils::format_set_disk_id;
|
||||
use rustfs_heal_contracts::heal_channel::{
|
||||
@@ -35,13 +36,11 @@ use rustfs_heal_contracts::heal_channel::{
|
||||
};
|
||||
use rustfs_policy::policy::action::{Action, AdminAction};
|
||||
use rustfs_scanner::scanner::{BackgroundHealInfo, read_background_heal_info};
|
||||
use rustfs_utils::path::path_join;
|
||||
use s3s::header::{CONTENT_LENGTH, CONTENT_TYPE};
|
||||
use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::{BTreeMap, BTreeSet, HashSet};
|
||||
use std::future::Future;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
use tokio::time::{Duration, timeout};
|
||||
@@ -71,9 +70,17 @@ struct HealInitParams {
|
||||
}
|
||||
|
||||
fn extract_heal_init_params(body: &Bytes, uri: &Uri, params: Params<'_, '_>) -> S3Result<HealInitParams> {
|
||||
// matchit captures the original URI bytes. Decode once before validation
|
||||
// so literal %2F keys remain distinct from actual path separators.
|
||||
let mut hip = HealInitParams {
|
||||
bucket: params.get("bucket").map(|s| s.to_string()).unwrap_or_default(),
|
||||
obj_prefix: params.get("prefix").map(|s| s.to_string()).unwrap_or_default(),
|
||||
bucket: percent_decode_str(params.get("bucket").unwrap_or_default())
|
||||
.decode_utf8()
|
||||
.map_err(|_| s3_error!(InvalidRequest, "invalid bucket name encoding"))?
|
||||
.into_owned(),
|
||||
obj_prefix: percent_decode_str(params.get("prefix").unwrap_or_default())
|
||||
.decode_utf8()
|
||||
.map_err(|_| s3_error!(InvalidRequest, "invalid object name encoding"))?
|
||||
.into_owned(),
|
||||
..Default::default()
|
||||
};
|
||||
validate_heal_target(&hip.bucket, &hip.obj_prefix)?;
|
||||
@@ -164,13 +171,13 @@ fn validate_heal_target(bucket: &str, obj_prefix: &str) -> S3Result<()> {
|
||||
}
|
||||
|
||||
fn encode_heal_control_path(bucket: &str, obj_prefix: &str) -> String {
|
||||
if bucket.is_empty() && obj_prefix.is_empty() {
|
||||
return String::new();
|
||||
if obj_prefix.is_empty() {
|
||||
return bucket.to_owned();
|
||||
}
|
||||
|
||||
path_join(&[PathBuf::from(bucket), PathBuf::from(obj_prefix)])
|
||||
.to_string_lossy()
|
||||
.into_owned()
|
||||
// This identifies an S3 target, not a filesystem path. In particular,
|
||||
// a leading slash in the object must not alias the sibling without it.
|
||||
format!("{bucket}/{obj_prefix}")
|
||||
}
|
||||
|
||||
fn heal_control_response_id(heal_path: &str, client_token: &str) -> String {
|
||||
@@ -200,7 +207,7 @@ pub fn register_heal_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<
|
||||
|
||||
r.insert(
|
||||
Method::POST,
|
||||
format!("{}{}", ADMIN_PREFIX, "/v3/heal/{bucket}/{prefix}").as_str(),
|
||||
format!("{}{}", ADMIN_PREFIX, "/v3/heal/{bucket}/{*prefix}").as_str(),
|
||||
AdminOperation(&HealHandler {}),
|
||||
)?;
|
||||
|
||||
@@ -1616,6 +1623,135 @@ mod tests {
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::time::Duration;
|
||||
|
||||
fn parse_registered_heal_request(uri: &Uri) -> s3s::S3Result<HealInitParams> {
|
||||
let mut registered = super::S3Router::new(false);
|
||||
super::register_heal_route(&mut registered).expect("register production Heal routes");
|
||||
let mut router = Router::new();
|
||||
for route in registered.registered_routes() {
|
||||
router.insert(route.clone(), ()).expect("replay production route");
|
||||
}
|
||||
let path = format!("POST|{}", uri.path());
|
||||
let matched = router.at(&path).expect("request must match a production Heal route");
|
||||
let body = Bytes::from_static(
|
||||
br#"{"recursive":false,"dryRun":true,"remove":false,"recreate":false,"scanMode":2,"updateParity":false,"nolock":false,"readRepair":false,"pool":0,"set":0}"#,
|
||||
);
|
||||
extract_heal_init_params(&body, uri, matched.params)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_routes_accept_nested_and_encoded_object_paths() {
|
||||
let mut router = super::S3Router::new(false);
|
||||
super::register_heal_route(&mut router).expect("register production Heal routes");
|
||||
for prefix in ["/rustfs/admin", "/minio/admin"] {
|
||||
for target in [
|
||||
"",
|
||||
"test-bucket",
|
||||
"test-bucket/object.bin",
|
||||
"test-bucket/dir/sub/object.bin",
|
||||
"test-bucket/dir%2Fobject.bin",
|
||||
] {
|
||||
let path = format!("{prefix}/v3/heal/{target}");
|
||||
assert!(router.contains_compatible_route(http::Method::POST, &path), "{path}");
|
||||
assert!(!router.contains_compatible_route(http::Method::GET, &path), "{path}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_target_decodes_once_and_keeps_start_status_stop_identity() {
|
||||
for (wire, object) in [
|
||||
("object.bin", "object.bin"),
|
||||
("dir/sub/object.bin", "dir/sub/object.bin"),
|
||||
("dir%2Fsub%2Fobject.bin", "dir/sub/object.bin"),
|
||||
("dir%2fsub/object.bin", "dir/sub/object.bin"),
|
||||
("%2Fobject.bin", "/object.bin"),
|
||||
("dir/", "dir/"),
|
||||
("dir%2F", "dir/"),
|
||||
("literal%252Fslash", "literal%2Fslash"),
|
||||
("space%20key%2Bplus", "space key+plus"),
|
||||
("literal+plus", "literal+plus"),
|
||||
("%E4%B8%AD%E6%96%87%2F%E6%96%87%E4%BB%B6", "中文/文件"),
|
||||
("query%3Fhash%23percent%25", "query?hash#percent%"),
|
||||
] {
|
||||
for query in ["", "?clientToken=task", "?clientToken=task&forceStop=true"] {
|
||||
let uri = format!("/rustfs/admin/v3/heal/test%2Dbucket/{wire}{query}")
|
||||
.parse()
|
||||
.expect("valid encoded URI");
|
||||
let parsed = parse_registered_heal_request(&uri).expect("valid Heal target");
|
||||
assert_eq!(parsed.bucket, "test-bucket");
|
||||
assert_eq!(parsed.obj_prefix, object, "wire target: {wire}");
|
||||
assert_eq!(
|
||||
encode_heal_control_path(&parsed.bucket, &parsed.obj_prefix),
|
||||
format!("test-bucket/{object}")
|
||||
);
|
||||
assert_eq!(parsed.client_token, if query.is_empty() { "" } else { "task" });
|
||||
assert_eq!(parsed.force_stop, query.ends_with("forceStop=true"));
|
||||
if query.is_empty() {
|
||||
let request = build_heal_channel_request(&parsed);
|
||||
assert_eq!(request.bucket, "test-bucket");
|
||||
assert_eq!(request.object_prefix.as_deref(), Some(object));
|
||||
assert_eq!(request.pool_index, Some(0));
|
||||
assert_eq!(request.set_index, Some(0));
|
||||
assert_eq!(request.dry_run, Some(true));
|
||||
assert_eq!(request.scan_mode, Some(HealScanMode::Deep));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_target_validates_decoded_paths_before_admission() {
|
||||
for target in [
|
||||
"test%2Fbucket/object",
|
||||
"test%00bucket/object",
|
||||
"test%FFbucket/object",
|
||||
"test-bucket/dir%2F..%2Fobject",
|
||||
"test-bucket/dir/%2e/object",
|
||||
"test-bucket/dir%5C..%5Cobject",
|
||||
"test-bucket/dir%2F%2Fobject",
|
||||
"test-bucket/object%00",
|
||||
"test-bucket/object%FF",
|
||||
] {
|
||||
let uri = format!("/rustfs/admin/v3/heal/{target}").parse().expect("encoded URI");
|
||||
let err = parse_registered_heal_request(&uri).expect_err("decoded invalid target must fail closed");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest, "target: {target}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_nested_heal_routes_still_require_authentication() {
|
||||
use s3s::route::S3Route;
|
||||
|
||||
let mut router = super::S3Router::new(false);
|
||||
super::register_heal_route(&mut router).expect("register production Heal routes");
|
||||
for prefix in ["/rustfs/admin", "/minio/admin"] {
|
||||
for object in ["dir/object.bin", "dir%2Fobject.bin", "literal%252Fslash"] {
|
||||
let mut req = s3s::S3Request {
|
||||
input: s3s::Body::empty(),
|
||||
method: http::Method::POST,
|
||||
uri: format!("{prefix}/v3/heal/test-bucket/{object}").parse().expect("Heal URI"),
|
||||
headers: http::HeaderMap::new(),
|
||||
extensions: http::Extensions::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
};
|
||||
let err = router
|
||||
.check_access(&mut req)
|
||||
.await
|
||||
.expect_err("router must require a signature");
|
||||
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
|
||||
let err = router
|
||||
.call(req)
|
||||
.await
|
||||
.expect_err("handler must independently require authentication");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
assert!(err.to_string().contains("authentication required"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn replacement_record(task_id: &str) -> rustfs_heal::ReplacementRecoveryRecord {
|
||||
rustfs_heal::ReplacementRecoveryRecord {
|
||||
task_id: task_id.to_string(),
|
||||
|
||||
@@ -198,7 +198,7 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
|
||||
admin_route(Method::POST, "/v3/rebalance/stop"),
|
||||
admin_route(Method::POST, "/v3/heal/"),
|
||||
admin_route_sample(Method::POST, "/v3/heal/{bucket}", "/v3/heal/test-bucket"),
|
||||
admin_route_sample(Method::POST, "/v3/heal/{bucket}/{prefix}", "/v3/heal/test-bucket/prefix"),
|
||||
admin_route_sample(Method::POST, "/v3/heal/{bucket}/{*prefix}", "/v3/heal/test-bucket/prefix"),
|
||||
admin_route(Method::POST, "/v3/background-heal/status"),
|
||||
admin_route(Method::GET, "/v4/heal/replacement-recovery"),
|
||||
admin_route(Method::GET, "/v3/tier"),
|
||||
|
||||
@@ -377,6 +377,10 @@ impl FS {
|
||||
|
||||
pub(crate) fn parse_object_version_id(version_id: Option<String>) -> S3Result<Option<Uuid>> {
|
||||
if let Some(vid) = version_id {
|
||||
if vid == "null" {
|
||||
// A nil UUID selects the stored null version; None selects latest.
|
||||
return Ok(Some(Uuid::nil()));
|
||||
}
|
||||
let uuid = Uuid::parse_str(&vid).map_err(|e| {
|
||||
error!("Invalid version ID: {}", e);
|
||||
s3_error!(InvalidArgument, "Invalid version ID")
|
||||
@@ -1183,7 +1187,11 @@ impl S3 for FS {
|
||||
error = %e,
|
||||
"Object tags not found"
|
||||
);
|
||||
return Err(s3_error!(NoSuchKey));
|
||||
return Err(if opts.version_id.is_some() {
|
||||
s3_error!(NoSuchVersion)
|
||||
} else {
|
||||
s3_error!(NoSuchKey)
|
||||
});
|
||||
}
|
||||
error!(
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
|
||||
@@ -17,7 +17,9 @@ mod tests {
|
||||
use crate::config::WorkloadProfile;
|
||||
use crate::server::cors;
|
||||
use crate::storage::StorageError;
|
||||
use crate::storage::ecfs::{FS, propagate_object_lock_peer_reload, validate_object_lock_configuration_input};
|
||||
use crate::storage::ecfs::{
|
||||
FS, parse_object_version_id, propagate_object_lock_peer_reload, validate_object_lock_configuration_input,
|
||||
};
|
||||
use crate::storage::ecfs_extend::{apply_bucket_default_lock_retention, map_bucket_object_lock_config_state};
|
||||
use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner};
|
||||
use crate::storage::storage_api::ecstore_bucket::metadata_sys::ObjectLockConfigState;
|
||||
@@ -641,6 +643,36 @@ mod tests {
|
||||
assert_eq!(metadata.get("content-type"), Some(&"application/octet-stream".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tagging_version_id_preserves_explicit_null_and_latest_selection() {
|
||||
assert_eq!(parse_object_version_id(None).expect("latest version selector"), None);
|
||||
assert_eq!(
|
||||
parse_object_version_id(Some("null".to_owned())).expect("explicit null version selector"),
|
||||
Some(uuid::Uuid::nil())
|
||||
);
|
||||
for version in [uuid::Uuid::nil(), uuid::Uuid::new_v4()] {
|
||||
assert_eq!(
|
||||
parse_object_version_id(Some(version.to_string())).expect("UUID version selector"),
|
||||
Some(version)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tagging_version_id_rejects_invalid_values_instead_of_selecting_latest() {
|
||||
for version in [
|
||||
"",
|
||||
"NULL",
|
||||
" null ",
|
||||
"not-a-version",
|
||||
"null/other",
|
||||
"00000000-0000-0000-0000-00000000000g",
|
||||
] {
|
||||
let err = parse_object_version_id(Some(version.to_owned())).expect_err("invalid version must fail closed");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument, "version: {version:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_object_tagging_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
|
||||
@@ -5939,8 +5939,11 @@ mod tests {
|
||||
})
|
||||
.await
|
||||
.expect_err("mismatched kms context should fail");
|
||||
assert_eq!(err.code, S3ErrorCode::InternalError);
|
||||
assert_eq!(err.message, ApiError::error_code_to_message(&S3ErrorCode::InternalError));
|
||||
assert_eq!(err.code, S3ErrorCode::InvalidRequest);
|
||||
assert_eq!(
|
||||
err.message,
|
||||
"Encryption context mismatch: Context mismatch for key 'tenant': expected 'alpha', got 'beta'"
|
||||
);
|
||||
assert_eq!(super::kms_data_plane_error_class(&err), "context_mismatch");
|
||||
|
||||
manager.stop().await.expect("kms service should stop cleanly");
|
||||
|
||||
@@ -955,7 +955,10 @@ pub(crate) async fn get_local_server_property() -> rustfs_madmin::ServerProperti
|
||||
|
||||
pub(crate) async fn init_background_replication(store: Arc<ECStore>) {
|
||||
let durable_dirty_usage_journal = super::scanner_dirty_journal::start_durable_dirty_usage_journal(store.clone()).await;
|
||||
let journal_writer = durable_dirty_usage_journal.clone();
|
||||
let mutation_journal = durable_dirty_usage_journal.clone();
|
||||
rustfs_scanner::set_scanner_dirty_usage_mutation_observer(Some(Arc::new(move |bucket, object, producer| {
|
||||
mutation_journal.record_committed_mutation(bucket, object, producer);
|
||||
})));
|
||||
ecstore_bucket::replication::set_scanner_dirty_usage_mutation_observer(Some(Arc::new(move |bucket, object, source| {
|
||||
let producer = match source {
|
||||
ecstore_bucket::replication::ScannerDirtyUsageMutationSource::Replication => {
|
||||
@@ -966,7 +969,6 @@ pub(crate) async fn init_background_replication(store: Arc<ECStore>) {
|
||||
}
|
||||
};
|
||||
rustfs_scanner::record_dirty_usage_object_from_producer(bucket, object, producer);
|
||||
journal_writer.record_committed_mutation(bucket, object, producer);
|
||||
})));
|
||||
rustfs_scanner::set_scanner_dirty_usage_clear_observer(Some(Arc::new(move |cleared| {
|
||||
durable_dirty_usage_journal.clear_confirmed_buckets(cleared);
|
||||
|
||||
@@ -61,10 +61,12 @@ their issue closes.
|
||||
| `run_scanner_heal_checkpoint_crash_evidence.py` | dev-tool | Assembles measured Scanner/Heal G02/R-E checkpoint and restart release descriptors from scanner restart diagnostic reports | `diagnose_scanner_enumeration_restart.py`; `test_scanner_heal_checkpoint_crash_evidence.sh` |
|
||||
| `run_scanner_heal_g14_multiset_evidence.py` | dev-tool | Assembles measured Scanner/Heal G14 same-window EC8+4 multi-set/multi-pool release descriptors from e2e case directories or an operator-collected proof | `.config/scanner-heal-required-tests.json`; `test_scanner_heal_g14_multiset_evidence.sh` |
|
||||
| `run_scanner_heal_g09_upgrade_evidence.sh` | dev-tool | Runs the G09 mixed-version and rollback upgrade E2E lanes against a pinned previous release and verifies the raw evidence artifacts | `docs/testing/ci-gates.md`; `.github/workflows/e2e-upgrade.yml`; `test_scanner_heal_g09_upgrade_evidence.sh` |
|
||||
| `run_scanner_heal_linux_evidence_plan.py` | dev-tool | Writes the unified Scanner/Heal Linux release-evidence execution manifest and can run lightweight preflight checks without producing measured evidence | `docs/testing/ci-gates.md`; `test_scanner_heal_linux_evidence_plan.sh` |
|
||||
| `run_scanner_heal_scoped_ack_evidence.py` | dev-tool | Assembles measured Scanner/Heal G03 scoped ACK publication and mixed-peer fallback release descriptors | `.config/scanner-heal-required-tests.json`; `test_scanner_heal_scoped_ack_evidence.sh` |
|
||||
| `run_scanner_heal_legacy_rollback_evidence.py` | dev-tool | Assembles measured Scanner/Heal R-L legacy source-conflict, migration-gap, and source-retirement release descriptors | `.config/scanner-heal-required-tests.json`; `test_scanner_heal_legacy_rollback_evidence.sh` |
|
||||
| `run_scanner_heal_mrf_evidence.py` | dev-tool | Assembles measured Scanner/Heal G07/G08/P4 MRF release descriptors from W13 raw artifacts | `.config/scanner-heal-required-tests.json`; `test_scanner_heal_w13_mrf_evidence.sh` |
|
||||
| `run_scanner_heal_scheduler_pressure_evidence.py` | dev-tool | Assembles measured Scanner/Heal G10/P1/P3 scheduler-pressure release descriptors from a completed measured ABBA run, recovery-window proof, and profile artifacts | `docs/operations/scanner-benchmark-runbook.md`; `test_scanner_heal_scheduler_pressure_evidence.sh` |
|
||||
| `run_scanner_heal_status_outcome_probe.py` | dev-tool | Normalizes live Scanner/Heal status/outcome observations into the measured G05/G06/R-D raw artifacts consumed by the descriptor producer | `.config/scanner-heal-required-tests.json`; `test_scanner_heal_status_outcome_evidence.sh` |
|
||||
| `run_scanner_heal_status_outcome_evidence.py` | dev-tool | Assembles measured Scanner/Heal G05/G06/R-D status-and-outcome release descriptors from same-run status, compatibility, and disposition artifacts | `.config/scanner-heal-required-tests.json`; `test_scanner_heal_status_outcome_evidence.sh` |
|
||||
| `run_scanner_heal_maintenance_evidence.py` | dev-tool | Assembles measured Scanner/Heal G11/G13 maintenance-producer release descriptors from operator-collected proof JSON | `.config/scanner-heal-required-tests.json`; `test_scanner_heal_maintenance_evidence.sh` |
|
||||
| `run_scanner_heal_w13_mrf_evidence.sh` | dev-tool | Runs the W13 durable MRF replay lanes and writes G07/G08/P4 bundle-ready evidence descriptors | `docs/testing/ci-gates.md`; `test_scanner_heal_w13_mrf_evidence.sh` |
|
||||
|
||||
@@ -428,6 +428,7 @@ SCANNER_HEAL_RELEASE_SCOPED_ACK_CASES = {
|
||||
"exact-generation",
|
||||
"scanner-instance",
|
||||
"participating-peer-set",
|
||||
"cleared-count-bound",
|
||||
),
|
||||
"participating_peer_capability_snapshot": (
|
||||
"supports-scoped-ack",
|
||||
@@ -452,6 +453,7 @@ SCANNER_HEAL_RELEASE_G03_REQUIRED_TRUE_FIELDS = {
|
||||
"scanner_instance_observed",
|
||||
"participating_peer_set_observed",
|
||||
"whole_cycle_fallback_observed",
|
||||
"cleared_count_bound_observed",
|
||||
),
|
||||
"participating_peer_capability_snapshot": (
|
||||
"capability_probe_observed",
|
||||
@@ -1682,6 +1684,25 @@ def check_scanner_heal_evidence(root: Path, directory: Path, case_id: str) -> li
|
||||
expected_outage_target_required = requirement.get("outage_target_manifest_required", True)
|
||||
require(oracle.get("outage_target_manifest_required", True) is expected_outage_target_required,
|
||||
"oracle outage target-manifest contract mismatch")
|
||||
outage_write = oracle.get("outage_write")
|
||||
outage_write_deferred = oracle.get("outage_write_deferred_until_rejoin", False)
|
||||
require(type(outage_write_deferred) is bool, "invalid outage-write deferred flag")
|
||||
if not expected_outage_target_required and outage_write is not None:
|
||||
require(isinstance(outage_write, dict), "invalid optional outage-write diagnostic")
|
||||
require(outage_write.get("attempted") is True, "optional outage-write diagnostic was not attempted")
|
||||
require(outage_write.get("required") is False, "optional outage-write diagnostic required flag mismatch")
|
||||
require(outage_write.get("accepted") is True, "optional outage-write final S3 body was not accepted")
|
||||
outage_write_attempts = evidence_integer(outage_write.get("attempts"), "optional outage-write attempts", 1, 1024)
|
||||
outage_write_service_unavailable = evidence_integer(
|
||||
outage_write.get("service_unavailable"), "optional outage-write ServiceUnavailable count", 0, 1024
|
||||
)
|
||||
if outage_write_deferred:
|
||||
require(
|
||||
outage_write_service_unavailable == outage_write_attempts,
|
||||
"deferred outage write must account for every down-window attempt",
|
||||
)
|
||||
else:
|
||||
require(outage_write_deferred is False, "outage-write deferred flag requires optional outage-write diagnostic")
|
||||
if requirement.get("sets", 1) > 1 or requirement.get("pools", 1) > 1:
|
||||
require(oracle.get("distributed_ec_invalidation") is True,
|
||||
"oracle missing distributed EC invalidation proof")
|
||||
@@ -1708,8 +1729,8 @@ def check_scanner_heal_evidence(root: Path, directory: Path, case_id: str) -> li
|
||||
require(isinstance(objects, list) and requirement["min_objects"] <= len(objects) <= requirement["max_objects"],
|
||||
"incomplete/oversized object oracle")
|
||||
require(len({obj["key"] for obj in objects}) == len(objects), "duplicate object identity")
|
||||
require(sum(obj["expected_physical"] is None for obj in objects) == 1,
|
||||
"only the outage object may lack a pre-fault target manifest")
|
||||
outage_object_count = sum(obj["expected_physical"] is None for obj in objects)
|
||||
require(outage_object_count == 1, "only the outage object may lack a pre-fault target manifest")
|
||||
for obj in objects:
|
||||
require(isinstance(obj["key"], str) and 0 < len(obj["key"].encode()) <= 1024, "invalid object identity")
|
||||
require(obj["version_id"] is None, "this case only covers unversioned objects")
|
||||
@@ -1827,6 +1848,7 @@ def release_bundle_json_artifact_mirrored_fields(gate: str, field: str) -> tuple
|
||||
"raw_entry_budget",
|
||||
"max_raw_entries_per_round",
|
||||
"max_objects_processed_per_round",
|
||||
"bounded_work_quantum_observed",
|
||||
"durable_checkpoint_committed",
|
||||
"no_unbounded_tail",
|
||||
))
|
||||
@@ -2255,6 +2277,8 @@ def validate_release_bundle_domain_evidence(gate: str, field: str, evidence: dic
|
||||
f"{gate}.{field}.max_objects_processed_per_round", 1, 4096)
|
||||
require(max_raw <= budget, f"{gate}.{field} raw entries exceed fixed budget")
|
||||
require(max_objects <= budget, f"{gate}.{field} processed objects exceed fixed budget")
|
||||
release_bundle_bool_true(evidence.get("bounded_work_quantum_observed"),
|
||||
f"{gate}.{field}.bounded_work_quantum_observed")
|
||||
release_bundle_bool_true(evidence.get("durable_checkpoint_committed"),
|
||||
f"{gate}.{field}.durable_checkpoint_committed")
|
||||
release_bundle_bool_true(evidence.get("no_unbounded_tail"), f"{gate}.{field}.no_unbounded_tail")
|
||||
@@ -2772,10 +2796,10 @@ def validate_release_bundle_artifact(bundle_path: Path, source_revision: str, ga
|
||||
require(evidence.get("stale_journals_after_gc") == 0,
|
||||
f"{gate}.{field} requires zero stale journals after GC")
|
||||
if field == "segment_activation_preflight":
|
||||
require(evidence.get("production_activation") is False,
|
||||
f"{gate}.{field} must keep production activation disabled")
|
||||
require(evidence.get("scanner_segment_reuse_activated") is False,
|
||||
f"{gate}.{field} must prove the runtime activation gate is disabled")
|
||||
require(evidence.get("production_activation") is True,
|
||||
f"{gate}.{field} must prove production activation is enabled")
|
||||
require(evidence.get("scanner_segment_reuse_activated") is True,
|
||||
f"{gate}.{field} must prove the runtime activation gate is enabled")
|
||||
evidence_exact_strings(evidence.get("proof_inputs"),
|
||||
SCANNER_HEAL_SEGMENT_ACTIVATION_PROOF_INPUTS,
|
||||
f"{gate}.{field}.proof_inputs")
|
||||
@@ -3319,6 +3343,7 @@ def write_scanner_heal_release_bundle_fixture(root: Path, directory: Path) -> Pa
|
||||
"raw_entry_budget": 8,
|
||||
"max_raw_entries_per_round": 8,
|
||||
"max_objects_processed_per_round": 8,
|
||||
"bounded_work_quantum_observed": True,
|
||||
"durable_checkpoint_committed": True,
|
||||
"no_unbounded_tail": True,
|
||||
})
|
||||
@@ -3739,6 +3764,7 @@ class SelfTests(unittest.TestCase):
|
||||
"raw_entry_budget": 8,
|
||||
"max_raw_entries_per_round": 8,
|
||||
"max_objects_processed_per_round": 8,
|
||||
"bounded_work_quantum_observed": True,
|
||||
"durable_checkpoint_committed": True,
|
||||
"no_unbounded_tail": True,
|
||||
})
|
||||
@@ -3944,8 +3970,8 @@ class SelfTests(unittest.TestCase):
|
||||
},
|
||||
]
|
||||
if field == "segment_activation_preflight":
|
||||
evidence["production_activation"] = False
|
||||
evidence["scanner_segment_reuse_activated"] = False
|
||||
evidence["production_activation"] = True
|
||||
evidence["scanner_segment_reuse_activated"] = True
|
||||
evidence["proof_inputs"] = list(SCANNER_HEAL_SEGMENT_ACTIVATION_PROOF_INPUTS)
|
||||
evidence["fail_closed_checks"] = list(SCANNER_HEAL_SEGMENT_ACTIVATION_FAIL_CLOSED_CHECKS)
|
||||
if field == "cold_segment_reuse_measurement":
|
||||
@@ -4447,7 +4473,14 @@ class SelfTests(unittest.TestCase):
|
||||
"zero stale journals",
|
||||
),
|
||||
("same-window-fields", "G14", "same_window_field_evidence", lambda item: item.update({"same_window_fields": ["ec8_4_evidence", "multi_set_evidence"]}), "JSON artifact same_window_fields mismatch"),
|
||||
("activation-enabled", "G11", "segment_activation_preflight", lambda item: item.update({"production_activation": True}), "production activation disabled"),
|
||||
("activation-disabled", "G11", "segment_activation_preflight", lambda item: item.update({"production_activation": False}), "production activation is enabled"),
|
||||
(
|
||||
"activation-runtime-disabled",
|
||||
"G11",
|
||||
"segment_activation_preflight",
|
||||
lambda item: item.update({"scanner_segment_reuse_activated": False}),
|
||||
"runtime activation gate is enabled",
|
||||
),
|
||||
(
|
||||
"activation-missing-fail-closed",
|
||||
"G11",
|
||||
@@ -5156,6 +5189,35 @@ class SelfTests(unittest.TestCase):
|
||||
errors,
|
||||
)
|
||||
|
||||
def test_scanner_heal_multipool_case_accepts_deferred_optional_outage_write(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root, run_dir = self.scanner_heal_fixture(Path(tmp))
|
||||
path = run_dir / "background-target-crash-ec8-4-multi-pool.json"
|
||||
oracle = read_json(path)
|
||||
oracle["outage_write_deferred_until_rejoin"] = True
|
||||
oracle["outage_write"] = {
|
||||
"attempted": True,
|
||||
"required": False,
|
||||
"accepted": True,
|
||||
"attempts": 36,
|
||||
"service_unavailable": 36,
|
||||
}
|
||||
write_json(path, oracle)
|
||||
(run_dir / "execution.json").unlink()
|
||||
finish_scanner_heal_receipt(run_dir, 0, root)
|
||||
|
||||
self.assertEqual(check_scanner_heal_evidence(root, run_dir, "background-target-crash-ec8-4-multi-pool"), [])
|
||||
|
||||
oracle["outage_write"]["accepted"] = False
|
||||
write_json(path, oracle)
|
||||
(run_dir / "execution.json").unlink()
|
||||
finish_scanner_heal_receipt(run_dir, 0, root)
|
||||
errors = check_scanner_heal_evidence(root, run_dir, "background-target-crash-ec8-4-multi-pool")
|
||||
self.assertTrue(
|
||||
any("optional outage-write final S3 body was not accepted" in error for error in errors),
|
||||
errors,
|
||||
)
|
||||
|
||||
def test_scanner_heal_pending_gate_cannot_map_to_implemented_lane(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root, run_dir = self.scanner_heal_fixture(Path(tmp))
|
||||
|
||||
+22
-1
@@ -59,7 +59,7 @@ def expected_results(mode: str, event: str, ref: str) -> dict[str, str]:
|
||||
expected.update({job: "success" if mode == "full" else "skipped" for job in CODE_JOBS})
|
||||
rio = mode == "full" and event in ("schedule", "workflow_dispatch")
|
||||
expected.update({job: "success" if rio else "skipped" for job in OPTIONAL_JOBS[:2]})
|
||||
full = mode == "full" and (event in ("merge_group", "workflow_dispatch") or (event == "push" and ref == "refs/heads/main"))
|
||||
full = mode == "full" and (event in ("merge_group", "workflow_dispatch") or (event == "push" and ref in ("refs/heads/main", "refs/heads/release")))
|
||||
expected["e2e-full"] = "success" if full else "skipped"
|
||||
return expected
|
||||
|
||||
@@ -219,6 +219,27 @@ class SelfTests(unittest.TestCase):
|
||||
bad = {**good, "classify-changes": {"result": "success", "outputs": selection}}
|
||||
self.assertTrue(verify_results(bad, event, "refs/heads/main"))
|
||||
|
||||
def test_full_e2e_gate_preserves_workflow_branch_and_event_scope(self):
|
||||
for event, ref, required in (
|
||||
("push", "refs/heads/main", "success"),
|
||||
("push", "refs/heads/release", "success"),
|
||||
("push", "refs/heads/feature", "skipped"),
|
||||
("push", "refs/heads/release-candidate", "skipped"),
|
||||
("push", "refs/tags/release", "skipped"),
|
||||
("pull_request", "refs/pull/1/merge", "skipped"),
|
||||
("schedule", "refs/heads/release", "skipped"),
|
||||
("workflow_dispatch", "refs/heads/feature", "success"),
|
||||
("merge_group", "refs/heads/gh-readonly-queue/release/pr-1", "success"),
|
||||
):
|
||||
with self.subTest(event=event, ref=ref):
|
||||
expected = expected_results("full", event, ref)
|
||||
self.assertEqual(expected["e2e-full"], required)
|
||||
needs = {job: {"result": result} for job, result in expected.items()}
|
||||
needs["classify-changes"]["outputs"] = {"mode": "full"}
|
||||
for result in ("success", "skipped", "failure", "cancelled"):
|
||||
needs["e2e-full"]["result"] = result
|
||||
self.assertEqual(verify_results(needs, event, ref) == [], result == required)
|
||||
|
||||
def test_repository_wiring_and_missing_dependency_regression(self):
|
||||
self.assertEqual(check_workflow(ROOT), [])
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
|
||||
@@ -74,10 +74,31 @@ def converged(report, objects):
|
||||
|
||||
|
||||
def replays_raw_window(previous, current):
|
||||
if (previous["raw_entries"] == 0 or current["raw_entries"] == 0
|
||||
or previous["raw_first_entry"] is None or current["raw_first_entry"] is None
|
||||
or previous["raw_last_entry"] is None or current["raw_last_entry"] is None):
|
||||
return False
|
||||
raw_page_index_progressed = (
|
||||
previous["raw_page_index_parent"] == current["raw_page_index_parent"]
|
||||
and (
|
||||
current["raw_page_index_committed_entries"] > previous["raw_page_index_committed_entries"]
|
||||
or (current["raw_page_index_complete"] and not previous["raw_page_index_complete"])
|
||||
)
|
||||
)
|
||||
return (previous["raw_first_entry"] == current["raw_first_entry"]
|
||||
and previous["raw_last_entry"] == current["raw_last_entry"]
|
||||
and previous["objects_retained"] == current["objects_before"]
|
||||
and current["objects_retained"] == previous["objects_retained"])
|
||||
and current["objects_retained"] == previous["objects_retained"]
|
||||
and not raw_page_index_progressed)
|
||||
|
||||
|
||||
def fully_retained(report, objects):
|
||||
return all(report[key] == objects for key in
|
||||
("objects_retained", "versions_retained", "bytes_retained"))
|
||||
|
||||
|
||||
def final_complete_recheck_after_full_retention(previous, current, objects):
|
||||
return fully_retained(previous, objects) and converged(current, objects)
|
||||
|
||||
|
||||
def validate_recoverable_quantum(reports, *, objects, budget, require_converged):
|
||||
@@ -97,7 +118,8 @@ def validate_recoverable_quantum(reports, *, objects, budget, require_converged)
|
||||
raise ValueError("durable retained coverage did not survive process restart")
|
||||
if report["objects_retained"] < previous["objects_retained"]:
|
||||
raise ValueError("durable retained coverage regressed across restart")
|
||||
if replays_raw_window(previous, report):
|
||||
if (not final_complete_recheck_after_full_retention(previous, report, objects)
|
||||
and replays_raw_window(previous, report)):
|
||||
raise ValueError("raw enumeration window replayed without durable coverage")
|
||||
if (report["raw_page_index_parent"] == previous["raw_page_index_parent"]
|
||||
and report["raw_page_index_committed_entries"] < previous["raw_page_index_committed_entries"]
|
||||
|
||||
@@ -32,6 +32,11 @@ def positive_int(value: Any, name: str, minimum: int = 1) -> int:
|
||||
return value
|
||||
|
||||
|
||||
def reject_non_measured_markers(payload: dict[str, Any], label: str) -> None:
|
||||
for marker in ("fixture", "fixture_only", "dry_run", "synthetic"):
|
||||
require(payload.get(marker) is not True, f"{label} is {marker}")
|
||||
|
||||
|
||||
def timestamp(value: Any, name: str) -> str:
|
||||
require(isinstance(value, str) and value.endswith("Z"), f"invalid {name}")
|
||||
datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
@@ -63,6 +68,7 @@ def load_reports(directory: Path) -> list[dict[str, Any]]:
|
||||
"raw_page_index_indexed_entries",
|
||||
}
|
||||
for index, report in enumerate(reports):
|
||||
reject_non_measured_markers(report, f"round {index}")
|
||||
require(report.get("schema") == 1, f"round {index} has wrong schema")
|
||||
require(report.get("round") == index, f"round {index} order mismatch")
|
||||
for key in (
|
||||
@@ -88,8 +94,7 @@ def load_reports(directory: Path) -> list[dict[str, Any]]:
|
||||
|
||||
def require_measured_manifest(path: Path, source_revision: str) -> dict[str, Any]:
|
||||
manifest = read_json(path)
|
||||
for marker in ("fixture", "fixture_only", "dry_run", "synthetic"):
|
||||
require(manifest.get(marker) is not True, f"checkpoint/crash manifest is {marker}")
|
||||
reject_non_measured_markers(manifest, "checkpoint/crash manifest")
|
||||
require(manifest.get("schema") == 1, "unsupported manifest schema")
|
||||
require(manifest.get("evidence_type") == "measured", "manifest must be measured")
|
||||
require(manifest.get("source_revision") == source_revision, "manifest source revision mismatch")
|
||||
@@ -108,8 +113,7 @@ def derived_measured_manifest(directory: Path, source_revision: str, reports: li
|
||||
request_path = directory / "request.json"
|
||||
request = read_json(request_path)
|
||||
require(isinstance(request, dict), "diagnostic request must be a JSON object")
|
||||
for marker in ("fixture", "fixture_only", "dry_run", "synthetic"):
|
||||
require(request.get(marker) is not True, f"diagnostic request is {marker}")
|
||||
reject_non_measured_markers(request, "diagnostic request")
|
||||
objects = positive_int(request.get("objects"), "request.objects")
|
||||
raw_entry_budget = positive_int(request.get("raw_entry_budget"), "request.raw_entry_budget")
|
||||
final_round = positive_int(request.get("round"), "request.round", 0)
|
||||
@@ -251,6 +255,7 @@ def build_descriptor(args: argparse.Namespace) -> Path:
|
||||
"raw_entry_budget": summary["raw_entry_budget"],
|
||||
"max_raw_entries_per_round": summary["max_raw_entries_per_round"],
|
||||
"max_objects_processed_per_round": summary["max_objects_processed_per_round"],
|
||||
"bounded_work_quantum_observed": True,
|
||||
"durable_checkpoint_committed": True,
|
||||
"no_unbounded_tail": True,
|
||||
}),
|
||||
@@ -422,6 +427,24 @@ def run_self_test() -> None:
|
||||
else:
|
||||
raise ValueError("self-test accepted non-converged diagnostic")
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
source_revision = git_head()
|
||||
manifest, diagnostic = write_self_test_inputs(root, source_revision)
|
||||
report = read_json(diagnostic / "round-0.json")
|
||||
report["synthetic"] = True
|
||||
write_json(diagnostic / "round-0.json", report)
|
||||
try:
|
||||
build_descriptor(parse_args([
|
||||
"--manifest", str(manifest),
|
||||
"--diagnostic-dir", str(diagnostic),
|
||||
"--out-dir", str(root / "out"),
|
||||
]))
|
||||
except ValueError as err:
|
||||
require("round 0 is synthetic" in str(err), "wrong self-test failure for synthetic round report")
|
||||
else:
|
||||
raise ValueError("self-test accepted synthetic diagnostic round report")
|
||||
|
||||
|
||||
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
|
||||
@@ -28,6 +28,9 @@ The script intentionally runs a single case, not the release pseudo-case. After
|
||||
a successful case run it verifies that the release gate still remains blocked.
|
||||
Set RUSTFS_E2E_TEST_PORT_MIN and RUSTFS_E2E_TEST_PORT_RANGE to move the e2e
|
||||
port allocator when the default 20000..30000 test range is unavailable.
|
||||
Set RUSTFS_SCANNER_HEAL_SKIP_CLEAN=1 to reuse an existing cargo target directory
|
||||
while narrowing a case locally; release evidence should keep the default clean
|
||||
build.
|
||||
USAGE
|
||||
}
|
||||
|
||||
@@ -109,7 +112,7 @@ apply_runtime_profile() {
|
||||
export RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS="${RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS:-180}"
|
||||
;;
|
||||
background-ec8-4-multi-pool)
|
||||
export RUSTFS_HEAL_CHAOS_OBJECT_COUNT="${RUSTFS_HEAL_CHAOS_OBJECT_COUNT:-16}"
|
||||
export RUSTFS_HEAL_CHAOS_OBJECT_COUNT="${RUSTFS_HEAL_CHAOS_OBJECT_COUNT:-64}"
|
||||
export RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES="${RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES:-4194304}"
|
||||
export RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS="${RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS:-240}"
|
||||
;;
|
||||
@@ -162,6 +165,40 @@ if not status.get("pending_gates"):
|
||||
PY
|
||||
}
|
||||
|
||||
nextest_junit_candidates() {
|
||||
local target_dir="$1"
|
||||
local profile="$2"
|
||||
local primary="$target_dir/nextest/$profile/junit.xml"
|
||||
local fallback="$ROOT/target/nextest/$profile/junit.xml"
|
||||
printf '%s\n' "$primary"
|
||||
if [[ "$fallback" != "$primary" ]]; then
|
||||
printf '%s\n' "$fallback"
|
||||
fi
|
||||
}
|
||||
|
||||
remove_nextest_junit_candidates() {
|
||||
local target_dir="$1"
|
||||
local profile="$2"
|
||||
local candidate
|
||||
while IFS= read -r candidate; do
|
||||
rm -f "$candidate"
|
||||
done < <(nextest_junit_candidates "$target_dir" "$profile")
|
||||
}
|
||||
|
||||
copy_nextest_junit() {
|
||||
local target_dir="$1"
|
||||
local profile="$2"
|
||||
local run_dir="$3"
|
||||
local candidate
|
||||
while IFS= read -r candidate; do
|
||||
if [[ -f "$candidate" ]]; then
|
||||
cp "$candidate" "$run_dir/junit.xml"
|
||||
return 0
|
||||
fi
|
||||
done < <(nextest_junit_candidates "$target_dir" "$profile")
|
||||
return 1
|
||||
}
|
||||
|
||||
run_self_test() {
|
||||
if "$0" --case release --plan-only >/dev/null 2>&1; then
|
||||
echo "self-test failed: release pseudo-case must not be runnable" >&2
|
||||
@@ -183,6 +220,24 @@ run_self_test() {
|
||||
return 1
|
||||
fi
|
||||
done < <(case_ids)
|
||||
|
||||
local junit_profile="scanner-heal-junit-self-test-$$"
|
||||
local junit_run_dir="$ROOT/target/scanner-heal-junit-self-test-$$"
|
||||
local junit_target_dir="$ROOT/target/scanner-heal-junit-custom-target-$$"
|
||||
local junit_fallback="$ROOT/target/nextest/$junit_profile/junit.xml"
|
||||
mkdir -p "$(dirname "$junit_fallback")" "$junit_run_dir"
|
||||
printf '<testsuites />\n' >"$junit_fallback"
|
||||
if ! copy_nextest_junit "$junit_target_dir" "$junit_profile" "$junit_run_dir"; then
|
||||
echo "self-test failed: nextest junit fallback was not copied" >&2
|
||||
rm -rf "$junit_run_dir" "$junit_target_dir" "$(dirname "$junit_fallback")"
|
||||
return 1
|
||||
fi
|
||||
if ! cmp -s "$junit_fallback" "$junit_run_dir/junit.xml"; then
|
||||
echo "self-test failed: copied nextest junit fallback changed content" >&2
|
||||
rm -rf "$junit_run_dir" "$junit_target_dir" "$(dirname "$junit_fallback")"
|
||||
return 1
|
||||
fi
|
||||
rm -rf "$junit_run_dir" "$junit_target_dir" "$(dirname "$junit_fallback")"
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
@@ -255,18 +310,31 @@ if [[ -n "$(git status --porcelain --untracked-files=no)" ]]; then
|
||||
echo "commit tracked source changes before creating evidence" >&2
|
||||
exit 1
|
||||
fi
|
||||
NOFILE_SOFT="$(ulimit -Sn)"
|
||||
NOFILE_HARD="$(ulimit -Hn)"
|
||||
if [[ "$NOFILE_SOFT" =~ ^[0-9]+$ && "$NOFILE_HARD" =~ ^[0-9]+$ && "$NOFILE_SOFT" -lt 65535 ]]; then
|
||||
if [[ "$NOFILE_HARD" -ge 65535 ]]; then
|
||||
ulimit -n 65535 || true
|
||||
elif [[ "$NOFILE_HARD" -gt "$NOFILE_SOFT" ]]; then
|
||||
ulimit -n "$NOFILE_HARD" || true
|
||||
fi
|
||||
fi
|
||||
mkdir -p "$(dirname "$RUN_DIR")"
|
||||
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/rustfs-scanner-heal-evidence.XXXXXX")"
|
||||
trap 'rm -rf "$TMP_DIR"' EXIT
|
||||
|
||||
BUILD_FEATURES="${RUSTFS_BUILD_FEATURES:-}"
|
||||
cargo clean -p rustfs
|
||||
TARGET_DIR="${CARGO_TARGET_DIR:-$ROOT/target}"
|
||||
DEBUG_DIR="$TARGET_DIR/debug"
|
||||
if [[ "${RUSTFS_SCANNER_HEAL_SKIP_CLEAN:-0}" != "1" ]]; then
|
||||
cargo clean -p rustfs
|
||||
fi
|
||||
if [[ -n "$BUILD_FEATURES" ]]; then
|
||||
cargo build --locked -p rustfs --bins --features "$BUILD_FEATURES"
|
||||
else
|
||||
cargo build --locked -p rustfs --bins
|
||||
fi
|
||||
printf '%s' "$BUILD_FEATURES" >"$ROOT/target/debug/rustfs.features"
|
||||
printf '%s' "$BUILD_FEATURES" >"$DEBUG_DIR/rustfs.features"
|
||||
|
||||
LISTING_TMP="$TMP_DIR/listing.json"
|
||||
NO_PROXY="${NO_PROXY:-127.0.0.1,localhost}" \
|
||||
@@ -278,13 +346,13 @@ cargo nextest list --profile "$PROFILE" -p e2e_test -E "$TEST_FILTER" --message-
|
||||
TEST_BINARY="$(test_binary_from_listing "$LISTING_TMP" "$CASE_ID")"
|
||||
|
||||
export RUSTFS_E2E_EXPECTED_FEATURES="${RUSTFS_E2E_EXPECTED_FEATURES:-default}"
|
||||
"$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --begin-scanner-heal "$RUN_DIR" "$ROOT/target/debug/rustfs" "$TEST_BINARY"
|
||||
"$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --begin-scanner-heal "$RUN_DIR" "$DEBUG_DIR/rustfs" "$TEST_BINARY"
|
||||
cp "$LISTING_TMP" "$RUN_DIR/listing.json"
|
||||
export RUSTFS_E2E_LOG_DIR="${RUSTFS_E2E_LOG_DIR:-$RUN_DIR/e2e-logs}"
|
||||
export RUSTFS_HEAL_CHAOS_LOG_DIR="${RUSTFS_HEAL_CHAOS_LOG_DIR:-$RUSTFS_E2E_LOG_DIR}"
|
||||
mkdir -p "$RUSTFS_E2E_LOG_DIR"
|
||||
|
||||
JUNIT_PATH="$ROOT/target/nextest/$PROFILE/junit.xml"
|
||||
rm -f "$JUNIT_PATH"
|
||||
remove_nextest_junit_candidates "$TARGET_DIR" "$PROFILE"
|
||||
set +e
|
||||
NO_PROXY="${NO_PROXY:-127.0.0.1,localhost}" \
|
||||
HTTP_PROXY= \
|
||||
@@ -294,9 +362,7 @@ cargo nextest run --profile "$PROFILE" -p e2e_test -E "$TEST_FILTER" --no-tests=
|
||||
STATUS=$?
|
||||
set -e
|
||||
|
||||
if [[ -f "$JUNIT_PATH" ]]; then
|
||||
cp "$JUNIT_PATH" "$RUN_DIR/junit.xml"
|
||||
fi
|
||||
copy_nextest_junit "$TARGET_DIR" "$PROFILE" "$RUN_DIR" || true
|
||||
"$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --finish-scanner-heal "$RUN_DIR" "$STATUS"
|
||||
|
||||
if [[ "$STATUS" -ne 0 ]]; then
|
||||
|
||||
@@ -48,6 +48,11 @@ def positive_int(value: Any, name: str, minimum: int = 1) -> int:
|
||||
return value
|
||||
|
||||
|
||||
def reject_non_measured_markers(payload: dict[str, Any], label: str) -> None:
|
||||
for marker in ("fixture", "fixture_only", "dry_run", "synthetic"):
|
||||
require(payload.get(marker) is not True, f"{label} is {marker}")
|
||||
|
||||
|
||||
def parse_case_dir_arg(value: str) -> tuple[str | None, Path]:
|
||||
if "=" in value:
|
||||
case_id, raw_path = value.split("=", 1)
|
||||
@@ -72,8 +77,7 @@ def proof_case_artifact_path(proof_path: Path, sample: dict[str, Any], index: in
|
||||
|
||||
def load_proof(path: Path, source_revision: str) -> dict[str, Any]:
|
||||
proof = read_json(path)
|
||||
for marker in ("fixture", "fixture_only", "dry_run", "synthetic"):
|
||||
require(proof.get(marker) is not True, f"G14 proof is {marker}")
|
||||
reject_non_measured_markers(proof, "G14 proof")
|
||||
require(proof.get("schema") == 1, "unsupported G14 proof schema")
|
||||
require(proof.get("evidence_type") == "measured", "G14 proof must be measured")
|
||||
require(proof.get("source_revision") == source_revision, "G14 proof source revision mismatch")
|
||||
@@ -110,8 +114,7 @@ def load_proof(path: Path, source_revision: str) -> dict[str, Any]:
|
||||
f"G14 case evidence {index} measurement window mismatch")
|
||||
artifact_payload = read_json(artifact)
|
||||
require(isinstance(artifact_payload, dict), f"G14 case evidence {index} artifact must be a JSON object")
|
||||
for marker in ("fixture", "fixture_only", "dry_run", "synthetic"):
|
||||
require(artifact_payload.get(marker) is not True, f"G14 case evidence {index} artifact is {marker}")
|
||||
reject_non_measured_markers(artifact_payload, f"G14 case evidence {index} artifact")
|
||||
require(artifact_payload.get("case") == sample["case"], f"G14 case evidence {index} artifact case mismatch")
|
||||
if "source_revision" in artifact_payload:
|
||||
require(artifact_payload["source_revision"] == source_revision,
|
||||
@@ -127,6 +130,10 @@ def load_case_directory(raw_value: str, source_revision: str) -> dict[str, Any]:
|
||||
require(directory.is_dir(), f"G14 case directory is missing: {directory}")
|
||||
run = read_json(directory / "run.json")
|
||||
execution = read_json(directory / "execution.json")
|
||||
require(isinstance(run, dict), "G14 case run must be a JSON object")
|
||||
require(isinstance(execution, dict), "G14 case execution must be a JSON object")
|
||||
reject_non_measured_markers(run, "G14 case run")
|
||||
reject_non_measured_markers(execution, "G14 case execution")
|
||||
require(run.get("schema") == 1, "G14 case run schema mismatch")
|
||||
require(isinstance(run.get("run_id"), str) and re.fullmatch(r"[0-9a-f]{32}", run["run_id"]),
|
||||
"invalid G14 case run id")
|
||||
@@ -156,6 +163,7 @@ def load_case_directory(raw_value: str, source_revision: str) -> dict[str, Any]:
|
||||
if expected_case is not None:
|
||||
require(oracle.get("case") == expected_case, "G14 case directory case id mismatch")
|
||||
require(oracle.get("source_revision") == source_revision, "G14 oracle source revision mismatch")
|
||||
reject_non_measured_markers(oracle, "G14 oracle")
|
||||
require(oracle.get("evidence") in {"process-restart", "process-crash-restart"}, "G14 oracle has wrong evidence type")
|
||||
topology = oracle.get("topology")
|
||||
require(isinstance(topology, dict), "G14 oracle missing topology")
|
||||
@@ -504,6 +512,27 @@ def run_self_test() -> None:
|
||||
else:
|
||||
raise ValueError("self-test accepted case evidence without multi-pool proof")
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
source_revision = git_head()
|
||||
multi_pool = write_self_test_case_dir(root, source_revision, "background-target-crash-ec8-4-multi-pool", 3, 3)
|
||||
oracle_path = multi_pool / "background-target-crash-ec8-4-multi-pool.json"
|
||||
oracle = read_json(oracle_path)
|
||||
oracle["fixture_only"] = True
|
||||
write_json(oracle_path, oracle)
|
||||
execution = read_json(multi_pool / "execution.json")
|
||||
execution["artifacts"][oracle_path.name] = digest(oracle_path)
|
||||
write_json(multi_pool / "execution.json", execution)
|
||||
try:
|
||||
build_descriptor(parse_args([
|
||||
"--case-dir", f"background-target-crash-ec8-4-multi-pool={multi_pool}",
|
||||
"--out-dir", str(root / "out"),
|
||||
]))
|
||||
except ValueError as err:
|
||||
require("G14 oracle is fixture_only" in str(err), "wrong self-test failure for fixture oracle")
|
||||
else:
|
||||
raise ValueError("self-test accepted fixture-only G14 case oracle")
|
||||
|
||||
|
||||
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
|
||||
+715
@@ -0,0 +1,715 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Plan the Scanner/Heal Linux release-evidence validation flow.
|
||||
|
||||
The planner is intentionally not a release-evidence producer. It writes a
|
||||
machine-readable execution manifest for the existing measured runners and can
|
||||
run only their lightweight preflight checks. Long-running Linux, distributed,
|
||||
mixed-version, ABBA, and profile lanes remain explicit operator actions.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
REGISTRY = ROOT / ".config" / "scanner-heal-required-tests.json"
|
||||
DEFAULT_OUTPUT_ROOT = ROOT / "target" / "scanner-heal-linux-evidence-plan"
|
||||
REQUIRED_GATES = {
|
||||
"G01",
|
||||
"G02",
|
||||
"G03",
|
||||
"G04",
|
||||
"G05",
|
||||
"G06",
|
||||
"G07",
|
||||
"G08",
|
||||
"G09",
|
||||
"G10",
|
||||
"G11",
|
||||
"G12",
|
||||
"G13",
|
||||
"G14",
|
||||
"P1",
|
||||
"P2",
|
||||
"P3",
|
||||
"P4",
|
||||
"R-D",
|
||||
"R-E",
|
||||
"R-L",
|
||||
}
|
||||
|
||||
|
||||
def command(*parts: str) -> list[str]:
|
||||
return list(parts)
|
||||
|
||||
|
||||
def expected_outputs(*parts: str) -> list[str]:
|
||||
return list(parts)
|
||||
|
||||
|
||||
def git_output(*args: str) -> str:
|
||||
return subprocess.check_output(command("git", *args), cwd=ROOT, text=True).strip()
|
||||
|
||||
|
||||
def source_revision(explicit: str | None) -> str:
|
||||
if explicit:
|
||||
if len(explicit) != 40 or any(char not in "0123456789abcdef" for char in explicit):
|
||||
raise ValueError("--source-revision must be a 40-character lowercase Git SHA")
|
||||
return explicit
|
||||
return git_output("rev-parse", "HEAD")
|
||||
|
||||
|
||||
def load_registry() -> dict[str, Any]:
|
||||
with REGISTRY.open() as stream:
|
||||
registry = json.load(stream)
|
||||
if registry.get("schema") != 2:
|
||||
raise ValueError("Scanner/Heal release registry must use schema 2")
|
||||
return registry
|
||||
|
||||
|
||||
def validate_registry(registry: dict[str, Any]) -> None:
|
||||
gates = {item["gate"] for item in registry.get("release_requirements", [])}
|
||||
missing = sorted(REQUIRED_GATES - gates)
|
||||
if missing:
|
||||
raise ValueError(f"release registry is missing gates: {', '.join(missing)}")
|
||||
lanes = registry.get("release_lanes")
|
||||
if not isinstance(lanes, dict) or not lanes:
|
||||
raise ValueError("release registry is missing release_lanes")
|
||||
lane_gates = set()
|
||||
for lane_name, lane in lanes.items():
|
||||
if not isinstance(lane, dict):
|
||||
raise ValueError(f"release lane {lane_name} must be an object")
|
||||
lane_gates.update(lane.get("gates", []))
|
||||
missing_from_lanes = sorted(gates - lane_gates)
|
||||
if missing_from_lanes:
|
||||
raise ValueError(f"release lanes do not cover gates: {', '.join(missing_from_lanes)}")
|
||||
|
||||
|
||||
def existing_script_command(*parts: str) -> dict[str, Any]:
|
||||
for part in parts:
|
||||
if part.startswith("scripts/") and "$" not in part and not (ROOT / part).is_file():
|
||||
raise ValueError(f"missing planned script: {part}")
|
||||
return {"command": command(*parts), "script": parts[0]}
|
||||
|
||||
|
||||
def preflight_steps() -> list[dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
"id": "repo-source-clean",
|
||||
"description": "Verify the checkout revision and tracked-source cleanliness before measured evidence.",
|
||||
"commands": [
|
||||
{"command": command("git", "rev-parse", "HEAD")},
|
||||
{"command": command("git", "status", "--porcelain", "--untracked-files=no")},
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "release-registry-self-test",
|
||||
"description": "Exercise the release registry, descriptor, bundle, and negative parser cases.",
|
||||
"commands": [existing_script_command("scripts/python_bin.sh", "scripts/check_test_wiring.py", "--self-test")],
|
||||
"preflight_runnable": True,
|
||||
},
|
||||
{
|
||||
"id": "runner-self-tests",
|
||||
"description": "Run lightweight CLI/schema tests for every Scanner/Heal evidence runner.",
|
||||
"commands": [
|
||||
existing_script_command("scripts/run_scanner_heal_evidence_case.sh", "--self-test"),
|
||||
existing_script_command("scripts/test_scanner_heal_authority_evidence.sh"),
|
||||
existing_script_command("scripts/test_scanner_heal_checkpoint_crash_evidence.sh"),
|
||||
existing_script_command("scripts/test_scanner_heal_status_outcome_evidence.sh"),
|
||||
existing_script_command("scripts/test_scanner_heal_scoped_ack_evidence.sh"),
|
||||
existing_script_command("scripts/test_scanner_heal_legacy_rollback_evidence.sh"),
|
||||
existing_script_command("scripts/test_scanner_heal_g14_multiset_evidence.sh"),
|
||||
existing_script_command("scripts/test_scanner_heal_g09_upgrade_evidence.sh"),
|
||||
existing_script_command("scripts/test_scanner_heal_scheduler_pressure_evidence.sh"),
|
||||
existing_script_command("scripts/test_scanner_heal_maintenance_evidence.sh"),
|
||||
existing_script_command("scripts/test_scanner_heal_w13_mrf_evidence.sh"),
|
||||
existing_script_command("scripts/test_scanner_heal_w16_recovery_evidence.sh"),
|
||||
],
|
||||
"preflight_runnable": True,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def concrete_case_steps(registry: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
cases = registry["cases"]
|
||||
ordered_cases = [
|
||||
"background-target-restart",
|
||||
"background-target-crash",
|
||||
"background-target-restart-ec8-4",
|
||||
"background-target-crash-ec8-4",
|
||||
"ec84-target-drive-restart",
|
||||
"background-target-restart-ec8-4-multi-set",
|
||||
"background-target-crash-ec8-4-multi-pool",
|
||||
]
|
||||
missing = [case for case in ordered_cases if case not in cases]
|
||||
if missing:
|
||||
raise ValueError(f"release registry is missing concrete cases: {', '.join(missing)}")
|
||||
return [
|
||||
{
|
||||
"id": f"case-{case_id}",
|
||||
"description": cases[case_id]["scope"],
|
||||
"covers": {
|
||||
"gate": cases[case_id]["gate"],
|
||||
"task": cases[case_id]["task"],
|
||||
"lane": cases[case_id]["lane"],
|
||||
"evidence": cases[case_id]["evidence"],
|
||||
},
|
||||
"commands": [
|
||||
existing_script_command(
|
||||
"scripts/run_scanner_heal_evidence_case.sh",
|
||||
"--case",
|
||||
case_id,
|
||||
"--run-dir",
|
||||
f"$RUN_ROOT/cases/{case_id}",
|
||||
)
|
||||
],
|
||||
"expected_outputs": expected_outputs(
|
||||
f"$RUN_ROOT/cases/{case_id}/run.json",
|
||||
f"$RUN_ROOT/cases/{case_id}/execution.json",
|
||||
f"$RUN_ROOT/cases/{case_id}/listing.json",
|
||||
f"$RUN_ROOT/cases/{case_id}/junit.xml",
|
||||
f"$RUN_ROOT/cases/{case_id}/{cases[case_id]['oracle']}",
|
||||
f"$RUN_ROOT/cases/{case_id}/release-status.json",
|
||||
),
|
||||
}
|
||||
for case_id in ordered_cases
|
||||
]
|
||||
|
||||
|
||||
def descriptor_steps() -> list[dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
"id": "authority-coverage",
|
||||
"description": "Assemble the G01 authority descriptor from operator-collected root and quota authority artifacts.",
|
||||
"covers": {"gates": ["G01"], "issues": ["2270"]},
|
||||
"commands": [
|
||||
existing_script_command(
|
||||
"scripts/python_bin.sh",
|
||||
"scripts/run_scanner_heal_authority_evidence.py",
|
||||
"--root-authority-json",
|
||||
"$RUN_ROOT/raw/authority/root-authority.json",
|
||||
"--quota-authority-json",
|
||||
"$RUN_ROOT/raw/authority/quota-authority.json",
|
||||
"--out-dir",
|
||||
"$RUN_ROOT/descriptors/authority",
|
||||
)
|
||||
],
|
||||
"expected_outputs": expected_outputs("$RUN_ROOT/descriptors/authority/release-bundle-authority.json"),
|
||||
},
|
||||
{
|
||||
"id": "checkpoint-and-restart",
|
||||
"description": "Assemble G02/R-E bounded checkpoint and restart descriptors from measured diagnostic reports.",
|
||||
"covers": {"gates": ["G02", "R-E"], "issues": ["2269"]},
|
||||
"commands": [
|
||||
existing_script_command(
|
||||
"scripts/python_bin.sh",
|
||||
"scripts/run_scanner_heal_checkpoint_crash_evidence.py",
|
||||
"--diagnostic-dir",
|
||||
"$RUN_ROOT/raw/checkpoint",
|
||||
"--out-dir",
|
||||
"$RUN_ROOT/descriptors/checkpoint",
|
||||
)
|
||||
],
|
||||
"expected_outputs": expected_outputs("$RUN_ROOT/descriptors/checkpoint/release-bundle-checkpoint-crash.json"),
|
||||
},
|
||||
{
|
||||
"id": "status-and-outcome",
|
||||
"description": "Collect live G05/G06/R-D raw observations, then assemble their descriptor.",
|
||||
"covers": {"gates": ["G05", "G06", "R-D"], "issues": ["2278"]},
|
||||
"commands": [
|
||||
existing_script_command(
|
||||
"scripts/python_bin.sh",
|
||||
"scripts/run_scanner_heal_status_outcome_probe.py",
|
||||
"--observations-json",
|
||||
"$RUN_ROOT/raw/status-outcome/observations.json",
|
||||
"--out-dir",
|
||||
"$RUN_ROOT/raw/status-outcome",
|
||||
),
|
||||
existing_script_command(
|
||||
"scripts/python_bin.sh",
|
||||
"scripts/run_scanner_heal_status_outcome_evidence.py",
|
||||
"--status-outcome-json",
|
||||
"$RUN_ROOT/raw/status-outcome/status-outcome.json",
|
||||
"--status-compat-json",
|
||||
"$RUN_ROOT/raw/status-outcome/status-compat.json",
|
||||
"--disposition-json",
|
||||
"$RUN_ROOT/raw/status-outcome/disposition.json",
|
||||
"--out-dir",
|
||||
"$RUN_ROOT/descriptors/status-outcome",
|
||||
),
|
||||
],
|
||||
"expected_outputs": expected_outputs(
|
||||
"$RUN_ROOT/raw/status-outcome/status-outcome.json",
|
||||
"$RUN_ROOT/raw/status-outcome/status-compat.json",
|
||||
"$RUN_ROOT/raw/status-outcome/disposition.json",
|
||||
"$RUN_ROOT/descriptors/status-outcome/release-bundle-status-outcome.json",
|
||||
),
|
||||
},
|
||||
{
|
||||
"id": "ec8-4-multiset-descriptor",
|
||||
"description": "Assemble G14 EC8+4 multi-set and multi-pool descriptors from the measured case directories.",
|
||||
"covers": {"gates": ["G14"], "issues": ["2266", "2269"]},
|
||||
"commands": [
|
||||
existing_script_command(
|
||||
"scripts/python_bin.sh",
|
||||
"scripts/run_scanner_heal_g14_multiset_evidence.py",
|
||||
"--case-dir",
|
||||
"multi-set=$RUN_ROOT/cases/background-target-restart-ec8-4-multi-set",
|
||||
"--case-dir",
|
||||
"multi-pool=$RUN_ROOT/cases/background-target-crash-ec8-4-multi-pool",
|
||||
"--out-dir",
|
||||
"$RUN_ROOT/descriptors/g14",
|
||||
)
|
||||
],
|
||||
"expected_outputs": expected_outputs("$RUN_ROOT/descriptors/g14/release-bundle-g14.json"),
|
||||
},
|
||||
{
|
||||
"id": "w16-recovery-intent",
|
||||
"description": "Run the G04/G12 recovery-intent and quota-authority lanes.",
|
||||
"covers": {"gates": ["G04", "G12"], "issues": ["2279"]},
|
||||
"commands": [
|
||||
existing_script_command(
|
||||
"scripts/run_scanner_heal_w16_recovery_evidence.sh",
|
||||
"--run-dir",
|
||||
"$RUN_ROOT/w16",
|
||||
)
|
||||
],
|
||||
"expected_outputs": expected_outputs("$RUN_ROOT/w16/release-bundle-w16.json"),
|
||||
},
|
||||
{
|
||||
"id": "mrf-responsibility",
|
||||
"description": "Run or package G07/G08/P4 durable MRF responsibility evidence.",
|
||||
"covers": {"gates": ["G07", "G08", "P4"], "issues": ["2277", "2278"]},
|
||||
"commands": [
|
||||
existing_script_command(
|
||||
"scripts/run_scanner_heal_w13_mrf_evidence.sh",
|
||||
"--run-dir",
|
||||
"$RUN_ROOT/w13",
|
||||
)
|
||||
],
|
||||
"expected_outputs": expected_outputs("$RUN_ROOT/w13/release-bundle-w13.json"),
|
||||
},
|
||||
{
|
||||
"id": "mixed-version-rollback",
|
||||
"description": "Run G09 mixed-version/rollback, then assemble G03 and R-L operator descriptors.",
|
||||
"covers": {"gates": ["G03", "G09", "R-L"], "issues": ["2269", "2281"]},
|
||||
"commands": [
|
||||
existing_script_command(
|
||||
"scripts/run_scanner_heal_g09_upgrade_evidence.sh",
|
||||
"--run-dir",
|
||||
"$RUN_ROOT/g09",
|
||||
),
|
||||
existing_script_command(
|
||||
"scripts/python_bin.sh",
|
||||
"scripts/run_scanner_heal_scoped_ack_evidence.py",
|
||||
"--proof-json",
|
||||
"$RUN_ROOT/raw/scoped-ack/scoped-ack-proof.json",
|
||||
"--out-dir",
|
||||
"$RUN_ROOT/descriptors/scoped-ack",
|
||||
),
|
||||
existing_script_command(
|
||||
"scripts/python_bin.sh",
|
||||
"scripts/run_scanner_heal_legacy_rollback_evidence.py",
|
||||
"--proof-json",
|
||||
"$RUN_ROOT/raw/legacy-rollback/legacy-rollback-proof.json",
|
||||
"--out-dir",
|
||||
"$RUN_ROOT/descriptors/legacy-rollback",
|
||||
),
|
||||
],
|
||||
"expected_outputs": expected_outputs(
|
||||
"$RUN_ROOT/g09/release-bundle-g09.json",
|
||||
"$RUN_ROOT/descriptors/scoped-ack/release-bundle-scoped-ack.json",
|
||||
"$RUN_ROOT/descriptors/legacy-rollback/release-bundle-legacy-rollback.json",
|
||||
),
|
||||
},
|
||||
{
|
||||
"id": "maintenance-producers",
|
||||
"description": "Assemble G11/G13 producer coverage, quorum-minus-one, and remount descriptors.",
|
||||
"covers": {"gates": ["G11", "G13"], "issues": ["2272", "2280"]},
|
||||
"commands": [
|
||||
existing_script_command(
|
||||
"scripts/python_bin.sh",
|
||||
"scripts/run_scanner_heal_maintenance_evidence.py",
|
||||
"--proof-json",
|
||||
"$RUN_ROOT/raw/maintenance/maintenance-proof.json",
|
||||
"--out-dir",
|
||||
"$RUN_ROOT/descriptors/maintenance",
|
||||
)
|
||||
],
|
||||
"expected_outputs": expected_outputs("$RUN_ROOT/descriptors/maintenance/release-bundle-maintenance.json"),
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def performance_steps() -> list[dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
"id": "scanner-heal-abba",
|
||||
"description": "Run the isolated EC8+4 Scanner/Heal ABBA matrix through the deployment adapter.",
|
||||
"covers": {"gates": ["G10", "P1", "P2", "P3"], "issues": ["2266", "2273", "2274", "2275"]},
|
||||
"commands": [
|
||||
existing_script_command(
|
||||
"scripts/python_bin.sh",
|
||||
"scripts/scanner_abba.py",
|
||||
"--manifest",
|
||||
"$RUN_ROOT/abba/manifest.json",
|
||||
"--adapter",
|
||||
"$RUN_ROOT/abba/adapter.py",
|
||||
"--out-dir",
|
||||
"$RUN_ROOT/abba/out",
|
||||
"--data-root",
|
||||
"$RUN_ROOT/abba/data",
|
||||
)
|
||||
],
|
||||
"expected_outputs": expected_outputs("$RUN_ROOT/abba/out/report.json"),
|
||||
},
|
||||
{
|
||||
"id": "scheduler-pressure",
|
||||
"description": "Assemble scheduler-pressure, profile, RSS, throughput, and latency descriptors.",
|
||||
"covers": {"gates": ["G10", "P1", "P2", "P3"], "issues": ["2266", "2274", "2275"]},
|
||||
"commands": [
|
||||
existing_script_command(
|
||||
"scripts/python_bin.sh",
|
||||
"scripts/run_scanner_heal_scheduler_pressure_evidence.py",
|
||||
"--abba-dir",
|
||||
"$RUN_ROOT/abba/out",
|
||||
"--recovery-window-json",
|
||||
"$RUN_ROOT/raw/scheduler/recovery-window.json",
|
||||
"--profile-artifact",
|
||||
"allocation-profile=$RUN_ROOT/profile/allocation-profile.json",
|
||||
"--profile-artifact",
|
||||
"flamegraph=$RUN_ROOT/profile/flamegraph.svg",
|
||||
"--profile-artifact",
|
||||
"rss-samples=$RUN_ROOT/profile/rss-samples.json",
|
||||
"--profile-artifact",
|
||||
"save-frequency=$RUN_ROOT/profile/save-frequency.json",
|
||||
"--out-dir",
|
||||
"$RUN_ROOT/descriptors/scheduler-pressure",
|
||||
)
|
||||
],
|
||||
"expected_outputs": expected_outputs("$RUN_ROOT/descriptors/scheduler-pressure/release-bundle-scheduler-pressure.json"),
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def bundle_steps() -> list[dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
"id": "assemble-release-bundle",
|
||||
"description": "Assemble every measured lane descriptor into the final release bundle.",
|
||||
"covers": {"gates": sorted(REQUIRED_GATES), "issues": ["2240", "2428"]},
|
||||
"commands": [
|
||||
existing_script_command(
|
||||
"scripts/python_bin.sh",
|
||||
"scripts/check_test_wiring.py",
|
||||
"--assemble-scanner-heal-release-bundle",
|
||||
"$RUN_ROOT/descriptors/authority/release-bundle-authority.json",
|
||||
"$RUN_ROOT/descriptors/checkpoint/release-bundle-checkpoint-crash.json",
|
||||
"$RUN_ROOT/descriptors/status-outcome/release-bundle-status-outcome.json",
|
||||
"$RUN_ROOT/w16/release-bundle-w16.json",
|
||||
"$RUN_ROOT/w13/release-bundle-w13.json",
|
||||
"$RUN_ROOT/g09/release-bundle-g09.json",
|
||||
"$RUN_ROOT/descriptors/scoped-ack/release-bundle-scoped-ack.json",
|
||||
"$RUN_ROOT/descriptors/legacy-rollback/release-bundle-legacy-rollback.json",
|
||||
"$RUN_ROOT/descriptors/maintenance/release-bundle-maintenance.json",
|
||||
"$RUN_ROOT/descriptors/g14/release-bundle-g14.json",
|
||||
"$RUN_ROOT/descriptors/scheduler-pressure/release-bundle-scheduler-pressure.json",
|
||||
"$RUN_ROOT/release-bundle",
|
||||
),
|
||||
existing_script_command(
|
||||
"scripts/python_bin.sh",
|
||||
"scripts/check_test_wiring.py",
|
||||
"--check-scanner-heal-release-bundle",
|
||||
"$RUN_ROOT/release-bundle/release-evidence.json",
|
||||
),
|
||||
],
|
||||
"expected_outputs": expected_outputs("$RUN_ROOT/release-bundle/release-evidence.json"),
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def build_plan(registry: dict[str, Any], revision: str, phases: set[str]) -> dict[str, Any]:
|
||||
stage_defs = [
|
||||
("preflight", "Preflight", preflight_steps()),
|
||||
("functional", "Functional And Durable Evidence", concrete_case_steps(registry) + descriptor_steps()),
|
||||
("performance", "ABBA And Profile Evidence", performance_steps()),
|
||||
("bundle", "Release Bundle Assembly", bundle_steps()),
|
||||
]
|
||||
stages = []
|
||||
for key, title, steps in stage_defs:
|
||||
if "all" not in phases and key not in phases:
|
||||
continue
|
||||
stages.append({"id": key, "title": title, "steps": steps})
|
||||
return {
|
||||
"schema": 1,
|
||||
"kind": "scanner-heal-linux-evidence-plan",
|
||||
"evidence_type": "plan_only",
|
||||
"source_revision": revision,
|
||||
"registry": str(REGISTRY.relative_to(ROOT)),
|
||||
"run_root_env": "RUN_ROOT",
|
||||
"requirements": {
|
||||
"base_branch": "release",
|
||||
"platform": "Linux",
|
||||
"tracked_source_clean": True,
|
||||
"measured_evidence_required": True,
|
||||
"synthetic_evidence_rejected": True,
|
||||
"stop_on_product_failure": True,
|
||||
},
|
||||
"stages": stages,
|
||||
}
|
||||
|
||||
|
||||
def write_plan(out_dir: Path, plan: dict[str, Any]) -> Path:
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
path = out_dir / "scanner-heal-linux-evidence-plan.json"
|
||||
path.write_text(json.dumps(plan, indent=2, sort_keys=True) + "\n")
|
||||
return path
|
||||
|
||||
|
||||
def text_plan(plan: dict[str, Any]) -> str:
|
||||
lines = [
|
||||
f"kind={plan['kind']}",
|
||||
f"source_revision={plan['source_revision']}",
|
||||
f"registry={plan['registry']}",
|
||||
"evidence_type=plan_only",
|
||||
]
|
||||
for stage in plan["stages"]:
|
||||
lines.append(f"stage={stage['id']} steps={len(stage['steps'])}")
|
||||
for step in stage["steps"]:
|
||||
lines.append(f" step={step['id']}")
|
||||
for entry in step["commands"]:
|
||||
lines.append(" command=" + " ".join(entry["command"]))
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def iter_preflight_commands(plan: dict[str, Any]) -> list[list[str]]:
|
||||
commands: list[list[str]] = []
|
||||
for stage in plan["stages"]:
|
||||
if stage["id"] != "preflight":
|
||||
continue
|
||||
for step in stage["steps"]:
|
||||
if not step.get("preflight_runnable"):
|
||||
continue
|
||||
commands.extend(entry["command"] for entry in step["commands"])
|
||||
return commands
|
||||
|
||||
|
||||
def render_run_root_path(value: str, run_root: Path) -> Path:
|
||||
rendered = value.replace("$RUN_ROOT", str(run_root))
|
||||
return Path(rendered)
|
||||
|
||||
|
||||
def check_expected_output(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {"path": str(path), "status": "missing"}
|
||||
if not path.is_file():
|
||||
return {"path": str(path), "status": "invalid", "error": "expected a file"}
|
||||
size = path.stat().st_size
|
||||
if size <= 0:
|
||||
return {"path": str(path), "status": "empty", "bytes": size}
|
||||
return {"path": str(path), "status": "present", "bytes": size}
|
||||
|
||||
|
||||
def covered_gates(step: dict[str, Any]) -> list[str]:
|
||||
covers = step.get("covers")
|
||||
if not isinstance(covers, dict):
|
||||
return []
|
||||
gate = covers.get("gate")
|
||||
gates = covers.get("gates")
|
||||
if isinstance(gate, str):
|
||||
return [gate]
|
||||
if isinstance(gates, list):
|
||||
return [item for item in gates if isinstance(item, str)]
|
||||
return []
|
||||
|
||||
|
||||
def build_status(plan: dict[str, Any], run_root: Path) -> dict[str, Any]:
|
||||
run_root = run_root.resolve()
|
||||
stage_statuses = []
|
||||
totals = {"present": 0, "missing": 0, "empty": 0, "invalid": 0, "expected": 0}
|
||||
pending_gates: set[str] = set()
|
||||
next_step: dict[str, Any] | None = None
|
||||
for stage in plan["stages"]:
|
||||
step_statuses = []
|
||||
for step in stage["steps"]:
|
||||
outputs = [
|
||||
check_expected_output(render_run_root_path(path, run_root))
|
||||
for path in step.get("expected_outputs", [])
|
||||
]
|
||||
counts = {"present": 0, "missing": 0, "empty": 0, "invalid": 0}
|
||||
for output in outputs:
|
||||
counts[output["status"]] += 1
|
||||
for key in counts:
|
||||
totals[key] += counts[key]
|
||||
totals["expected"] += len(outputs)
|
||||
if not outputs:
|
||||
status = "not_tracked"
|
||||
elif counts["invalid"] or counts["empty"]:
|
||||
status = "invalid"
|
||||
elif counts["missing"]:
|
||||
status = "pending" if counts["present"] == 0 else "partial"
|
||||
else:
|
||||
status = "complete"
|
||||
if status in {"pending", "partial", "invalid"}:
|
||||
pending_gates.update(covered_gates(step))
|
||||
if next_step is None:
|
||||
next_step = {
|
||||
"stage": stage["id"],
|
||||
"step": step["id"],
|
||||
"status": status,
|
||||
"commands": step["commands"],
|
||||
}
|
||||
step_statuses.append({
|
||||
"id": step["id"],
|
||||
"status": status,
|
||||
"covers": step.get("covers", {}),
|
||||
"outputs": outputs,
|
||||
})
|
||||
tracked = [step for step in step_statuses if step["status"] != "not_tracked"]
|
||||
if not tracked:
|
||||
stage_state = "not_tracked"
|
||||
elif all(step["status"] == "complete" for step in tracked):
|
||||
stage_state = "complete"
|
||||
elif any(step["status"] == "invalid" for step in tracked):
|
||||
stage_state = "invalid"
|
||||
elif any(step["status"] in {"complete", "partial"} for step in tracked):
|
||||
stage_state = "partial"
|
||||
else:
|
||||
stage_state = "pending"
|
||||
stage_statuses.append({"id": stage["id"], "status": stage_state, "steps": step_statuses})
|
||||
if totals["invalid"] or totals["empty"]:
|
||||
decision = "invalid"
|
||||
elif totals["missing"]:
|
||||
decision = "blocked"
|
||||
else:
|
||||
decision = "complete"
|
||||
return {
|
||||
"schema": 1,
|
||||
"kind": "scanner-heal-linux-evidence-status",
|
||||
"decision": decision,
|
||||
"release_approved": False,
|
||||
"source_revision": plan["source_revision"],
|
||||
"run_root": str(run_root),
|
||||
"artifact_totals": totals,
|
||||
"pending_gates": sorted(pending_gates),
|
||||
"next_step": next_step,
|
||||
"stages": stage_statuses,
|
||||
}
|
||||
|
||||
|
||||
def run_preflight(plan: dict[str, Any]) -> int:
|
||||
commands = iter_preflight_commands(plan)
|
||||
if not commands:
|
||||
raise ValueError("--run-preflight requires the preflight stage")
|
||||
failures = []
|
||||
for args in commands:
|
||||
result = subprocess.run(args, cwd=ROOT)
|
||||
if result.returncode != 0:
|
||||
failures.append({"command": args, "exit_code": result.returncode})
|
||||
break
|
||||
if failures:
|
||||
print(json.dumps({"status": "failed", "failures": failures}, indent=2), file=sys.stderr)
|
||||
return 1
|
||||
print(json.dumps({"status": "passed", "commands": len(commands)}, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
def self_test() -> None:
|
||||
registry = load_registry()
|
||||
validate_registry(registry)
|
||||
revision = git_output("rev-parse", "HEAD")
|
||||
plan = build_plan(registry, revision, {"all"})
|
||||
assert plan["evidence_type"] == "plan_only"
|
||||
stages = {stage["id"]: stage for stage in plan["stages"]}
|
||||
assert set(stages) == {"preflight", "functional", "performance", "bundle"}
|
||||
commands = [
|
||||
" ".join(entry["command"])
|
||||
for stage in plan["stages"]
|
||||
for step in stage["steps"]
|
||||
for entry in step["commands"]
|
||||
]
|
||||
assert not any("--case release" in item for item in commands)
|
||||
assert any("run_scanner_heal_g09_upgrade_evidence.sh" in item for item in commands)
|
||||
assert any("scanner_abba.py" in item for item in commands)
|
||||
assert any("--check-scanner-heal-release-bundle" in item for item in commands)
|
||||
preflight = iter_preflight_commands(plan)
|
||||
assert preflight
|
||||
assert all(args[0].startswith("scripts/") for args in preflight)
|
||||
out_dir = Path(os.environ.get("TMPDIR", "/tmp")) / "rustfs-scanner-heal-linux-plan-self-test"
|
||||
path = write_plan(out_dir, plan)
|
||||
loaded = json.loads(path.read_text())
|
||||
assert loaded["source_revision"] == revision
|
||||
status = build_status(plan, out_dir / "empty-run")
|
||||
assert status["decision"] == "blocked"
|
||||
assert status["release_approved"] is False
|
||||
assert status["pending_gates"]
|
||||
complete_root = out_dir / "complete-run"
|
||||
for stage in plan["stages"]:
|
||||
for step in stage["steps"]:
|
||||
for output in step.get("expected_outputs", []):
|
||||
path = render_run_root_path(output, complete_root)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text("{}\n")
|
||||
complete = build_status(plan, complete_root)
|
||||
assert complete["decision"] == "complete"
|
||||
assert complete["release_approved"] is False
|
||||
print("PASS: scanner/heal Linux evidence plan self-test")
|
||||
|
||||
|
||||
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--phase", action="append", choices=("all", "preflight", "functional", "performance", "bundle"), default=[])
|
||||
parser.add_argument("--source-revision")
|
||||
parser.add_argument("--out-dir", type=Path, default=DEFAULT_OUTPUT_ROOT)
|
||||
parser.add_argument("--write-plan", action="store_true")
|
||||
parser.add_argument("--format", choices=("text", "json"), default="text")
|
||||
parser.add_argument("--run-preflight", action="store_true")
|
||||
parser.add_argument("--status-root", type=Path)
|
||||
parser.add_argument("--self-test", action="store_true")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = parse_args(argv)
|
||||
if args.self_test:
|
||||
self_test()
|
||||
return 0
|
||||
try:
|
||||
registry = load_registry()
|
||||
validate_registry(registry)
|
||||
phases = set(args.phase or ["all"])
|
||||
if "all" in phases and len(phases) > 1:
|
||||
raise ValueError("--phase all cannot be combined with another phase")
|
||||
if args.status_root is not None and (args.write_plan or args.run_preflight):
|
||||
raise ValueError("--status-root cannot be combined with --write-plan or --run-preflight")
|
||||
plan = build_plan(registry, source_revision(args.source_revision), phases)
|
||||
if args.status_root is not None:
|
||||
status = build_status(plan, args.status_root)
|
||||
print(json.dumps(status, indent=2, sort_keys=True))
|
||||
return 0 if status["decision"] == "complete" else 3
|
||||
if args.write_plan:
|
||||
path = write_plan(args.out_dir.resolve(), plan)
|
||||
print(path)
|
||||
elif args.run_preflight:
|
||||
pass
|
||||
elif args.format == "json":
|
||||
print(json.dumps(plan, indent=2, sort_keys=True))
|
||||
else:
|
||||
print(text_plan(plan))
|
||||
if args.run_preflight:
|
||||
return run_preflight(plan)
|
||||
return 0
|
||||
except (AssertionError, ValueError, OSError, subprocess.CalledProcessError) as error:
|
||||
print(f"ERROR: {error}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -146,13 +146,16 @@ def build_descriptor(args: argparse.Namespace) -> Path:
|
||||
"gates": gates,
|
||||
})
|
||||
for gate in ("G11", "G13"):
|
||||
subprocess.check_call([
|
||||
check = subprocess.run([
|
||||
sys.executable,
|
||||
str(ROOT / "scripts/check_test_wiring.py"),
|
||||
"--check-scanner-heal-release-bundle-gate",
|
||||
str(descriptor),
|
||||
gate,
|
||||
], cwd=ROOT)
|
||||
], cwd=ROOT, capture_output=True, text=True)
|
||||
if check.returncode != 0:
|
||||
details = "\n".join(part for part in (check.stdout.strip(), check.stderr.strip()) if part)
|
||||
raise ValueError(details or f"{gate} release bundle gate check failed")
|
||||
return descriptor
|
||||
|
||||
|
||||
@@ -188,8 +191,8 @@ def write_self_test_proof(path: Path, source_revision: str) -> None:
|
||||
"unknown_producer_excluded": True,
|
||||
},
|
||||
"segment_activation_preflight": {
|
||||
"production_activation": False,
|
||||
"scanner_segment_reuse_activated": False,
|
||||
"production_activation": True,
|
||||
"scanner_segment_reuse_activated": True,
|
||||
"proof_inputs": list(wiring.SCANNER_HEAL_SEGMENT_ACTIVATION_PROOF_INPUTS),
|
||||
"fail_closed_checks": list(wiring.SCANNER_HEAL_SEGMENT_ACTIVATION_FAIL_CLOSED_CHECKS),
|
||||
},
|
||||
@@ -243,6 +246,21 @@ def run_self_test() -> None:
|
||||
else:
|
||||
raise ValueError("self-test accepted synthetic proof")
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
source_revision = git_head()
|
||||
proof = root / "maintenance-proof.json"
|
||||
write_self_test_proof(proof, source_revision)
|
||||
payload = wiring.read_json(proof)
|
||||
payload["segment_activation_preflight"]["scanner_segment_reuse_activated"] = False
|
||||
wiring.write_json(proof, payload)
|
||||
try:
|
||||
build_descriptor(parse_args(["--proof-json", str(proof), "--out-dir", str(root / "out")]))
|
||||
except ValueError as err:
|
||||
wiring.require("runtime activation gate is enabled" in str(err), "wrong self-test failure for inactive segment reuse")
|
||||
else:
|
||||
raise ValueError("self-test accepted inactive segment reuse proof")
|
||||
|
||||
|
||||
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
|
||||
+470
@@ -0,0 +1,470 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Collect Scanner/Heal status-and-outcome raw evidence from live observations.
|
||||
|
||||
This helper normalizes operator-collected live observation JSON into the three
|
||||
measured raw artifacts consumed by run_scanner_heal_status_outcome_evidence.py.
|
||||
It does not approve a release bundle by itself.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import check_test_wiring as wiring
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def git_head() -> str:
|
||||
return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip()
|
||||
|
||||
|
||||
def timestamp(value: Any, name: str) -> str:
|
||||
wiring.require(isinstance(value, str) and value.endswith("Z"), f"invalid {name}")
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
wiring.require(parsed.tzinfo is not None, f"{name} must include timezone")
|
||||
return parsed.isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def identity_string(value: Any, name: str) -> str:
|
||||
wiring.require(
|
||||
isinstance(value, str) and re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._:-]{7,127}", value),
|
||||
f"invalid {name}",
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
def measured_observation(path: Path, source_revision: str) -> dict[str, Any]:
|
||||
payload = wiring.read_json(path.resolve())
|
||||
wiring.require(isinstance(payload, dict), "observation must be a JSON object")
|
||||
for marker in ("fixture", "fixture_only", "dry_run", "synthetic"):
|
||||
wiring.require(payload.get(marker) is not True, f"observation is {marker}")
|
||||
wiring.require(payload.get("schema") == 1, "observation schema must be 1")
|
||||
wiring.require(payload.get("evidence_type") == "measured", "observation must be measured")
|
||||
wiring.require(payload.get("source_revision") == source_revision, "observation source revision mismatch")
|
||||
run_id = identity_string(payload.get("run_id"), "run_id")
|
||||
window_id = identity_string(payload.get("measurement_window_id"), "measurement_window_id")
|
||||
wiring.require(run_id != window_id, "run/window identities must differ")
|
||||
started = timestamp(payload.get("started_at"), "started_at")
|
||||
finished = timestamp(payload.get("finished_at"), "finished_at")
|
||||
wiring.require(
|
||||
datetime.fromisoformat(finished.replace("Z", "+00:00"))
|
||||
>= datetime.fromisoformat(started.replace("Z", "+00:00")),
|
||||
"finished_at precedes started_at",
|
||||
)
|
||||
wiring.require(isinstance(payload.get("command"), list) and payload["command"], "missing command provenance")
|
||||
return payload
|
||||
|
||||
|
||||
def object_items(payload: dict[str, Any], key: str) -> list[dict[str, Any]]:
|
||||
items = payload.get(key)
|
||||
wiring.require(isinstance(items, list) and items, f"missing {key}")
|
||||
for index, item in enumerate(items):
|
||||
wiring.require(isinstance(item, dict), f"{key}[{index}] must be an object")
|
||||
return items
|
||||
|
||||
|
||||
def case_map(items: list[dict[str, Any]], key: str, expected: tuple[str, ...]) -> dict[str, dict[str, Any]]:
|
||||
observed: dict[str, dict[str, Any]] = {}
|
||||
for item in items:
|
||||
case = item.get("case")
|
||||
wiring.require(isinstance(case, str) and case, f"{key} item missing case")
|
||||
wiring.require(case not in observed, f"{key} duplicate case: {case}")
|
||||
observed[case] = item
|
||||
missing = [case for case in expected if case not in observed]
|
||||
unknown = [case for case in observed if case not in expected]
|
||||
wiring.require(not missing, f"{key} missing cases: {', '.join(missing)}")
|
||||
wiring.require(not unknown, f"{key} unknown cases: {', '.join(unknown)}")
|
||||
return observed
|
||||
|
||||
|
||||
def bool_true(value: Any, name: str) -> None:
|
||||
wiring.release_bundle_bool_true(value, name)
|
||||
|
||||
|
||||
def status_outcome(payload: dict[str, Any], common: dict[str, Any]) -> dict[str, Any]:
|
||||
outcome_items = case_map(
|
||||
object_items(payload, "status_outcomes"),
|
||||
"status_outcomes",
|
||||
wiring.SCANNER_HEAL_RELEASE_G05_PER_OBJECT_OUTCOME_CASES,
|
||||
)
|
||||
counts = {"repaired": 0, "healthy": 0, "skipped": 0, "failed": 0}
|
||||
for case, item in outcome_items.items():
|
||||
outcome = item.get("outcome")
|
||||
wiring.require(outcome in counts, f"status_outcomes.{case} has invalid outcome")
|
||||
bool_true(item.get("status_matches_object_oracle"), f"status_outcomes.{case}.status_matches_object_oracle")
|
||||
counts[outcome] += 1
|
||||
for outcome, count in counts.items():
|
||||
wiring.require(count > 0, f"missing measured {outcome} outcome")
|
||||
|
||||
retention = case_map(
|
||||
object_items(payload, "terminal_retention_samples"),
|
||||
"terminal_retention_samples",
|
||||
wiring.SCANNER_HEAL_RELEASE_G05_TERMINAL_RETENTION_CASES,
|
||||
)
|
||||
window = wiring.evidence_integer(
|
||||
payload.get("terminal_retention_window_seconds"),
|
||||
"terminal_retention_window_seconds",
|
||||
1,
|
||||
86400,
|
||||
)
|
||||
max_age = 0
|
||||
pruned = 0
|
||||
for case, item in retention.items():
|
||||
bool_true(item.get("retained_until_window"), f"terminal_retention_samples.{case}.retained_until_window")
|
||||
max_age = max(
|
||||
max_age,
|
||||
wiring.evidence_integer(
|
||||
item.get("max_age_seconds"),
|
||||
f"terminal_retention_samples.{case}.max_age_seconds",
|
||||
0,
|
||||
86400,
|
||||
),
|
||||
)
|
||||
if item.get("pruned_after_window") is True:
|
||||
pruned += 1
|
||||
wiring.require(max_age <= window, "terminal retention max age exceeds window")
|
||||
wiring.require(pruned > 0, "no terminal records were pruned after the retention window")
|
||||
|
||||
return {
|
||||
**common,
|
||||
"per_object_outcome_cases": list(wiring.SCANNER_HEAL_RELEASE_G05_PER_OBJECT_OUTCOME_CASES),
|
||||
"outcome_counts": counts,
|
||||
"status_matches_object_oracle": True,
|
||||
"terminal_retention_cases": list(wiring.SCANNER_HEAL_RELEASE_G05_TERMINAL_RETENTION_CASES),
|
||||
"terminal_retention_window_seconds": window,
|
||||
"max_terminal_record_age_seconds": max_age,
|
||||
"terminal_records_pruned_after_window": pruned,
|
||||
}
|
||||
|
||||
|
||||
def status_compat(payload: dict[str, Any], common: dict[str, Any]) -> dict[str, Any]:
|
||||
status = case_map(
|
||||
object_items(payload, "status_samples"),
|
||||
"status_samples",
|
||||
wiring.SCANNER_HEAL_RELEASE_G06_CONCURRENT_STATUS_CASES,
|
||||
)
|
||||
status_samples = 0
|
||||
degraded = False
|
||||
for case, item in status.items():
|
||||
bool_true(item.get("http_success"), f"status_samples.{case}.http_success")
|
||||
status_samples += wiring.evidence_integer(item.get("samples"), f"status_samples.{case}.samples", 1, 2**31 - 1)
|
||||
degraded = degraded or item.get("degraded") is True
|
||||
|
||||
legacy = case_map(
|
||||
object_items(payload, "legacy_client_samples"),
|
||||
"legacy_client_samples",
|
||||
wiring.SCANNER_HEAL_RELEASE_G06_LEGACY_CLIENT_CASES,
|
||||
)
|
||||
for case, item in legacy.items():
|
||||
bool_true(item.get("accepted"), f"legacy_client_samples.{case}.accepted")
|
||||
rustfs_and_minio = (
|
||||
legacy["rustfs-admin-v3-background-heal-status"].get("path_compatible") is True
|
||||
and legacy["minio-admin-v3-background-heal-status"].get("path_compatible") is True
|
||||
)
|
||||
empty_body = legacy["heal-client-token-empty-body"].get("empty_body_accepted") is True
|
||||
bool_true(rustfs_and_minio, "rustfs and minio status path compatibility")
|
||||
bool_true(empty_body, "empty body heal status request")
|
||||
|
||||
truncation = case_map(
|
||||
object_items(payload, "truncation_samples"),
|
||||
"truncation_samples",
|
||||
wiring.SCANNER_HEAL_RELEASE_G06_TRUNCATION_CASES,
|
||||
)
|
||||
max_payload = 1
|
||||
for case, item in truncation.items():
|
||||
bool_true(item.get("rejected"), f"truncation_samples.{case}.rejected")
|
||||
max_payload = max(
|
||||
max_payload,
|
||||
wiring.evidence_integer(item.get("payload_bytes"), f"truncation_samples.{case}.payload_bytes", 1, 2**20),
|
||||
)
|
||||
|
||||
return {
|
||||
**common,
|
||||
"concurrent_status_cases": list(wiring.SCANNER_HEAL_RELEASE_G06_CONCURRENT_STATUS_CASES),
|
||||
"status_samples": status_samples,
|
||||
"all_status_responses_http_success": True,
|
||||
"partial_status_reports_degraded": degraded,
|
||||
"legacy_client_cases": list(wiring.SCANNER_HEAL_RELEASE_G06_LEGACY_CLIENT_CASES),
|
||||
"rustfs_and_minio_paths_compatible": rustfs_and_minio,
|
||||
"empty_body_status_requests_accepted": empty_body,
|
||||
"truncation_cases": list(wiring.SCANNER_HEAL_RELEASE_G06_TRUNCATION_CASES),
|
||||
"truncated_payloads_rejected": True,
|
||||
"max_status_payload_bytes": max_payload,
|
||||
}
|
||||
|
||||
|
||||
def disposition(payload: dict[str, Any], common: dict[str, Any]) -> dict[str, Any]:
|
||||
managers = case_map(
|
||||
object_items(payload, "manager_dispositions"),
|
||||
"manager_dispositions",
|
||||
wiring.SCANNER_HEAL_RELEASE_RD_MANAGER_CASES,
|
||||
)
|
||||
for case, item in managers.items():
|
||||
bool_true(item.get("terminal"), f"manager_dispositions.{case}.terminal")
|
||||
|
||||
events = case_map(
|
||||
object_items(payload, "event_dispositions"),
|
||||
"event_dispositions",
|
||||
wiring.SCANNER_HEAL_RELEASE_RD_EVENT_CASES,
|
||||
)
|
||||
for case, item in events.items():
|
||||
bool_true(item.get("correlates_to_manager"), f"event_dispositions.{case}.correlates_to_manager")
|
||||
|
||||
ledgers = case_map(
|
||||
object_items(payload, "ledger_dispositions"),
|
||||
"ledger_dispositions",
|
||||
wiring.SCANNER_HEAL_RELEASE_RD_LEDGER_CASES,
|
||||
)
|
||||
for case, item in ledgers.items():
|
||||
bool_true(item.get("correlates_to_events"), f"ledger_dispositions.{case}.correlates_to_events")
|
||||
bool_true(item.get("replay_preserves_terminal"), f"ledger_dispositions.{case}.replay_preserves_terminal")
|
||||
|
||||
grace = case_map(
|
||||
object_items(payload, "grace_samples"),
|
||||
"grace_samples",
|
||||
wiring.SCANNER_HEAL_RELEASE_RD_GRACE_CASES,
|
||||
)
|
||||
grace_window = wiring.evidence_integer(payload.get("grace_window_seconds"), "grace_window_seconds", 1, 86400)
|
||||
retained = False
|
||||
pruned = False
|
||||
for item in grace.values():
|
||||
retained = retained or item.get("retention_observed") is True
|
||||
pruned = pruned or item.get("expiry_pruned_terminal_records") is True
|
||||
bool_true(retained, "grace retention observed")
|
||||
bool_true(pruned, "grace expiry pruned terminal records")
|
||||
|
||||
return {
|
||||
**common,
|
||||
"manager_disposition_cases": list(wiring.SCANNER_HEAL_RELEASE_RD_MANAGER_CASES),
|
||||
"manager_dispositions_are_terminal": True,
|
||||
"event_disposition_cases": list(wiring.SCANNER_HEAL_RELEASE_RD_EVENT_CASES),
|
||||
"events_correlate_to_manager_dispositions": True,
|
||||
"ledger_disposition_cases": list(wiring.SCANNER_HEAL_RELEASE_RD_LEDGER_CASES),
|
||||
"ledger_correlates_to_events": True,
|
||||
"ledger_replay_preserves_terminal_disposition": True,
|
||||
"grace_cases": list(wiring.SCANNER_HEAL_RELEASE_RD_GRACE_CASES),
|
||||
"grace_window_seconds": grace_window,
|
||||
"grace_retention_observed": retained,
|
||||
"grace_expiry_pruned_terminal_records": pruned,
|
||||
}
|
||||
|
||||
|
||||
def common_raw(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"schema": 1,
|
||||
"evidence_type": "measured",
|
||||
"source_revision": payload["source_revision"],
|
||||
"run_id": payload["run_id"],
|
||||
"measurement_window_id": payload["measurement_window_id"],
|
||||
"started_at": payload["started_at"],
|
||||
"finished_at": payload["finished_at"],
|
||||
"command": payload["command"],
|
||||
}
|
||||
|
||||
|
||||
def collect(args: argparse.Namespace) -> tuple[Path, Path, Path]:
|
||||
out_dir = args.out_dir.resolve()
|
||||
wiring.require(not out_dir.exists(), "output directory must be new")
|
||||
source_revision = args.source_revision or git_head()
|
||||
observed = measured_observation(args.observations_json, source_revision)
|
||||
common = common_raw(observed)
|
||||
out_dir.mkdir(parents=True)
|
||||
status_outcome_path = out_dir / "status-outcome.json"
|
||||
status_compat_path = out_dir / "status-compat.json"
|
||||
disposition_path = out_dir / "disposition.json"
|
||||
wiring.write_json(status_outcome_path, status_outcome(observed, common))
|
||||
wiring.write_json(status_compat_path, status_compat(observed, common))
|
||||
wiring.write_json(disposition_path, disposition(observed, common))
|
||||
return status_outcome_path, status_compat_path, disposition_path
|
||||
|
||||
|
||||
def write_self_test_observation(path: Path, source_revision: str) -> None:
|
||||
now = datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
||||
payload = {
|
||||
"schema": 1,
|
||||
"evidence_type": "measured",
|
||||
"source_revision": source_revision,
|
||||
"run_id": f"status-probe-{source_revision[:12]}",
|
||||
"measurement_window_id": f"status-probe-window-{source_revision[:12]}",
|
||||
"started_at": now,
|
||||
"finished_at": now,
|
||||
"command": ["scripts/run_scanner_heal_status_outcome_probe.py", "--observations-json", "<live-observations>"],
|
||||
"terminal_retention_window_seconds": 3600,
|
||||
"grace_window_seconds": 300,
|
||||
"status_outcomes": [
|
||||
{"case": "object-repaired", "outcome": "repaired", "status_matches_object_oracle": True},
|
||||
{"case": "object-already-healthy", "outcome": "healthy", "status_matches_object_oracle": True},
|
||||
{"case": "object-skipped-by-policy", "outcome": "skipped", "status_matches_object_oracle": True},
|
||||
{"case": "object-failed-and-retained", "outcome": "failed", "status_matches_object_oracle": True},
|
||||
],
|
||||
"terminal_retention_samples": [
|
||||
{"case": "finished-retained-until-window", "retained_until_window": True, "max_age_seconds": 1200},
|
||||
{"case": "failed-retained-until-window", "retained_until_window": True, "max_age_seconds": 1300},
|
||||
{"case": "canceled-retained-until-window", "retained_until_window": True, "max_age_seconds": 1400},
|
||||
{
|
||||
"case": "expired-terminal-pruned-after-window",
|
||||
"retained_until_window": True,
|
||||
"max_age_seconds": 3599,
|
||||
"pruned_after_window": True,
|
||||
},
|
||||
],
|
||||
"status_samples": [
|
||||
{"case": "status-during-admin-heal", "samples": 2, "http_success": True},
|
||||
{"case": "status-during-background-heal", "samples": 2, "http_success": True},
|
||||
{"case": "status-while-peer-down", "samples": 2, "http_success": True, "degraded": True},
|
||||
{"case": "status-after-peer-rejoin", "samples": 2, "http_success": True},
|
||||
],
|
||||
"legacy_client_samples": [
|
||||
{"case": "rustfs-admin-v3-background-heal-status", "accepted": True, "path_compatible": True},
|
||||
{"case": "minio-admin-v3-background-heal-status", "accepted": True, "path_compatible": True},
|
||||
{"case": "heal-client-token-empty-body", "accepted": True, "empty_body_accepted": True},
|
||||
{"case": "node-heal-status-v1-wire", "accepted": True},
|
||||
],
|
||||
"truncation_samples": [
|
||||
{"case": "oversize-node-status-reject", "rejected": True, "payload_bytes": 1048576},
|
||||
{"case": "truncated-node-status-reject", "rejected": True, "payload_bytes": 4096},
|
||||
{"case": "trailing-data-node-status-reject", "rejected": True, "payload_bytes": 4096},
|
||||
],
|
||||
"manager_dispositions": [
|
||||
{"case": "accepted", "terminal": True},
|
||||
{"case": "coalesced-duplicate", "terminal": True},
|
||||
{"case": "rejected-policy", "terminal": True},
|
||||
{"case": "terminal-retained", "terminal": True},
|
||||
],
|
||||
"event_dispositions": [
|
||||
{"case": "event-repaired", "correlates_to_manager": True},
|
||||
{"case": "event-failed", "correlates_to_manager": True},
|
||||
{"case": "event-skipped", "correlates_to_manager": True},
|
||||
{"case": "event-grace-retained", "correlates_to_manager": True},
|
||||
],
|
||||
"ledger_dispositions": [
|
||||
{"case": "ledger-recorded", "correlates_to_events": True, "replay_preserves_terminal": True},
|
||||
{"case": "ledger-replayed", "correlates_to_events": True, "replay_preserves_terminal": True},
|
||||
{"case": "ledger-discharged", "correlates_to_events": True, "replay_preserves_terminal": True},
|
||||
{"case": "ledger-pruned-after-grace", "correlates_to_events": True, "replay_preserves_terminal": True},
|
||||
],
|
||||
"grace_samples": [
|
||||
{"case": "grace-open-retains-disposition", "retention_observed": True},
|
||||
{"case": "grace-expired-prunes-terminal", "expiry_pruned_terminal_records": True},
|
||||
{"case": "restart-preserves-grace-clock", "retention_observed": True},
|
||||
],
|
||||
}
|
||||
wiring.write_json(path, payload)
|
||||
|
||||
|
||||
def run_self_test() -> None:
|
||||
import tempfile
|
||||
|
||||
source_revision = git_head()
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
observation = root / "observation.json"
|
||||
write_self_test_observation(observation, source_revision)
|
||||
raw_paths = collect(parse_args([
|
||||
"--observations-json",
|
||||
str(observation),
|
||||
"--out-dir",
|
||||
str(root / "raw"),
|
||||
]))
|
||||
descriptor = root / "descriptor"
|
||||
subprocess.check_call([
|
||||
sys.executable,
|
||||
str(ROOT / "scripts/run_scanner_heal_status_outcome_evidence.py"),
|
||||
"--status-outcome-json",
|
||||
str(raw_paths[0]),
|
||||
"--status-compat-json",
|
||||
str(raw_paths[1]),
|
||||
"--disposition-json",
|
||||
str(raw_paths[2]),
|
||||
"--out-dir",
|
||||
str(descriptor),
|
||||
], cwd=ROOT)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
observation = root / "observation.json"
|
||||
write_self_test_observation(observation, source_revision)
|
||||
payload = wiring.read_json(observation)
|
||||
payload["status_outcomes"].pop()
|
||||
wiring.write_json(observation, payload)
|
||||
try:
|
||||
collect(parse_args(["--observations-json", str(observation), "--out-dir", str(root / "raw")]))
|
||||
except ValueError as err:
|
||||
wiring.require("status_outcomes missing cases" in str(err), "wrong self-test failure for missing outcome")
|
||||
else:
|
||||
raise ValueError("self-test accepted incomplete status outcome observations")
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
observation = root / "observation.json"
|
||||
write_self_test_observation(observation, source_revision)
|
||||
payload = wiring.read_json(observation)
|
||||
payload["synthetic"] = True
|
||||
wiring.write_json(observation, payload)
|
||||
try:
|
||||
collect(parse_args(["--observations-json", str(observation), "--out-dir", str(root / "raw")]))
|
||||
except ValueError as err:
|
||||
wiring.require("observation is synthetic" in str(err), "wrong self-test failure for synthetic observation")
|
||||
else:
|
||||
raise ValueError("self-test accepted synthetic status/outcome observations")
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
observation = root / "observation.json"
|
||||
write_self_test_observation(observation, source_revision)
|
||||
payload = wiring.read_json(observation)
|
||||
payload["measurement_window_id"] = payload["run_id"]
|
||||
wiring.write_json(observation, payload)
|
||||
try:
|
||||
collect(parse_args(["--observations-json", str(observation), "--out-dir", str(root / "raw")]))
|
||||
except ValueError as err:
|
||||
wiring.require("run/window identities must differ" in str(err), "wrong self-test failure for identity reuse")
|
||||
else:
|
||||
raise ValueError("self-test accepted reused run/window identities")
|
||||
|
||||
|
||||
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--observations-json", type=Path)
|
||||
parser.add_argument("--out-dir", type=Path)
|
||||
parser.add_argument("--source-revision")
|
||||
parser.add_argument("--self-test", action="store_true")
|
||||
args = parser.parse_args(argv)
|
||||
if not args.self_test:
|
||||
if args.observations_json is None:
|
||||
parser.error("--observations-json is required unless --self-test is used")
|
||||
if args.out_dir is None:
|
||||
parser.error("--out-dir is required unless --self-test is used")
|
||||
return args
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
if args.self_test:
|
||||
run_self_test()
|
||||
return
|
||||
paths = collect(args)
|
||||
json.dump(
|
||||
{
|
||||
"status_outcome_json": str(paths[0]),
|
||||
"status_compat_json": str(paths[1]),
|
||||
"disposition_json": str(paths[2]),
|
||||
},
|
||||
sys.stdout,
|
||||
indent=2,
|
||||
allow_nan=False,
|
||||
)
|
||||
sys.stdout.write("\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -4,6 +4,8 @@ import unittest
|
||||
|
||||
from diagnose_scanner_enumeration_restart import (
|
||||
converged,
|
||||
final_complete_recheck_after_full_retention,
|
||||
fully_retained,
|
||||
replays_raw_window,
|
||||
validate_recoverable_quantum,
|
||||
validate_report,
|
||||
@@ -29,6 +31,7 @@ class ReportTests(unittest.TestCase):
|
||||
report = self.report()
|
||||
self.validate(report)
|
||||
self.assertTrue(converged(report, 4))
|
||||
self.assertTrue(fully_retained(report, 4))
|
||||
|
||||
def test_incomplete_or_inexact_coverage_cannot_pass(self):
|
||||
for key, value in (("snapshot_complete", False), ("objects_retained", 3),
|
||||
@@ -132,6 +135,9 @@ class ReportTests(unittest.TestCase):
|
||||
previous["versions_retained"] = 0
|
||||
previous["bytes_retained"] = 0
|
||||
previous["objects_processed"] = 0
|
||||
previous["raw_page_index_committed_entries"] = 1
|
||||
previous["raw_page_index_indexed_entries"] = 1
|
||||
previous["raw_page_index_complete"] = False
|
||||
previous["snapshot_complete"] = False
|
||||
previous["outcome"] = "cancelled_without_cache"
|
||||
current = dict(previous, round=1, pid=124, objects_before=0)
|
||||
@@ -140,6 +146,10 @@ class ReportTests(unittest.TestCase):
|
||||
advanced = dict(current, objects_retained=1)
|
||||
self.assertFalse(replays_raw_window(previous, advanced))
|
||||
|
||||
indexed = dict(current, raw_page_index_committed_entries=2,
|
||||
raw_page_index_indexed_entries=2)
|
||||
self.assertFalse(replays_raw_window(previous, indexed))
|
||||
|
||||
def test_recoverable_quantum_rejects_replayed_raw_window(self):
|
||||
previous = self.report()
|
||||
previous.update(objects_retained=0, versions_retained=0, bytes_retained=0,
|
||||
@@ -149,6 +159,35 @@ class ReportTests(unittest.TestCase):
|
||||
with self.assertRaisesRegex(ValueError, "raw enumeration window replayed"):
|
||||
validate_recoverable_quantum([previous, current], objects=4, budget=16, require_converged=False)
|
||||
|
||||
def test_recoverable_quantum_allows_final_complete_round_after_full_retention(self):
|
||||
previous = self.report()
|
||||
previous.update(raw_entries=8, raw_page_index_committed_entries=4,
|
||||
raw_page_index_indexed_entries=4, objects_before=2,
|
||||
objects_processed=2, objects_retained=4,
|
||||
versions_retained=4, bytes_retained=4,
|
||||
snapshot_complete=False, outcome="partial")
|
||||
current = dict(previous, round=1, pid=124, objects_before=4,
|
||||
snapshot_complete=True, outcome="complete")
|
||||
|
||||
self.assertTrue(replays_raw_window(previous, current))
|
||||
self.assertTrue(final_complete_recheck_after_full_retention(previous, current, 4))
|
||||
validate_recoverable_quantum([previous, current], objects=4, budget=16, require_converged=True)
|
||||
|
||||
def test_recoverable_quantum_rejects_partial_replay_after_full_retention(self):
|
||||
previous = self.report()
|
||||
previous.update(raw_entries=8, raw_page_index_committed_entries=4,
|
||||
raw_page_index_indexed_entries=4, objects_before=2,
|
||||
objects_processed=2, objects_retained=4,
|
||||
versions_retained=4, bytes_retained=4,
|
||||
snapshot_complete=False, outcome="partial")
|
||||
current = dict(previous, round=1, pid=124, objects_before=4,
|
||||
snapshot_complete=False, outcome="partial")
|
||||
|
||||
self.assertTrue(replays_raw_window(previous, current))
|
||||
self.assertFalse(final_complete_recheck_after_full_retention(previous, current, 4))
|
||||
with self.assertRaisesRegex(ValueError, "raw enumeration window replayed"):
|
||||
validate_recoverable_quantum([previous, current], objects=4, budget=16, require_converged=False)
|
||||
|
||||
def test_recoverable_quantum_requires_three_stage_progress_and_convergence(self):
|
||||
first = self.report()
|
||||
first.update(round=0, pid=123, raw_entries=2, raw_page_index_committed_entries=2,
|
||||
|
||||
+123
@@ -0,0 +1,123 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
RUNNER="$SCRIPT_DIR/run_scanner_heal_linux_evidence_plan.py"
|
||||
TMP_DIR="$(mktemp -d)"
|
||||
|
||||
cleanup() {
|
||||
rm -rf "$TMP_DIR"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
"${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" --self-test
|
||||
|
||||
"${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" --phase preflight >"$TMP_DIR/preflight.out"
|
||||
rg -q "stage=preflight" "$TMP_DIR/preflight.out"
|
||||
rg -q "scripts/check_test_wiring.py --self-test" "$TMP_DIR/preflight.out"
|
||||
|
||||
"${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" \
|
||||
--phase functional \
|
||||
--source-revision aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
|
||||
--format json >"$TMP_DIR/functional.json"
|
||||
|
||||
"${RUSTFS_PYTHON_BIN:-python3}" - "$TMP_DIR/functional.json" <<'PY'
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
plan = json.loads(pathlib.Path(sys.argv[1]).read_text())
|
||||
assert plan["schema"] == 1
|
||||
assert plan["source_revision"] == "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
assert [stage["id"] for stage in plan["stages"]] == ["functional"]
|
||||
commands = [
|
||||
" ".join(entry["command"])
|
||||
for stage in plan["stages"]
|
||||
for step in stage["steps"]
|
||||
for entry in step["commands"]
|
||||
]
|
||||
assert any("background-target-crash-ec8-4-multi-pool" in command for command in commands)
|
||||
assert any("run_scanner_heal_status_outcome_probe.py" in command for command in commands)
|
||||
assert all("--case release" not in command for command in commands)
|
||||
PY
|
||||
|
||||
"${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" --write-plan --out-dir "$TMP_DIR/plan" >"$TMP_DIR/path.out"
|
||||
PLAN_PATH="$(tr -d '\n' <"$TMP_DIR/path.out")"
|
||||
test -s "$PLAN_PATH"
|
||||
rg -q '"evidence_type": "plan_only"' "$PLAN_PATH"
|
||||
rg -q '"stop_on_product_failure": true' "$PLAN_PATH"
|
||||
|
||||
if "${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" \
|
||||
--phase functional \
|
||||
--source-revision aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
|
||||
--status-root "$TMP_DIR/missing-run" >"$TMP_DIR/status-missing.json"; then
|
||||
echo "missing evidence status should fail closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
rg -q '"decision": "blocked"' "$TMP_DIR/status-missing.json"
|
||||
rg -q '"release_approved": false' "$TMP_DIR/status-missing.json"
|
||||
rg -q '"next_step"' "$TMP_DIR/status-missing.json"
|
||||
|
||||
if "${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" \
|
||||
--phase functional \
|
||||
--status-root "$TMP_DIR/missing-run" \
|
||||
--run-preflight >/dev/null 2>"$TMP_DIR/status-mode.err"; then
|
||||
echo "status mode should reject preflight execution" >&2
|
||||
exit 1
|
||||
fi
|
||||
rg -q "status-root cannot be combined" "$TMP_DIR/status-mode.err"
|
||||
|
||||
"${RUSTFS_PYTHON_BIN:-python3}" - "$RUNNER" "$TMP_DIR/complete-run" <<'PY'
|
||||
import json
|
||||
import pathlib
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
runner = pathlib.Path(sys.argv[1])
|
||||
run_root = pathlib.Path(sys.argv[2])
|
||||
plan = json.loads(subprocess.check_output([
|
||||
sys.executable,
|
||||
str(runner),
|
||||
"--phase",
|
||||
"functional",
|
||||
"--source-revision",
|
||||
"a" * 40,
|
||||
"--format",
|
||||
"json",
|
||||
], text=True))
|
||||
for stage in plan["stages"]:
|
||||
for step in stage["steps"]:
|
||||
for output in step.get("expected_outputs", []):
|
||||
path = pathlib.Path(output.replace("$RUN_ROOT", str(run_root)))
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text("{}\n")
|
||||
status = json.loads(subprocess.check_output([
|
||||
sys.executable,
|
||||
str(runner),
|
||||
"--phase",
|
||||
"functional",
|
||||
"--source-revision",
|
||||
"a" * 40,
|
||||
"--status-root",
|
||||
str(run_root),
|
||||
], text=True))
|
||||
assert status["decision"] == "complete"
|
||||
assert status["release_approved"] is False
|
||||
assert status["artifact_totals"]["missing"] == 0
|
||||
PY
|
||||
|
||||
if "${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" \
|
||||
--phase performance \
|
||||
--run-preflight >/dev/null 2>"$TMP_DIR/no-preflight.err"; then
|
||||
echo "preflight execution without the preflight stage should fail" >&2
|
||||
exit 1
|
||||
fi
|
||||
rg -q "requires the preflight stage" "$TMP_DIR/no-preflight.err"
|
||||
|
||||
if "${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" --source-revision bad >/dev/null 2>"$TMP_DIR/bad.err"; then
|
||||
echo "invalid source revision should fail" >&2
|
||||
exit 1
|
||||
fi
|
||||
rg -q "source-revision" "$TMP_DIR/bad.err"
|
||||
@@ -3,5 +3,7 @@ set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
RUNNER="$SCRIPT_DIR/run_scanner_heal_status_outcome_evidence.py"
|
||||
PROBE="$SCRIPT_DIR/run_scanner_heal_status_outcome_probe.py"
|
||||
|
||||
"${RUSTFS_PYTHON_BIN:-python3}" "$PROBE" --self-test
|
||||
"${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" --self-test
|
||||
|
||||
Reference in New Issue
Block a user