mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-11 13:29:12 +00:00
Compare commits
14 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| b5cdfd496a | |||
| 853ae63b6a | |||
| f02bc947cd | |||
| 0cbc3ffe61 | |||
| b1cc286cac | |||
| 50b31bc75b | |||
| 5cd58319ed | |||
| c677590d50 | |||
| 94d875ea07 | |||
| 2f5db4e8f2 | |||
| 5c17012fe2 | |||
| aeaba86d73 | |||
| 8f150d1d8e | |||
| da762e0b02 |
@@ -1,2 +1,2 @@
|
||||
sha256-linux=563bff8f1171d6dbe166ff8440310dbe98430e466aa3ecd8dc39e3c872b320f7
|
||||
sha256-darwin=563bff8f1171d6dbe166ff8440310dbe98430e466aa3ecd8dc39e3c872b320f7
|
||||
sha256-linux=775825dcb2b4997c4fa24bd9ba9c0546316c503f4d5e369c39abff0678c95e8e
|
||||
sha256-darwin=775825dcb2b4997c4fa24bd9ba9c0546316c503f4d5e369c39abff0678c95e8e
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=874c881d7b45f12378a5817c7f42c95c4981960a2ec9ce12dcf4af239ae1f9d5
|
||||
sha256-linux=9515861be899ceb10e2e0ef93c34208bb7a7a8a7f8067a02db4cfba23270ebd6
|
||||
sha256-linux=9351e25b45bf7dfce18b951a5e3740225f457cacc53b8bf9f500f6947763ec0e
|
||||
|
||||
@@ -114,7 +114,7 @@
|
||||
"sets": 3,
|
||||
"pools": 3,
|
||||
"outage_target_manifest_required": false,
|
||||
"scope": "Target process crash during partial background rebuild on three single-node EC8+4 pools; exact baseline S3 bodies and replacement-drive shards, with outage object verified through S3 but not forced onto the replaced target drive."
|
||||
"scope": "Target process crash during partial background rebuild on three single-node EC8+4 pools; exact baseline S3 bodies and replacement-drive shards, with down-window outage PUT refusal recorded and a deferred post-rejoin outage object verified through S3 when the full target pool was offline."
|
||||
}
|
||||
},
|
||||
"release_lanes": {
|
||||
|
||||
@@ -259,9 +259,6 @@ jobs:
|
||||
} > artifacts/test-and-lint/doctest-diagnostics.txt
|
||||
exit "${status}"
|
||||
|
||||
- name: Check offline enrollment E2E root boundary
|
||||
run: ./scripts/check_offline_enrollment_e2e.sh
|
||||
|
||||
- name: Upload test reports and diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
@@ -296,6 +293,36 @@ jobs:
|
||||
- name: Run rebalance/decommission migration proofs
|
||||
run: ./scripts/check_migration_gate_count.sh
|
||||
|
||||
# The root boundary requires fresh CLI and integration-test builds. Give it
|
||||
# its own time budget instead of sharing the workspace lint/test budget.
|
||||
offline-enrollment-root-boundary:
|
||||
name: Offline Enrollment Root Boundary
|
||||
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-dev
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Protect Connect test home
|
||||
run: chmod go-w "$(realpath "$HOME")"
|
||||
|
||||
- name: Check offline enrollment E2E root boundary
|
||||
run: ./scripts/check_offline_enrollment_e2e.sh
|
||||
|
||||
# Dedicated serial lane for the ILM / lifecycle integration tests. These tests
|
||||
# drive the object layer through process-global singletons (the GLOBAL_ENV
|
||||
# ECStore, the global tier-config manager, background-expiry workers) and bind
|
||||
@@ -970,6 +997,7 @@ jobs:
|
||||
# debug binary; each test spawns its own rustfs server on a random port.
|
||||
- name: Run e2e full suite
|
||||
env:
|
||||
CARGO_BIN_EXE_rustfs: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs
|
||||
RUSTFS_E2E_STARTUP_CAS_BINARY: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs
|
||||
RUSTFS_E2E_STARTUP_CAS_BUILD_MANIFEST: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs.e2e-startup-cas-build.json
|
||||
RUSTFS_E2E_STARTUP_CAS_ARTIFACT_DIR: ${{ runner.temp }}/rustfs-startup-cas-evidence
|
||||
@@ -1189,6 +1217,7 @@ jobs:
|
||||
- typos
|
||||
- quick-checks
|
||||
- test-and-lint
|
||||
- offline-enrollment-root-boundary
|
||||
- test-ilm-integration-serial
|
||||
- test-and-lint-rio-v2
|
||||
- connect-short-credential-boundary
|
||||
@@ -1222,6 +1251,7 @@ jobs:
|
||||
- typos
|
||||
- quick-checks
|
||||
- test-and-lint
|
||||
- offline-enrollment-root-boundary
|
||||
- test-ilm-integration-serial
|
||||
- test-and-lint-rio-v2
|
||||
- test-and-lint-protocols
|
||||
|
||||
@@ -156,6 +156,7 @@ jobs:
|
||||
|
||||
- name: Run cluster fault e2e nightly suite
|
||||
env:
|
||||
CARGO_BIN_EXE_rustfs: ${{ github.workspace }}/target/debug/rustfs
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-nightly-logs
|
||||
run: cargo nextest run --profile e2e-nightly -p e2e_test
|
||||
|
||||
|
||||
Generated
+97
-97
@@ -271,7 +271,7 @@ version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -282,7 +282,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"once_cell_polyfill",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1641,9 +1641,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.13.1"
|
||||
version = "2.13.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da"
|
||||
checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
@@ -1906,7 +1906,7 @@ dependencies = [
|
||||
"maybe-owned",
|
||||
"rustix",
|
||||
"rustix-linux-procfs",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
"winx",
|
||||
]
|
||||
|
||||
@@ -2270,9 +2270,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "console"
|
||||
version = "0.16.4"
|
||||
version = "0.16.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4fe5f465a4f6fee88fad41b85d990f84c835335e85b5d9e6e63e0d06d28cba7c"
|
||||
checksum = "e96a4956774c13c126a8b5af4daa79384f4d826534c95a02d76afb39e2ab64e3"
|
||||
dependencies = [
|
||||
"encode_unicode",
|
||||
"libc",
|
||||
@@ -3942,7 +3942,7 @@ dependencies = [
|
||||
"libc",
|
||||
"option-ext",
|
||||
"redox_users",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3951,7 +3951,7 @@ version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"objc2",
|
||||
]
|
||||
|
||||
@@ -3986,7 +3986,7 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
|
||||
|
||||
[[package]]
|
||||
name = "e2e_test"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"astral-tokio-tar",
|
||||
@@ -4287,7 +4287,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4416,7 +4416,7 @@ version = "25.12.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"rustc_version",
|
||||
]
|
||||
|
||||
@@ -5706,7 +5706,7 @@ version = "0.7.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed3bd0ecfbb87805f538bb7b32e5239ca0763890c623e349860ecba69469f2bb"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"cfg-if",
|
||||
"libc",
|
||||
]
|
||||
@@ -6212,7 +6212,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c9f8ff371890db2cf65a0758dba9a79f9cd965de369f6dbdc6581a22780af45e"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"dashmap",
|
||||
@@ -6794,7 +6794,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0f27695f286b461da077b8c2f72f47feaa04ce3c3f9c0976257410e90e21208a"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"btoi",
|
||||
"byteorder",
|
||||
"bytes",
|
||||
@@ -6826,7 +6826,7 @@ version = "0.7.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "22f9786d56d972959e1408b6a93be6af13b9c1392036c5c1fafa08a1b0c6ee87"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"byteorder",
|
||||
"derive_builder",
|
||||
"getset",
|
||||
@@ -6874,7 +6874,7 @@ version = "0.29.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"cfg-if",
|
||||
"cfg_aliases",
|
||||
"libc",
|
||||
@@ -6887,7 +6887,7 @@ version = "0.30.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"cfg-if",
|
||||
"cfg_aliases",
|
||||
"libc",
|
||||
@@ -6899,7 +6899,7 @@ version = "0.31.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"cfg-if",
|
||||
"cfg_aliases",
|
||||
"libc",
|
||||
@@ -6963,7 +6963,7 @@ version = "0.50.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -7100,7 +7100,7 @@ version = "0.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d164abbde0b3c03edb9edb9cb8d31a7f5b79015c692b7c771f6e0840e9106b9f"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"libloading",
|
||||
"nvml-wrapper-sys",
|
||||
"static_assertions",
|
||||
@@ -7151,7 +7151,7 @@ version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"dispatch2",
|
||||
"objc2",
|
||||
]
|
||||
@@ -7168,7 +7168,7 @@ version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"objc2",
|
||||
]
|
||||
|
||||
@@ -7721,7 +7721,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e33c6dbf1a8fb7f71742cd70f5e9f0986e60b2d19dc0b28d9ca0d1323259274a"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"thiserror 2.0.20",
|
||||
"zerocopy",
|
||||
"zerocopy-derive",
|
||||
@@ -7777,7 +7777,7 @@ version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "575828d9d7d205188048eb1508560607a03d21eafdbba47b8cade1736c1c28e1"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"c-enum",
|
||||
"perf-event-open-sys2",
|
||||
]
|
||||
@@ -8294,7 +8294,7 @@ checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744"
|
||||
dependencies = [
|
||||
"bit-set",
|
||||
"bit-vec 0.8.0",
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"num-traits",
|
||||
"rand 0.9.5",
|
||||
"rand_chacha 0.9.0",
|
||||
@@ -8436,7 +8436,7 @@ version = "0.13.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e9f068eba8e7071c5f9511831b44f32c740d5adf574e990f946ddb53db2f314e"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"memchr",
|
||||
"unicase",
|
||||
]
|
||||
@@ -8712,7 +8712,7 @@ dependencies = [
|
||||
"once_cell",
|
||||
"socket2",
|
||||
"tracing",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -8874,7 +8874,7 @@ version = "11.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -8983,7 +8983,7 @@ version = "0.5.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -9310,7 +9310,7 @@ checksum = "036204edbd199552a5b3832f63c60dcdf395dc44c7f06b4af1c0e8139cc11bce"
|
||||
dependencies = [
|
||||
"aes 0.9.3",
|
||||
"aws-lc-rs",
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"block-padding 0.4.2",
|
||||
"byteorder",
|
||||
"bytes",
|
||||
@@ -9391,7 +9391,7 @@ version = "3.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "093197e526668d92bba562e2bbbe98d1af9831bf080b619c736316ca1fa35101"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"dashmap",
|
||||
@@ -9476,7 +9476,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"anyhow",
|
||||
@@ -9624,7 +9624,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-audit"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"const-str",
|
||||
"futures",
|
||||
@@ -9646,7 +9646,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-checksums"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -9662,7 +9662,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-common"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"metrics",
|
||||
@@ -9675,7 +9675,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-concurrency"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"insta",
|
||||
@@ -9688,7 +9688,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-config"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"const-str",
|
||||
"hotpath",
|
||||
@@ -9698,7 +9698,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-credentials"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"hmac 0.13.0",
|
||||
@@ -9712,7 +9712,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-crypto"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"argon2",
|
||||
@@ -9733,7 +9733,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-data-usage"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"rmp-serde",
|
||||
@@ -9743,7 +9743,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-ecstore"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-channel",
|
||||
@@ -9879,7 +9879,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-extension-schema"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"serde",
|
||||
@@ -9889,7 +9889,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-filemeta"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"byteorder",
|
||||
@@ -9917,7 +9917,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-heal"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64-simd",
|
||||
@@ -9954,7 +9954,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-heal-contracts"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -9964,7 +9964,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-iam"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
@@ -10013,7 +10013,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-io-core"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"hotpath",
|
||||
@@ -10025,7 +10025,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-io-metrics"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"criterion",
|
||||
"hotpath",
|
||||
@@ -10089,7 +10089,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-keystone"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures",
|
||||
@@ -10116,7 +10116,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-kms"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"anyhow",
|
||||
@@ -10166,14 +10166,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-license"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"thiserror 2.0.20",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-lifecycle"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"hotpath",
|
||||
@@ -10195,7 +10195,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-lock"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"compact_str",
|
||||
@@ -10218,7 +10218,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-log-analyzer"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"flate2",
|
||||
@@ -10237,7 +10237,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-madmin"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"http 1.5.0",
|
||||
@@ -10275,7 +10275,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-notify"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
@@ -10310,7 +10310,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-object-capacity"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"criterion",
|
||||
"futures",
|
||||
@@ -10329,7 +10329,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-object-data-cache"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"criterion",
|
||||
@@ -10346,7 +10346,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-obs"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"crossbeam-channel",
|
||||
@@ -10404,7 +10404,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-policy"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64-simd",
|
||||
@@ -10435,7 +10435,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-protocols"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"astral-tokio-tar",
|
||||
"async-compression",
|
||||
@@ -10497,7 +10497,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-protos"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"flatbuffers",
|
||||
"hotpath",
|
||||
@@ -10522,7 +10522,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-replication"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"bytes",
|
||||
@@ -10540,7 +10540,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-rio"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"arc-swap",
|
||||
@@ -10584,7 +10584,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-rio-v2"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"bytes",
|
||||
@@ -10607,7 +10607,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3-client"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -10651,7 +10651,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3-ops"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"rustfs-s3-types",
|
||||
@@ -10659,7 +10659,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3-types"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"serde",
|
||||
@@ -10668,7 +10668,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3select-api"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-compression",
|
||||
@@ -10703,7 +10703,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3select-query"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"async-recursion",
|
||||
"async-trait",
|
||||
@@ -10724,7 +10724,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-scanner"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
@@ -10770,7 +10770,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-scanner-metrics"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"jiff",
|
||||
@@ -10785,7 +10785,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-security-governance"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"thiserror 2.0.20",
|
||||
@@ -10793,7 +10793,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-signer"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -10811,7 +10811,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-storage-api"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"hotpath",
|
||||
@@ -10826,7 +10826,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-targets"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-nats",
|
||||
@@ -10880,7 +10880,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-test-utils"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"rustfs-data-usage",
|
||||
@@ -10896,7 +10896,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-tls-runtime"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"hotpath",
|
||||
@@ -10917,7 +10917,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-trusted-proxies"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
@@ -10954,7 +10954,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-utils"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"blake2",
|
||||
@@ -10996,7 +10996,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-zip"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"astral-tokio-tar",
|
||||
"async-compression",
|
||||
@@ -11061,11 +11061,11 @@ version = "1.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11148,7 +11148,7 @@ dependencies = [
|
||||
"security-framework",
|
||||
"security-framework-sys",
|
||||
"webpki-root-certs",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11431,7 +11431,7 @@ version = "3.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"core-foundation 0.10.1",
|
||||
"core-foundation-sys",
|
||||
"libc",
|
||||
@@ -11949,7 +11949,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -12100,7 +12100,7 @@ dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"psm",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -12321,7 +12321,7 @@ version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"core-foundation 0.9.4",
|
||||
"system-configuration-sys",
|
||||
]
|
||||
@@ -12405,10 +12405,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.4.3",
|
||||
"getrandom 0.3.4",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -12876,7 +12876,7 @@ version = "0.6.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"http 1.5.0",
|
||||
@@ -12895,7 +12895,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "08a05a66a4fdd61cbbe0a1d755ffe0ca6aba159dd4820936a0ff8a8278245b9c"
|
||||
dependencies = [
|
||||
"async-compression",
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
@@ -13244,9 +13244,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "uuid"
|
||||
version = "1.26.0"
|
||||
version = "1.26.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812"
|
||||
checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce"
|
||||
dependencies = [
|
||||
"getrandom 0.4.3",
|
||||
"js-sys",
|
||||
@@ -13510,7 +13510,7 @@ version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -13820,7 +13820,7 @@ version = "0.36.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bitflags 2.13.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
|
||||
+52
-52
@@ -73,7 +73,7 @@ edition = "2024"
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/rustfs/rustfs"
|
||||
rust-version = "1.98.0"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
homepage = "https://rustfs.com"
|
||||
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
||||
keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
|
||||
@@ -90,56 +90,56 @@ redundant_clone = "warn"
|
||||
|
||||
[workspace.dependencies]
|
||||
# RustFS Internal Crates
|
||||
rustfs = { path = "./rustfs", version = "1.0.0-rc.5" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.5" }
|
||||
rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.0-rc.5" }
|
||||
rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.0-rc.5" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.5" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.5" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0-rc.5" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.5" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0-rc.5" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.5" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.5" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.5" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.5" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.5" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.5" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.5" }
|
||||
rustfs-license = { path = "crates/license", version = "1.0.0-rc.5" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.5" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.5" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.5" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.5" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.5" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.5" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.5" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.5" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.5", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.5" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.5" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.5" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.5" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.5" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.5" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.5" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.5" }
|
||||
rustfs-s3-client = { path = "crates/s3-client", version = "1.0.0-rc.5" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.5" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.5" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.5" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.5" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.5" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.5" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.5" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.5" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.5" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.5" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.5" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.5" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.5" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.5" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.5" }
|
||||
rustfs = { path = "./rustfs", version = "1.0.0-rc.6" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.6" }
|
||||
rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.0-rc.6" }
|
||||
rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.0-rc.6" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.6" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.6" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0-rc.6" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.6" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0-rc.6" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.6" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.6" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.6" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.6" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.6" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.6" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.6" }
|
||||
rustfs-license = { path = "crates/license", version = "1.0.0-rc.6" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.6" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.6" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.6" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.6" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.6" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.6" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.6" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.6" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.6", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.6" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.6" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.6" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.6" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.6" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.6" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.6" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.6" }
|
||||
rustfs-s3-client = { path = "crates/s3-client", version = "1.0.0-rc.6" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.6" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.6" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.6" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.6" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.6" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.6" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.6" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.6" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.6" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.6" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.6" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.6" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.6" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.6" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.6" }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
@@ -335,7 +335,7 @@ tracing-subscriber = { version = "0.3.23" }
|
||||
transform-stream = "0.3.1"
|
||||
url = "2.5.8"
|
||||
urlencoding = "2.1.3"
|
||||
uuid = { version = "1.26.0" }
|
||||
uuid = { version = "1.26.1" }
|
||||
vaultrs = { version = "0.8.0" }
|
||||
tar = "0.4.46"
|
||||
walkdir = "2.5.0"
|
||||
|
||||
@@ -141,7 +141,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# Using specific version
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.5
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.6
|
||||
```
|
||||
|
||||
If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
|
||||
|
||||
+1
-1
@@ -138,7 +138,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# 使用指定版本运行
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.5
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.6
|
||||
```
|
||||
|
||||
如果您通过绑定挂载启用 TLS 证书目录,也请用同样方式准备该目录:
|
||||
|
||||
@@ -62,6 +62,37 @@ Current guidance:
|
||||
- `RUSTFS_CORS_ALLOWED_ORIGINS` defaults to empty, so the S3 endpoint emits no generic CORS headers unless configured. Set `*` for wildcard origins without credentials, or a comma-separated allow-list for credentialed explicit origins.
|
||||
- `RUSTFS_CONSOLE_CORS_ALLOWED_ORIGINS` defaults to `*` for the console service.
|
||||
|
||||
## Console URL prefix
|
||||
|
||||
`RUSTFS_CONSOLE_PREFIX` changes the embedded console URL prefix. The default is
|
||||
`/rustfs/console`. For example, `RUSTFS_CONSOLE_PREFIX=/console` serves the UI at
|
||||
`http://localhost:9001/console/`. Nested prefixes such as `/management/console`
|
||||
are supported; one trailing slash is removed. Restart the server after changing it.
|
||||
|
||||
The prefix must be a non-root absolute path of at most 256 bytes, with nonempty
|
||||
segments containing only ASCII letters, digits, `-`, `_`, `.`, or `~`. Dot
|
||||
segments, encoded characters, and overlaps with reserved admin, RPC, health,
|
||||
profiling, browser entry, and icon routes are rejected at startup. `/` is not supported.
|
||||
Choose a prefix that does not collide with S3 bucket paths.
|
||||
|
||||
The console routes, embedded frontend asset URLs, browser redirects, and OIDC
|
||||
console redirects use this prefix. Admin API paths and the identity provider's
|
||||
`/rustfs/admin/v3/oidc/callback/...` URL remain unchanged. `RUSTFS_CONSOLE_ADDRESS`
|
||||
continues to control only the listening address and port. The server adapts bundled
|
||||
console asset references from their build-time base path to the runtime prefix.
|
||||
|
||||
OEM builds can set `RUSTFS_CONSOLE_BASE_PATH` when compiling RustFS to embed a
|
||||
different default, such as `/nuofans/console`. Build the bundled console with the
|
||||
same `NEXT_PUBLIC_BASE_PATH`. An unset or empty build variable retains
|
||||
`/rustfs/console`. The build path must satisfy the validation rules above and must
|
||||
not have a trailing slash.
|
||||
|
||||
At startup, `RUSTFS_CONSOLE_PREFIX` takes precedence over the compiled default.
|
||||
Changing `RUSTFS_CONSOLE_BASE_PATH` when starting an existing binary has no effect;
|
||||
rebuild both components to change the embedded default. If a runtime prefix is
|
||||
configured, asset adaptation uses the compiled base path as its source, including
|
||||
when restoring `/rustfs/console` for a custom OEM build.
|
||||
|
||||
## Browser redirect environment variables
|
||||
|
||||
- `RUSTFS_BROWSER_REDIRECT_URL` sets the externally reachable browser origin used for OIDC callback, console success redirect, and logout fallback URLs. Configure it to the public scheme and authority without a path, for example `https://console.example.com`. In load-balancer deployments, keep OIDC authorize and callback requests on the same backend node because the in-flight OIDC `state` is local to the RustFS node.
|
||||
|
||||
@@ -213,6 +213,11 @@ pub const ENV_RUSTFS_CONSOLE_ENABLE: &str = "RUSTFS_CONSOLE_ENABLE";
|
||||
/// Environment variable for console server address.
|
||||
pub const ENV_RUSTFS_CONSOLE_ADDRESS: &str = "RUSTFS_CONSOLE_ADDRESS";
|
||||
|
||||
/// URL path prefix for the embedded console, read once at server startup.
|
||||
pub const ENV_RUSTFS_CONSOLE_PREFIX: &str = "RUSTFS_CONSOLE_PREFIX";
|
||||
/// Default embedded console URL path prefix.
|
||||
pub const DEFAULT_CONSOLE_PREFIX: &str = "/rustfs/console";
|
||||
|
||||
/// Public browser entrypoint used to build OIDC callback and console redirects.
|
||||
///
|
||||
/// This should be the externally reachable scheme and authority, without a path.
|
||||
|
||||
@@ -64,6 +64,14 @@ pub const MAX_HEAL_REQUEST_SIZE: usize = 1024 * 1024; // 1 MB
|
||||
/// memory exhaustion from malicious or misconfigured remote services.
|
||||
pub const MAX_S3_CLIENT_RESPONSE_SIZE: usize = 10 * 1024 * 1024; // 10 MB
|
||||
|
||||
/// Maximum body size accepted by a single `PutObject` or `UploadPart` request (5 GiB).
|
||||
/// Used for: the s3s streaming-body limit and the request-header admission check.
|
||||
/// Rationale: matches the AWS S3 single-PUT / single-part ceiling. Larger objects
|
||||
/// must use multipart upload. The header check rejects an oversize
|
||||
/// `Content-Length` before any body byte is read so the client gets
|
||||
/// `EntityTooLarge` immediately instead of streaming 5 GiB into a mid-stream failure.
|
||||
pub const MAX_SINGLE_PUT_OBJECT_SIZE: u64 = 5 * 1024 * 1024 * 1024; // 5 GiB
|
||||
|
||||
/// Maximum size for OIDC provider response bodies (1 MB)
|
||||
/// Used for: discovery documents, JWKS documents and token endpoint responses
|
||||
/// Rationale: a hostile or compromised identity provider must not be able to exhaust
|
||||
|
||||
@@ -38,6 +38,13 @@ All commands assume repo root. `cargo test` triggers an on-demand build of the
|
||||
`rustfs` binary from [`src/common.rs`](src/common.rs) (`rustfs_binary_path`) on
|
||||
first use — the first invocation is slow, later ones reuse the binary.
|
||||
|
||||
Root-heal interruption scenarios use a test-only commit barrier. Prebuild with `e2e-test-hooks` and pin that binary so concurrent cases do not replace it through on-demand builds:
|
||||
|
||||
```bash
|
||||
cargo build -p rustfs --bin rustfs --features e2e-test-hooks
|
||||
CARGO_BIN_EXE_rustfs="$PWD/target/debug/rustfs" cargo nextest run -p e2e_test -E 'test(heal_erasure_disk_rebuild_test)'
|
||||
```
|
||||
|
||||
```bash
|
||||
# Whole crate (default = ignored tests skipped)
|
||||
cargo nextest run -p e2e_test
|
||||
|
||||
@@ -0,0 +1,642 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Standard S3 deletion permissions and the explicit recursive-delete extension.
|
||||
|
||||
use crate::common::{
|
||||
AdminTransport, RustFSTestEnvironment, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user,
|
||||
init_logging,
|
||||
};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
|
||||
use aws_sdk_s3::operation::delete_object::{DeleteObjectError, DeleteObjectOutput};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, Delete, ObjectIdentifier, VersioningConfiguration};
|
||||
use futures::{StreamExt, TryStreamExt, stream};
|
||||
use serde_json::{Value, json};
|
||||
use std::collections::BTreeSet;
|
||||
use std::error::Error;
|
||||
use uuid::Uuid;
|
||||
|
||||
type TestResult<T = ()> = Result<T, Box<dyn Error + Send + Sync>>;
|
||||
type VersionSnapshot = BTreeSet<(String, String, bool)>;
|
||||
|
||||
async fn set_policy(env: &RustFSTestEnvironment, name: &str, policy: &Value) -> TestResult {
|
||||
admin_add_canned_policy_via(
|
||||
AdminTransport::Signed,
|
||||
&env.url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
name,
|
||||
&policy.to_string(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn policy_user(env: &RustFSTestEnvironment, policy_name: &str, policy: Option<Value>) -> TestResult<Client> {
|
||||
let username = Uuid::new_v4().simple().to_string();
|
||||
let secret = Uuid::new_v4().simple().to_string();
|
||||
admin_create_user(env, &username, &secret).await?;
|
||||
if let Some(policy) = policy {
|
||||
set_policy(env, policy_name, &policy).await?;
|
||||
}
|
||||
admin_attach_user_policy_via(AdminTransport::Signed, &env.url, &env.access_key, &env.secret_key, policy_name, &username)
|
||||
.await?;
|
||||
Ok(env.create_s3_client_with_credentials(&username, &secret))
|
||||
}
|
||||
|
||||
async fn versioning(client: &Client, bucket: &str, status: BucketVersioningStatus) -> TestResult {
|
||||
client
|
||||
.put_bucket_versioning()
|
||||
.bucket(bucket)
|
||||
.versioning_configuration(VersioningConfiguration::builder().status(status).build())
|
||||
.send()
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn put(client: &Client, bucket: &str, key: &str) -> TestResult<String> {
|
||||
let result = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from_static(b"delete authorization fixture"))
|
||||
.send()
|
||||
.await?;
|
||||
Ok(result.version_id().unwrap_or("null").to_string())
|
||||
}
|
||||
|
||||
async fn versions(client: &Client, bucket: &str, prefix: &str) -> TestResult<VersionSnapshot> {
|
||||
let mut result = BTreeSet::new();
|
||||
let mut markers = (None, None);
|
||||
loop {
|
||||
let page = client
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.prefix(prefix)
|
||||
.set_key_marker(markers.0.clone())
|
||||
.set_version_id_marker(markers.1.clone())
|
||||
.send()
|
||||
.await?;
|
||||
for version in page.versions() {
|
||||
result.insert((
|
||||
version.key().ok_or("listed version missing key")?.to_string(),
|
||||
version.version_id().ok_or("listed version missing ID")?.to_string(),
|
||||
false,
|
||||
));
|
||||
}
|
||||
for marker in page.delete_markers() {
|
||||
result.insert((
|
||||
marker.key().ok_or("listed delete marker missing key")?.to_string(),
|
||||
marker.version_id().ok_or("listed delete marker missing ID")?.to_string(),
|
||||
true,
|
||||
));
|
||||
}
|
||||
if page.is_truncated() != Some(true) {
|
||||
return Ok(result);
|
||||
}
|
||||
let next = (
|
||||
Some(
|
||||
page.next_key_marker()
|
||||
.ok_or("truncated versions page missing next key marker")?
|
||||
.to_string(),
|
||||
),
|
||||
page.next_version_id_marker().map(str::to_string),
|
||||
);
|
||||
assert_ne!(markers, next, "ListObjectVersions pagination must advance");
|
||||
markers = next;
|
||||
}
|
||||
}
|
||||
|
||||
async fn force_delete(client: &Client, bucket: &str, prefix: &str) -> Result<DeleteObjectOutput, SdkError<DeleteObjectError>> {
|
||||
client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key(prefix)
|
||||
.customize()
|
||||
.mutate_request(|request| {
|
||||
request.headers_mut().insert("x-rustfs-force-delete", "true");
|
||||
})
|
||||
.send()
|
||||
.await
|
||||
}
|
||||
|
||||
async fn replica_force_delete(
|
||||
client: &Client,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
) -> Result<DeleteObjectOutput, SdkError<DeleteObjectError>> {
|
||||
client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key(prefix)
|
||||
.customize()
|
||||
.mutate_request(|request| {
|
||||
request.headers_mut().insert("x-rustfs-force-delete", "true");
|
||||
request.headers_mut().insert("x-amz-replication-status", "REPLICA");
|
||||
})
|
||||
.send()
|
||||
.await
|
||||
}
|
||||
|
||||
fn assert_denied<T, E>(result: Result<T, SdkError<E>>)
|
||||
where
|
||||
T: std::fmt::Debug,
|
||||
E: ProvideErrorMetadata + std::fmt::Debug,
|
||||
{
|
||||
let error = result.expect_err("request must be denied by its S3 permission");
|
||||
assert_eq!(
|
||||
error.as_service_error().and_then(ProvideErrorMetadata::code),
|
||||
Some("AccessDenied"),
|
||||
"expected an S3 authorization denial, got {error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sdk_version_deletion_requires_only_delete_object_version() -> TestResult {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
let root = env.create_s3_client();
|
||||
let bucket = "delete-version-permissions";
|
||||
root.create_bucket().bucket(bucket).send().await?;
|
||||
put(&root, bucket, "single-null.txt").await?;
|
||||
put(&root, bucket, "batch-null.txt").await?;
|
||||
versioning(&root, bucket, BucketVersioningStatus::Enabled).await?;
|
||||
let old = put(&root, bucket, "single.txt").await?;
|
||||
let current = put(&root, bucket, "single.txt").await?;
|
||||
let batch_version = put(&root, bucket, "batch.txt").await?;
|
||||
let ordinary_version = put(&root, bucket, "ordinary.txt").await?;
|
||||
let user = policy_user(
|
||||
&env,
|
||||
"version-deleter",
|
||||
Some(json!({"Version":"2012-10-17","Statement":[
|
||||
{"Effect":"Allow","Action":"s3:DeleteObjectVersion","Resource":format!("arn:aws:s3:::{bucket}/*")},
|
||||
{"Effect":"Deny","Action":"s3:DeleteObject","Resource":format!("arn:aws:s3:::{bucket}/*")}
|
||||
]})),
|
||||
)
|
||||
.await?;
|
||||
|
||||
user.delete_object()
|
||||
.bucket(bucket)
|
||||
.key("single.txt")
|
||||
.version_id(&old)
|
||||
.send()
|
||||
.await?;
|
||||
user.delete_object()
|
||||
.bucket(bucket)
|
||||
.key("single-null.txt")
|
||||
.version_id("null")
|
||||
.send()
|
||||
.await?;
|
||||
assert_denied(user.delete_object().bucket(bucket).key("ordinary.txt").send().await);
|
||||
|
||||
let batch = user
|
||||
.delete_objects()
|
||||
.bucket(bucket)
|
||||
.delete(
|
||||
Delete::builder()
|
||||
.objects(
|
||||
ObjectIdentifier::builder()
|
||||
.key("batch.txt")
|
||||
.version_id(&batch_version)
|
||||
.build()?,
|
||||
)
|
||||
.objects(ObjectIdentifier::builder().key("batch-null.txt").version_id("null").build()?)
|
||||
.objects(ObjectIdentifier::builder().key("ordinary.txt").build()?)
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(batch.deleted().len(), 2, "both explicit version items must succeed");
|
||||
assert_eq!(batch.errors().len(), 1, "only the unversioned item must be denied");
|
||||
assert_eq!(batch.errors()[0].key(), Some("ordinary.txt"));
|
||||
assert_eq!(batch.errors()[0].code(), Some("AccessDenied"));
|
||||
assert_eq!(
|
||||
versions(&root, bucket, "").await?,
|
||||
BTreeSet::from([
|
||||
("single.txt".into(), current, false),
|
||||
("ordinary.txt".into(), ordinary_version, false)
|
||||
]),
|
||||
"version-only deletion must preserve the current single-object version and denied object"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sdk_list_bucket_and_list_bucket_versions_permissions_are_independent() -> TestResult {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
let root = env.create_s3_client();
|
||||
let bucket = "list-version-permissions";
|
||||
root.create_bucket().bucket(bucket).send().await?;
|
||||
versioning(&root, bucket, BucketVersioningStatus::Enabled).await?;
|
||||
put(&root, bucket, "visible.txt").await?;
|
||||
for (action, name) in [
|
||||
("s3:ListBucket", "object-lister"),
|
||||
("s3:ListBucketVersions", "version-lister"),
|
||||
] {
|
||||
let user = policy_user(
|
||||
&env,
|
||||
name,
|
||||
Some(json!({"Version":"2012-10-17","Statement":[
|
||||
{"Effect":"Allow","Action":action,"Resource":format!("arn:aws:s3:::{bucket}")}
|
||||
]})),
|
||||
)
|
||||
.await?;
|
||||
if action == "s3:ListBucket" {
|
||||
assert_eq!(user.list_objects_v2().bucket(bucket).send().await?.contents().len(), 1);
|
||||
assert_denied(user.list_object_versions().bucket(bucket).send().await);
|
||||
} else {
|
||||
assert_eq!(user.list_object_versions().bucket(bucket).send().await?.versions().len(), 1);
|
||||
assert_denied(user.list_objects_v2().bucket(bucket).send().await);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn console_admin_force_delete_removes_prefix_versions_and_delete_markers() -> TestResult {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
let root = env.create_s3_client();
|
||||
let bucket = "force-console-admin";
|
||||
root.create_bucket().bucket(bucket).send().await?;
|
||||
put(&root, bucket, "folder/null.txt").await?;
|
||||
versioning(&root, bucket, BucketVersioningStatus::Enabled).await?;
|
||||
for key in ["folder/a.txt", "folder/deep/b.txt", "single.txt"] {
|
||||
put(&root, bucket, key).await?;
|
||||
put(&root, bucket, key).await?;
|
||||
root.delete_object().bucket(bucket).key(key).send().await?;
|
||||
}
|
||||
put(&root, bucket, "keep.txt").await?;
|
||||
put(&root, bucket, "folder-sibling/keep.txt").await?;
|
||||
let keep = versions(&root, bucket, "keep.txt").await?;
|
||||
let sibling = versions(&root, bucket, "folder-sibling/").await?;
|
||||
let user = policy_user(&env, "consoleAdmin", None).await?;
|
||||
|
||||
force_delete(&user, bucket, "folder/").await?;
|
||||
assert!(
|
||||
versions(&root, bucket, "folder/").await?.is_empty(),
|
||||
"force prefix deletion must remove null versions and markers"
|
||||
);
|
||||
assert_eq!(versions(&root, bucket, "folder-sibling/").await?, sibling);
|
||||
assert_eq!(
|
||||
versions(&root, bucket, "single.txt").await?.len(),
|
||||
3,
|
||||
"the separate key must survive folder deletion"
|
||||
);
|
||||
force_delete(&user, bucket, "single.txt").await?;
|
||||
assert!(
|
||||
versions(&root, bucket, "single.txt").await?.is_empty(),
|
||||
"explicit force deletion must remove every version of the selected key"
|
||||
);
|
||||
assert_eq!(versions(&root, bucket, "keep.txt").await?, keep);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn force_delete_authorizes_only_its_path_scope_without_list_permissions() -> TestResult {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
let root = env.create_s3_client();
|
||||
let bucket = "force-delete-only";
|
||||
root.create_bucket().bucket(bucket).send().await?;
|
||||
versioning(&root, bucket, BucketVersioningStatus::Enabled).await?;
|
||||
put(&root, bucket, "selected.txt").await?;
|
||||
put(&root, bucket, "selected.txt/child.txt").await?;
|
||||
put(&root, bucket, "selected.txt-sibling").await?;
|
||||
let sibling = versions(&root, bucket, "selected.txt-sibling").await?;
|
||||
let user = policy_user(
|
||||
&env,
|
||||
"delete-only",
|
||||
Some(json!({"Version":"2012-10-17","Statement":[
|
||||
{"Effect":"Allow","Action":["s3:DeleteObject","s3:DeleteObjectVersion"],"Resource":[
|
||||
format!("arn:aws:s3:::{bucket}/selected.txt"), format!("arn:aws:s3:::{bucket}/selected.txt/*")
|
||||
]},
|
||||
{"Effect":"Deny","Action":["s3:DeleteObject","s3:DeleteObjectVersion"],"Resource":format!("arn:aws:s3:::{bucket}/selected.txt-sibling")}
|
||||
]})),
|
||||
)
|
||||
.await?;
|
||||
assert_denied(user.list_objects_v2().bucket(bucket).send().await);
|
||||
assert_denied(user.list_object_versions().bucket(bucket).send().await);
|
||||
force_delete(&user, bucket, "selected.txt").await?;
|
||||
assert_eq!(
|
||||
versions(&root, bucket, "selected.txt").await?,
|
||||
sibling,
|
||||
"force deletion must remove the selected path and descendants without authorizing or deleting its similarly prefixed sibling"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn force_directory_delete_cannot_remove_an_unauthorized_colliding_parent() -> TestResult {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
let root = env.create_s3_client();
|
||||
let bucket = "force-directory-collision";
|
||||
root.create_bucket().bucket(bucket).send().await?;
|
||||
versioning(&root, bucket, BucketVersioningStatus::Enabled).await?;
|
||||
let protected_parent_version = put(&root, bucket, "collision.txt").await?;
|
||||
for key in ["collision.txt/child", "collision.txt-sibling"] {
|
||||
put(&root, bucket, key).await?;
|
||||
}
|
||||
put(&root, bucket, "collision.txt").await?;
|
||||
root.delete_object().bucket(bucket).key("collision.txt").send().await?;
|
||||
let user = policy_user(
|
||||
&env,
|
||||
"parent-denier",
|
||||
Some(json!({"Version":"2012-10-17","Statement":[
|
||||
{"Effect":"Allow","Action":["s3:DeleteObject","s3:DeleteObjectVersion"],"Resource":format!("arn:aws:s3:::{bucket}/*")},
|
||||
{"Effect":"Deny","Action":"s3:DeleteObjectVersion","Resource":format!("arn:aws:s3:::{bucket}/collision.txt"),
|
||||
"Condition":{"StringEquals":{"s3:VersionId":protected_parent_version}}}
|
||||
]})),
|
||||
)
|
||||
.await?;
|
||||
let mut expected = versions(&root, bucket, "").await?;
|
||||
expected.retain(|(key, _, _)| key != "collision.txt/child");
|
||||
force_delete(&user, bucket, "collision.txt/").await?;
|
||||
assert_eq!(
|
||||
versions(&root, bucket, "").await?,
|
||||
expected,
|
||||
"folder deletion must preserve the denied parent's historical versions and delete marker, plus its sibling"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn force_unversioned_directory_requires_only_delete_object() -> TestResult {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
let root = env.create_s3_client();
|
||||
let bucket = "force-unversioned-permissions";
|
||||
root.create_bucket().bucket(bucket).send().await?;
|
||||
for key in ["folder/", "folder/child.txt", "outside.txt"] {
|
||||
put(&root, bucket, key).await?;
|
||||
}
|
||||
let outside = versions(&root, bucket, "outside.txt").await?;
|
||||
let user = policy_user(
|
||||
&env,
|
||||
"unversioned-deleter",
|
||||
Some(json!({"Version":"2012-10-17","Statement":[
|
||||
{"Effect":"Allow","Action":"s3:DeleteObject","Resource":format!("arn:aws:s3:::{bucket}/*")},
|
||||
{"Effect":"Deny","Action":"s3:DeleteObjectVersion","Resource":format!("arn:aws:s3:::{bucket}/*")}
|
||||
]})),
|
||||
)
|
||||
.await?;
|
||||
force_delete(&user, bucket, "folder/").await?;
|
||||
assert_eq!(
|
||||
versions(&root, bucket, "").await?,
|
||||
outside,
|
||||
"unversioned force deletion, including a synthetic nil directory marker, must use DeleteObject permission"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn force_delete_denied_child_preserves_every_object_despite_bucket_allow() -> TestResult {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
let root = env.create_s3_client();
|
||||
let bucket = "force-child-denial";
|
||||
root.create_bucket().bucket(bucket).send().await?;
|
||||
for key in ["folder/a-allowed.txt", "folder/z-denied.txt"] {
|
||||
put(&root, bucket, key).await?;
|
||||
}
|
||||
let user = policy_user(
|
||||
&env,
|
||||
"child-denier",
|
||||
Some(json!({"Version":"2012-10-17","Statement":[
|
||||
{"Effect":"Allow","Action":["s3:DeleteObject","s3:DeleteObjectVersion","s3:ReplicateDelete"],"Resource":format!("arn:aws:s3:::{bucket}/*")},
|
||||
{"Effect":"Deny","Action":["s3:DeleteObject","s3:DeleteObjectVersion","s3:ReplicateDelete"],"Resource":format!("arn:aws:s3:::{bucket}/folder/z-denied.txt")}
|
||||
]})),
|
||||
)
|
||||
.await?;
|
||||
root.put_bucket_policy()
|
||||
.bucket(bucket)
|
||||
.policy(json!({"Version":"2012-10-17","Statement":[
|
||||
{"Effect":"Allow","Principal":"*","Action":["s3:DeleteObject","s3:DeleteObjectVersion","s3:ReplicateDelete"],"Resource":format!("arn:aws:s3:::{bucket}/*")}
|
||||
]}).to_string())
|
||||
.send().await?;
|
||||
let before = versions(&root, bucket, "folder/").await?;
|
||||
assert_denied(force_delete(&user, bucket, "folder/").await);
|
||||
assert_eq!(
|
||||
versions(&root, bucket, "folder/").await?,
|
||||
before,
|
||||
"a denied descendant must prevent every mutation in the force scope"
|
||||
);
|
||||
assert_denied(replica_force_delete(&user, bucket, "folder/").await);
|
||||
assert_eq!(
|
||||
versions(&root, bucket, "folder/").await?,
|
||||
before,
|
||||
"the REPLICA header must not bypass a descendant's ReplicateDelete denial"
|
||||
);
|
||||
|
||||
root.delete_bucket_policy().bucket(bucket).send().await?;
|
||||
let replica_user = policy_user(
|
||||
&env,
|
||||
"replica-deleter",
|
||||
Some(json!({"Version":"2012-10-17","Statement":[
|
||||
{"Effect":"Allow","Action":"s3:DeleteObject","Resource":format!("arn:aws:s3:::{bucket}/*")},
|
||||
{"Effect":"Allow","Action":"s3:ReplicateDelete","Resource":format!("arn:aws:s3:::{bucket}/*")}
|
||||
]})),
|
||||
)
|
||||
.await?;
|
||||
replica_force_delete(&replica_user, bucket, "folder/").await?;
|
||||
assert!(
|
||||
versions(&root, bucket, "folder/").await?.is_empty(),
|
||||
"an authorized replica force request must check ReplicateDelete for its descendants"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn force_delete_denied_historical_version_preserves_versions_and_markers() -> TestResult {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
let root = env.create_s3_client();
|
||||
let bucket = "force-version-denial";
|
||||
root.create_bucket().bucket(bucket).send().await?;
|
||||
put(&root, bucket, "folder/null.txt").await?;
|
||||
versioning(&root, bucket, BucketVersioningStatus::Enabled).await?;
|
||||
let protected_version = put(&root, bucket, "folder/versioned.txt").await?;
|
||||
put(&root, bucket, "folder/versioned.txt").await?;
|
||||
let marker = root.delete_object().bucket(bucket).key("folder/versioned.txt").send().await?;
|
||||
let marker_version = marker
|
||||
.version_id()
|
||||
.ok_or("versioned delete must return a marker version ID")?;
|
||||
put(&root, bucket, "folder/a-allowed.txt").await?;
|
||||
let policy = |version: &str| {
|
||||
json!({"Version":"2012-10-17","Statement":[
|
||||
{"Effect":"Allow","Action":["s3:DeleteObject","s3:DeleteObjectVersion"],"Resource":format!("arn:aws:s3:::{bucket}/*")},
|
||||
{"Effect":"Deny","Action":"s3:DeleteObjectVersion","Resource":format!("arn:aws:s3:::{bucket}/folder/*"),
|
||||
"Condition":{"StringEquals":{"s3:VersionId":version}}}
|
||||
]})
|
||||
};
|
||||
let user = policy_user(&env, "version-denier", Some(policy(&protected_version))).await?;
|
||||
let before = versions(&root, bucket, "folder/").await?;
|
||||
for version in [protected_version.as_str(), "null", marker_version] {
|
||||
set_policy(&env, "version-denier", &policy(version)).await?;
|
||||
assert_denied(force_delete(&user, bucket, "folder/").await);
|
||||
assert_eq!(
|
||||
versions(&root, bucket, "folder/").await?,
|
||||
before,
|
||||
"denial of a historical, null, or delete-marker version must prevent recursive deletion"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sdk_ordinary_deletion_preserves_versions_and_directory_children() -> TestResult {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
let root = env.create_s3_client();
|
||||
let user = policy_user(
|
||||
&env,
|
||||
"ordinary-deleter",
|
||||
Some(json!({"Version":"2012-10-17","Statement":[
|
||||
{"Effect":"Allow","Action":"s3:DeleteObject","Resource":"arn:aws:s3:::*/*"},
|
||||
{"Effect":"Deny","Action":"s3:DeleteObjectVersion","Resource":"arn:aws:s3:::*/*"}
|
||||
]})),
|
||||
)
|
||||
.await?;
|
||||
for state in ["unversioned", "enabled", "suspended"] {
|
||||
let bucket = format!("ordinary-directory-{state}");
|
||||
root.create_bucket().bucket(&bucket).send().await?;
|
||||
if state != "unversioned" {
|
||||
versioning(&root, &bucket, BucketVersioningStatus::Enabled).await?;
|
||||
}
|
||||
let historical = put(&root, &bucket, "object.txt").await?;
|
||||
if state == "suspended" {
|
||||
versioning(&root, &bucket, BucketVersioningStatus::Suspended).await?;
|
||||
put(&root, &bucket, "object.txt").await?;
|
||||
}
|
||||
put(&root, &bucket, "folder/").await?;
|
||||
put(&root, &bucket, "folder/child.txt").await?;
|
||||
let child_before = versions(&root, &bucket, "folder/child.txt").await?;
|
||||
user.delete_object().bucket(&bucket).key("folder/").send().await?;
|
||||
assert_eq!(
|
||||
versions(&root, &bucket, "folder/").await?,
|
||||
child_before,
|
||||
"ordinary {state} directory-key deletion must remove only its synthetic marker and preserve children"
|
||||
);
|
||||
let deleted = user.delete_object().bucket(&bucket).key("object.txt").send().await?;
|
||||
let object_versions = versions(&root, &bucket, "object.txt").await?;
|
||||
if state == "unversioned" {
|
||||
assert!(object_versions.is_empty());
|
||||
} else {
|
||||
assert_eq!(deleted.delete_marker(), Some(true));
|
||||
assert_eq!(
|
||||
object_versions.len(),
|
||||
2,
|
||||
"ordinary {state} deletion must retain its historical data version"
|
||||
);
|
||||
assert!(object_versions.contains(&("object.txt".into(), historical, false)));
|
||||
if state == "suspended" {
|
||||
assert!(object_versions.contains(&("object.txt".into(), "null".into(), true)));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sdk_delete_objects_force_header_keeps_explicit_item_scope() -> TestResult {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
let root = env.create_s3_client();
|
||||
let bucket = "batch-force-explicit-scope";
|
||||
root.create_bucket().bucket(bucket).send().await?;
|
||||
put(&root, bucket, "folder/").await?;
|
||||
put(&root, bucket, "folder/child.txt").await?;
|
||||
let child = versions(&root, bucket, "folder/child.txt").await?;
|
||||
let user = policy_user(&env, "consoleAdmin", None).await?;
|
||||
let result = user
|
||||
.delete_objects()
|
||||
.bucket(bucket)
|
||||
.delete(
|
||||
Delete::builder()
|
||||
.objects(ObjectIdentifier::builder().key("folder/").build()?)
|
||||
.build()?,
|
||||
)
|
||||
.customize()
|
||||
.mutate_request(|request| {
|
||||
request.headers_mut().insert("x-rustfs-force-delete", "true");
|
||||
})
|
||||
.send()
|
||||
.await?;
|
||||
assert!(result.errors().is_empty());
|
||||
assert_eq!(result.deleted().len(), 1);
|
||||
assert_eq!(
|
||||
versions(&root, bucket, "folder/").await?,
|
||||
child,
|
||||
"batch deletion must remove only the explicit directory marker even with the force header"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn force_delete_checks_every_version_page_before_mutation() -> TestResult {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
let root = env.create_s3_client();
|
||||
let bucket = "force-delete-pagination";
|
||||
root.create_bucket().bucket(bucket).send().await?;
|
||||
versioning(&root, bucket, BucketVersioningStatus::Enabled).await?;
|
||||
stream::iter(0..1000)
|
||||
.map(|index| {
|
||||
let root = &root;
|
||||
async move { put(root, bucket, &format!("folder/{index:04}.txt")).await.map(|_| ()) }
|
||||
})
|
||||
.buffer_unordered(16)
|
||||
.try_collect::<Vec<_>>()
|
||||
.await?;
|
||||
put(&root, bucket, "folder/z-denied.txt").await?;
|
||||
let allow = json!({"Effect":"Allow","Action":["s3:DeleteObject","s3:DeleteObjectVersion"],"Resource":format!("arn:aws:s3:::{bucket}/*")});
|
||||
let user = policy_user(
|
||||
&env,
|
||||
"paged-deleter",
|
||||
Some(json!({"Version":"2012-10-17","Statement":[allow.clone(),
|
||||
{"Effect":"Deny","Action":"s3:DeleteObjectVersion","Resource":format!("arn:aws:s3:::{bucket}/folder/z-denied.txt")}
|
||||
]})),
|
||||
)
|
||||
.await?;
|
||||
let before = versions(&root, bucket, "folder/").await?;
|
||||
assert_eq!(before.len(), 1001, "the denied key must be beyond one default versions page");
|
||||
assert_denied(force_delete(&user, bucket, "folder/").await);
|
||||
assert_eq!(
|
||||
versions(&root, bucket, "folder/").await?,
|
||||
before,
|
||||
"a denial on the second page must preserve the first page too"
|
||||
);
|
||||
|
||||
set_policy(&env, "paged-deleter", &json!({"Version":"2012-10-17","Statement":[allow]})).await?;
|
||||
force_delete(&user, bucket, "folder/").await?;
|
||||
assert!(
|
||||
versions(&root, bucket, "folder/").await?.is_empty(),
|
||||
"authorized recursive deletion must cover all pages"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -28,6 +28,7 @@ mod harness;
|
||||
mod heal_test;
|
||||
mod object_lock_test;
|
||||
mod observability_test;
|
||||
mod replication_delete_marker_test;
|
||||
mod replication_quota_test;
|
||||
mod s3_basic_test;
|
||||
mod s3_during_data_movement_test;
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Functional REP-105 (rustfs/backlog#2195 item 4): a delete marker created
|
||||
//! on a multi-node source cluster must replicate to the bucket-replication
|
||||
//! target. Objects converged in seconds while delete markers did not arrive
|
||||
//! within 180 s on the shared 3-node functional environment; the single-node
|
||||
//! e2e never saw it.
|
||||
|
||||
use super::harness::{
|
||||
DistCluster, DistLayout, TestResult, enable_versioning, put_bucket_replication, put_object, set_remote_target, unique_bucket,
|
||||
wait_for_replicated_bytes, wait_until,
|
||||
};
|
||||
use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, RustFSTestEnvironment, init_logging, replication_fast_env, signed_request};
|
||||
use crate::replication_extension_test::LOOPBACK_REPLICATION_TARGET_ENV;
|
||||
use aws_sdk_s3::Client;
|
||||
use http::{Method, StatusCode};
|
||||
use std::time::Duration;
|
||||
|
||||
async fn target_has_delete_marker(client: &Client, bucket: &str, key: &str) -> TestResult<bool> {
|
||||
let versions = client.list_object_versions().bucket(bucket).prefix(key).send().await?;
|
||||
Ok(versions.delete_markers().iter().any(|marker| marker.key() == Some(key)))
|
||||
}
|
||||
|
||||
async fn delete_marker_replicates(
|
||||
source: &DistCluster,
|
||||
source_bucket: &str,
|
||||
target_client: &Client,
|
||||
target_bucket: &str,
|
||||
) -> TestResult {
|
||||
let key = "delete-marker/object.bin";
|
||||
let body = b"delete marker replication payload".to_vec();
|
||||
// Write through one node, delete through another: behind a load
|
||||
// balancer consecutive requests land on different nodes.
|
||||
put_object(&source.client(1)?, source_bucket, key, body.clone()).await?;
|
||||
wait_for_replicated_bytes(target_client, target_bucket, key, &body, Duration::from_secs(60)).await?;
|
||||
|
||||
let delete = source
|
||||
.client(2)?
|
||||
.delete_object()
|
||||
.bucket(source_bucket)
|
||||
.key(key)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
delete.delete_marker(),
|
||||
Some(true),
|
||||
"a versioned DELETE without versionId must create a marker"
|
||||
);
|
||||
|
||||
wait_until(
|
||||
Duration::from_secs(90),
|
||||
|| async { target_has_delete_marker(target_client, target_bucket, key).await },
|
||||
"delete marker replicated to the target bucket",
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_bucket_replication_replicates_delete_marker_to_peer_cluster() -> TestResult {
|
||||
init_logging();
|
||||
let (source, target) = DistCluster::start_replication_pair().await?;
|
||||
let source_bucket = unique_bucket("dm-src");
|
||||
let target_bucket = unique_bucket("dm-dst");
|
||||
source.create_bucket(&source_bucket).await?;
|
||||
target.create_bucket(&target_bucket).await?;
|
||||
enable_versioning(&source.client(0)?, &source_bucket).await?;
|
||||
enable_versioning(&target.client(0)?, &target_bucket).await?;
|
||||
|
||||
let arn = set_remote_target(&source.cluster, &source_bucket, &target.cluster, &target_bucket).await?;
|
||||
put_bucket_replication(&source.cluster, &source_bucket, &arn).await?;
|
||||
|
||||
delete_marker_replicates(&source, &source_bucket, &target.client(0)?, &target_bucket).await
|
||||
}
|
||||
|
||||
/// The functional environment replicates from a 3-node site to a single-node
|
||||
/// target; keep that shape as its own case.
|
||||
#[tokio::test]
|
||||
async fn four_node_bucket_replication_replicates_delete_marker_to_single_node_target() -> TestResult {
|
||||
init_logging();
|
||||
let mut extra: Vec<(&str, &str)> = replication_fast_env();
|
||||
extra.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
extra.extend_from_slice(FAST_DATA_USAGE_SCANNER_ENV);
|
||||
let source = DistCluster::start_with_env(DistLayout::FourNodeFourDisk, &extra).await?;
|
||||
let mut target = RustFSTestEnvironment::new().await?;
|
||||
target.start_rustfs_server_without_cleanup(vec![]).await?;
|
||||
|
||||
let source_bucket = unique_bucket("dm-src");
|
||||
let target_bucket = unique_bucket("dm-dst");
|
||||
source.create_bucket(&source_bucket).await?;
|
||||
let target_client = target.create_s3_client();
|
||||
target_client.create_bucket().bucket(&target_bucket).send().await?;
|
||||
enable_versioning(&source.client(0)?, &source_bucket).await?;
|
||||
enable_versioning(&target_client, &target_bucket).await?;
|
||||
|
||||
let body = serde_json::json!({
|
||||
"endpoint": target.address,
|
||||
"credentials": { "accessKey": target.access_key, "secretKey": target.secret_key },
|
||||
"targetbucket": target_bucket,
|
||||
"secure": false,
|
||||
"type": "replication"
|
||||
});
|
||||
let url = format!(
|
||||
"{}/rustfs/admin/v3/set-remote-target?bucket={}",
|
||||
source.cluster.nodes[0].url,
|
||||
urlencoding::encode(&source_bucket)
|
||||
);
|
||||
let response = signed_request(
|
||||
Method::PUT,
|
||||
&url,
|
||||
&source.cluster.access_key,
|
||||
&source.cluster.secret_key,
|
||||
Some(body.to_string().into_bytes()),
|
||||
Some("application/json"),
|
||||
)
|
||||
.await?;
|
||||
if response.status() != StatusCode::OK {
|
||||
let status = response.status();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
return Err(format!("set remote target failed: {status} {body}").into());
|
||||
}
|
||||
let arn: String = serde_json::from_slice(&response.bytes().await?)?;
|
||||
put_bucket_replication(&source.cluster, &source_bucket, &arn).await?;
|
||||
|
||||
delete_marker_replicates(&source, &source_bucket, &target_client, &target_bucket).await
|
||||
}
|
||||
@@ -126,3 +126,165 @@ async fn four_node_site_replication_replicates_object_to_peer_site() -> TestResu
|
||||
wait_for_replicated_bytes(&site_a.client(3)?, &bucket, reverse_key, &reverse_body, Duration::from_secs(60)).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn node_admin(
|
||||
cluster: &crate::common::RustFSTestClusterEnvironment,
|
||||
node_idx: usize,
|
||||
method: Method,
|
||||
path_and_query: &str,
|
||||
body: Option<String>,
|
||||
) -> TestResult<(StatusCode, String)> {
|
||||
crate::common::admin_request(
|
||||
&cluster.nodes[node_idx].url,
|
||||
method,
|
||||
path_and_query,
|
||||
body,
|
||||
&cluster.access_key,
|
||||
&cluster.secret_key,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Pair two clusters through site A's first node and wait until both report
|
||||
/// the two-site topology as enabled.
|
||||
async fn pair_sites(site_a: &DistCluster, site_b: &DistCluster) -> TestResult {
|
||||
let sites = vec![
|
||||
PeerSite {
|
||||
name: "site-a".to_string(),
|
||||
endpoint: site_a.cluster.nodes[0].url.clone(),
|
||||
access_key: site_a.cluster.access_key.clone(),
|
||||
secret_key: site_a.cluster.secret_key.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
PeerSite {
|
||||
name: "site-b".to_string(),
|
||||
endpoint: site_b.cluster.nodes[0].url.clone(),
|
||||
access_key: site_b.cluster.access_key.clone(),
|
||||
secret_key: site_b.cluster.secret_key.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
];
|
||||
let add_status = site_replication_add(&site_a.cluster, &sites).await?;
|
||||
assert!(
|
||||
add_status.success && add_status.err_detail.is_empty() && add_status.initial_sync_error_message.is_empty(),
|
||||
"site replication add reported failure: {add_status:?}"
|
||||
);
|
||||
wait_for_site_replication_enabled(&site_a.cluster).await?;
|
||||
wait_for_site_replication_enabled(&site_b.cluster).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn list_users_contains(
|
||||
cluster: &crate::common::RustFSTestClusterEnvironment,
|
||||
node_idx: usize,
|
||||
access_key: &str,
|
||||
) -> TestResult<bool> {
|
||||
let (status, body) = node_admin(cluster, node_idx, Method::GET, "/rustfs/admin/v3/list-users", None).await?;
|
||||
if !status.is_success() {
|
||||
return Err(format!("list-users on node {node_idx} failed: {status} {body}").into());
|
||||
}
|
||||
let users: serde_json::Value = serde_json::from_str(&body)?;
|
||||
Ok(users.get(access_key).is_some())
|
||||
}
|
||||
|
||||
/// backlog#2367 A-7 / functional SITE-102: an IAM change handled by a node
|
||||
/// other than the one that ran `site-replication/add` must still reach the
|
||||
/// peer site. Behind a load balancer every admin call may land on a
|
||||
/// different node, so the coordinator node is not special.
|
||||
#[tokio::test]
|
||||
async fn four_node_site_replication_converges_iam_user_created_on_a_non_coordinator_node() -> TestResult {
|
||||
init_logging();
|
||||
let (site_a, site_b) = DistCluster::start_replication_pair().await?;
|
||||
pair_sites(&site_a, &site_b).await?;
|
||||
|
||||
let user = format!("siteuser-{}", &uuid::Uuid::new_v4().simple().to_string()[..8]);
|
||||
let body = serde_json::json!({ "secretKey": "siteuser-secret-key-1234", "status": "enabled" }).to_string();
|
||||
let (status, response) = node_admin(
|
||||
&site_a.cluster,
|
||||
1,
|
||||
Method::PUT,
|
||||
&format!("/rustfs/admin/v3/add-user?accessKey={user}"),
|
||||
Some(body),
|
||||
)
|
||||
.await?;
|
||||
assert!(status.is_success(), "add-user on site A node 1 failed: {status} {response}");
|
||||
|
||||
let site_b_cluster = &site_b.cluster;
|
||||
let user_ref = user.as_str();
|
||||
wait_until(
|
||||
Duration::from_secs(90),
|
||||
|| async move { list_users_contains(site_b_cluster, 0, user_ref).await },
|
||||
"user created on site A node 1 visible on site B",
|
||||
)
|
||||
.await?;
|
||||
assert!(
|
||||
list_users_contains(&site_a.cluster, 2, &user).await?,
|
||||
"the user must be visible on every site A node"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// backlog#2367 A-5 / functional SITE-105: a resync started right after
|
||||
/// pairing must not report buckets as failed. The bucket carrying an
|
||||
/// operator-configured bucket-replication target to the peer (the shape the
|
||||
/// functional suite leaves behind) and a plain versioned bucket are both
|
||||
/// wired by the pairing itself.
|
||||
#[tokio::test]
|
||||
async fn four_node_site_replication_resync_start_right_after_pairing_reports_no_failed_bucket() -> TestResult {
|
||||
init_logging();
|
||||
let (site_a, site_b) = DistCluster::start_replication_pair().await?;
|
||||
|
||||
let pre_src = unique_bucket("pre-src");
|
||||
let pre_dst = unique_bucket("pre-dst");
|
||||
let plain = unique_bucket("plain");
|
||||
site_a.create_bucket(&pre_src).await?;
|
||||
site_b.create_bucket(&pre_dst).await?;
|
||||
site_a.create_bucket(&plain).await?;
|
||||
enable_versioning(&site_a.client(0)?, &pre_src).await?;
|
||||
enable_versioning(&site_b.client(0)?, &pre_dst).await?;
|
||||
enable_versioning(&site_a.client(0)?, &plain).await?;
|
||||
let arn = super::harness::set_remote_target(&site_a.cluster, &pre_src, &site_b.cluster, &pre_dst).await?;
|
||||
super::harness::put_bucket_replication(&site_a.cluster, &pre_src, &arn).await?;
|
||||
|
||||
pair_sites(&site_a, &site_b).await?;
|
||||
|
||||
let (status, info) = node_admin(&site_a.cluster, 1, Method::GET, "/rustfs/admin/v3/site-replication/info", None).await?;
|
||||
assert!(status.is_success(), "site-replication/info failed: {status} {info}");
|
||||
let info: serde_json::Value = serde_json::from_str(&info)?;
|
||||
let peer = info["sites"]
|
||||
.as_array()
|
||||
.and_then(|sites| sites.iter().find(|site| site["name"] == "site-b"))
|
||||
.cloned()
|
||||
.ok_or_else(|| format!("site-b peer missing from info: {info}"))?;
|
||||
|
||||
// Through a non-coordinator node, like a load-balanced admin call.
|
||||
let (status, response) = node_admin(
|
||||
&site_a.cluster,
|
||||
1,
|
||||
Method::PUT,
|
||||
"/rustfs/admin/v3/site-replication/resync/op?operation=start",
|
||||
Some(peer.to_string()),
|
||||
)
|
||||
.await?;
|
||||
assert!(status.is_success(), "resync start failed: {status} {response}");
|
||||
let resync: rustfs_madmin::SRResyncOpStatus = serde_json::from_str(&response)?;
|
||||
let failed: Vec<String> = resync
|
||||
.buckets
|
||||
.iter()
|
||||
.filter(|bucket| bucket.status == "failed")
|
||||
.map(|bucket| format!("{}: {}", bucket.bucket, bucket.err_detail))
|
||||
.collect();
|
||||
assert!(
|
||||
failed.is_empty(),
|
||||
"resync right after pairing reported failed buckets: {failed:?} (status={}, detail={})",
|
||||
resync.status,
|
||||
resync.err_detail
|
||||
);
|
||||
assert!(
|
||||
resync.buckets.iter().any(|bucket| bucket.bucket == pre_src)
|
||||
&& resync.buckets.iter().any(|bucket| bucket.bucket == plain),
|
||||
"both buckets must be part of the resync: {:?}",
|
||||
resync.buckets
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -22,7 +22,11 @@ mod tests {
|
||||
init_logging, rustfs_binary_path,
|
||||
};
|
||||
use crate::storage_api::RUSTFS_META_BUCKET;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::{
|
||||
error::{ProvideErrorMetadata, SdkError},
|
||||
operation::put_object::PutObjectError,
|
||||
primitives::ByteStream,
|
||||
};
|
||||
use http::Method;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashSet;
|
||||
@@ -39,6 +43,13 @@ mod tests {
|
||||
|
||||
const POOL_METADATA_OBJECT: &str = "pool.bin";
|
||||
|
||||
struct ReplacementDriveSelection {
|
||||
replaced_disk: PathBuf,
|
||||
replacement_format_path: PathBuf,
|
||||
replacement_format: Vec<u8>,
|
||||
expected_pool_metadata: Option<VersionShardCensus>,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct EvidenceBuild {
|
||||
sha256: String,
|
||||
@@ -586,6 +597,54 @@ mod tests {
|
||||
&& operations["activeBySource"]["admin"].as_u64() == Some(1)
|
||||
}
|
||||
|
||||
fn is_service_unavailable_put(error: &SdkError<PutObjectError>) -> bool {
|
||||
error.as_service_error().and_then(ProvideErrorMetadata::code) == Some("ServiceUnavailable")
|
||||
}
|
||||
|
||||
fn select_replacement_drive(
|
||||
cluster: &RustFSTestClusterEnvironment,
|
||||
node_index: usize,
|
||||
require_pool_metadata: bool,
|
||||
) -> Result<ReplacementDriveSelection, Box<dyn Error + Send + Sync>> {
|
||||
let node = cluster
|
||||
.nodes
|
||||
.get(node_index)
|
||||
.ok_or_else(|| format!("replacement node {node_index} is absent"))?;
|
||||
let mut incomplete_pool_metadata = Vec::new();
|
||||
|
||||
for drive in &node.data_dirs {
|
||||
let replaced_disk = PathBuf::from(drive);
|
||||
let replacement_format_path = replaced_disk.join(".rustfs.sys").join("format.json");
|
||||
let replacement_format = std::fs::read(&replacement_format_path).map_err(|err| {
|
||||
format!("failed to capture target format before replacement wipe at {replacement_format_path:?}: {err}")
|
||||
})?;
|
||||
if !require_pool_metadata {
|
||||
return Ok(ReplacementDriveSelection {
|
||||
replaced_disk,
|
||||
replacement_format_path,
|
||||
replacement_format,
|
||||
expected_pool_metadata: None,
|
||||
});
|
||||
}
|
||||
|
||||
let census = census_object_version_on_disk(&replaced_disk, RUSTFS_META_BUCKET, POOL_METADATA_OBJECT, None)?;
|
||||
if census.is_complete() {
|
||||
return Ok(ReplacementDriveSelection {
|
||||
replaced_disk,
|
||||
replacement_format_path,
|
||||
replacement_format,
|
||||
expected_pool_metadata: Some(census),
|
||||
});
|
||||
}
|
||||
incomplete_pool_metadata.push(census);
|
||||
}
|
||||
|
||||
Err(format!(
|
||||
"no replacement drive on node {node_index} held complete pool metadata before the fault: {incomplete_pool_metadata:?}"
|
||||
)
|
||||
.into())
|
||||
}
|
||||
|
||||
async fn replacement_recovery_status(
|
||||
cluster: &RustFSTestClusterEnvironment,
|
||||
) -> Result<serde_json::Value, Box<dyn Error + Send + Sync>> {
|
||||
@@ -1008,7 +1067,7 @@ mod tests {
|
||||
let mut versions_observed = false;
|
||||
let mut observations = Vec::with_capacity(cluster.nodes.len());
|
||||
for (node_index, node) in cluster.nodes.iter().enumerate() {
|
||||
let (status, body) = timeout(
|
||||
let response = timeout(
|
||||
Duration::from_secs(5),
|
||||
admin_request(
|
||||
&node.url,
|
||||
@@ -1019,8 +1078,22 @@ mod tests {
|
||||
&cluster.secret_key,
|
||||
),
|
||||
)
|
||||
.await??;
|
||||
assert_eq!(status, 200, "scanner status must be available: {body}");
|
||||
.await;
|
||||
let (status, body) = match response {
|
||||
Ok(Ok(response)) => response,
|
||||
Ok(Err(error)) => {
|
||||
observations.push(format!("node{node_index}: scanner status request failed: {error}"));
|
||||
continue;
|
||||
}
|
||||
Err(_) => {
|
||||
observations.push(format!("node{node_index}: scanner status request exceeded 5s"));
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if status != 200 {
|
||||
observations.push(format!("node{node_index}: scanner status returned {status}: {body}"));
|
||||
continue;
|
||||
}
|
||||
let status: serde_json::Value = serde_json::from_str(&body)?;
|
||||
assert_eq!(status["enabled"].as_bool(), Some(true), "scanner must stay enabled: {status}");
|
||||
let metrics = &status["metrics"];
|
||||
@@ -1257,16 +1330,18 @@ mod tests {
|
||||
let bucket = "heal-restart-during-rebuild";
|
||||
clients[0].create_bucket().bucket(bucket).send().await?;
|
||||
|
||||
let replaced_disk = PathBuf::from(&cluster.nodes[1].data_dir);
|
||||
let replacement_format_path = replaced_disk.join(".rustfs.sys").join("format.json");
|
||||
let replacement_format = std::fs::read(&replacement_format_path).map_err(|err| {
|
||||
format!("failed to capture target format before replacement wipe at {replacement_format_path:?}: {err}")
|
||||
})?;
|
||||
let ReplacementDriveSelection {
|
||||
replaced_disk,
|
||||
replacement_format_path,
|
||||
replacement_format,
|
||||
expected_pool_metadata,
|
||||
} = select_replacement_drive(&cluster, 1, background_enabled)?;
|
||||
let default_online_object_count = if !outage_target_manifest_required { 64 } else { 24 };
|
||||
let online_object_count = std::env::var("RUSTFS_HEAL_CHAOS_OBJECT_COUNT")
|
||||
.ok()
|
||||
.and_then(|value| value.parse::<usize>().ok())
|
||||
.unwrap_or(24)
|
||||
.clamp(8, 64);
|
||||
.unwrap_or(default_online_object_count)
|
||||
.clamp(8, 128);
|
||||
let object_size_bytes = std::env::var("RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES")
|
||||
.ok()
|
||||
.and_then(|value| value.parse::<usize>().ok())
|
||||
@@ -1317,17 +1392,9 @@ mod tests {
|
||||
attempt_count += 1;
|
||||
}
|
||||
|
||||
let expected_pool_metadata = if background_enabled {
|
||||
let census = census_object_version_on_disk(&replaced_disk, RUSTFS_META_BUCKET, POOL_METADATA_OBJECT, None)?;
|
||||
assert!(
|
||||
census.is_complete(),
|
||||
"target must hold complete pool metadata before the fault: {census:?}"
|
||||
);
|
||||
if background_enabled {
|
||||
wait_for_scanner_cycle_after(&cluster, 0).await?;
|
||||
Some(census)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
}
|
||||
|
||||
cluster.stop_node(1)?;
|
||||
std::fs::remove_dir_all(&replaced_disk)?;
|
||||
@@ -1342,48 +1409,87 @@ mod tests {
|
||||
"replacement target must retain only its preformatted topology identity"
|
||||
);
|
||||
|
||||
let outage_key = "cluster/written-while-node-down.bin";
|
||||
let outage_payload_seed = 0xf1;
|
||||
timeout(
|
||||
Duration::from_secs(30),
|
||||
clients[2]
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(outage_key)
|
||||
.body(ByteStream::from(deterministic_object_body(object_size_bytes, outage_payload_seed)))
|
||||
.send(),
|
||||
)
|
||||
.await??;
|
||||
let max_outage_write_attempts = topology.total_drives().max(1);
|
||||
let mut outage_key = None;
|
||||
let mut outage_write_deferred_until_rejoin = false;
|
||||
let mut service_unavailable_outage_writes = 0usize;
|
||||
let mut last_service_unavailable = None;
|
||||
for attempt in 0..max_outage_write_attempts {
|
||||
let candidate_key = format!("cluster/written-while-node-down-{attempt:04}.bin");
|
||||
let put_result = timeout(
|
||||
Duration::from_secs(30),
|
||||
clients[2]
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(&candidate_key)
|
||||
.body(ByteStream::from(deterministic_object_body(object_size_bytes, outage_payload_seed)))
|
||||
.send(),
|
||||
)
|
||||
.await;
|
||||
match put_result {
|
||||
Ok(Ok(_)) => {
|
||||
outage_key = Some(candidate_key);
|
||||
break;
|
||||
}
|
||||
Ok(Err(error)) if is_service_unavailable_put(&error) => {
|
||||
service_unavailable_outage_writes += 1;
|
||||
last_service_unavailable = Some(format!("{error:?}"));
|
||||
}
|
||||
Ok(Err(error)) => return Err(error.into()),
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
}
|
||||
let outage_key = match outage_key {
|
||||
Some(key) => key,
|
||||
None if !outage_target_manifest_required => {
|
||||
outage_write_deferred_until_rejoin = true;
|
||||
"cluster/written-after-target-rejoin.bin".to_string()
|
||||
}
|
||||
None => {
|
||||
return Err(format!(
|
||||
"no online pool accepted an outage object after {max_outage_write_attempts} candidates; \
|
||||
observed {service_unavailable_outage_writes} ServiceUnavailable responses; \
|
||||
last ServiceUnavailable: {last_service_unavailable:?}"
|
||||
)
|
||||
.into());
|
||||
}
|
||||
};
|
||||
|
||||
let mut outage_peer_erasure_indices = HashSet::new();
|
||||
for (node_index, node) in cluster.nodes.iter().enumerate() {
|
||||
if node_index == 1 {
|
||||
continue;
|
||||
}
|
||||
for (drive_index, drive) in node.data_dirs.iter().enumerate() {
|
||||
let census = census_object_version_on_disk(Path::new(drive), bucket, outage_key, None)?;
|
||||
if !census.has_xl_meta {
|
||||
if !outage_write_deferred_until_rejoin {
|
||||
for (node_index, node) in cluster.nodes.iter().enumerate() {
|
||||
if node_index == 1 {
|
||||
continue;
|
||||
}
|
||||
assert!(
|
||||
census.is_complete(),
|
||||
"online node {node_index} drive {drive_index} must hold a complete outage-object shard: {census:?}"
|
||||
);
|
||||
let erasure_index = census.erasure_index.ok_or_else(|| {
|
||||
format!("online node {node_index} drive {drive_index} outage-object shard has no erasure index: {census:?}")
|
||||
})?;
|
||||
assert!(
|
||||
(1..=erasure_set_drive_count).contains(&erasure_index),
|
||||
"online node {node_index} drive {drive_index} outage-object erasure index is out of range: {census:?}"
|
||||
);
|
||||
assert!(
|
||||
outage_peer_erasure_indices.insert(erasure_index),
|
||||
"outage-object erasure index {erasure_index} is duplicated across online drives"
|
||||
);
|
||||
for (drive_index, drive) in node.data_dirs.iter().enumerate() {
|
||||
let census = census_object_version_on_disk(Path::new(drive), bucket, &outage_key, None)?;
|
||||
if !census.has_xl_meta {
|
||||
continue;
|
||||
}
|
||||
assert!(
|
||||
census.is_complete(),
|
||||
"online node {node_index} drive {drive_index} must hold a complete outage-object shard: {census:?}"
|
||||
);
|
||||
let erasure_index = census.erasure_index.ok_or_else(|| {
|
||||
format!(
|
||||
"online node {node_index} drive {drive_index} outage-object shard has no erasure index: {census:?}"
|
||||
)
|
||||
})?;
|
||||
assert!(
|
||||
(1..=erasure_set_drive_count).contains(&erasure_index),
|
||||
"online node {node_index} drive {drive_index} outage-object erasure index is out of range: {census:?}"
|
||||
);
|
||||
assert!(
|
||||
outage_peer_erasure_indices.insert(erasure_index),
|
||||
"outage-object erasure index {erasure_index} is duplicated across online drives"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
!outage_peer_erasure_indices.is_empty() && outage_peer_erasure_indices.len() <= erasure_set_drive_count,
|
||||
outage_write_deferred_until_rejoin
|
||||
|| (!outage_peer_erasure_indices.is_empty() && outage_peer_erasure_indices.len() <= erasure_set_drive_count),
|
||||
"outage-object must occupy one non-empty erasure set"
|
||||
);
|
||||
if outage_target_manifest_required {
|
||||
@@ -1424,6 +1530,16 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
// Keep the partial-repair checkpoint stable across readiness and admin
|
||||
// requests. Endpoint-blackhole tests must prove their own network stall.
|
||||
let commit_barrier = if scenario != InterruptionScenario::TargetEndpointBlackhole {
|
||||
let barrier = replaced_disk.join(".rustfs.sys/e2e-heal-commit-barrier");
|
||||
std::fs::create_dir_all(barrier.parent().ok_or("commit barrier has no parent")?)?;
|
||||
std::fs::write(&barrier, format!("{bucket}/cluster/online/"))?;
|
||||
Some(barrier)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
cluster.start_node_from_binary(1, &server_binary).await?;
|
||||
|
||||
let status_url = format!("{}/rustfs/admin/v3/background-heal/status", cluster.nodes[0].url);
|
||||
@@ -1436,7 +1552,12 @@ mod tests {
|
||||
);
|
||||
let recovered: serde_json::Value = serde_json::from_str(&status_body)
|
||||
.map_err(|err| format!("background heal status is not JSON ({err}): {status_body}"))?;
|
||||
let ready = if background_enabled {
|
||||
let ready = if background_enabled && outage_write_deferred_until_rejoin {
|
||||
// Whole-pool outage writes are deferred, so start the admin
|
||||
// root heal as soon as the target answers instead of waiting
|
||||
// for background convergence to consume the interruption window.
|
||||
true
|
||||
} else if background_enabled {
|
||||
recovered["clusterStatusComplete"] == serde_json::Value::Bool(true)
|
||||
} else {
|
||||
cluster_heal_is_idle(&recovered)
|
||||
@@ -1457,7 +1578,7 @@ mod tests {
|
||||
"non-admin Heal is disabled, so the replacement target must remain empty before the explicit root heal"
|
||||
);
|
||||
assert!(
|
||||
!census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?.has_xl_meta,
|
||||
!census_object_version_on_disk(&replaced_disk, bucket, &outage_key, None)?.has_xl_meta,
|
||||
"the object written during the outage must be absent before the explicit root heal"
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -1467,38 +1588,50 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
let background_rejoin_heal_evidence = background_enabled && outage_write_deferred_until_rejoin;
|
||||
let heal_url = format!("{}/rustfs/admin/v3/heal/?forceStart=true", cluster.nodes[0].url);
|
||||
let heal_start_body = signed_admin_post(&heal_url, Some(heal_body), &cluster.access_key, &cluster.secret_key).await?;
|
||||
let heal_start: serde_json::Value = serde_json::from_str(&heal_start_body)
|
||||
.map_err(|err| format!("heal start response is not JSON ({err}): {heal_start_body}"))?;
|
||||
let client_token = heal_start["clientToken"]
|
||||
.as_str()
|
||||
.filter(|token| !token.is_empty())
|
||||
.ok_or_else(|| format!("heal start response has no client token: {heal_start}"))?;
|
||||
let task_status_url = format!("{}/rustfs/admin/v3/heal/?clientToken={client_token}", cluster.nodes[0].url);
|
||||
let (mut client_token, mut task_status_url) = if background_rejoin_heal_evidence {
|
||||
(String::new(), String::new())
|
||||
} else {
|
||||
let heal_start_body = signed_admin_post(&heal_url, Some(heal_body), &cluster.access_key, &cluster.secret_key).await?;
|
||||
let heal_start: serde_json::Value = serde_json::from_str(&heal_start_body)
|
||||
.map_err(|err| format!("heal start response is not JSON ({err}): {heal_start_body}"))?;
|
||||
let client_token = heal_start["clientToken"]
|
||||
.as_str()
|
||||
.filter(|token| !token.is_empty())
|
||||
.ok_or_else(|| format!("heal start response has no client token: {heal_start}"))?
|
||||
.to_string();
|
||||
let task_status_url = format!("{}/rustfs/admin/v3/heal/?clientToken={client_token}", cluster.nodes[0].url);
|
||||
(client_token, task_status_url)
|
||||
};
|
||||
let restart_recovery_admin_after_failure =
|
||||
scenario == InterruptionScenario::BackgroundTargetCrashEc84MultiPool && !background_rejoin_heal_evidence;
|
||||
let mut recovery_admin_task_restarted = false;
|
||||
|
||||
let partial_timeout_secs = std::env::var("RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS")
|
||||
.ok()
|
||||
.and_then(|value| value.parse::<u64>().ok())
|
||||
.unwrap_or(60);
|
||||
let partial_deadline = Instant::now() + Duration::from_secs(partial_timeout_secs);
|
||||
loop {
|
||||
let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
|
||||
let active_status: serde_json::Value = serde_json::from_str(&status_body)
|
||||
.map_err(|err| format!("background heal status is not JSON ({err}): {status_body}"))?;
|
||||
let active = if background_enabled {
|
||||
active_status["state"].as_str() == Some("active")
|
||||
&& active_status["healOperations"]["activeBySource"]["admin"].as_u64() == Some(1)
|
||||
} else {
|
||||
only_admin_heal_is_active(&active_status)
|
||||
};
|
||||
if active {
|
||||
break;
|
||||
if !background_rejoin_heal_evidence {
|
||||
loop {
|
||||
let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
|
||||
let active_status: serde_json::Value = serde_json::from_str(&status_body)
|
||||
.map_err(|err| format!("background heal status is not JSON ({err}): {status_body}"))?;
|
||||
let active = if background_enabled {
|
||||
active_status["state"].as_str() == Some("active")
|
||||
&& active_status["healOperations"]["activeBySource"]["admin"].as_u64() == Some(1)
|
||||
} else {
|
||||
only_admin_heal_is_active(&active_status)
|
||||
};
|
||||
if active {
|
||||
break;
|
||||
}
|
||||
if Instant::now() >= partial_deadline {
|
||||
return Err(format!("root heal never became active within {partial_timeout_secs}s: {active_status}").into());
|
||||
}
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
if Instant::now() >= partial_deadline {
|
||||
return Err(format!("root heal never became active within {partial_timeout_secs}s: {active_status}").into());
|
||||
}
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
let (partial_count, partial_manifest) = loop {
|
||||
// Hash one committed shard to prove progress without letting a
|
||||
@@ -1516,43 +1649,74 @@ mod tests {
|
||||
}
|
||||
if materialized == expected_manifests.len() {
|
||||
return Err(format!(
|
||||
"root heal rebuilt all {} baseline objects before the target could be interrupted",
|
||||
"{} rebuilt all {} baseline objects before the target could be interrupted",
|
||||
if background_rejoin_heal_evidence {
|
||||
"background rejoin heal"
|
||||
} else {
|
||||
"root heal"
|
||||
},
|
||||
expected_manifests.len()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
if Instant::now() >= partial_deadline {
|
||||
return Err(format!(
|
||||
"root heal made no observable partial progress on the replacement target within {partial_timeout_secs}s"
|
||||
"{} made no observable partial progress on the replacement target within {partial_timeout_secs}s",
|
||||
if background_rejoin_heal_evidence {
|
||||
"background rejoin heal"
|
||||
} else {
|
||||
"root heal"
|
||||
}
|
||||
)
|
||||
.into());
|
||||
}
|
||||
sleep(Duration::from_millis(10)).await;
|
||||
};
|
||||
|
||||
if let Some(barrier) = &commit_barrier {
|
||||
assert!(
|
||||
barrier.with_extension("admitted").is_file(),
|
||||
"interruption tests require a server built with e2e-test-hooks"
|
||||
);
|
||||
}
|
||||
let pre_interrupt_status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
|
||||
let pre_interrupt_status: serde_json::Value = serde_json::from_str(&pre_interrupt_status_body)
|
||||
.map_err(|err| format!("pre-interrupt background heal status is not JSON ({err}): {pre_interrupt_status_body}"))?;
|
||||
let pre_interrupt_replacement = replacement_recovery_status(&cluster).await?;
|
||||
let coordinator_log = std::fs::read_to_string(format!("{log_dir}/node0.log"))?;
|
||||
assert!(
|
||||
coordinator_log
|
||||
.lines()
|
||||
.filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
|
||||
.any(|event| {
|
||||
event["event"] == "heal_task_state"
|
||||
&& event["task_id"] == client_token
|
||||
&& event["heal_type"] == "cluster"
|
||||
&& event["state"] == "started"
|
||||
}),
|
||||
"node 0 must have started the exact admin task before interruption"
|
||||
);
|
||||
if background_rejoin_heal_evidence {
|
||||
let target_log = std::fs::read_to_string(format!("{log_dir}/node1.log"))?;
|
||||
assert!(
|
||||
target_log
|
||||
.lines()
|
||||
.filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
|
||||
.any(|event| {
|
||||
event["event"] == "heal_task_state" && event["heal_type"] == "erasure_set" && event["state"] == "started"
|
||||
}),
|
||||
"node 1 must have started a background erasure-set heal before interruption"
|
||||
);
|
||||
} else {
|
||||
let coordinator_log = std::fs::read_to_string(format!("{log_dir}/node0.log"))?;
|
||||
assert!(
|
||||
coordinator_log
|
||||
.lines()
|
||||
.filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
|
||||
.any(|event| {
|
||||
event["event"] == "heal_task_state"
|
||||
&& event["task_id"].as_str() == Some(client_token.as_str())
|
||||
&& event["heal_type"] == "cluster"
|
||||
&& event["state"] == "started"
|
||||
}),
|
||||
"node 0 must have started the exact admin task before interruption"
|
||||
);
|
||||
}
|
||||
let pre_interrupt_operations = &pre_interrupt_status["healOperations"];
|
||||
assert_eq!(
|
||||
pre_interrupt_operations["activeBySource"]["admin"].as_u64(),
|
||||
Some(1),
|
||||
"interruption must occur while the single admin task is active: {pre_interrupt_status}"
|
||||
);
|
||||
if !background_rejoin_heal_evidence {
|
||||
assert_eq!(
|
||||
pre_interrupt_operations["activeBySource"]["admin"].as_u64(),
|
||||
Some(1),
|
||||
"interruption must occur while the single admin task is active: {pre_interrupt_status}"
|
||||
);
|
||||
}
|
||||
if !background_enabled {
|
||||
assert!(
|
||||
only_admin_heal_is_active(&pre_interrupt_status),
|
||||
@@ -1675,6 +1839,7 @@ mod tests {
|
||||
scenario,
|
||||
InterruptionScenario::BackgroundTargetRestart
|
||||
| InterruptionScenario::BackgroundTargetRestartEc84
|
||||
| InterruptionScenario::BackgroundTargetRestartEc84MultiSet
|
||||
| InterruptionScenario::BackgroundCoordinatorRestart
|
||||
) {
|
||||
cluster.stop_node_gracefully(interruption_node).await?;
|
||||
@@ -1719,6 +1884,9 @@ mod tests {
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(barrier) = &commit_barrier {
|
||||
std::fs::remove_file(barrier)?;
|
||||
}
|
||||
cluster.start_node_from_binary(interruption_node, &server_binary).await?;
|
||||
if interruption_node == 0 {
|
||||
let target = cluster.nodes[1]
|
||||
@@ -1742,12 +1910,43 @@ mod tests {
|
||||
.unwrap_or(180);
|
||||
let heal_deadline = Instant::now() + Duration::from_secs(heal_timeout_secs);
|
||||
loop {
|
||||
if restart_recovery_admin_after_failure && !recovery_admin_task_restarted {
|
||||
let task_state = timeout(
|
||||
Duration::from_secs(2),
|
||||
signed_admin_post(&task_status_url, None, &cluster.access_key, &cluster.secret_key),
|
||||
)
|
||||
.await;
|
||||
if let Ok(Ok(body)) = task_state
|
||||
&& let Ok(status) = serde_json::from_str::<serde_json::Value>(&body)
|
||||
&& status["summary"].as_str() == Some("failed")
|
||||
{
|
||||
let heal_start_body =
|
||||
signed_admin_post(&heal_url, Some(heal_body), &cluster.access_key, &cluster.secret_key).await?;
|
||||
let heal_start: serde_json::Value = serde_json::from_str(&heal_start_body)
|
||||
.map_err(|err| format!("recovery heal start response is not JSON ({err}): {heal_start_body}"))?;
|
||||
client_token = heal_start["clientToken"]
|
||||
.as_str()
|
||||
.filter(|token| !token.is_empty())
|
||||
.ok_or_else(|| format!("recovery heal start response has no client token: {heal_start}"))?
|
||||
.to_string();
|
||||
task_status_url = format!("{}/rustfs/admin/v3/heal/?clientToken={client_token}", cluster.nodes[0].url);
|
||||
recovery_admin_task_restarted = true;
|
||||
info!(
|
||||
event = "heal_interruption_recovery_task_restarted",
|
||||
component = "e2e_test",
|
||||
subsystem = "heal",
|
||||
interruption_kind,
|
||||
recovery_client_token = client_token,
|
||||
"Restarted admin root heal after interrupted target-crash task failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
let baseline_recovered = metadata_count(&replaced_disk, bucket, &expected_manifests) == expected_manifests.len();
|
||||
let outage_recovered =
|
||||
!outage_target_manifest_required || object_metadata_exists_on_disk(&replaced_disk, bucket, outage_key);
|
||||
!outage_target_manifest_required || object_metadata_exists_on_disk(&replaced_disk, bucket, &outage_key);
|
||||
if baseline_recovered && outage_recovered {
|
||||
let matching = matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?;
|
||||
let outage_census = census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?;
|
||||
let outage_census = census_object_version_on_disk(&replaced_disk, bucket, &outage_key, None)?;
|
||||
let pool_metadata_matches = match &expected_pool_metadata {
|
||||
Some(expected) => {
|
||||
census_object_version_on_disk(&replaced_disk, RUSTFS_META_BUCKET, POOL_METADATA_OBJECT, None)?
|
||||
@@ -1764,21 +1963,25 @@ mod tests {
|
||||
}
|
||||
if Instant::now() >= heal_deadline {
|
||||
let matching = matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?;
|
||||
let outage_census = census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?;
|
||||
let outage_census = census_object_version_on_disk(&replaced_disk, bucket, &outage_key, None)?;
|
||||
let pool_metadata =
|
||||
census_object_version_on_disk(&replaced_disk, RUSTFS_META_BUCKET, POOL_METADATA_OBJECT, None)?;
|
||||
let final_status = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key)
|
||||
.await
|
||||
.unwrap_or_else(|err| format!("status request failed: {err}"));
|
||||
let task_status = match timeout(
|
||||
Duration::from_secs(5),
|
||||
signed_admin_post(&task_status_url, None, &cluster.access_key, &cluster.secret_key),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Ok(body)) => heal_task_status_diagnostic(&body),
|
||||
Ok(Err(err)) => format!("task status request failed: {err}"),
|
||||
Err(_) => "task status request exceeded 5s diagnostic budget".to_string(),
|
||||
let task_status = if background_rejoin_heal_evidence {
|
||||
"background erasure-set heal has no admin root-heal task token".to_string()
|
||||
} else {
|
||||
match timeout(
|
||||
Duration::from_secs(5),
|
||||
signed_admin_post(&task_status_url, None, &cluster.access_key, &cluster.secret_key),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Ok(body)) => heal_task_status_diagnostic(&body),
|
||||
Ok(Err(err)) => format!("task status request failed: {err}"),
|
||||
Err(_) => "task status request exceeded 5s diagnostic budget".to_string(),
|
||||
}
|
||||
};
|
||||
let replacement_status = match timeout(Duration::from_secs(5), replacement_recovery_status(&cluster)).await {
|
||||
Ok(Ok(status)) => status.to_string(),
|
||||
@@ -1802,7 +2005,7 @@ mod tests {
|
||||
expected.key
|
||||
);
|
||||
}
|
||||
let outage_census = census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?;
|
||||
let outage_census = census_object_version_on_disk(&replaced_disk, bucket, &outage_key, None)?;
|
||||
if outage_target_manifest_required {
|
||||
assert!(
|
||||
outage_census.is_complete(),
|
||||
@@ -1821,11 +2024,43 @@ mod tests {
|
||||
wait_for_scanner_cycle_after(&cluster, cycle_end).await?;
|
||||
}
|
||||
|
||||
if outage_write_deferred_until_rejoin {
|
||||
let deferred_deadline = Instant::now() + Duration::from_secs(60);
|
||||
loop {
|
||||
let put_result = timeout(
|
||||
Duration::from_secs(30),
|
||||
clients[0]
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(&outage_key)
|
||||
.body(ByteStream::from(deterministic_object_body(object_size_bytes, outage_payload_seed)))
|
||||
.send(),
|
||||
)
|
||||
.await;
|
||||
match put_result {
|
||||
Ok(Ok(_)) => break,
|
||||
Ok(Err(error)) if is_service_unavailable_put(&error) && Instant::now() < deferred_deadline => {
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
Ok(Err(error)) => return Err(error.into()),
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
}
|
||||
info!(
|
||||
event = "heal_interruption_outage_write_deferred",
|
||||
component = "e2e_test",
|
||||
subsystem = "heal",
|
||||
interruption_kind,
|
||||
outage_key,
|
||||
"Deferred whole-pool outage write until the target pool rejoined"
|
||||
);
|
||||
}
|
||||
|
||||
let mut expected_keys = created_online_objects
|
||||
.iter()
|
||||
.map(|(key, _)| key.clone())
|
||||
.collect::<HashSet<_>>();
|
||||
assert!(expected_keys.insert(outage_key.to_string()));
|
||||
assert!(expected_keys.insert(outage_key.clone()));
|
||||
let node_listings = assert_all_nodes_list_exact_keys(&clients, bucket, &expected_keys).await?;
|
||||
|
||||
let target_client = cluster.create_s3_client(1)?;
|
||||
@@ -1849,7 +2084,7 @@ mod tests {
|
||||
}));
|
||||
}
|
||||
}
|
||||
let response = target_client.get_object().bucket(bucket).key(outage_key).send().await?;
|
||||
let response = target_client.get_object().bucket(bucket).key(&outage_key).send().await?;
|
||||
let actual = response.body.collect().await?.into_bytes();
|
||||
let expected_outage_body = deterministic_object_body(object_size_bytes, outage_payload_seed);
|
||||
assert_eq!(actual.as_ref(), expected_outage_body.as_slice(), "object body changed for {outage_key}");
|
||||
@@ -1860,7 +2095,7 @@ mod tests {
|
||||
"expected_sha256": sha256_hex(&expected_outage_body),
|
||||
"actual_sha256": sha256_hex(&actual),
|
||||
"expected_physical": null,
|
||||
"physical": census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?,
|
||||
"physical": census_object_version_on_disk(&replaced_disk, bucket, &outage_key, None)?,
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -1878,11 +2113,13 @@ mod tests {
|
||||
sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
|
||||
let task_status_body = signed_admin_post(&task_status_url, None, &cluster.access_key, &cluster.secret_key).await?;
|
||||
let task_status: serde_json::Value = serde_json::from_str(&task_status_body)
|
||||
.map_err(|err| format!("heal task status is not JSON ({err}): {task_status_body}"))?;
|
||||
if task_status["summary"].as_str() != Some("finished") {
|
||||
return Err(format!("heal data rebuilt but task did not finish successfully: {task_status}").into());
|
||||
if !background_rejoin_heal_evidence {
|
||||
let task_status_body = signed_admin_post(&task_status_url, None, &cluster.access_key, &cluster.secret_key).await?;
|
||||
let task_status: serde_json::Value = serde_json::from_str(&task_status_body)
|
||||
.map_err(|err| format!("heal task status is not JSON ({err}): {task_status_body}"))?;
|
||||
if task_status["summary"].as_str() != Some("finished") {
|
||||
return Err(format!("heal data rebuilt but task did not finish successfully: {task_status}").into());
|
||||
}
|
||||
}
|
||||
if interruption_node == 0 {
|
||||
// Restart recovery must finish the original durable root request.
|
||||
@@ -1906,7 +2143,21 @@ mod tests {
|
||||
evidence_context.run.binary.sha256,
|
||||
"server build changed during restart"
|
||||
);
|
||||
let evidence = serde_json::json!({
|
||||
let outage_write_diagnostic = (!outage_target_manifest_required).then(|| {
|
||||
serde_json::json!({
|
||||
"attempted": true,
|
||||
"required": false,
|
||||
"accepted": true,
|
||||
"attempts": if outage_write_deferred_until_rejoin {
|
||||
max_outage_write_attempts
|
||||
} else {
|
||||
service_unavailable_outage_writes + 1
|
||||
},
|
||||
"service_unavailable": service_unavailable_outage_writes,
|
||||
"deferred_until_rejoin": outage_write_deferred_until_rejoin,
|
||||
})
|
||||
});
|
||||
let mut evidence = serde_json::json!({
|
||||
"schema": 1, "case": evidence_context.case.id, "evidence": evidence_context.case.evidence,
|
||||
"run_id": evidence_context.run.run_id, "source_revision": evidence_context.run.source_revision,
|
||||
"test_build": compiled_test_identity(),
|
||||
@@ -1916,7 +2167,11 @@ mod tests {
|
||||
"erasure_set_drive_count": erasure_set_drive_count,
|
||||
"sets": topology.set_count(evidence_context.case.erasure_set_drive_count),
|
||||
"pools": topology.pool_count(),
|
||||
"rebuild_owner": if background_rejoin_heal_evidence { "background-erasure-set" } else { "admin-root-heal" },
|
||||
"outage_target_manifest_required": outage_target_manifest_required,
|
||||
"outage_write_deferred_until_rejoin": outage_write_deferred_until_rejoin,
|
||||
"admin_root_heal_takeover": !background_rejoin_heal_evidence,
|
||||
"recovery_admin_task_restarted": recovery_admin_task_restarted,
|
||||
"distributed_ec_invalidation": true,
|
||||
"peer_count": cluster.nodes.len(),
|
||||
"same_window_remote_proof": true,
|
||||
@@ -1925,6 +2180,9 @@ mod tests {
|
||||
"unclean_shutdown_marker": unclean_shutdown_marker_observed.unwrap_or(false),
|
||||
"objects": evidence_objects, "node_listings": node_listings,
|
||||
});
|
||||
if let Some(outage_write) = outage_write_diagnostic {
|
||||
evidence["outage_write"] = outage_write;
|
||||
}
|
||||
let data = serde_json::to_vec(&evidence)?;
|
||||
if data.len() > 1024 * 1024 {
|
||||
return Err("scanner/heal oracle exceeds the 1 MiB artifact budget".into());
|
||||
|
||||
@@ -179,6 +179,9 @@ mod compression_test;
|
||||
#[cfg(test)]
|
||||
mod delete_objects_versioning_test;
|
||||
|
||||
#[cfg(test)]
|
||||
mod delete_authorization_test;
|
||||
|
||||
// Regression test for signed DELETE Object?versionId requests without Content-Length.
|
||||
#[cfg(test)]
|
||||
mod delete_object_no_content_length_test;
|
||||
|
||||
@@ -6383,6 +6383,18 @@ async fn test_site_replication_edit_and_status_peer_state_real_three_node() -> R
|
||||
let relayed_key = "after-edit-from-relay.txt";
|
||||
let relayed_payload = b"site replication after endpoint edit from relay".to_vec();
|
||||
|
||||
// The first joining receiver owns data before the third site has the
|
||||
// shared account. Initial probes and backfill must wait for every join.
|
||||
target_client.create_bucket().bucket(bucket).send().await?;
|
||||
enable_bucket_versioning(&target_env, bucket).await?;
|
||||
target_client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(baseline_key)
|
||||
.body(ByteStream::from(baseline_payload.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let add_status = site_replication_add(
|
||||
&source_env,
|
||||
&[
|
||||
@@ -6410,7 +6422,10 @@ async fn test_site_replication_edit_and_status_peer_state_real_three_node() -> R
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
assert!(add_status.success, "unexpected site add result: {:?}", add_status);
|
||||
assert!(
|
||||
add_status.success && add_status.err_detail.is_empty() && add_status.initial_sync_error_message.is_empty(),
|
||||
"unexpected site add result: {add_status:?}"
|
||||
);
|
||||
|
||||
let source_info = wait_for_site_replication_enabled(&source_env, 3).await?;
|
||||
let _target_info = wait_for_site_replication_enabled(&target_env, 3).await?;
|
||||
@@ -6421,19 +6436,11 @@ async fn test_site_replication_edit_and_status_peer_state_real_three_node() -> R
|
||||
.find(|peer| peer.endpoint == target_env.url)
|
||||
.ok_or("target peer missing from source site replication info")?;
|
||||
|
||||
source_client.create_bucket().bucket(bucket).send().await?;
|
||||
enable_bucket_versioning(&source_env, bucket).await?;
|
||||
wait_for_bucket_on_target(&target_client, bucket).await?;
|
||||
wait_for_bucket_on_target(&relay_client, bucket).await?;
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(baseline_key)
|
||||
.body(ByteStream::from(baseline_payload.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
let replicated_baseline = wait_for_object_on_target(&target_client, bucket, baseline_key).await?;
|
||||
assert_eq!(replicated_baseline, baseline_payload);
|
||||
for client in [&source_client, &relay_client] {
|
||||
wait_for_bucket_on_target(client, bucket).await?;
|
||||
let backfilled = wait_for_object_on_target(client, bucket, baseline_key).await?;
|
||||
assert_eq!(backfilled, baseline_payload);
|
||||
}
|
||||
|
||||
let old_target_address = target_env.address.clone();
|
||||
let new_target_port = RustFSTestEnvironment::find_available_port().await?;
|
||||
|
||||
@@ -202,7 +202,9 @@ pub mod bucket {
|
||||
}
|
||||
|
||||
pub mod migration {
|
||||
pub use crate::bucket::migration::{LegacyBlobDecryptFn, try_migrate_bucket_metadata, try_migrate_iam_config};
|
||||
pub use crate::bucket::migration::{
|
||||
LegacyBlobDecryptFn, migration_startup_error, try_migrate_bucket_metadata, try_migrate_iam_config,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod object_lock {
|
||||
|
||||
@@ -1220,7 +1220,7 @@ impl ExpiryState {
|
||||
|
||||
while state.tasks_tx.len() < n {
|
||||
let (tx, rx) = mpsc::channel(EXPIRY_WORKER_QUEUE_CAPACITY);
|
||||
let api = api.clone();
|
||||
let api = Arc::downgrade(&api);
|
||||
let rx = Arc::new(tokio::sync::Mutex::new(rx));
|
||||
let stats = Arc::clone(&state.stats);
|
||||
let recovery_notify = Arc::clone(&state.recovery_notify);
|
||||
@@ -1248,14 +1248,18 @@ impl ExpiryState {
|
||||
|
||||
async fn worker(
|
||||
rx: &mut Receiver<Option<ExpiryOpType>>,
|
||||
api: Arc<ECStore>,
|
||||
api: Weak<ECStore>,
|
||||
stats: Arc<ExpiryStats>,
|
||||
recovery_notify: Arc<Notify>,
|
||||
) {
|
||||
let cancel_token = api.ctx.background_cancel_token().unwrap_or_else(|| {
|
||||
let Some(initial_api) = api.upgrade() else {
|
||||
return;
|
||||
};
|
||||
let cancel_token = initial_api.ctx.background_cancel_token().unwrap_or_else(|| {
|
||||
static FALLBACK: std::sync::OnceLock<tokio_util::sync::CancellationToken> = std::sync::OnceLock::new();
|
||||
FALLBACK.get_or_init(tokio_util::sync::CancellationToken::new).clone()
|
||||
});
|
||||
drop(initial_api);
|
||||
|
||||
loop {
|
||||
select! {
|
||||
@@ -1284,6 +1288,9 @@ impl ExpiryState {
|
||||
let v = v.expect("received None after None check");
|
||||
stats.decrement_pending_tasks();
|
||||
let _active_task = ExpiryActiveTask::begin(Arc::clone(&stats));
|
||||
let Some(api) = api.upgrade() else {
|
||||
return;
|
||||
};
|
||||
if v.as_any().is::<ExpiryTask>() {
|
||||
let v = v.as_any().downcast_ref::<ExpiryTask>().expect("ExpiryTask downcast failed");
|
||||
//debug!("lifecycle expiry worker received task: {:?}", v.obj_info);
|
||||
@@ -7759,8 +7766,9 @@ mod tests {
|
||||
let (tx, mut rx) = tokio::sync::mpsc::channel(2);
|
||||
let worker_stats = Arc::clone(&stats);
|
||||
let worker_notify = Arc::clone(&recovery_notify);
|
||||
let worker_store = Arc::downgrade(&ecstore);
|
||||
let worker = tokio::spawn(async move {
|
||||
ExpiryState::worker(&mut rx, ecstore, worker_stats, worker_notify).await;
|
||||
ExpiryState::worker(&mut rx, worker_store, worker_stats, worker_notify).await;
|
||||
});
|
||||
let oi = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
@@ -7861,8 +7869,9 @@ mod tests {
|
||||
let (tx, mut rx) = tokio::sync::mpsc::channel(2);
|
||||
let worker_stats = Arc::clone(&stats);
|
||||
let worker_notify = Arc::clone(&recovery_notify);
|
||||
let worker_store = Arc::downgrade(&ecstore);
|
||||
let worker = tokio::spawn(async move {
|
||||
ExpiryState::worker(&mut rx, ecstore, worker_stats, worker_notify).await;
|
||||
ExpiryState::worker(&mut rx, worker_store, worker_stats, worker_notify).await;
|
||||
});
|
||||
|
||||
stats.increment_pending_tasks();
|
||||
@@ -8031,7 +8040,7 @@ mod tests {
|
||||
let (tx, mut rx) = tokio::sync::mpsc::channel(2);
|
||||
let worker_stats = Arc::clone(&stats);
|
||||
let worker_notify = Arc::clone(&recovery_notify);
|
||||
let worker_store = Arc::clone(&ecstore);
|
||||
let worker_store = Arc::downgrade(&ecstore);
|
||||
let worker = tokio::spawn(async move {
|
||||
ExpiryState::worker(&mut rx, worker_store, worker_stats, worker_notify).await;
|
||||
});
|
||||
@@ -8118,7 +8127,7 @@ mod tests {
|
||||
let (tx, mut rx) = tokio::sync::mpsc::channel(2);
|
||||
let worker_stats = Arc::clone(&stats);
|
||||
let worker_notify = Arc::clone(&recovery_notify);
|
||||
let worker_store = Arc::clone(&ecstore);
|
||||
let worker_store = Arc::downgrade(&ecstore);
|
||||
let worker = tokio::spawn(async move {
|
||||
ExpiryState::worker(&mut rx, worker_store, worker_stats, worker_notify).await;
|
||||
});
|
||||
@@ -8225,7 +8234,7 @@ mod tests {
|
||||
let (tx, mut rx) = tokio::sync::mpsc::channel(2);
|
||||
let worker_stats = Arc::clone(&stats);
|
||||
let worker_notify = Arc::clone(&recovery_notify);
|
||||
let worker_store = Arc::clone(&ecstore);
|
||||
let worker_store = Arc::downgrade(&ecstore);
|
||||
let worker = tokio::spawn(async move {
|
||||
ExpiryState::worker(&mut rx, worker_store, worker_stats, worker_notify).await;
|
||||
});
|
||||
@@ -8279,8 +8288,9 @@ mod tests {
|
||||
let (tx, mut rx) = tokio::sync::mpsc::channel(2);
|
||||
let worker_stats = Arc::clone(&stats);
|
||||
let worker_notify = Arc::clone(&recovery_notify);
|
||||
let worker_store = Arc::downgrade(&ecstore);
|
||||
let worker = tokio::spawn(async move {
|
||||
ExpiryState::worker(&mut rx, ecstore, worker_stats, worker_notify).await;
|
||||
ExpiryState::worker(&mut rx, worker_store, worker_stats, worker_notify).await;
|
||||
});
|
||||
let oi = ObjectInfo {
|
||||
bucket: format!("missing-bucket-{}", Uuid::new_v4()),
|
||||
|
||||
@@ -248,10 +248,9 @@ fn validate_authoritative_object_lock_config(config: &ObjectLockConfiguration) -
|
||||
}
|
||||
|
||||
pub async fn init_bucket_metadata_sys(api: Arc<ECStore>, buckets: Vec<String>) {
|
||||
// The metadata system is inherently per-store (it holds the store handle
|
||||
// and that store's bucket cache), so it lives on the store's own instance
|
||||
// context (backlog#1052 S3) — a second instance initializes its own cell
|
||||
// instead of panicking on the process-global one.
|
||||
// The metadata system is inherently per-store, so it lives on the store's
|
||||
// own instance context (backlog#1052 S3). It resolves the store through a
|
||||
// weak handle so the context cache cannot keep the store and disks alive.
|
||||
let instance_ctx = api.ctx.clone();
|
||||
let is_dist_erasure = instance_ctx.is_dist_erasure().await;
|
||||
|
||||
@@ -317,18 +316,22 @@ fn start_refresh_buckets_metadata_loop(sys: Arc<RwLock<BucketMetadataSys>>) {
|
||||
warn!("bucket metadata refresh loop skipped because background cancellation token is not initialized");
|
||||
return;
|
||||
};
|
||||
let sys = Arc::downgrade(&sys);
|
||||
|
||||
tokio::spawn(async move {
|
||||
refresh_buckets_metadata_loop(sys, cancel_token).await;
|
||||
});
|
||||
}
|
||||
|
||||
async fn refresh_buckets_metadata_loop(sys: Arc<RwLock<BucketMetadataSys>>, cancel_token: CancellationToken) {
|
||||
async fn refresh_buckets_metadata_loop(sys: Weak<RwLock<BucketMetadataSys>>, cancel_token: CancellationToken) {
|
||||
loop {
|
||||
if !wait_refresh_interval_or_cancel(&cancel_token, BUCKET_METADATA_REFRESH_INTERVAL).await {
|
||||
break;
|
||||
}
|
||||
refresh_buckets_metadata_once(sys.clone()).await;
|
||||
let Some(sys) = sys.upgrade() else {
|
||||
break;
|
||||
};
|
||||
refresh_buckets_metadata_once(sys).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -455,7 +458,7 @@ pub(crate) async fn object_store_in(ctx: &crate::runtime::instance::InstanceCont
|
||||
|
||||
pub(crate) async fn object_store_if_initialized_in(ctx: &crate::runtime::instance::InstanceContext) -> Option<Arc<ECStore>> {
|
||||
let sys = ctx.bucket_metadata_sys().or_else(get_global_bucket_metadata_sys)?;
|
||||
Some(sys.read().await.api.clone())
|
||||
sys.read().await.object_store_if_live()
|
||||
}
|
||||
|
||||
pub(crate) async fn get_in(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result<Arc<BucketMetadata>> {
|
||||
@@ -475,7 +478,7 @@ pub(crate) async fn get_config_from_disk_with_presence_in(
|
||||
bucket: &str,
|
||||
) -> Result<(BucketMetadata, bool)> {
|
||||
let sys = bucket_metadata_sys_of(ctx)?;
|
||||
let api = sys.read().await.api.clone();
|
||||
let api = sys.read().await.object_store();
|
||||
load_bucket_metadata_parse_with_presence(api, bucket, true).await
|
||||
}
|
||||
|
||||
@@ -738,7 +741,7 @@ pub async fn acquire_scanner_bucket_incarnation_fence(
|
||||
) -> Result<BucketMetadataMutationGuard> {
|
||||
super::utils::check_valid_bucket_name(bucket)?;
|
||||
let sys = get_bucket_metadata_sys()?;
|
||||
if expected_owner_id.is_nil() || sys.read().await.api.id != expected_owner_id || expected_incarnation_id.is_nil() {
|
||||
if expected_owner_id.is_nil() || sys.read().await.object_store().id != expected_owner_id || expected_incarnation_id.is_nil() {
|
||||
return Err(Error::other("scanner bucket incarnation owner does not match"));
|
||||
}
|
||||
acquire_config_write_guard_with_migration(sys, bucket, Some(expected_incarnation_id), false).await
|
||||
@@ -751,7 +754,7 @@ async fn acquire_config_write_guard_with_migration(
|
||||
migrate: bool,
|
||||
) -> Result<BucketMetadataMutationGuard> {
|
||||
let metadata_sys = sys.read().await.clone();
|
||||
let lifecycle_guard = metadata_sys.api.acquire_bucket_lifecycle_read_lock(bucket).await?;
|
||||
let lifecycle_guard = metadata_sys.object_store().acquire_bucket_lifecycle_read_lock(bucket).await?;
|
||||
|
||||
// Legacy buckets are migrated while the lifecycle fence prevents a
|
||||
// same-name replacement. The second read under the write transaction is
|
||||
@@ -782,7 +785,7 @@ async fn acquire_config_write_guard_with_migration(
|
||||
"bucket config existence transaction validation",
|
||||
async {
|
||||
match metadata_sys
|
||||
.api
|
||||
.object_store()
|
||||
.get_bucket_info_from_sets(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
|
||||
.await
|
||||
{
|
||||
@@ -802,7 +805,7 @@ async fn acquire_config_write_guard_with_migration(
|
||||
Some(&transaction_guard),
|
||||
bucket,
|
||||
"bucket config incarnation transaction validation",
|
||||
load_bucket_incarnation(metadata_sys.api.clone(), bucket),
|
||||
load_bucket_incarnation(metadata_sys.object_store(), bucket),
|
||||
),
|
||||
)
|
||||
.await?
|
||||
@@ -1461,7 +1464,7 @@ pub struct BucketMetadataSys {
|
||||
/// Physically missing names are TTL-bounded to limit memory under bogus
|
||||
/// name floods while avoiding repeated namespace and erasure reads.
|
||||
missing_buckets: moka::future::Cache<String, ()>,
|
||||
api: Arc<ECStore>,
|
||||
api: Weak<ECStore>,
|
||||
}
|
||||
|
||||
impl BucketMetadataSys {
|
||||
@@ -1489,12 +1492,17 @@ impl BucketMetadataSys {
|
||||
.max_capacity(MISSING_BUCKET_MAX_ENTRIES)
|
||||
.time_to_live(MISSING_BUCKET_TTL)
|
||||
.build(),
|
||||
api,
|
||||
api: Arc::downgrade(&api),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn object_store(&self) -> Arc<ECStore> {
|
||||
self.api.clone()
|
||||
self.object_store_if_live()
|
||||
.expect("bucket metadata object store should still be live")
|
||||
}
|
||||
|
||||
fn object_store_if_live(&self) -> Option<Arc<ECStore>> {
|
||||
self.api.upgrade()
|
||||
}
|
||||
|
||||
fn metadata_publish_lock(&self, bucket: &str) -> Arc<Mutex<MetadataPublishLockState>> {
|
||||
@@ -1549,7 +1557,7 @@ impl BucketMetadataSys {
|
||||
) -> Result<bool> {
|
||||
await_bucket_namespace_operation(Some(namespace_guard), bucket, operation, async {
|
||||
match self
|
||||
.api
|
||||
.object_store()
|
||||
.get_bucket_info_from_sets(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
|
||||
.await
|
||||
{
|
||||
@@ -1566,7 +1574,7 @@ impl BucketMetadataSys {
|
||||
}
|
||||
async fn init_internal(&self, buckets: Vec<String>) -> Result<()> {
|
||||
let count = self
|
||||
.api
|
||||
.object_store()
|
||||
.pools
|
||||
.iter()
|
||||
.map(|pool| pool.disk_set.len())
|
||||
@@ -1598,7 +1606,7 @@ impl BucketMetadataSys {
|
||||
let mut futures = Vec::new();
|
||||
|
||||
for bucket in buckets.iter() {
|
||||
let api = self.api.clone();
|
||||
let api = self.object_store();
|
||||
let bucket = bucket.clone();
|
||||
futures.push(async move {
|
||||
sleep(Duration::from_millis(30)).await;
|
||||
@@ -1644,7 +1652,9 @@ impl BucketMetadataSys {
|
||||
let bucket = bucket.clone();
|
||||
futures.push(async move {
|
||||
sleep(Duration::from_millis(30)).await;
|
||||
let api = sys.read().await.api.clone();
|
||||
let Some(api) = sys.read().await.object_store_if_live() else {
|
||||
return Ok(());
|
||||
};
|
||||
let namespace_lock = api.new_ns_lock(&bucket, &bucket).await?;
|
||||
let namespace_guard = namespace_lock
|
||||
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
@@ -1685,7 +1695,7 @@ impl BucketMetadataSys {
|
||||
Some(namespace_guard),
|
||||
bucket,
|
||||
"bucket metadata heal existence check",
|
||||
self.api.bucket_exists_for_heal(bucket),
|
||||
self.object_store().bucket_exists_for_heal(bucket),
|
||||
)
|
||||
.await?
|
||||
{
|
||||
@@ -1709,7 +1719,7 @@ impl BucketMetadataSys {
|
||||
Some(namespace_guard),
|
||||
bucket,
|
||||
"bucket metadata heal",
|
||||
self.api.heal_bucket(
|
||||
self.object_store().heal_bucket(
|
||||
bucket,
|
||||
&HealOpts {
|
||||
recreate: true,
|
||||
@@ -1723,7 +1733,7 @@ impl BucketMetadataSys {
|
||||
Some(namespace_guard),
|
||||
bucket,
|
||||
"bucket metadata load",
|
||||
load_bucket_metadata_parse_with_presence(self.api.clone(), bucket, true),
|
||||
load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
|
||||
)
|
||||
.await?;
|
||||
match mode {
|
||||
@@ -1903,7 +1913,7 @@ impl BucketMetadataSys {
|
||||
// (backlog#1052 S7). Reading from the ambient handle instead made the
|
||||
// read and the write of a single read-modify-write able to target
|
||||
// different instances.
|
||||
let mut bm = Box::pin(Self::load_bucket_metadata_for_update(self.api.clone(), bucket, parse)).await?;
|
||||
let mut bm = Box::pin(Self::load_bucket_metadata_for_update(self.object_store(), bucket, parse)).await?;
|
||||
if !bm.bucket_incarnation_sidecar || bm.bucket_incarnation_id != expected_incarnation_id {
|
||||
return Err(Error::BucketNotFound(bucket.to_string()));
|
||||
}
|
||||
@@ -1942,7 +1952,7 @@ impl BucketMetadataSys {
|
||||
where
|
||||
F: FnOnce(&BucketMetadata) -> Result<Vec<u8>> + Send,
|
||||
{
|
||||
let mut bm = Box::pin(Self::load_bucket_metadata_for_update(self.api.clone(), bucket, true)).await?;
|
||||
let mut bm = Box::pin(Self::load_bucket_metadata_for_update(self.object_store(), bucket, true)).await?;
|
||||
if !bm.bucket_incarnation_sidecar || bm.bucket_incarnation_id != expected_incarnation_id {
|
||||
return Err(Error::BucketNotFound(bucket.to_string()));
|
||||
}
|
||||
@@ -1993,7 +2003,7 @@ impl BucketMetadataSys {
|
||||
/// server's metadata never leaks into the ambient (first) instance.
|
||||
pub(crate) async fn persist_and_set(&self, bm: BucketMetadata) -> Result<()> {
|
||||
let mut bm = bm;
|
||||
bm.save_with_store(self.api.clone()).await?;
|
||||
bm.save_with_store(self.object_store()).await?;
|
||||
|
||||
self.set(bm.name.clone(), Arc::new(bm)).await;
|
||||
|
||||
@@ -2001,8 +2011,8 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
async fn persist_new_and_set(&self, mut bm: BucketMetadata) -> Result<()> {
|
||||
bm.save_with_store(self.api.clone()).await?;
|
||||
save_bucket_incarnation(self.api.clone(), &bm.name, bm.bucket_incarnation_id).await?;
|
||||
bm.save_with_store(self.object_store()).await?;
|
||||
save_bucket_incarnation(self.object_store(), &bm.name, bm.bucket_incarnation_id).await?;
|
||||
bm.bucket_incarnation_sidecar = true;
|
||||
self.set(bm.name.clone(), Arc::new(bm)).await;
|
||||
Ok(())
|
||||
@@ -2019,7 +2029,7 @@ impl BucketMetadataSys {
|
||||
return Err(Error::other("errInvalidArgument"));
|
||||
}
|
||||
|
||||
load_bucket_metadata(self.api.clone(), bucket).await
|
||||
load_bucket_metadata(self.object_store(), bucket).await
|
||||
}
|
||||
|
||||
/// Reload persisted metadata under the bucket namespace generation fence.
|
||||
@@ -2033,7 +2043,7 @@ impl BucketMetadataSys {
|
||||
return Err(Error::other("errInvalidArgument"));
|
||||
}
|
||||
|
||||
let namespace_lock = self.api.new_ns_lock(bucket, bucket).await?;
|
||||
let namespace_lock = self.object_store().new_ns_lock(bucket, bucket).await?;
|
||||
let namespace_guard = namespace_lock
|
||||
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await?;
|
||||
@@ -2056,7 +2066,7 @@ impl BucketMetadataSys {
|
||||
Some(namespace_guard),
|
||||
bucket,
|
||||
"peer bucket metadata load",
|
||||
load_bucket_metadata_parse_with_presence(self.api.clone(), bucket, true),
|
||||
load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
|
||||
)
|
||||
.await?;
|
||||
if !persisted {
|
||||
@@ -2101,11 +2111,11 @@ impl BucketMetadataSys {
|
||||
#[cfg(test)]
|
||||
self.lazy_disk_loads.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
|
||||
let lock = self.api.new_ns_lock(bucket, bucket).await?;
|
||||
let lock = self.object_store().new_ns_lock(bucket, bucket).await?;
|
||||
let guard = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
|
||||
#[cfg(test)]
|
||||
if self.lazy_load_lock_probe.load(std::sync::atomic::Ordering::Relaxed) {
|
||||
let competing = self.api.new_ns_lock(bucket, bucket).await?;
|
||||
let competing = self.object_store().new_ns_lock(bucket, bucket).await?;
|
||||
assert!(
|
||||
competing.get_write_lock(Duration::from_millis(20)).await.is_err(),
|
||||
"lazy metadata IO must start while the bucket namespace read lock is held"
|
||||
@@ -2115,7 +2125,7 @@ impl BucketMetadataSys {
|
||||
Some(&guard),
|
||||
bucket,
|
||||
"lazy bucket metadata load",
|
||||
Box::pin(load_bucket_metadata_parse_with_presence(self.api.clone(), bucket, true)),
|
||||
Box::pin(load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true)),
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -2127,7 +2137,7 @@ impl BucketMetadataSys {
|
||||
bucket,
|
||||
"lazy bucket metadata existence check",
|
||||
Box::pin(async {
|
||||
self.api
|
||||
self.object_store()
|
||||
.get_bucket_info_from_sets(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
|
||||
.await
|
||||
.map(|_| ())
|
||||
@@ -2334,13 +2344,13 @@ impl BucketMetadataSys {
|
||||
|
||||
async fn get_bucket_incarnation_id_from_disk(&self, bucket: &str) -> Result<Uuid> {
|
||||
let transaction_lock = self
|
||||
.api
|
||||
.object_store()
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
|
||||
.await?;
|
||||
let _transaction_guard = transaction_lock
|
||||
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await?;
|
||||
let incarnation_id = load_bucket_incarnation(self.api.clone(), bucket).await?;
|
||||
let incarnation_id = load_bucket_incarnation(self.object_store(), bucket).await?;
|
||||
if _transaction_guard.is_lock_lost() {
|
||||
return Err(Error::other(format!("bucket incarnation metadata transaction lock was lost: {bucket}")));
|
||||
}
|
||||
@@ -2376,7 +2386,7 @@ impl BucketMetadataSys {
|
||||
|
||||
async fn migrate_legacy_metadata(&self, bucket: &str) -> Result<BucketMetadataAuthority> {
|
||||
let transaction_lock = self
|
||||
.api
|
||||
.object_store()
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
|
||||
.await?;
|
||||
let _transaction_guard = transaction_lock
|
||||
@@ -2401,7 +2411,7 @@ impl BucketMetadataSys {
|
||||
return Err(Error::other(format!("injected Object Lock metadata disk read failure: {bucket}")));
|
||||
}
|
||||
|
||||
let namespace_lock = self.api.new_ns_lock(bucket, bucket).await?;
|
||||
let namespace_lock = self.object_store().new_ns_lock(bucket, bucket).await?;
|
||||
let namespace_guard = namespace_lock
|
||||
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await?;
|
||||
@@ -2411,7 +2421,7 @@ impl BucketMetadataSys {
|
||||
bucket,
|
||||
"legacy bucket metadata existence check",
|
||||
async {
|
||||
self.api
|
||||
self.object_store()
|
||||
.get_bucket_info_from_sets(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
|
||||
.await
|
||||
},
|
||||
@@ -2435,7 +2445,7 @@ impl BucketMetadataSys {
|
||||
Some(&namespace_guard),
|
||||
bucket,
|
||||
"legacy bucket metadata confirmation",
|
||||
load_bucket_metadata_parse_with_presence(self.api.clone(), bucket, true),
|
||||
load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
|
||||
)
|
||||
.await?;
|
||||
if persisted && !metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() {
|
||||
@@ -2457,20 +2467,20 @@ impl BucketMetadataSys {
|
||||
}
|
||||
#[cfg(test)]
|
||||
if self.legacy_migration_lock_probe.load(std::sync::atomic::Ordering::Relaxed) {
|
||||
let competing = self.api.new_ns_lock(bucket, bucket).await?;
|
||||
let competing = self.object_store().new_ns_lock(bucket, bucket).await?;
|
||||
assert!(
|
||||
competing.get_write_lock(Duration::from_millis(20)).await.is_err(),
|
||||
"bucket delete/recreate must not cross the legacy metadata migration fence"
|
||||
);
|
||||
}
|
||||
save_bucket_incarnation(self.api.clone(), bucket, metadata.bucket_incarnation_id).await?;
|
||||
save_bucket_incarnation(self.object_store(), bucket, metadata.bucket_incarnation_id).await?;
|
||||
metadata.bucket_incarnation_sidecar = true;
|
||||
if !persisted {
|
||||
await_bucket_namespace_operation(
|
||||
Some(&namespace_guard),
|
||||
bucket,
|
||||
"legacy bucket metadata migration",
|
||||
metadata.save_with_store(self.api.clone()),
|
||||
metadata.save_with_store(self.object_store()),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -2506,7 +2516,7 @@ impl BucketMetadataSys {
|
||||
return Err(Error::other(format!("injected Object Lock metadata disk read failure: {bucket}")));
|
||||
}
|
||||
|
||||
let namespace_lock = self.api.new_ns_lock(bucket, bucket).await?;
|
||||
let namespace_lock = self.object_store().new_ns_lock(bucket, bucket).await?;
|
||||
let namespace_guard = namespace_lock
|
||||
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await?;
|
||||
@@ -2515,8 +2525,11 @@ impl BucketMetadataSys {
|
||||
bucket,
|
||||
"bucket metadata snapshot existence check",
|
||||
async {
|
||||
self.api
|
||||
.get_bucket_info_from_sets(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
|
||||
self.object_store()
|
||||
.get_bucket_info_from_sets_at_read_quorum(
|
||||
bucket,
|
||||
&crate::storage_api_contracts::bucket::BucketOptions::default(),
|
||||
)
|
||||
.await
|
||||
},
|
||||
)
|
||||
@@ -2531,7 +2544,7 @@ impl BucketMetadataSys {
|
||||
Some(&namespace_guard),
|
||||
bucket,
|
||||
"bucket metadata authoritative snapshot",
|
||||
load_bucket_metadata_parse_with_presence(self.api.clone(), bucket, true),
|
||||
load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
|
||||
)
|
||||
.await?;
|
||||
if persisted {
|
||||
@@ -3695,7 +3708,7 @@ mod tests {
|
||||
let mut stale = BucketMetadata::new("recreated-bucket");
|
||||
stale.policy_config_json = b"old-generation".to_vec();
|
||||
let namespace_lock = sys
|
||||
.api
|
||||
.object_store()
|
||||
.new_ns_lock("recreated-bucket", "recreated-bucket")
|
||||
.await
|
||||
.expect("namespace lock should be created");
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
use crate::bucket::metadata::BUCKET_METADATA_FILE;
|
||||
use crate::bucket::replication::ReplicationMigrationBridge;
|
||||
use crate::disk::{BUCKET_META_PREFIX, MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
|
||||
use crate::error::Error;
|
||||
use crate::error::{Error, Result, is_err_strict_not_found, is_err_strict_volume_not_found};
|
||||
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
|
||||
use crate::storage_api_contracts::{
|
||||
bucket::{BucketOperations, BucketOptions},
|
||||
@@ -33,7 +33,7 @@ use rustfs_utils::path::SLASH_SEPARATOR;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::sync::Arc;
|
||||
use time::OffsetDateTime;
|
||||
use tracing::{debug, info, warn};
|
||||
use tracing::{debug, info};
|
||||
|
||||
/// IAM config prefix under meta bucket (e.g. config/iam/).
|
||||
const IAM_CONFIG_PREFIX: &str = "config/iam";
|
||||
@@ -53,6 +53,39 @@ type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
|
||||
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
|
||||
|
||||
#[derive(Clone, Debug, thiserror::Error)]
|
||||
enum MigrationMetadataError {
|
||||
#[error("empty legacy metadata: {0}")]
|
||||
Empty(String),
|
||||
#[error("incompatible legacy metadata: {0}")]
|
||||
Incompatible(String),
|
||||
}
|
||||
|
||||
impl From<MigrationMetadataError> for Error {
|
||||
fn from(error: MigrationMetadataError) -> Self {
|
||||
let message = match &error {
|
||||
MigrationMetadataError::Empty(_) => "empty legacy metadata",
|
||||
MigrationMetadataError::Incompatible(_) => "incompatible legacy metadata",
|
||||
};
|
||||
// Keep the record path in the typed source, not in the quorum grouping key.
|
||||
Self::other_with_context(message, error)
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts a migration failure at the startup boundary, rendering the safe
|
||||
/// record path while leaving storage-layer error grouping stable.
|
||||
pub fn migration_startup_error(error: Error) -> std::io::Error {
|
||||
if let Error::Io(io_error) = &error
|
||||
&& let Some(metadata_error) = io_error
|
||||
.get_ref()
|
||||
.and_then(|context| context.source())
|
||||
.and_then(|source| source.downcast_ref::<MigrationMetadataError>())
|
||||
{
|
||||
return std::io::Error::other(metadata_error.clone());
|
||||
}
|
||||
std::io::Error::other(error)
|
||||
}
|
||||
|
||||
/// Callback used to decrypt an at-rest config blob during MinIO -> RustFS migration.
|
||||
///
|
||||
/// MinIO encrypts IAM identity/service-account files and the server config at rest
|
||||
@@ -211,7 +244,7 @@ fn normalize_bucket_meta_blob(path: &str, data: &[u8]) -> std::result::Result<Op
|
||||
/// Uses list_bucket (from disk volumes) to get bucket names, since list_objects_v2 on the legacy
|
||||
/// meta bucket may not work (legacy format differs from object layer expectations).
|
||||
/// Skips buckets that already exist in RustFS (idempotent).
|
||||
pub async fn try_migrate_bucket_metadata<S>(store: Arc<S>)
|
||||
pub async fn try_migrate_bucket_metadata<S>(store: Arc<S>) -> Result<()>
|
||||
where
|
||||
S: BucketOperations<Error = crate::error::Error>
|
||||
+ ObjectIO<
|
||||
@@ -231,25 +264,18 @@ where
|
||||
DeletedObject = DeletedObject,
|
||||
>,
|
||||
{
|
||||
let buckets_list = match store
|
||||
let buckets_list = store
|
||||
.list_bucket(&BucketOptions {
|
||||
no_metadata: true,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(b) => b,
|
||||
Err(e) => {
|
||||
warn!("list buckets failed (skip migration): {e}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
.await?;
|
||||
|
||||
let buckets: Vec<String> = buckets_list.into_iter().map(|b| b.name).collect();
|
||||
|
||||
if buckets.is_empty() {
|
||||
debug!("No migrating bucket metadata found");
|
||||
return;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
debug!("Found {} migrating bucket metadata, migrating...", buckets.len());
|
||||
@@ -263,26 +289,40 @@ where
|
||||
|
||||
for bucket in buckets {
|
||||
let meta_path = format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}{bucket}{SLASH_SEPARATOR}{BUCKET_METADATA_FILE}");
|
||||
migrate_one_if_missing(store.clone(), &opts, &h, &meta_path, &format!("bucket metadata: {bucket}")).await;
|
||||
migrate_one_if_missing(store.clone(), &opts, &h, &meta_path, &format!("bucket metadata: {bucket}")).await?;
|
||||
|
||||
let resync_path = format!(
|
||||
"{BUCKET_META_PREFIX}{SLASH_SEPARATOR}{bucket}{SLASH_SEPARATOR}{REPLICATION_META_DIR}{SLASH_SEPARATOR}{RESYNC_META_FILE}"
|
||||
);
|
||||
migrate_one_if_missing(store.clone(), &opts, &h, &resync_path, &format!("bucket replication resync: {bucket}")).await;
|
||||
migrate_one_if_missing(store.clone(), &opts, &h, &resync_path, &format!("bucket replication resync: {bucket}")).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn migration_target_exists<S: EcstoreObjectOperations>(store: &S, path: &str) -> Result<bool> {
|
||||
match store
|
||||
.get_object_info(RUSTFS_META_BUCKET, path, &ObjectOptions::default())
|
||||
.await
|
||||
{
|
||||
Ok(_) => Ok(true),
|
||||
Err(err) if is_err_strict_not_found(&err) || is_err_strict_volume_not_found(&err) => Ok(false),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
async fn migrate_one_if_missing<S>(store: Arc<S>, opts: &ObjectOptions, headers: &HeaderMap, path: &str, label: &str)
|
||||
async fn migrate_one_if_missing<S>(
|
||||
store: Arc<S>,
|
||||
opts: &ObjectOptions,
|
||||
headers: &HeaderMap,
|
||||
path: &str,
|
||||
label: &str,
|
||||
) -> Result<()>
|
||||
where
|
||||
S: EcstoreObjectIO + EcstoreObjectOperations,
|
||||
{
|
||||
if store
|
||||
.get_object_info(RUSTFS_META_BUCKET, path, &ObjectOptions::default())
|
||||
.await
|
||||
.is_ok()
|
||||
{
|
||||
if migration_target_exists(store.as_ref(), path).await? {
|
||||
debug!("{label} already exists in RustFS, skip");
|
||||
return;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut rd = match store
|
||||
@@ -290,43 +330,31 @@ where
|
||||
.await
|
||||
{
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
debug!("read migrating {label}: {e}");
|
||||
return;
|
||||
}
|
||||
// Ordinary RustFS deployments have no legacy bucket, and optional
|
||||
// legacy settings (such as replication resync) may not exist.
|
||||
Err(err) if is_err_strict_not_found(&err) || is_err_strict_volume_not_found(&err) => return Ok(()),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
|
||||
let data = match rd.read_all().await {
|
||||
Ok(d) if !d.is_empty() => d,
|
||||
Ok(_) => return,
|
||||
Err(e) => {
|
||||
debug!("read migrating {label} body: {e}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let data = match normalize_bucket_meta_blob(path, &data) {
|
||||
Ok(Some(normalized)) => normalized,
|
||||
Ok(None) => data,
|
||||
Err(e) => {
|
||||
warn!("skip {label} migration due to incompatible format: {e}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let data = rd.read_all().await?;
|
||||
if data.is_empty() {
|
||||
return Err(MigrationMetadataError::Empty(path.to_owned()).into());
|
||||
}
|
||||
let data = normalize_bucket_meta_blob(path, &data)
|
||||
.map_err(|_| MigrationMetadataError::Incompatible(path.to_owned()))?
|
||||
.unwrap_or(data);
|
||||
|
||||
let mut put_data = PutObjReader::from_vec(data);
|
||||
if let Err(e) = store.put_object(RUSTFS_META_BUCKET, path, &mut put_data, opts).await {
|
||||
warn!("write {label}: {e}");
|
||||
} else {
|
||||
info!("Migrated {label}");
|
||||
}
|
||||
store.put_object(RUSTFS_META_BUCKET, path, &mut put_data, opts).await?;
|
||||
info!("Migrated {label}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Migrates IAM config from legacy meta bucket `config/iam/` to RustFS meta bucket.
|
||||
/// Lists all objects under the IAM prefix in the source, copies each to the target if not present.
|
||||
/// Skips objects that already exist in RustFS (idempotent).
|
||||
/// If list_objects_v2 on the legacy bucket fails (e.g. format differs), migration is skipped.
|
||||
pub async fn try_migrate_iam_config<S>(store: Arc<S>, decrypt_fn: Option<LegacyBlobDecryptFn>)
|
||||
/// An absent legacy bucket is a no-op; migration errors prevent startup readiness.
|
||||
pub async fn try_migrate_iam_config<S>(store: Arc<S>, decrypt_fn: Option<LegacyBlobDecryptFn>) -> Result<()>
|
||||
where
|
||||
S: ListOperations<
|
||||
Error = crate::error::Error,
|
||||
@@ -366,47 +394,36 @@ where
|
||||
loop {
|
||||
let list_result = match store
|
||||
.clone()
|
||||
.list_objects_v2(MIGRATING_META_BUCKET, &prefix, continuation, None, 500, false, None, false)
|
||||
.list_objects_v2(MIGRATING_META_BUCKET, &prefix, continuation.clone(), None, 500, false, None, false)
|
||||
.await
|
||||
{
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
debug!("list IAM config from legacy bucket failed (skip migration): {e}");
|
||||
return;
|
||||
}
|
||||
Err(err) if is_err_strict_volume_not_found(&err) => return Ok(()),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
|
||||
for obj in list_result.objects {
|
||||
let path = &obj.name;
|
||||
if path.is_empty() || path.ends_with('/') {
|
||||
// Unsupported records must not trigger target lookups, reads, or decryption.
|
||||
if path != IAM_FORMAT_FILE_PATH
|
||||
&& !is_identity_path(path)
|
||||
&& !is_group_path(path)
|
||||
&& !is_policy_doc_path(path)
|
||||
&& !is_policy_mapping_path(path)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if store
|
||||
.get_object_info(RUSTFS_META_BUCKET, path, &ObjectOptions::default())
|
||||
.await
|
||||
.is_ok()
|
||||
{
|
||||
if migration_target_exists(store.as_ref(), path).await? {
|
||||
debug!("IAM config already exists in RustFS, skip: {path}");
|
||||
continue;
|
||||
}
|
||||
let mut rd = match store
|
||||
let mut rd = store
|
||||
.get_object_reader(MIGRATING_META_BUCKET, path, None, h.clone(), &opts)
|
||||
.await
|
||||
{
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
debug!("read migrating IAM config {path}: {e}");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let data = match rd.read_all().await {
|
||||
Ok(d) if !d.is_empty() => d,
|
||||
Ok(_) => continue,
|
||||
Err(e) => {
|
||||
debug!("read migrating IAM config {path} body: {e}");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
.await?;
|
||||
let data = rd.read_all().await?;
|
||||
if data.is_empty() {
|
||||
return Err(MigrationMetadataError::Empty(path.to_owned()).into());
|
||||
}
|
||||
// MinIO encrypts IAM identity/service-account files at rest. Decrypt
|
||||
// before normalizing; fall back to the raw bytes when no key applies
|
||||
// (plaintext blobs, or nothing to decrypt) so existing behavior holds.
|
||||
@@ -420,22 +437,17 @@ where
|
||||
debug!("skip unsupported IAM config path during migration: {path}");
|
||||
continue;
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("skip IAM config migration due to incompatible format, path: {path}, err: {e}");
|
||||
continue;
|
||||
}
|
||||
// Parser errors may contain credential data. Report only the path.
|
||||
Err(_) => return Err(MigrationMetadataError::Incompatible(path.to_owned()).into()),
|
||||
};
|
||||
let mut put_data = PutObjReader::from_vec(data);
|
||||
if let Err(e) = store.put_object(RUSTFS_META_BUCKET, path, &mut put_data, &opts).await {
|
||||
warn!("write IAM config {path}: {e}");
|
||||
} else {
|
||||
info!("Migrated IAM config: {path}");
|
||||
total_migrated += 1;
|
||||
}
|
||||
store.put_object(RUSTFS_META_BUCKET, path, &mut put_data, &opts).await?;
|
||||
info!("Migrated IAM config: {path}");
|
||||
total_migrated += 1;
|
||||
}
|
||||
|
||||
continuation = list_result.next_continuation_token.or(list_result.continuation_token);
|
||||
if !list_result.is_truncated || continuation.is_none() {
|
||||
continuation = next_iam_migration_page(list_result.is_truncated, continuation, list_result.next_continuation_token)?;
|
||||
if continuation.is_none() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -443,10 +455,74 @@ where
|
||||
if total_migrated > 0 {
|
||||
info!("IAM migration complete: {} object(s) migrated", total_migrated);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn next_iam_migration_page(truncated: bool, previous: Option<String>, next: Option<String>) -> Result<Option<String>> {
|
||||
if !truncated {
|
||||
return Ok(None);
|
||||
}
|
||||
let next = next.filter(|token| !token.is_empty());
|
||||
if next.is_none() || next == previous {
|
||||
return Err(Error::other("legacy IAM migration listing did not advance"));
|
||||
}
|
||||
Ok(next)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn migration_errors_group_by_cause_and_retain_typed_record_context() {
|
||||
use super::{Error, MigrationMetadataError};
|
||||
|
||||
for (make_error, message) in [
|
||||
(
|
||||
MigrationMetadataError::Empty as fn(String) -> MigrationMetadataError,
|
||||
"empty legacy metadata",
|
||||
),
|
||||
(MigrationMetadataError::Incompatible, "incompatible legacy metadata"),
|
||||
] {
|
||||
let first: Error = make_error("buckets/first/.metadata.bin".into()).into();
|
||||
let second: Error = make_error("buckets/second/.metadata.bin".into()).into();
|
||||
assert_eq!(first, second, "record paths must not fragment error grouping");
|
||||
assert_eq!(first.clone(), second, "cloning must preserve error grouping");
|
||||
|
||||
let io_error = std::io::Error::from(first);
|
||||
let detail = io_error
|
||||
.get_ref()
|
||||
.and_then(|context| context.source())
|
||||
.expect("record context must remain in the error source");
|
||||
assert!(detail.downcast_ref::<MigrationMetadataError>().is_some());
|
||||
assert!(detail.to_string().contains("buckets/first/.metadata.bin"));
|
||||
|
||||
let startup_error = super::migration_startup_error(make_error("buckets/startup/.metadata.bin".into()).into());
|
||||
assert!(
|
||||
startup_error
|
||||
.get_ref()
|
||||
.is_some_and(|source| source.is::<MigrationMetadataError>())
|
||||
);
|
||||
assert_eq!(startup_error.to_string(), format!("{message}: buckets/startup/.metadata.bin"));
|
||||
}
|
||||
assert_ne!(
|
||||
Error::from(MigrationMetadataError::Empty("record".into())),
|
||||
Error::from(MigrationMetadataError::Incompatible("record".into())),
|
||||
"different migration failures must remain distinguishable"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncated_iam_listing_cannot_report_completed_migration() {
|
||||
use super::next_iam_migration_page;
|
||||
assert_eq!(next_iam_migration_page(false, Some("old".into()), None).expect("final page"), None);
|
||||
assert_eq!(
|
||||
next_iam_migration_page(true, Some("old".into()), Some("next".into())).expect("advancing page"),
|
||||
Some("next".into())
|
||||
);
|
||||
for next in [None, Some(String::new()), Some("old".into())] {
|
||||
assert!(next_iam_migration_page(true, Some("old".into()), next).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
use super::{normalize_bucket_meta_blob, normalize_iam_config_blob};
|
||||
use crate::bucket::replication::{
|
||||
BucketReplicationResyncStatus, ReplicationMigrationBridge, ResyncStatusType, TargetReplicationResyncStatus,
|
||||
@@ -659,6 +735,13 @@ mod tests {
|
||||
.collect();
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), existing).await;
|
||||
|
||||
super::try_migrate_bucket_metadata(ecstore.clone())
|
||||
.await
|
||||
.expect("fresh stores do not require a legacy metadata bucket");
|
||||
super::try_migrate_iam_config(ecstore.clone(), None)
|
||||
.await
|
||||
.expect("fresh stores do not require a legacy IAM bucket");
|
||||
|
||||
let meta_path = format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}interop{SLASH_SEPARATOR}{BUCKET_METADATA_FILE}");
|
||||
let put_opts = ObjectOptions::default();
|
||||
|
||||
@@ -680,8 +763,31 @@ mod tests {
|
||||
.await
|
||||
.expect("seed .minio.sys bucket metadata");
|
||||
|
||||
// --- Run the real startup migration. ---
|
||||
super::try_migrate_bucket_metadata(ecstore.clone()).await;
|
||||
// A partial import must report failure, even if the main bucket
|
||||
// metadata copied successfully before an incompatible resync record.
|
||||
let resync_path = format!("{BUCKET_META_PREFIX}/interop/.replication/resync.bin");
|
||||
ecstore
|
||||
.put_object(
|
||||
MIGRATING_META_BUCKET,
|
||||
&resync_path,
|
||||
&mut PutObjReader::from_vec(b"invalid resync metadata".to_vec()),
|
||||
&put_opts,
|
||||
)
|
||||
.await
|
||||
.expect("seed malformed legacy resync metadata");
|
||||
assert!(
|
||||
super::try_migrate_bucket_metadata(ecstore.clone()).await.is_err(),
|
||||
"incompatible native metadata must not be reported as a completed migration"
|
||||
);
|
||||
ecstore
|
||||
.delete_object(MIGRATING_META_BUCKET, &resync_path, ObjectOptions::default())
|
||||
.await
|
||||
.expect("remove invalid optional legacy resync record");
|
||||
|
||||
// Retry the real startup migration after repairing the source.
|
||||
super::try_migrate_bucket_metadata(ecstore.clone())
|
||||
.await
|
||||
.expect("native bucket metadata migration completes");
|
||||
|
||||
// --- The migrated `.rustfs.sys` blob must carry every MinIO config, ---
|
||||
// byte-identical to the source (typed XML/JSON parsing of these fields is
|
||||
|
||||
@@ -303,8 +303,16 @@ fn scanner_scoped_dirty_usage_ack_response_matches(
|
||||
&& cleared_within_request
|
||||
}
|
||||
|
||||
fn scanner_scoped_dirty_usage_ack_reconciled(activity: &ScannerPeerActivity, expected_instance_id: &str) -> bool {
|
||||
activity.instance_id == expected_instance_id && activity.dirty_usage_pending == Some(false)
|
||||
fn scanner_scoped_dirty_usage_ack_reconciled(
|
||||
activity: &ScannerPeerActivity,
|
||||
expected_instance_id: &str,
|
||||
expected_generation: u64,
|
||||
) -> bool {
|
||||
activity.instance_id == expected_instance_id
|
||||
&& activity.dirty_usage_pending == Some(false)
|
||||
&& activity
|
||||
.dirty_usage_generation
|
||||
.is_some_and(|generation| generation >= expected_generation)
|
||||
}
|
||||
|
||||
fn scanner_instance_id_is_valid(instance_id: &str) -> bool {
|
||||
@@ -2290,6 +2298,7 @@ impl PeerRestClient {
|
||||
entries: Vec<ScannerScopedDirtyUsageAckEntry>,
|
||||
) -> Result<ScannerPeerActivity> {
|
||||
use rustfs_protos::scoped_dirty_usage::*;
|
||||
let expected_generation = entries.iter().map(|entry| entry.generation).max().unwrap_or(0);
|
||||
let payloads = scanner_scoped_dirty_usage_ack_payloads(owner_id, instance_id.clone(), false, entries)?;
|
||||
let ack_attempt = async {
|
||||
let mut client = super::client::scanner_control_time_out_client(
|
||||
@@ -2337,7 +2346,9 @@ impl PeerRestClient {
|
||||
.await;
|
||||
}
|
||||
match self.scanner_scoped_dirty_usage_activity_confirmation().await {
|
||||
Ok(activity) if scanner_scoped_dirty_usage_ack_reconciled(&activity, &instance_id) => Ok(activity),
|
||||
Ok(activity) if scanner_scoped_dirty_usage_ack_reconciled(&activity, &instance_id, expected_generation) => {
|
||||
Ok(activity)
|
||||
}
|
||||
_ => Err(err),
|
||||
}
|
||||
}
|
||||
@@ -3176,30 +3187,48 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn scanner_scoped_dirty_usage_ack_reconciliation_requires_same_clean_instance() {
|
||||
let activity = |instance_id: &str, pending| ScannerPeerActivity {
|
||||
let activity = |instance_id: &str, generation, pending| ScannerPeerActivity {
|
||||
instance_id: instance_id.to_string(),
|
||||
namespace_generation: 1,
|
||||
maintenance_generation: 1,
|
||||
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
topology_digest: Some([1; 32]),
|
||||
data_movement_active: Some(false),
|
||||
dirty_usage_generation: Some(9),
|
||||
dirty_usage_generation: generation,
|
||||
dirty_usage_pending: pending,
|
||||
movement_generation: Some(1),
|
||||
publication_blocked: Some(false),
|
||||
};
|
||||
|
||||
assert!(scanner_scoped_dirty_usage_ack_reconciled(
|
||||
&activity("0123456789abcdef0123456789abcdef", Some(false)),
|
||||
"0123456789abcdef0123456789abcdef"
|
||||
&activity("0123456789abcdef0123456789abcdef", Some(9), Some(false)),
|
||||
"0123456789abcdef0123456789abcdef",
|
||||
9
|
||||
));
|
||||
assert!(scanner_scoped_dirty_usage_ack_reconciled(
|
||||
&activity("0123456789abcdef0123456789abcdef", Some(10), Some(false)),
|
||||
"0123456789abcdef0123456789abcdef",
|
||||
9
|
||||
));
|
||||
assert!(!scanner_scoped_dirty_usage_ack_reconciled(
|
||||
&activity("0123456789abcdef0123456789abcdef", Some(true)),
|
||||
"0123456789abcdef0123456789abcdef"
|
||||
&activity("0123456789abcdef0123456789abcdef", Some(8), Some(false)),
|
||||
"0123456789abcdef0123456789abcdef",
|
||||
9
|
||||
));
|
||||
assert!(!scanner_scoped_dirty_usage_ack_reconciled(
|
||||
&activity("fedcba9876543210fedcba9876543210", Some(false)),
|
||||
"0123456789abcdef0123456789abcdef"
|
||||
&activity("0123456789abcdef0123456789abcdef", None, Some(false)),
|
||||
"0123456789abcdef0123456789abcdef",
|
||||
9
|
||||
));
|
||||
assert!(!scanner_scoped_dirty_usage_ack_reconciled(
|
||||
&activity("0123456789abcdef0123456789abcdef", Some(9), Some(true)),
|
||||
"0123456789abcdef0123456789abcdef",
|
||||
9
|
||||
));
|
||||
assert!(!scanner_scoped_dirty_usage_ack_reconciled(
|
||||
&activity("fedcba9876543210fedcba9876543210", Some(9), Some(false)),
|
||||
"0123456789abcdef0123456789abcdef",
|
||||
9
|
||||
));
|
||||
}
|
||||
|
||||
|
||||
@@ -152,6 +152,7 @@ pub(crate) const DECOMMISSION_VERSION_COPY_ATTEMPTS: usize = 3;
|
||||
const DECOMMISSION_COPY_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(50);
|
||||
const DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT: usize = 100;
|
||||
const DECOMMISSION_TERMINAL_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(1);
|
||||
const DECOMMISSION_CANCEL_TARGET_LOCK_MAX_ATTEMPTS: usize = 3;
|
||||
const DECOMMISSION_DURABLE_ILM_RECEIPT_ROOT: &str = "decommission/ilm-receipts";
|
||||
const DECOMMISSION_DURABLE_ILM_MANIFEST_ROOT: &str = "decommission/ilm-manifests";
|
||||
const DECOMMISSION_DURABLE_ILM_RECEIPT_SCHEMA: &str = "v2";
|
||||
@@ -9624,6 +9625,10 @@ struct DecommissionCapacityLockOrderBarrierState {
|
||||
cancel_before_start_entered: tokio::sync::Notify,
|
||||
cancel_before_start_release: tokio::sync::Notify,
|
||||
cancel_before_start_paused: AtomicBool,
|
||||
cancel_target_timeout_entered: tokio::sync::Notify,
|
||||
cancel_target_timeout_release: tokio::sync::Notify,
|
||||
cancel_target_timeout_paused: AtomicBool,
|
||||
cancel_target_timeouts: AtomicUsize,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -9663,6 +9668,10 @@ impl DecommissionCapacityLockOrderBarrier {
|
||||
cancel_before_start_entered: tokio::sync::Notify::new(),
|
||||
cancel_before_start_release: tokio::sync::Notify::new(),
|
||||
cancel_before_start_paused: AtomicBool::new(false),
|
||||
cancel_target_timeout_entered: tokio::sync::Notify::new(),
|
||||
cancel_target_timeout_release: tokio::sync::Notify::new(),
|
||||
cancel_target_timeout_paused: AtomicBool::new(false),
|
||||
cancel_target_timeouts: AtomicUsize::new(0),
|
||||
});
|
||||
let mut slot = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
@@ -9784,6 +9793,21 @@ impl DecommissionCapacityLockOrderBarrier {
|
||||
self.state.cancel_before_start_release.notify_one();
|
||||
}
|
||||
|
||||
fn pause_cancel_target_timeout(&self) {
|
||||
self.state.cancel_target_timeout_paused.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
async fn wait_until_cancel_target_timeout(&self) {
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30), self.state.cancel_target_timeout_entered.notified())
|
||||
.await
|
||||
.expect("cancel should observe contention on its target capacity fence");
|
||||
}
|
||||
|
||||
fn release_cancel_target_timeout(&self) {
|
||||
self.state.cancel_target_timeout_paused.store(false, Ordering::Release);
|
||||
self.state.cancel_target_timeout_release.notify_one();
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub(crate) fn release_owner(&self) {
|
||||
self.state.owner_release.notify_one();
|
||||
@@ -9826,6 +9850,7 @@ impl Drop for DecommissionCapacityLockOrderBarrier {
|
||||
self.state.external_object_capacity_probe_release.notify_one();
|
||||
self.state.external_object_commit_phase_release.notify_one();
|
||||
self.state.cancel_before_start_release.notify_one();
|
||||
self.state.cancel_target_timeout_release.notify_one();
|
||||
let mut slot = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
@@ -9884,6 +9909,24 @@ async fn pause_decommission_cancel_before_start_gate(store_id: uuid::Uuid) {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn pause_decommission_cancel_target_timeout(store_id: uuid::Uuid) {
|
||||
let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("decommission capacity lock-order barrier should not be poisoned")
|
||||
.as_ref()
|
||||
.filter(|state| state.owner_store_id == store_id)
|
||||
.cloned();
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.cancel_target_timeouts.fetch_add(1, Ordering::AcqRel);
|
||||
barrier.cancel_target_timeout_entered.notify_one();
|
||||
if barrier.cancel_target_timeout_paused.load(Ordering::Acquire) {
|
||||
barrier.cancel_target_timeout_release.notified().await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn notify_decommission_target_gate_retry(store_id: uuid::Uuid) {
|
||||
let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
|
||||
@@ -10521,6 +10564,7 @@ impl ECStore {
|
||||
|
||||
async fn acquire_decommission_capacity_terminal_guards(
|
||||
&self,
|
||||
source_pool_index: usize,
|
||||
plan: Option<&DecommissionCapacityTerminalFencePlan>,
|
||||
) -> Result<Vec<rustfs_lock::NamespaceLockGuard>> {
|
||||
let Some(plan) = plan else {
|
||||
@@ -10544,26 +10588,69 @@ impl ECStore {
|
||||
"no storage pools available".to_string(),
|
||||
)
|
||||
})?;
|
||||
let mut guards = Vec::with_capacity(plan.target_pool_indices.len());
|
||||
for &target_pool_index in &plan.target_pool_indices {
|
||||
let object = format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{target_pool_index}");
|
||||
let target_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
|
||||
let guard = target_lock
|
||||
.get_write_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(|err| match err {
|
||||
rustfs_lock::LockError::QuorumNotReached { required, achieved } => Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "write",
|
||||
bucket: RUSTFS_META_BUCKET.to_string(),
|
||||
object,
|
||||
required,
|
||||
achieved,
|
||||
},
|
||||
other => Error::Lock(other),
|
||||
})?;
|
||||
guards.push(guard);
|
||||
// Retry only target acquisition, never persistence. Keep the original
|
||||
// owner/cohort pinned so a remote Clear/start cannot retarget a cancel.
|
||||
let mut attempt = 1;
|
||||
'acquire_targets: loop {
|
||||
let mut guards = Vec::with_capacity(plan.target_pool_indices.len());
|
||||
for &target_pool_index in &plan.target_pool_indices {
|
||||
let object = format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{target_pool_index}");
|
||||
let target_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
|
||||
let started = std::time::Instant::now();
|
||||
match target_lock.get_write_lock(get_lock_acquire_timeout()).await {
|
||||
Ok(guard) => guards.push(guard),
|
||||
Err(err @ rustfs_lock::LockError::Timeout { .. }) => {
|
||||
// Release the entire partial cohort before backoff or
|
||||
// metadata reads; workers need these gates to settle I/O.
|
||||
drop(guards);
|
||||
#[cfg(test)]
|
||||
pause_decommission_cancel_target_timeout(self.id).await;
|
||||
if attempt >= DECOMMISSION_CANCEL_TARGET_LOCK_MAX_ATTEMPTS {
|
||||
return Err(Error::Lock(err));
|
||||
}
|
||||
warn!(
|
||||
event = EVENT_DECOMMISSION_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
state = "cancel_target_fence_retry",
|
||||
pool_index = source_pool_index,
|
||||
target_pool_index,
|
||||
operation_id = %plan.operation_id,
|
||||
generation = plan.generation,
|
||||
owner_nonce = %plan.owner_nonce,
|
||||
attempt,
|
||||
max_attempts = DECOMMISSION_CANCEL_TARGET_LOCK_MAX_ATTEMPTS,
|
||||
wait_ms = %started.elapsed().as_millis(),
|
||||
error = %err,
|
||||
"Decommission cancel will retry target capacity fencing"
|
||||
);
|
||||
tokio::time::sleep(DECOMMISSION_TERMINAL_RETRY_DELAY).await;
|
||||
let save_guard = self.pool_meta_save_gate.lock().await;
|
||||
let (_read_guard, snapshot) = self
|
||||
.acquire_pool_meta_read_guard(&save_guard, "decommission cancel fence retry failed")
|
||||
.await?;
|
||||
if decommission_capacity_terminal_fence_plan(&snapshot, source_pool_index)?.as_ref() != Some(plan) {
|
||||
return Err(decommission_capacity_blocked_error(
|
||||
"decommission capacity owner or target cohort changed while retrying terminal fences",
|
||||
));
|
||||
}
|
||||
attempt += 1;
|
||||
continue 'acquire_targets;
|
||||
}
|
||||
Err(rustfs_lock::LockError::QuorumNotReached { required, achieved }) => {
|
||||
return Err(Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "write",
|
||||
bucket: RUSTFS_META_BUCKET.to_string(),
|
||||
object,
|
||||
required,
|
||||
achieved,
|
||||
});
|
||||
}
|
||||
Err(err) => return Err(Error::Lock(err)),
|
||||
}
|
||||
}
|
||||
return Ok(guards);
|
||||
}
|
||||
Ok(guards)
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire_external_decommission_capacity_fence(
|
||||
@@ -12493,7 +12580,7 @@ impl ECStore {
|
||||
None
|
||||
};
|
||||
let _capacity_target_guards = if acquire_runtime_fence {
|
||||
self.acquire_decommission_capacity_terminal_guards(terminal_fence_plan.as_ref())
|
||||
self.acquire_decommission_capacity_terminal_guards(idx, terminal_fence_plan.as_ref())
|
||||
.await?
|
||||
} else {
|
||||
Vec::new()
|
||||
@@ -18736,6 +18823,429 @@ mod tests {
|
||||
assert_eq!(reservation.inflight_target_physical_bytes, 0);
|
||||
}
|
||||
|
||||
async fn start_target_fenced_cancel_test(
|
||||
store: &Arc<ECStore>,
|
||||
first_target_free: usize,
|
||||
) -> (DecommissionCapacityTerminalFencePlan, DecommissionCanceler) {
|
||||
crate::services::rebalance::promote_test_pool_meta_to_v2(store).await;
|
||||
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
|
||||
set_decommission_capacity_info_overrides_for_test(
|
||||
store.id,
|
||||
vec![vec![
|
||||
DecommissionPoolCapacityInfo::for_test(0, layout, 0, 10, 10),
|
||||
DecommissionPoolCapacityInfo::for_test(1, layout, first_target_free, 10, 10 - first_target_free),
|
||||
DecommissionPoolCapacityInfo::for_test(2, layout, 40, 40, 0),
|
||||
]],
|
||||
);
|
||||
store
|
||||
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
|
||||
.await
|
||||
.expect("persist an active target-fenced decommission");
|
||||
let plan = decommission_capacity_terminal_fence_plan(&*store.pool_meta.read().await, 0)
|
||||
.expect("active decommission should have a valid fence plan")
|
||||
.expect("active decommission should retain its reservation");
|
||||
assert_eq!(plan.model_version, DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION);
|
||||
let canceler = DecommissionCanceler::new(CancellationToken::new());
|
||||
*store.decommission_cancelers.write().await = vec![Some(canceler.clone()), None, None];
|
||||
(plan, canceler)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn decommission_cancel_waits_for_target_contention_past_one_lock_timeout() {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("5"))], async {
|
||||
let (_temp_dirs, store, _other_store) =
|
||||
crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
|
||||
let (plan, canceler) = start_target_fenced_cancel_test(&store, 0).await;
|
||||
assert_eq!(plan.target_pool_indices, vec![2]);
|
||||
let target_lock = store.pools[0]
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2"))
|
||||
.await
|
||||
.expect("create the active migration target gate");
|
||||
let target_guard = target_lock
|
||||
.get_write_lock(std::time::Duration::from_secs(30))
|
||||
.await
|
||||
.expect("hold the target gate across the first cancel acquisition timeout");
|
||||
let movement_gate = store.ctx.data_movement_operation_gate();
|
||||
let movement_guard = movement_gate.read().await;
|
||||
let cancel_store = Arc::clone(&store);
|
||||
let mut cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await });
|
||||
|
||||
tokio::time::timeout(get_lock_acquire_timeout() + std::time::Duration::from_secs(1), &mut cancel)
|
||||
.await
|
||||
.expect_err("one target-lock timeout must not end a legitimate cancel while its retry budget remains");
|
||||
assert!(!canceler.is_cancelled(), "cancel must not signal its worker before durable fencing");
|
||||
let mut durable = PoolMeta::default();
|
||||
durable
|
||||
.load_no_lock_from_replicas(store.pools.clone())
|
||||
.await
|
||||
.expect("the active reservation must remain readable while cancel waits");
|
||||
assert_eq!(
|
||||
decommission_capacity_terminal_fence_plan(&durable, 0).expect("read the durable active plan"),
|
||||
Some(plan),
|
||||
"a failed target acquisition must not release or replace the durable reservation"
|
||||
);
|
||||
assert!(!durable.pools[0].decommission.as_ref().expect("active decommission").canceled);
|
||||
|
||||
drop(target_guard);
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30), canceler.token().cancelled())
|
||||
.await
|
||||
.expect("cancel should persist and signal its worker after target contention is released");
|
||||
durable
|
||||
.load_no_lock_from_replicas(store.pools.clone())
|
||||
.await
|
||||
.expect("reload the committed cancellation from native replicas");
|
||||
let info = durable.pools[0]
|
||||
.decommission
|
||||
.as_ref()
|
||||
.expect("canceled state must remain durable");
|
||||
assert!(info.canceled);
|
||||
assert!(!info.complete && !info.failed);
|
||||
let reservation = info
|
||||
.capacity_reservation
|
||||
.as_ref()
|
||||
.expect("terminal capacity accounting must remain inspectable");
|
||||
assert!(!reservation.active());
|
||||
assert_eq!(reservation.pending_target_physical_bytes, 0);
|
||||
assert_eq!(reservation.inflight_target_physical_bytes, 0);
|
||||
tokio::time::timeout(std::time::Duration::from_millis(500), &mut cancel)
|
||||
.await
|
||||
.expect_err("the runtime-fenced cancel must wait for in-flight movement after signaling");
|
||||
drop(movement_guard);
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30), cancel)
|
||||
.await
|
||||
.expect("cancel should return after in-flight movement quiesces")
|
||||
.expect("cancel task should not panic")
|
||||
.expect("the same cancel request should finish its durable terminal transition");
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn decommission_cancel_target_contention_exhausts_bounded_attempts_without_committing() {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
|
||||
let (_temp_dirs, store, _other_store) =
|
||||
crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
|
||||
let (plan, canceler) = start_target_fenced_cancel_test(&store, 0).await;
|
||||
let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
|
||||
let target_lock = store.pools[0]
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2"))
|
||||
.await
|
||||
.expect("create the persistently contended target gate");
|
||||
let target_guard = target_lock
|
||||
.get_write_lock(std::time::Duration::from_secs(30))
|
||||
.await
|
||||
.expect("hold the target gate for every cancel attempt");
|
||||
|
||||
let err = tokio::time::timeout(std::time::Duration::from_secs(30), store.decommission_cancel(0))
|
||||
.await
|
||||
.expect("target contention must not retry indefinitely")
|
||||
.expect_err("exhausting the acquisition budget must not report cancellation success");
|
||||
match err {
|
||||
Error::Lock(rustfs_lock::LockError::Timeout { resource, timeout }) => {
|
||||
assert_eq!(resource, ".rustfs.sys/decommission/capacity-target/2@latest");
|
||||
assert_eq!(timeout, std::time::Duration::from_secs(1));
|
||||
}
|
||||
other => panic!("expected the final typed target-lock timeout, got {other:?}"),
|
||||
}
|
||||
assert_eq!(barrier.state.cancel_target_timeouts.load(Ordering::Acquire), 3);
|
||||
assert!(!canceler.is_cancelled());
|
||||
assert!(canceler.is_active());
|
||||
let mut durable = PoolMeta::default();
|
||||
durable
|
||||
.load_no_lock_from_replicas(store.pools.clone())
|
||||
.await
|
||||
.expect("reload the reservation after the canceled request exhausted its budget");
|
||||
assert_eq!(
|
||||
decommission_capacity_terminal_fence_plan(&durable, 0).expect("the active plan should remain valid"),
|
||||
Some(plan)
|
||||
);
|
||||
assert!(!durable.pools[0].decommission.as_ref().expect("active decommission").canceled);
|
||||
drop(target_guard);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn decommission_cancel_target_retry_releases_partial_cohort_and_rejects_remote_replacement() {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
|
||||
let (_temp_dirs, store, other_store) =
|
||||
crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
|
||||
let (plan, _old_canceler) = start_target_fenced_cancel_test(&store, 8).await;
|
||||
assert_eq!(plan.target_pool_indices, vec![1, 2]);
|
||||
let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
|
||||
barrier.pause_cancel_target_timeout();
|
||||
let target_lock = store.pools[0]
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2"))
|
||||
.await
|
||||
.expect("create the second target gate");
|
||||
let target_guard = target_lock
|
||||
.get_write_lock(std::time::Duration::from_secs(30))
|
||||
.await
|
||||
.expect("block cancellation after it acquires the first target");
|
||||
let cancel_store = Arc::clone(&store);
|
||||
let cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await });
|
||||
barrier.wait_until_cancel_target_timeout().await;
|
||||
|
||||
let first_target_lock = store.pools[0]
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/1"))
|
||||
.await
|
||||
.expect("create the partial-cohort release probe");
|
||||
let first_target_guard = first_target_lock
|
||||
.get_write_lock(std::time::Duration::from_secs(1))
|
||||
.await
|
||||
.expect("cancel must release its first target before retrying the blocked second target");
|
||||
drop(first_target_guard);
|
||||
drop(target_guard);
|
||||
|
||||
other_store
|
||||
.reload_pool_meta()
|
||||
.await
|
||||
.expect("load the active generation on the remote node");
|
||||
other_store
|
||||
.decommission_cancel(0)
|
||||
.await
|
||||
.expect("the remote node should cancel the old generation");
|
||||
other_store
|
||||
.clear_decommission(0)
|
||||
.await
|
||||
.expect("the remote node should clear the old generation");
|
||||
let (replacement_plan, replacement_canceler) = start_target_fenced_cancel_test(&other_store, 8).await;
|
||||
assert_ne!(replacement_plan.operation_id, plan.operation_id);
|
||||
|
||||
barrier.release_cancel_target_timeout();
|
||||
let err = tokio::time::timeout(std::time::Duration::from_secs(30), cancel)
|
||||
.await
|
||||
.expect("the stale request must finish without retrying a replacement operation")
|
||||
.expect("the stale cancel task should not panic")
|
||||
.expect_err("the original request must not cancel a remotely replaced generation");
|
||||
assert!(err.to_string().contains("changed while retrying terminal fences"));
|
||||
assert_eq!(barrier.state.cancel_target_timeouts.load(Ordering::Acquire), 1);
|
||||
assert!(!replacement_canceler.is_cancelled());
|
||||
let mut durable = PoolMeta::default();
|
||||
durable
|
||||
.load_no_lock_from_replicas(store.pools.clone())
|
||||
.await
|
||||
.expect("reload the replacement after the stale cancel returns");
|
||||
assert_eq!(
|
||||
decommission_capacity_terminal_fence_plan(&durable, 0).expect("the replacement must retain its valid plan"),
|
||||
Some(replacement_plan)
|
||||
);
|
||||
assert!(
|
||||
!durable.pools[0]
|
||||
.decommission
|
||||
.as_ref()
|
||||
.expect("replacement decommission")
|
||||
.canceled
|
||||
);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn decommission_cancel_target_retry_survives_caller_abort_and_settles_inflight_mutation() {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
|
||||
let (_temp_dirs, store, _other_store) =
|
||||
crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
|
||||
let (plan, canceler) = start_target_fenced_cancel_test(&store, 0).await;
|
||||
let owner = DecommissionCapacityOwner {
|
||||
source_pool_index: 0,
|
||||
operation_id: plan.operation_id,
|
||||
generation: plan.generation,
|
||||
owner_nonce: plan.owner_nonce,
|
||||
mutation_id: Some(uuid::Uuid::new_v4()),
|
||||
};
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
|
||||
let mutation_store = Arc::clone(&store);
|
||||
let mutation = tokio::spawn(async move {
|
||||
mutation_store
|
||||
.run_decommission_capacity_admitted_mutation(2, Some(owner), Some(1), || async {
|
||||
entered_tx
|
||||
.send(())
|
||||
.expect("the test should observe the admitted target mutation");
|
||||
release_rx.await.expect("the test should release the in-flight mutation");
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30), entered_rx)
|
||||
.await
|
||||
.expect("target mutation should reach its controlled I/O phase")
|
||||
.expect("target admission should succeed before cancel starts");
|
||||
let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
|
||||
let cancel_store = Arc::clone(&store);
|
||||
let cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await });
|
||||
barrier.wait_until_cancel_target_timeout().await;
|
||||
assert!(!canceler.is_cancelled(), "in-flight capacity settlement must precede the cancel signal");
|
||||
cancel.abort();
|
||||
assert!(cancel.await.expect_err("the RPC waiter should be aborted").is_cancelled());
|
||||
|
||||
release_tx
|
||||
.send(())
|
||||
.expect("the mutation must still be alive after the caller disconnects");
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30), mutation)
|
||||
.await
|
||||
.expect("the in-flight mutation must be able to settle without a metadata lock cycle")
|
||||
.expect("the mutation task should not panic")
|
||||
.expect("the admitted mutation must settle before the target gate is released");
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30), async {
|
||||
loop {
|
||||
if store.pool_meta.read().await.pools[0]
|
||||
.decommission
|
||||
.as_ref()
|
||||
.is_some_and(|info| info.canceled)
|
||||
{
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("the detached cancel transaction must finish after target contention clears");
|
||||
assert!(canceler.is_cancelled());
|
||||
let mut durable = PoolMeta::default();
|
||||
durable
|
||||
.load_no_lock_from_replicas(store.pools.clone())
|
||||
.await
|
||||
.expect("reload cancellation after the RPC waiter was dropped");
|
||||
let info = durable.pools[0].decommission.as_ref().expect("durable canceled decommission");
|
||||
assert!(info.canceled && !info.failed && !info.complete);
|
||||
let reservation = info
|
||||
.capacity_reservation
|
||||
.as_ref()
|
||||
.expect("inspect settled capacity accounting");
|
||||
assert!(!reservation.active());
|
||||
assert_eq!(reservation.pending_target_physical_bytes, 0);
|
||||
assert_eq!(reservation.inflight_target_physical_bytes, 0);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn decommission_cancel_target_retry_does_not_replay_a_failed_durable_save() {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
|
||||
let (_temp_dirs, store, _other_store) =
|
||||
crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
|
||||
let (plan, canceler) = start_target_fenced_cancel_test(&store, 0).await;
|
||||
let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
|
||||
barrier.pause_cancel_target_timeout();
|
||||
let target_lock = store.pools[0]
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2"))
|
||||
.await
|
||||
.expect("create the target gate preceding the failed save");
|
||||
let target_guard = target_lock
|
||||
.get_write_lock(std::time::Duration::from_secs(30))
|
||||
.await
|
||||
.expect("force one target acquisition retry before persistence");
|
||||
let save_calls = Arc::new(AtomicUsize::new(0));
|
||||
let calls = Arc::clone(&save_calls);
|
||||
let cancel_store = Arc::clone(&store);
|
||||
let owner = canceler.clone();
|
||||
let cancel = tokio::spawn(async move {
|
||||
cancel_store
|
||||
.decommission_cancel_transaction(0, Some(owner), true, move |_, _| async move {
|
||||
calls.fetch_add(1, Ordering::AcqRel);
|
||||
Err(Error::Timeout)
|
||||
})
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_cancel_target_timeout().await;
|
||||
assert_eq!(save_calls.load(Ordering::Acquire), 0, "persistence must wait for every target fence");
|
||||
drop(target_guard);
|
||||
barrier.release_cancel_target_timeout();
|
||||
let err = tokio::time::timeout(std::time::Duration::from_secs(30), cancel)
|
||||
.await
|
||||
.expect("a persistence failure must end the cancellation transaction")
|
||||
.expect("the failed-save task should not panic")
|
||||
.expect_err("the injected durable-save failure must reach the caller");
|
||||
assert!(matches!(err, Error::Timeout));
|
||||
assert_eq!(save_calls.load(Ordering::Acquire), 1);
|
||||
assert_eq!(barrier.state.cancel_target_timeouts.load(Ordering::Acquire), 1);
|
||||
assert!(!canceler.is_cancelled());
|
||||
assert!(canceler.is_active());
|
||||
store
|
||||
.ensure_pool_meta_side_effects_safe("verify ambiguous cancellation save blocks further writes")
|
||||
.await
|
||||
.expect_err("a failed durable save must retain the existing recovery gate");
|
||||
let mut durable = PoolMeta::default();
|
||||
durable
|
||||
.load_no_lock_from_replicas(store.pools.clone())
|
||||
.await
|
||||
.expect("the original reservation should remain readable after the injected save failure");
|
||||
assert_eq!(
|
||||
decommission_capacity_terminal_fence_plan(&durable, 0).expect("the old reservation should remain valid"),
|
||||
Some(plan)
|
||||
);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn decommission_cancel_target_retry_converges_after_successive_contenders() {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
|
||||
let (_temp_dirs, store, _other_store) =
|
||||
crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
|
||||
let (_plan, canceler) = start_target_fenced_cancel_test(&store, 0).await;
|
||||
let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
|
||||
barrier.pause_cancel_target_timeout();
|
||||
let target_lock = store.pools[0]
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2"))
|
||||
.await
|
||||
.expect("create the repeatedly contended target gate");
|
||||
let first_guard = target_lock
|
||||
.get_write_lock(std::time::Duration::from_secs(30))
|
||||
.await
|
||||
.expect("the first contender should own the target gate");
|
||||
let cancel_store = Arc::clone(&store);
|
||||
let cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await });
|
||||
barrier.wait_until_cancel_target_timeout().await;
|
||||
assert_eq!(barrier.state.cancel_target_timeouts.load(Ordering::Acquire), 1);
|
||||
drop(first_guard);
|
||||
let second_guard = target_lock
|
||||
.get_write_lock(std::time::Duration::from_secs(30))
|
||||
.await
|
||||
.expect("a second contender should be able to acquire between cancel attempts");
|
||||
barrier.release_cancel_target_timeout();
|
||||
barrier.pause_cancel_target_timeout();
|
||||
barrier.wait_until_cancel_target_timeout().await;
|
||||
assert_eq!(barrier.state.cancel_target_timeouts.load(Ordering::Acquire), 2);
|
||||
assert!(!canceler.is_cancelled());
|
||||
drop(second_guard);
|
||||
barrier.release_cancel_target_timeout();
|
||||
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30), cancel)
|
||||
.await
|
||||
.expect("cancel should converge after the repeated contention ends")
|
||||
.expect("the cancel task should not panic")
|
||||
.expect("the third target acquisition should permit a durable cancellation");
|
||||
assert!(canceler.is_cancelled());
|
||||
let mut durable = PoolMeta::default();
|
||||
durable
|
||||
.load_no_lock_from_replicas(store.pools.clone())
|
||||
.await
|
||||
.expect("the successful retried cancellation must survive a native metadata reload");
|
||||
assert!(
|
||||
durable.pools[0]
|
||||
.decommission
|
||||
.as_ref()
|
||||
.expect("canceled decommission")
|
||||
.canceled
|
||||
);
|
||||
assert!(
|
||||
decommission_capacity_terminal_fence_plan(&durable, 0)
|
||||
.expect("valid terminal metadata")
|
||||
.is_none()
|
||||
);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn stale_node_cancel_cannot_replace_a_new_durable_v2_operation() {
|
||||
|
||||
@@ -55,7 +55,7 @@ use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use rustfs_utils::{crc_hash, path::path_join_buf, sip_hash};
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
sync::Arc,
|
||||
sync::{Arc, Weak},
|
||||
};
|
||||
use tokio::sync::RwLock;
|
||||
use tokio::sync::broadcast::{Receiver, Sender};
|
||||
@@ -219,7 +219,10 @@ impl Sets {
|
||||
|
||||
let mut disk_set = Vec::with_capacity(set_count);
|
||||
|
||||
let lock_registry = runtime_sources::lock_registry();
|
||||
let pool_lockers = runtime_sources::lock_registry()
|
||||
.as_ref()
|
||||
.map(|registry| registry.clients_for_endpoints(endpoints.endpoints.as_ref()))
|
||||
.unwrap_or_default();
|
||||
|
||||
for i in 0..set_count {
|
||||
let mut set_drive = Vec::with_capacity(set_drive_count);
|
||||
@@ -270,10 +273,6 @@ impl Sets {
|
||||
}
|
||||
}
|
||||
|
||||
let lockers = lock_registry
|
||||
.as_ref()
|
||||
.map(|registry| registry.clients_for_endpoints(&set_endpoints))
|
||||
.unwrap_or_default();
|
||||
let set_disks = SetDisks::new_with_instance_ctx(
|
||||
runtime_sources::local_node_name().await,
|
||||
Arc::new(RwLock::new(set_drive)),
|
||||
@@ -283,7 +282,7 @@ impl Sets {
|
||||
pool_idx,
|
||||
set_endpoints,
|
||||
fm.clone(),
|
||||
lockers,
|
||||
pool_lockers.clone(),
|
||||
instance_ctx.clone(),
|
||||
)
|
||||
.await;
|
||||
@@ -308,10 +307,9 @@ impl Sets {
|
||||
ctx: instance_ctx,
|
||||
});
|
||||
|
||||
let asets = sets.clone();
|
||||
|
||||
let rx1 = rx.resubscribe();
|
||||
tokio::spawn(async move { asets.monitor_and_connect_endpoints(rx1).await });
|
||||
let weak_sets = Arc::downgrade(&sets);
|
||||
tokio::spawn(async move { Self::monitor_and_connect_endpoints_task(weak_sets, rx1).await });
|
||||
|
||||
Ok(sets)
|
||||
}
|
||||
@@ -326,12 +324,26 @@ impl Sets {
|
||||
&self.ctx
|
||||
}
|
||||
|
||||
pub async fn monitor_and_connect_endpoints(&self, mut rx: Receiver<()>) {
|
||||
tokio::time::sleep(Duration::from_secs(5)).await;
|
||||
async fn monitor_and_connect_endpoints_task(sets: Weak<Sets>, mut rx: Receiver<()>) {
|
||||
let startup_delay = tokio::time::sleep(Duration::from_secs(5));
|
||||
tokio::pin!(startup_delay);
|
||||
|
||||
tokio::select! {
|
||||
_ = &mut startup_delay => {}
|
||||
_ = rx.recv() => {
|
||||
warn!("monitor_and_connect_endpoints ctx cancelled");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
info!("start monitor_and_connect_endpoints");
|
||||
|
||||
self.connect_disks().await;
|
||||
let Some(current) = sets.upgrade() else {
|
||||
warn!("monitor_and_connect_endpoints exit");
|
||||
return;
|
||||
};
|
||||
current.connect_disks().await;
|
||||
drop(current);
|
||||
|
||||
// TODO(backlog): make monitor_and_connect interval configurable instead of hardcoded 15s
|
||||
let mut interval = tokio::time::interval(Duration::from_secs(15));
|
||||
@@ -339,7 +351,10 @@ impl Sets {
|
||||
tokio::select! {
|
||||
_= interval.tick()=>{
|
||||
// debug!("tick...");
|
||||
self.connect_disks().await;
|
||||
let Some(current) = sets.upgrade() else {
|
||||
break;
|
||||
};
|
||||
current.connect_disks().await;
|
||||
|
||||
interval.reset();
|
||||
},
|
||||
|
||||
@@ -6409,6 +6409,10 @@ impl LocalDisk {
|
||||
// A missing or still-populated directory is benign here; see
|
||||
// is_benign_object_rmdir_error (handles the illumos/Solaris EEXIST
|
||||
// convention, rustfs/rustfs#4978).
|
||||
if is_dir_not_empty_error(&err) {
|
||||
// A populated directory keeps its ancestors populated; no further pruning is needed.
|
||||
return Ok(());
|
||||
}
|
||||
if !is_benign_object_rmdir_error(&err) {
|
||||
warn!(
|
||||
event = EVENT_DISK_LOCAL_DELETE_FAILED,
|
||||
@@ -11222,6 +11226,176 @@ mod test {
|
||||
(disk, dir)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_pruning_stops_at_live_metadata_below_a_guarded_ancestor() {
|
||||
// Tuple fields drop in order, releasing the disk's root handle before the temporary directory.
|
||||
let fixture = new_disk().await;
|
||||
let (disk, _dir) = &fixture;
|
||||
let base = disk.get_bucket_path(RUSTFS_META_BUCKET).expect("resolve metadata volume");
|
||||
let shared = base.join("buckets");
|
||||
let guard = Arc::new(
|
||||
os::mkdir_all_below_existing_base_std(&shared, &base, &disk.publication_root)
|
||||
.expect("retain the shared publication directory"),
|
||||
);
|
||||
|
||||
for owned in [false, true] {
|
||||
for missing_backup in [false, true] {
|
||||
let transaction = Uuid::new_v4();
|
||||
let object = shared.join(".bloomcycle.bin");
|
||||
let rollback = object.join(transaction.to_string());
|
||||
let metadata = object.join(STORAGE_FORMAT_FILE);
|
||||
let backup = rollback.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
fs::create_dir_all(&rollback).await.expect("create rollback directory");
|
||||
fs::write(&metadata, b"committed metadata")
|
||||
.await
|
||||
.expect("write live metadata");
|
||||
if !missing_backup {
|
||||
fs::write(&backup, b"old metadata").await.expect("write rollback backup");
|
||||
}
|
||||
let owner: Option<Arc<dyn Send + Sync>> = if owned { Some(guard.clone()) } else { None };
|
||||
let result = disk
|
||||
.delete_with_namespace_owner(
|
||||
RUSTFS_META_BUCKET,
|
||||
&format!("buckets/.bloomcycle.bin/{transaction}/{STORAGE_FORMAT_FILE_BACKUP}"),
|
||||
DeleteOptions::default(),
|
||||
owner,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(!backup.exists(), "backup must be absent, owned={owned}, missing={missing_backup}");
|
||||
assert!(!rollback.exists(), "empty rollback directory must be pruned");
|
||||
assert_eq!(fs::read(&metadata).await.expect("read committed metadata"), b"committed metadata");
|
||||
result.expect("a nonempty object must stop pruning before the guarded ancestor");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_pruning_removes_empty_and_missing_ancestors_but_keeps_the_volume() {
|
||||
let fixture = new_disk().await;
|
||||
let (disk, _dir) = &fixture;
|
||||
ensure_test_volume(disk, "pruning").await;
|
||||
let base = disk.get_bucket_path("pruning").expect("resolve test volume");
|
||||
|
||||
for missing in [false, true] {
|
||||
let parent = base.join("parent");
|
||||
let rollback = parent.join("object/transaction");
|
||||
fs::create_dir_all(&rollback).await.expect("create empty ancestor chain");
|
||||
let path = if missing {
|
||||
"parent/object/transaction/missing/xl.meta.bkp"
|
||||
} else {
|
||||
fs::write(rollback.join(STORAGE_FORMAT_FILE_BACKUP), b"backup")
|
||||
.await
|
||||
.expect("create backup");
|
||||
"parent/object/transaction/xl.meta.bkp"
|
||||
};
|
||||
|
||||
disk.delete("pruning", path, DeleteOptions::default())
|
||||
.await
|
||||
.expect("empty and missing ancestors should be pruned");
|
||||
assert!(!parent.exists(), "the whole empty chain should be removed");
|
||||
assert!(base.is_dir(), "pruning must stop at the volume boundary");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_pruning_does_not_remove_the_base_or_an_outside_path() {
|
||||
let fixture = new_disk().await;
|
||||
let (disk, dir) = &fixture;
|
||||
let base = dir.path().join("base");
|
||||
let outside = dir.path().join("outside");
|
||||
fs::create_dir(&base).await.expect("create base");
|
||||
fs::write(&outside, b"outside data").await.expect("create outside file");
|
||||
|
||||
disk.delete_file(&base, &base, false, false)
|
||||
.await
|
||||
.expect("base path is protected");
|
||||
disk.delete_file(&base, &outside, false, false)
|
||||
.await
|
||||
.expect("outside path is protected");
|
||||
assert!(base.is_dir(), "the base must not be removed even when empty");
|
||||
assert_eq!(fs::read(&outside).await.expect("read outside file"), b"outside data");
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[tokio::test]
|
||||
async fn delete_pruning_propagates_a_locked_backup_error() {
|
||||
use std::os::windows::fs::OpenOptionsExt;
|
||||
use windows_sys::Win32::{Foundation::ERROR_SHARING_VIOLATION, Storage::FileSystem::FILE_SHARE_READ};
|
||||
|
||||
let fixture = new_disk().await;
|
||||
let (disk, _dir) = &fixture;
|
||||
ensure_test_volume(disk, "pruning").await;
|
||||
let base = disk.get_bucket_path("pruning").expect("resolve test volume");
|
||||
let backup = base.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
fs::write(&backup, b"backup").await.expect("write backup");
|
||||
let guard = std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.share_mode(FILE_SHARE_READ)
|
||||
.open(&backup)
|
||||
.expect("hold the backup without delete sharing");
|
||||
|
||||
let err = disk
|
||||
.delete("pruning", STORAGE_FORMAT_FILE_BACKUP, DeleteOptions::default())
|
||||
.await
|
||||
.expect_err("a genuine target-file deletion failure must propagate");
|
||||
let DiskError::Io(err) = err else {
|
||||
panic!("expected contextual I/O error, got {err:?}");
|
||||
};
|
||||
let context = err
|
||||
.get_ref()
|
||||
.and_then(|err| err.downcast_ref::<FileAccessDeniedWithContext>())
|
||||
.expect("preserve the failing path and original OS error");
|
||||
assert_eq!(context.path, backup);
|
||||
assert_eq!(
|
||||
context.source.raw_os_error(),
|
||||
Some(i32::try_from(ERROR_SHARING_VIOLATION).expect("OS code fits"))
|
||||
);
|
||||
assert_eq!(fs::read(&backup).await.expect("backup remains readable"), b"backup");
|
||||
drop(guard);
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[tokio::test]
|
||||
async fn delete_pruning_propagates_a_locked_empty_parent_error() {
|
||||
use windows_sys::Win32::Foundation::ERROR_SHARING_VIOLATION;
|
||||
|
||||
let fixture = new_disk().await;
|
||||
let (disk, _dir) = &fixture;
|
||||
ensure_test_volume(disk, "pruning").await;
|
||||
let base = disk.get_bucket_path("pruning").expect("resolve test volume");
|
||||
let parent = base.join("parent");
|
||||
let guard = os::mkdir_all_below_existing_base_std(&parent, &base, &disk.publication_root)
|
||||
.expect("retain an empty parent without delete sharing");
|
||||
let backup = parent.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
fs::write(&backup, b"backup").await.expect("write backup");
|
||||
|
||||
let err = disk
|
||||
.delete("pruning", "parent/xl.meta.bkp", DeleteOptions::default())
|
||||
.await
|
||||
.expect_err("a real parent failure without a nonempty boundary must still propagate");
|
||||
let DiskError::Io(err) = err else {
|
||||
panic!("expected contextual I/O error, got {err:?}");
|
||||
};
|
||||
let context = err
|
||||
.get_ref()
|
||||
.and_then(|err| err.downcast_ref::<FileAccessDeniedWithContext>())
|
||||
.expect("preserve parent failure context");
|
||||
assert_eq!(context.path, parent);
|
||||
assert_eq!(
|
||||
context.source.raw_os_error(),
|
||||
Some(i32::try_from(ERROR_SHARING_VIOLATION).expect("OS code fits"))
|
||||
);
|
||||
assert!(!backup.exists(), "the target was removed before the parent error");
|
||||
assert!(parent.is_dir(), "the guarded parent remains");
|
||||
drop(guard);
|
||||
disk.delete("pruning", "parent/xl.meta.bkp", DeleteOptions::default())
|
||||
.await
|
||||
.expect("pruning should succeed once the actual guard is released");
|
||||
assert!(!parent.exists());
|
||||
assert!(base.is_dir());
|
||||
}
|
||||
|
||||
// #948: a genuinely missing source is benign and must still return Ok.
|
||||
#[tokio::test]
|
||||
async fn windows_and_unix_move_to_trash_missing_source_is_ok() {
|
||||
@@ -11705,14 +11879,59 @@ mod test {
|
||||
/// stale deterministically, instead of sleeping and hoping the filesystem
|
||||
/// timestamp granularity (or a backward wall-clock step) cooperates.
|
||||
fn backdate_mtime(path: &Path, age: Duration) {
|
||||
use std::fs::{File, FileTimes};
|
||||
use std::fs::{FileTimes, OpenOptions};
|
||||
let mtime = std::time::SystemTime::now() - age;
|
||||
File::open(path)
|
||||
let mut options = OpenOptions::new();
|
||||
options.read(true);
|
||||
#[cfg(windows)]
|
||||
{
|
||||
use std::os::windows::fs::OpenOptionsExt;
|
||||
use windows_sys::Win32::Storage::FileSystem::{FILE_FLAG_BACKUP_SEMANTICS, FILE_WRITE_ATTRIBUTES};
|
||||
|
||||
// Directories need backup semantics, and changing mtime needs attribute-write access.
|
||||
options
|
||||
.access_mode(FILE_WRITE_ATTRIBUTES)
|
||||
.custom_flags(FILE_FLAG_BACKUP_SEMANTICS);
|
||||
}
|
||||
options
|
||||
.open(path)
|
||||
.expect("path should open to backdate its mtime")
|
||||
.set_times(FileTimes::new().set_modified(mtime))
|
||||
.expect("mtime should rewind into the past");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cleanup_tmp_on_startup_backdate_mtime_preserves_files_and_directory_contents() {
|
||||
use std::time::SystemTime;
|
||||
|
||||
let root = tempfile::tempdir().expect("create timestamp fixture root");
|
||||
let directory = root.path().join("directory");
|
||||
let file = directory.join("payload");
|
||||
std::fs::create_dir(&directory).expect("create timestamp fixture directory");
|
||||
std::fs::write(&file, b"unchanged payload").expect("write timestamp fixture payload");
|
||||
let age = Duration::from_secs(60);
|
||||
// Filesystems may round stored timestamps; do not require subsecond precision or sleep.
|
||||
let rounding = Duration::from_secs(2);
|
||||
|
||||
for path in [&file, &directory] {
|
||||
let earliest = SystemTime::now() - age - rounding;
|
||||
backdate_mtime(path, age);
|
||||
let latest = SystemTime::now() - age + rounding;
|
||||
let modified = std::fs::metadata(path)
|
||||
.expect("read backdated path metadata")
|
||||
.modified()
|
||||
.expect("read backdated modification time");
|
||||
assert!(modified >= earliest && modified <= latest, "mtime must be backdated for {path:?}");
|
||||
}
|
||||
|
||||
let moved = root.path().join("moved");
|
||||
std::fs::rename(&directory, &moved).expect("mtime helper must release its handles before cleanup");
|
||||
assert_eq!(
|
||||
std::fs::read(moved.join("payload")).expect("read preserved payload"),
|
||||
b"unchanged payload"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn startup_cleanup_barrier_and_tmp_trash_cleanup_cover_noop_and_delete_paths() {
|
||||
use tempfile::tempdir;
|
||||
|
||||
@@ -47,6 +47,43 @@ use tokio::fs;
|
||||
use tracing::{info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Hold later repair publications after admitting one baseline object. The
|
||||
/// fixture arms this on one replacement disk before rejoining the cluster.
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
async fn wait_for_heal_commit_test_barrier(root: &Path, bucket: &str, object: &str) -> Result<()> {
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
let barrier = root.join(".rustfs.sys/e2e-heal-commit-barrier");
|
||||
let prefix = match fs::read_to_string(&barrier).await {
|
||||
Ok(prefix) => prefix,
|
||||
Err(error) if error.kind() == ErrorKind::NotFound => return Ok(()),
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
let key = format!("{bucket}/{object}");
|
||||
if prefix.is_empty() || !key.starts_with(&prefix) {
|
||||
return Ok(());
|
||||
}
|
||||
let admitted = barrier.with_extension("admitted");
|
||||
match fs::OpenOptions::new().write(true).create_new(true).open(&admitted).await {
|
||||
Ok(mut file) => {
|
||||
file.write_all(key.as_bytes()).await?;
|
||||
return Ok(());
|
||||
}
|
||||
Err(error) if error.kind() == ErrorKind::AlreadyExists => {}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(120);
|
||||
loop {
|
||||
if !fs::try_exists(&barrier).await? || fs::read_to_string(&admitted).await? == key {
|
||||
return Ok(());
|
||||
}
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Err(std::io::Error::new(ErrorKind::TimedOut, "heal commit test barrier was not released").into());
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
}
|
||||
}
|
||||
|
||||
fn rollback_committed_rename_std(
|
||||
dst_file_path: &Path,
|
||||
new_data_path: Option<&Path>,
|
||||
@@ -253,6 +290,10 @@ impl LocalDisk {
|
||||
state: &mut RenameDataState,
|
||||
) -> Result<RenameDataResp> {
|
||||
crate::hp_guard!("LocalDisk::rename_data");
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
if fi.is_healing() {
|
||||
wait_for_heal_commit_test_barrier(&self.root, dst_volume, dst_path).await?;
|
||||
}
|
||||
let mut fi = fi;
|
||||
// A non-force DeleteBucket must not remove a directory while a local
|
||||
// object commit is publishing into it. The peer's empty scan remains
|
||||
|
||||
@@ -6433,9 +6433,48 @@ impl TierConfigMgr {
|
||||
}
|
||||
|
||||
pub(crate) async fn refresh_tier_config_handle(handle: Arc<RwLock<Self>>, api: Arc<ECStore>) {
|
||||
Self::refresh_tier_config_handle_with(handle, api).await;
|
||||
Self::refresh_tier_config_handle_with_weak(handle, Arc::downgrade(&api)).await;
|
||||
}
|
||||
|
||||
async fn refresh_tier_config_handle_with_weak(handle: Arc<RwLock<Self>>, api: Weak<ECStore>) {
|
||||
// The periodic refresh remains the recovery fallback; committed mutations
|
||||
// notify this worker so a successful peer commit converges immediately.
|
||||
let mutation_refresh = Self::mutation_refresh_notifier(&handle).await;
|
||||
let r = rand::rng().random_range(0.0..1.0);
|
||||
let rand_interval = || Duration::from_secs((r * 60_f64).round() as u64);
|
||||
|
||||
let refresh_interval = TIER_CFG_REFRESH + rand_interval();
|
||||
let mut t = delayed_tier_refresh_interval(refresh_interval);
|
||||
loop {
|
||||
select! {
|
||||
_ = t.tick() => {
|
||||
let Some(api) = Weak::upgrade(&api) else {
|
||||
return;
|
||||
};
|
||||
if let Err(err) = Self::reload_handle_with(&handle, api).await {
|
||||
warn!(
|
||||
event = EVENT_TIER_CONFIG_REFRESH,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_TIER,
|
||||
trigger = "periodic",
|
||||
result = "failed",
|
||||
error = ?err,
|
||||
"tier configuration refresh"
|
||||
);
|
||||
}
|
||||
}
|
||||
_ = mutation_refresh.notified() => {
|
||||
let Some(api) = Weak::upgrade(&api) else {
|
||||
return;
|
||||
};
|
||||
Self::reload_after_committed_mutation(&handle, api).await;
|
||||
}
|
||||
}
|
||||
t.reset();
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "used by focused tier refresh tests and non-ECStore generic harnesses")]
|
||||
pub(crate) async fn refresh_tier_config_handle_with<S>(handle: Arc<RwLock<Self>>, api: Arc<S>)
|
||||
where
|
||||
S: EcstoreObjectIO
|
||||
@@ -17540,6 +17579,66 @@ mod tests {
|
||||
assert!(current.tiers.contains_key("COLD-B"));
|
||||
}
|
||||
|
||||
async fn wait_for_reference_proof_barrier(
|
||||
barrier: &TierDriverBuildBarrier,
|
||||
update: &mut tokio::task::JoinHandle<std::result::Result<(), TierConfigUpdateError>>,
|
||||
) -> std::result::Result<(), String> {
|
||||
tokio::select! {
|
||||
biased;
|
||||
result = &mut *update => Err(format!("tier update exited before the reference proof barrier: {result:?}")),
|
||||
() = barrier.arrived.notified() => Ok(()),
|
||||
() = tokio::time::sleep(Duration::from_secs(30)) => {
|
||||
// Aborting the caller does not stop its owned mutation task.
|
||||
// Let a late arrival pass the test-only barrier.
|
||||
barrier.release.add_permits(1);
|
||||
update.abort();
|
||||
Err("timed out waiting for the reference proof barrier".to_string())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn reference_proof_barrier_reports_update_failure_before_arrival() {
|
||||
let manager = TierConfigMgr::new();
|
||||
let store = Arc::new(CasConfigStore::default());
|
||||
let mut persisted = empty_mgr();
|
||||
persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
|
||||
persisted
|
||||
.save_tiering_config_if_current(store.clone(), None)
|
||||
.await
|
||||
.expect("early update failure fixture should persist");
|
||||
let barrier = tier_reference_proof_test_barrier();
|
||||
let scoped_barrier = barrier.clone();
|
||||
let factory: TierDriverTestFactory =
|
||||
Arc::new(|_| Err(AdminError::msg("injected driver initialization failure before reference proof")));
|
||||
let mut update = tokio::spawn(async move {
|
||||
TIER_REFERENCE_PROOF_TEST_BARRIER
|
||||
.scope(
|
||||
scoped_barrier,
|
||||
TIER_DRIVER_TEST_FACTORY.scope(
|
||||
factory,
|
||||
TIER_MUTATION_TEST_PEERS.scope(
|
||||
Vec::new(),
|
||||
TierConfigMgr::update_candidate_with_config_lock(
|
||||
&manager,
|
||||
store,
|
||||
TierCandidateMutation::Remove("COLD-A".to_string(), true),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
.await
|
||||
});
|
||||
|
||||
let err = tokio::time::timeout(Duration::from_secs(5), wait_for_reference_proof_barrier(&barrier, &mut update))
|
||||
.await
|
||||
.expect("an early update failure should be observed without waiting for the barrier deadline")
|
||||
.expect_err("a failed update cannot reach the reference proof barrier");
|
||||
assert!(err.contains("Mutation"), "{err}");
|
||||
assert!(err.contains("injected driver initialization failure before reference proof"), "{err}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn reference_proof_rejects_a_changed_prepared_fence_revision_before_publish() {
|
||||
@@ -17560,7 +17659,7 @@ mod tests {
|
||||
let scoped_barrier = barrier.clone();
|
||||
let update_manager = manager.clone();
|
||||
let update_store = store.clone();
|
||||
let update = tokio::spawn(async move {
|
||||
let mut update = tokio::spawn(async move {
|
||||
TIER_REFERENCE_PROOF_TEST_BARRIER
|
||||
.scope(
|
||||
scoped_barrier,
|
||||
@@ -17575,7 +17674,9 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
});
|
||||
barrier.arrived.notified().await;
|
||||
wait_for_reference_proof_barrier(&barrier, &mut update)
|
||||
.await
|
||||
.expect("tier update should reach the reference proof barrier");
|
||||
|
||||
let unrelated = prepared_remove_intent("COLD-B", uuid::Uuid::from_u128(0x2237));
|
||||
TierConfigMgr::apply_prepared_mutation_intent_block(&manager, &unrelated)
|
||||
@@ -17583,8 +17684,9 @@ mod tests {
|
||||
.expect("an unrelated prepared fence should advance the runtime revision");
|
||||
barrier.release.add_permits(1);
|
||||
|
||||
let err = update
|
||||
let err = tokio::time::timeout(Duration::from_secs(30), update)
|
||||
.await
|
||||
.expect("tier update should finish after the reference proof barrier releases")
|
||||
.expect("tier update task should join")
|
||||
.expect_err("a reference proof cannot authorize publication across a fence revision change");
|
||||
let TierConfigUpdateError::Publish(err) = err else {
|
||||
|
||||
@@ -3852,7 +3852,7 @@ pub struct SetDisks {
|
||||
pub default_parity_count: usize,
|
||||
pub set_index: usize,
|
||||
pub pool_index: usize,
|
||||
/// Stable namespace shared by every object lock created for this set.
|
||||
/// Stable namespace shared by every object lock created for this pool.
|
||||
set_lock_namespace: Arc<str>,
|
||||
pub format: FormatV3,
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
@@ -4491,7 +4491,7 @@ impl SetDisks {
|
||||
instance_ctx: Arc<InstanceContext>,
|
||||
) -> Arc<Self> {
|
||||
let ctx = instance_ctx;
|
||||
let set_lock_namespace: Arc<str> = format!("set-{pool_index}-{set_index}").into();
|
||||
let set_lock_namespace: Arc<str> = format!("pool-{pool_index}").into();
|
||||
let shared_lockers = Arc::from(lockers.to_vec());
|
||||
Arc::new(SetDisks {
|
||||
locker_owner,
|
||||
@@ -4605,7 +4605,9 @@ impl SetDisks {
|
||||
pub(crate) async fn shares_namespace_lock_domain(&self, other: &Self) -> bool {
|
||||
match (self.ctx.is_dist_erasure().await, other.ctx.is_dist_erasure().await) {
|
||||
(false, false) => Arc::ptr_eq(&self.local_lock_manager, &other.local_lock_manager),
|
||||
(true, true) => same_distributed_lock_domain(&self.lockers, &other.lockers),
|
||||
(true, true) => {
|
||||
self.set_lock_namespace == other.set_lock_namespace && same_distributed_lock_domain(&self.lockers, &other.lockers)
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
@@ -7123,7 +7125,7 @@ mod tests {
|
||||
ctx.update_erasure_type(SetupType::Erasure).await;
|
||||
let set = make_test_set_disks_with_ctx(Vec::new(), ctx).await;
|
||||
|
||||
assert_eq!(&*set.set_lock_namespace, "set-0-0");
|
||||
assert_eq!(&*set.set_lock_namespace, "pool-0");
|
||||
let before = Arc::strong_count(&set.set_lock_namespace);
|
||||
let lock = set
|
||||
.new_ns_lock("bucket", "object")
|
||||
@@ -8348,6 +8350,78 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn test_new_ns_lock_distributed_write_succeeds_with_three_lockers_one_offline() {
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
|
||||
let manager_a = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let manager_b = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let healthy_a: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager_a));
|
||||
let healthy_b: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager_b));
|
||||
let failing_client: Arc<dyn LockClient> = Arc::new(FailingClient);
|
||||
let set_disks = make_test_set_disks(vec![healthy_a, failing_client, healthy_b]).await;
|
||||
|
||||
let guard = set_disks
|
||||
.new_ns_lock("bucket", "object")
|
||||
.await
|
||||
.expect("namespace lock should be created")
|
||||
.get_write_lock(Duration::from_millis(500))
|
||||
.await
|
||||
.expect("two healthy lockers should satisfy the three-locker write quorum");
|
||||
|
||||
match guard {
|
||||
NamespaceLockGuard::Standard(_) => {}
|
||||
NamespaceLockGuard::Fast(_) => panic!("Expected distributed guard for dist-erasure"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn namespace_lock_domain_includes_pool_namespace() {
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
|
||||
let first: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
let second: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
let lockers = vec![first, second];
|
||||
let same_pool_first_set = make_test_set_disks_with_ctx(lockers.clone(), bootstrap_ctx()).await;
|
||||
let same_pool_second_set = SetDisks::new_with_instance_ctx(
|
||||
"test-owner".to_string(),
|
||||
Arc::new(RwLock::new(vec![None, None])),
|
||||
2,
|
||||
1,
|
||||
1,
|
||||
0,
|
||||
same_pool_first_set.set_endpoints.clone(),
|
||||
FormatV3::new(2, 2),
|
||||
lockers.clone(),
|
||||
bootstrap_ctx(),
|
||||
)
|
||||
.await;
|
||||
let other_pool_set = SetDisks::new_with_instance_ctx(
|
||||
"test-owner".to_string(),
|
||||
Arc::new(RwLock::new(vec![None, None])),
|
||||
2,
|
||||
1,
|
||||
0,
|
||||
1,
|
||||
same_pool_first_set.set_endpoints.clone(),
|
||||
FormatV3::new(1, 2),
|
||||
lockers,
|
||||
bootstrap_ctx(),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
same_pool_first_set.shares_namespace_lock_domain(&same_pool_second_set).await,
|
||||
"sets in the same pool share the object namespace lock domain"
|
||||
);
|
||||
assert!(
|
||||
!same_pool_first_set.shares_namespace_lock_domain(&other_pool_set).await,
|
||||
"different pool namespaces must not be deduplicated solely by identical clients"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn streaming_reader_holds_read_lock_until_eof() {
|
||||
|
||||
@@ -13807,7 +13807,17 @@ mod transition_commit_failure_tests {
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn restore_failure_after_snapshot_cleans_exact_generation_and_returns_primary_error() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
assert_restore_failure_cleanup_boundary(true).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn restore_failure_after_snapshot_preserves_corrupt_known_transition_metadata() {
|
||||
assert_restore_failure_cleanup_boundary(false).await;
|
||||
}
|
||||
|
||||
async fn assert_restore_failure_cleanup_boundary(legacy_unknown: bool) {
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "restore-post-snapshot-cleanup-bucket";
|
||||
let object = "object.bin";
|
||||
for disk in &disk_stores {
|
||||
@@ -13816,7 +13826,15 @@ mod transition_commit_failure_tests {
|
||||
|
||||
let mut reader = PutObjReader::from_vec(b"post-snapshot cleanup source".repeat(1024));
|
||||
let original = set_disks
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("source object should be written");
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
@@ -13857,16 +13875,70 @@ mod transition_commit_failure_tests {
|
||||
.await
|
||||
.expect("transitioned metadata should be readable")
|
||||
.into_owned();
|
||||
let known_state = source_fi.transition_version_state;
|
||||
assert_ne!(known_state, rustfs_filemeta::TransitionVersionState::Unknown);
|
||||
source_fi.metadata.extend(restore_metadata(operation_id, true));
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut source_fi.metadata,
|
||||
rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
"invalid".to_string(),
|
||||
);
|
||||
set_disks
|
||||
.update_object_meta(bucket, object, source_fi, &online_disks)
|
||||
.await
|
||||
.expect("invalid backend identity fixture should be persisted");
|
||||
.expect("restore markers should be persisted");
|
||||
|
||||
// Normal writes reject damage to a reconciled binding. Model on-disk
|
||||
// corruption directly, with and without the legacy missing-state field.
|
||||
let mut corrupted_metadata = Vec::new();
|
||||
for temp_dir in &temp_dirs {
|
||||
let metadata_path = temp_dir.path().join(bucket).join(object).join(STORAGE_FORMAT_FILE);
|
||||
let encoded = tokio::fs::read(&metadata_path)
|
||||
.await
|
||||
.expect("transition metadata should be readable");
|
||||
let mut metadata = FileMeta::load(&encoded).expect("transition metadata should decode");
|
||||
let (version_index, mut version) = metadata
|
||||
.find_version(original.version_id)
|
||||
.expect("transitioned version should exist");
|
||||
let object_meta = version.object.as_mut().expect("transitioned version should be an object");
|
||||
rustfs_utils::http::insert_bytes(
|
||||
&mut object_meta.meta_sys,
|
||||
rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
b"invalid".to_vec(),
|
||||
);
|
||||
if legacy_unknown {
|
||||
rustfs_utils::http::remove_bytes(
|
||||
&mut object_meta.meta_sys,
|
||||
rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
);
|
||||
}
|
||||
metadata.versions[version_index] =
|
||||
rustfs_filemeta::FileMetaShallowVersion::try_from(version).expect("corrupt fixture should re-encode");
|
||||
tokio::fs::write(&metadata_path, metadata.marshal_msg().expect("corrupt fixture should encode"))
|
||||
.await
|
||||
.expect("corrupt fixture should be written");
|
||||
let persisted = tokio::fs::read(&metadata_path)
|
||||
.await
|
||||
.expect("corrupt fixture should be readable");
|
||||
let fixture = FileMeta::load(&persisted)
|
||||
.expect("corrupt fixture should decode")
|
||||
.find_version(original.version_id)
|
||||
.expect("corrupt version should exist")
|
||||
.1
|
||||
.into_fileinfo(bucket, object, true)
|
||||
.expect("corrupt version should decode");
|
||||
assert_eq!(
|
||||
fixture.transition_version_state,
|
||||
if legacy_unknown {
|
||||
rustfs_filemeta::TransitionVersionState::Unknown
|
||||
} else {
|
||||
known_state
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
rustfs_utils::http::get_str(&fixture.metadata, rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID),
|
||||
Some("invalid".to_string())
|
||||
);
|
||||
for (key, value) in restore_metadata(operation_id, true) {
|
||||
assert_eq!(fixture.metadata.get(&key), Some(&value), "fixture must retain restore marker {key}");
|
||||
}
|
||||
corrupted_metadata.push((metadata_path, persisted));
|
||||
}
|
||||
set_disks.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||
|
||||
let mut opts = ObjectOptions::default();
|
||||
@@ -13889,6 +13961,23 @@ mod transition_commit_failure_tests {
|
||||
.await
|
||||
.expect("cleanup should leave the transitioned object readable");
|
||||
assert_eq!(cleaned.transitioned_object.status, TRANSITION_COMPLETE);
|
||||
if !legacy_unknown {
|
||||
// Known bindings with corrupt identities must be repaired before
|
||||
// cleanup; rejection must preserve both the binding and markers.
|
||||
for (key, value) in restore_metadata(operation_id, true) {
|
||||
assert_eq!(cleaned.user_defined.get(&key), Some(&value), "cleanup must preserve restore marker {key}");
|
||||
}
|
||||
for (metadata_path, before) in corrupted_metadata {
|
||||
assert_eq!(
|
||||
tokio::fs::read(metadata_path)
|
||||
.await
|
||||
.expect("rejected cleanup metadata should remain readable"),
|
||||
before,
|
||||
"rejected cleanup must leave corrupt known metadata unchanged"
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
assert!(!cleaned.user_defined.contains_key(s3s::header::X_AMZ_RESTORE.as_str()));
|
||||
assert!(
|
||||
rustfs_utils::http::get_str(cleaned.user_defined.as_ref(), rustfs_utils::http::SUFFIX_RESTORE_OPERATION_ID,)
|
||||
|
||||
@@ -364,6 +364,19 @@ impl ECStore {
|
||||
Ok(pieces.into_guard(bucket, registration.token))
|
||||
}
|
||||
|
||||
/// Hold this guard through recursive-delete authorization and mutation so
|
||||
/// writers cannot introduce an unchecked object into the deletion scope.
|
||||
pub async fn lock_bucket_for_recursive_delete(&self, bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
|
||||
if self.ctx.lock_manager().is_disabled() {
|
||||
return Err(StorageError::InvalidArgument(
|
||||
bucket.to_owned(),
|
||||
String::new(),
|
||||
"Recursive deletion requires namespace locking".to_owned(),
|
||||
));
|
||||
}
|
||||
self.acquire_bucket_lifecycle_write_lock(bucket).await
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire_bucket_lifecycle_write_lock(&self, bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
|
||||
let lock = self.new_ns_lock(bucket, BUCKET_LIFECYCLE_LOCK_OBJECT).await?;
|
||||
lock.get_write_lock(get_lock_acquire_timeout())
|
||||
@@ -801,6 +814,15 @@ impl ECStore {
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn get_bucket_info_from_sets_at_read_quorum(
|
||||
&self,
|
||||
bucket: &str,
|
||||
opts: &BucketOptions,
|
||||
) -> Result<BucketInfo> {
|
||||
self.get_bucket_info_from_sets_with_quorum(bucket, opts, BucketInfoQuorum::Read)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get_bucket_info_from_sets_with_quorum(
|
||||
&self,
|
||||
bucket: &str,
|
||||
|
||||
@@ -3707,6 +3707,31 @@ mod tests {
|
||||
OfflineTestDisks { disks }
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn force_set_disk_range_offline_for_test(
|
||||
set: &Arc<crate::set_disk::SetDisks>,
|
||||
range: std::ops::Range<usize>,
|
||||
) -> OfflineTestDisks {
|
||||
let disks = set
|
||||
.disks
|
||||
.read()
|
||||
.await
|
||||
.get(range)
|
||||
.expect("offline test range must fit the set")
|
||||
.iter()
|
||||
.map(|disk| disk.clone().expect("fault-injection disk should start online"))
|
||||
.collect::<Vec<_>>();
|
||||
for disk in &disks {
|
||||
disk.close().await.expect("fault injection should stop per-disk monitoring");
|
||||
disk.force_runtime_state_for_test(crate::disk::health_state::RuntimeDriveHealthState::Offline);
|
||||
}
|
||||
set.connect_disks().await;
|
||||
for disk in &disks {
|
||||
assert_eq!(disk.runtime_state(), crate::disk::health_state::RuntimeDriveHealthState::Offline);
|
||||
}
|
||||
OfflineTestDisks { disks }
|
||||
}
|
||||
|
||||
fn active_rebalance_meta_for_pool(pool_count: usize, active_pool_idx: usize) -> RebalanceMeta {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let mut pool_stats = vec![RebalanceStats::default(); pool_count];
|
||||
@@ -15661,25 +15686,24 @@ mod tests {
|
||||
assert!(deleted[0].found, "the aggregate error must retain the committed pool result");
|
||||
drop(injection);
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
let mut metadata_absent = true;
|
||||
for pool in &store.pools {
|
||||
metadata_absent &= pool
|
||||
.get_disks_by_key(object)
|
||||
.load_file_info_versions_exact(bucket, object)
|
||||
.await
|
||||
.expect("aggregate-error cleanup metadata should remain readable")
|
||||
.is_none();
|
||||
}
|
||||
if metadata_absent && backend.remove_count().await == 1 {
|
||||
return;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("aggregate failure must not suppress committed receipt dispatch");
|
||||
// Exact reads can see subquorum metadata while workers remove each
|
||||
// disk's free version. Inspect the final state after cleanup drains.
|
||||
wait_for_expiry_workers_idle(&store).await;
|
||||
for pool in &store.pools {
|
||||
assert!(
|
||||
pool.get_disks_by_key(object)
|
||||
.load_file_info_versions_exact(bucket, object)
|
||||
.await
|
||||
.expect("aggregate-error cleanup metadata should remain readable")
|
||||
.is_none(),
|
||||
"aggregate failure must not suppress committed receipt cleanup"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
backend.remove_count().await,
|
||||
1,
|
||||
"committed receipts must remove the shared remote object once"
|
||||
);
|
||||
assert_eq!(backend.object_count().await, 0, "the shared remote object should be removed exactly once");
|
||||
store
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
@@ -17323,6 +17347,55 @@ mod tests {
|
||||
assert_eq!(body, original_body);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn object_lock_snapshot_uses_read_quorum_bucket_existence_probe() {
|
||||
let temp = tempfile::tempdir().expect("create degraded snapshot store dir");
|
||||
let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
|
||||
temp.path(),
|
||||
"degraded-object-lock-snapshot",
|
||||
&[(2, 12)],
|
||||
CancellationToken::new(),
|
||||
None,
|
||||
))
|
||||
.await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
|
||||
let bucket = format!("degraded-ol-{}", uuid::Uuid::new_v4());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create snapshot bucket");
|
||||
let expected_incarnation = store
|
||||
.bucket_incarnation_id(&bucket)
|
||||
.await
|
||||
.expect("read bucket incarnation before degrading sets");
|
||||
|
||||
let mut offline_disks = Vec::new();
|
||||
for set in store.all_set_disks() {
|
||||
offline_disks.push(force_set_disk_range_offline_for_test(&set, 6..12).await);
|
||||
}
|
||||
|
||||
let snapshot = store
|
||||
.object_lock_config_snapshot(&bucket)
|
||||
.await
|
||||
.expect("read-quorum bucket existence should admit guarded Object Lock snapshot");
|
||||
assert!(matches!(
|
||||
snapshot.state(),
|
||||
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent
|
||||
));
|
||||
assert!(snapshot.is_valid_for_destructive_put(store.id, &bucket, expected_incarnation));
|
||||
|
||||
let current_incarnation = crate::bucket::metadata_sys::get_object_lock_config_and_incarnation_from_disk_in(&ctx, &bucket)
|
||||
.await
|
||||
.expect("authoritative metadata read should also survive at read quorum")
|
||||
.1;
|
||||
assert_eq!(current_incarnation, expected_incarnation);
|
||||
|
||||
drop(offline_disks);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn force_create_existing_bucket_preserves_incarnation_and_inflight_request() {
|
||||
@@ -19057,27 +19130,34 @@ mod tests {
|
||||
.await
|
||||
.expect("transition metadata should be readable");
|
||||
let mut metadata = FileMeta::load(&encoded).expect("transition metadata should decode");
|
||||
let mut transitioned = metadata
|
||||
.get_all_file_info_versions(bucket, object, true)
|
||||
.expect("transitioned versions should decode")
|
||||
.versions
|
||||
.into_iter()
|
||||
.find(|version| version.version_id == history.version_id)
|
||||
let (version_index, mut transitioned) = metadata
|
||||
.find_version(history.version_id)
|
||||
.expect("transitioned history should exist");
|
||||
transitioned.transition_version_state = rustfs_filemeta::TransitionVersionState::Unknown;
|
||||
rustfs_utils::http::metadata_compat::remove_str(
|
||||
&mut transitioned.metadata,
|
||||
// Rewrite the serialized record to model legacy metadata;
|
||||
// ordinary writes preserve an already reconciled state.
|
||||
rustfs_utils::http::metadata_compat::remove_bytes(
|
||||
&mut transitioned.object.as_mut().expect("history should be an object").meta_sys,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
);
|
||||
metadata
|
||||
.add_version(transitioned)
|
||||
.expect("unknown state should replace the transitioned version");
|
||||
metadata.versions[version_index] = rustfs_filemeta::FileMetaShallowVersion::try_from(transitioned)
|
||||
.expect("legacy history should re-encode");
|
||||
tokio::fs::write(
|
||||
&metadata_path,
|
||||
metadata.marshal_msg().expect("unknown transition metadata should encode"),
|
||||
)
|
||||
.await
|
||||
.expect("unknown transition metadata should be written");
|
||||
let encoded = tokio::fs::read(&metadata_path)
|
||||
.await
|
||||
.expect("legacy transition metadata should be readable");
|
||||
let legacy = FileMeta::load(&encoded)
|
||||
.expect("legacy transition metadata should decode")
|
||||
.find_version(history.version_id)
|
||||
.expect("legacy history should exist")
|
||||
.1
|
||||
.into_fileinfo(bucket, object, true)
|
||||
.expect("legacy history should decode");
|
||||
assert_eq!(legacy.transition_version_state, rustfs_filemeta::TransitionVersionState::Unknown);
|
||||
}
|
||||
let lifecycle_event = crate::bucket::lifecycle::lifecycle::Event {
|
||||
action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction,
|
||||
|
||||
@@ -50,7 +50,7 @@ use crate::services::notification_sys::{
|
||||
use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease, tier_destination_id_from_metadata};
|
||||
use crate::set_disk::{
|
||||
SetDisks, get_lock_acquire_timeout, get_object_lock_diag_slow_acquire_threshold, get_object_lock_diag_slow_hold_threshold,
|
||||
is_lock_optimization_enabled, is_object_lock_diag_enabled, same_distributed_lock_domain,
|
||||
is_lock_optimization_enabled, is_object_lock_diag_enabled,
|
||||
};
|
||||
use crate::storage_api_contracts::{
|
||||
list::ListOperations as _,
|
||||
@@ -859,9 +859,78 @@ async fn delete_recursive_prefix_with_tier_delete_journal(
|
||||
}
|
||||
}
|
||||
}
|
||||
// A trailing slash selects a directory, not the object at its parent key.
|
||||
// Raw filesystem recursion would also remove that object's metadata and
|
||||
// data. Preserve it by purging the selected keys individually when they
|
||||
// share this physical directory. The bucket write lock covers both scans.
|
||||
if object.ends_with('/') && !is_meta_bucketname(bucket) {
|
||||
let parent = object.strip_suffix('/').unwrap_or(object);
|
||||
for pool in &store.pools {
|
||||
for set in &pool.disk_set {
|
||||
let page = set
|
||||
.clone()
|
||||
.inner_list_object_versions_for_recursive_delete(bucket, parent, None, None, 1)
|
||||
.await?;
|
||||
if page.objects.iter().any(|info| info.name == parent) {
|
||||
return delete_directory_keys_with_tier_delete_journal(store, bucket, object, opts, tier_journal_api).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
delete_prefix_with_tier_delete_journal(store, bucket, object, opts, tier_journal_api).await
|
||||
}
|
||||
|
||||
async fn delete_directory_keys_with_tier_delete_journal(
|
||||
store: &ECStore,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
opts: &ObjectOptions,
|
||||
tier_journal_api: Option<&Arc<ECStore>>,
|
||||
) -> Result<()> {
|
||||
for pool in &store.pools {
|
||||
for set in &pool.disk_set {
|
||||
let mut previous_keys = std::collections::BTreeSet::new();
|
||||
loop {
|
||||
// Restart after each bounded batch: its version markers have
|
||||
// been deleted, and the bucket write lock excludes new keys.
|
||||
let page = set
|
||||
.clone()
|
||||
.inner_list_object_versions_for_recursive_delete(
|
||||
bucket,
|
||||
prefix,
|
||||
None,
|
||||
None,
|
||||
RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE,
|
||||
)
|
||||
.await?;
|
||||
let keys = page
|
||||
.objects
|
||||
.into_iter()
|
||||
.map(|info| info.name)
|
||||
.filter(|key| key.starts_with(prefix))
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
if keys.is_empty() {
|
||||
break;
|
||||
}
|
||||
if keys == previous_keys {
|
||||
return Err(Error::other("directory deletion did not advance"));
|
||||
}
|
||||
for key in &keys {
|
||||
let encoded_key = encode_dir_object(key);
|
||||
let mut exact_opts = opts.clone();
|
||||
exact_opts.delete_prefix_object = true;
|
||||
let _guard = store
|
||||
.acquire_object_write_lock_if_needed("delete_object", bucket, &encoded_key, &mut exact_opts)
|
||||
.await?;
|
||||
delete_prefix_with_tier_delete_journal(store, bucket, &encoded_key, &exact_opts, tier_journal_api).await?;
|
||||
}
|
||||
previous_keys = keys;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A GET whose object identity has been resolved while its namespace read lock
|
||||
/// remains held, but whose body reader has not been constructed yet.
|
||||
///
|
||||
@@ -3440,10 +3509,15 @@ impl ECStore {
|
||||
|
||||
for pool in &self.pools {
|
||||
let hashed_set = pool.get_disks_by_key(object);
|
||||
let lock_domain_already_held = !distributed
|
||||
|| locked_sets
|
||||
.iter()
|
||||
.any(|locked_set| same_distributed_lock_domain(&locked_set.lockers, &hashed_set.lockers));
|
||||
let mut lock_domain_already_held = !distributed;
|
||||
if !lock_domain_already_held {
|
||||
for locked_set in &locked_sets {
|
||||
if locked_set.shares_namespace_lock_domain(&hashed_set).await {
|
||||
lock_domain_already_held = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if lock_domain_already_held {
|
||||
continue;
|
||||
}
|
||||
@@ -3500,10 +3574,15 @@ impl ECStore {
|
||||
let mut locked_sets = vec![fixed_set];
|
||||
for pool in &self.pools {
|
||||
for set in &pool.disk_set {
|
||||
let lock_domain_already_held = !distributed
|
||||
|| locked_sets
|
||||
.iter()
|
||||
.any(|locked_set| same_distributed_lock_domain(&locked_set.lockers, &set.lockers));
|
||||
let mut lock_domain_already_held = !distributed;
|
||||
if !lock_domain_already_held {
|
||||
for locked_set in &locked_sets {
|
||||
if locked_set.shares_namespace_lock_domain(set).await {
|
||||
lock_domain_already_held = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if lock_domain_already_held {
|
||||
continue;
|
||||
}
|
||||
@@ -3581,10 +3660,15 @@ impl ECStore {
|
||||
let mut locked_sets = vec![fixed_set];
|
||||
for pool in &self.pools {
|
||||
for set in &pool.disk_set {
|
||||
let lock_domain_already_held = !distributed
|
||||
|| locked_sets
|
||||
.iter()
|
||||
.any(|locked_set| same_distributed_lock_domain(&locked_set.lockers, &set.lockers));
|
||||
let mut lock_domain_already_held = !distributed;
|
||||
if !lock_domain_already_held {
|
||||
for locked_set in &locked_sets {
|
||||
if locked_set.shares_namespace_lock_domain(set).await {
|
||||
lock_domain_already_held = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if lock_domain_already_held {
|
||||
continue;
|
||||
}
|
||||
@@ -3667,11 +3751,15 @@ impl ECStore {
|
||||
.get(pool_idx)
|
||||
.ok_or_else(|| Error::other(format!("invalid data movement publication pool {pool_idx}")))?;
|
||||
let set = pool.get_disks_by_key(object);
|
||||
let lock_domain_already_held = !locked_sets.is_empty()
|
||||
&& (!distributed
|
||||
|| locked_sets.iter().any(|locked_set: &Arc<crate::set_disk::SetDisks>| {
|
||||
same_distributed_lock_domain(&locked_set.lockers, &set.lockers)
|
||||
}));
|
||||
let mut lock_domain_already_held = !locked_sets.is_empty() && !distributed;
|
||||
if !lock_domain_already_held {
|
||||
for locked_set in &locked_sets {
|
||||
if locked_set.shares_namespace_lock_domain(&set).await {
|
||||
lock_domain_already_held = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if lock_domain_already_held {
|
||||
continue;
|
||||
}
|
||||
@@ -4663,13 +4751,14 @@ impl ECStore {
|
||||
return Err(Error::other("lifecycle delete-all requires namespace locking"));
|
||||
}
|
||||
|
||||
let _bucket_lifecycle_guard = if is_meta_bucketname(bucket) {
|
||||
None
|
||||
} else if opts.delete_prefix {
|
||||
Some(self.acquire_bucket_lifecycle_write_lock(bucket).await?)
|
||||
} else {
|
||||
Some(self.acquire_bucket_lifecycle_read_lock(bucket).await?)
|
||||
};
|
||||
let _bucket_lifecycle_guard =
|
||||
if is_meta_bucketname(bucket) || (opts.delete_prefix && opts.bucket_lifecycle_lock_fence.is_some()) {
|
||||
None
|
||||
} else if opts.delete_prefix {
|
||||
Some(self.acquire_bucket_lifecycle_write_lock(bucket).await?)
|
||||
} else {
|
||||
Some(self.acquire_bucket_lifecycle_read_lock(bucket).await?)
|
||||
};
|
||||
let object = if opts.delete_prefix && !opts.delete_prefix_object {
|
||||
object.to_owned()
|
||||
} else {
|
||||
@@ -5717,6 +5806,7 @@ mod tests {
|
||||
GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, GetObjectBodySource, clear_get_object_body_cache_hook,
|
||||
lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
|
||||
};
|
||||
use crate::set_disk::same_distributed_lock_domain;
|
||||
use crate::set_disk::{SetDisks, disk_call_counters};
|
||||
use crate::storage_api_contracts::bucket::MakeBucketOptions;
|
||||
use crate::storage_api_contracts::lifecycle::TransitionedObject;
|
||||
|
||||
@@ -548,7 +548,10 @@ fn retry_budget_for_result(task: &HealTask, result: &Result<()>, retryable_batch
|
||||
}
|
||||
|
||||
let error = err.to_string();
|
||||
if !err.is_recoverable_heal() {
|
||||
// Batch aggregation preserves the typed classification in its counters,
|
||||
// while the returned task error retains only the first error's display text.
|
||||
let retryable_batch_result = retryable_batch_failure && matches!(err, Error::TaskExecutionFailed { .. });
|
||||
if !retryable_batch_result && !err.is_recoverable_heal() {
|
||||
return None;
|
||||
}
|
||||
|
||||
|
||||
@@ -2614,27 +2614,62 @@ fn test_retry_request_for_recoverable_error_stops_at_limit() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_retry_request_rescans_batch_when_all_exhausted_objects_are_retryable() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let task = HealTask::from_request(HealRequest::bucket("bucket".to_string()), storage);
|
||||
let result = Err(task
|
||||
.record_batch_failure(BatchHealFailure {
|
||||
scope: "bucket:bucket".to_string(),
|
||||
failed: 1,
|
||||
retryable: 1,
|
||||
permanent: 0,
|
||||
first_object: "object".to_string(),
|
||||
first_error: "Lock acquisition timeout".to_string(),
|
||||
})
|
||||
.await);
|
||||
for source_error in [
|
||||
Error::Disk(DiskError::FaultyDisk),
|
||||
Error::Disk(DiskError::FaultyRemoteDisk),
|
||||
Error::Storage(EcstoreError::SlowDown),
|
||||
Error::TaskExecutionFailed {
|
||||
message: "Lock acquisition timeout".to_string(),
|
||||
},
|
||||
] {
|
||||
assert!(source_error.is_recoverable_heal());
|
||||
let first_error = source_error.to_string();
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let task = HealTask::from_request(HealRequest::bucket("bucket".to_string()), storage);
|
||||
let result = Err(task
|
||||
.record_batch_failure(BatchHealFailure {
|
||||
scope: "bucket:bucket".to_string(),
|
||||
failed: 1,
|
||||
retryable: 1,
|
||||
permanent: 0,
|
||||
first_object: "object".to_string(),
|
||||
first_error: first_error.clone(),
|
||||
})
|
||||
.await);
|
||||
|
||||
let (retry_request, retry_delay, error) = retry_request_for_result_with_budget(&task, &result)
|
||||
.await
|
||||
.expect("all-retryable batch failure should rescan within the manager retry budget");
|
||||
let (retry_request, retry_delay, error) = retry_request_for_result_with_budget(&task, &result)
|
||||
.await
|
||||
.expect("all-retryable batch failure should rescan within the manager retry budget");
|
||||
|
||||
assert_eq!(retry_request.id, task.id);
|
||||
assert_eq!(retry_request.retry_attempts, 1);
|
||||
assert!(retry_delay > Duration::ZERO);
|
||||
assert!(error.contains("Lock acquisition timeout"));
|
||||
assert_eq!(retry_request.id, task.id);
|
||||
assert_eq!(retry_request.retry_attempts, 1);
|
||||
assert!(retry_delay > Duration::ZERO);
|
||||
assert!(error.contains(&first_error));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_retry_request_does_not_rescan_cancelled_or_timed_out_retryable_batch() {
|
||||
for terminal_error in [Error::TaskCancelled, Error::TaskTimeout] {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let task = HealTask::from_request(HealRequest::bucket("bucket".to_string()), storage);
|
||||
let _ = task
|
||||
.record_batch_failure(BatchHealFailure {
|
||||
scope: "bucket:bucket".to_string(),
|
||||
failed: 1,
|
||||
retryable: 1,
|
||||
permanent: 0,
|
||||
first_object: "object".to_string(),
|
||||
first_error: Error::Disk(DiskError::FaultyDisk).to_string(),
|
||||
})
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
retry_request_for_result_with_budget(&task, &Err(terminal_error))
|
||||
.await
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -100,6 +100,34 @@ async fn minio_permanent_identities_survive_migration_and_repeated_iam_loads() {
|
||||
.await;
|
||||
env.make_bucket(LEGACY_META_BUCKET, false).await;
|
||||
|
||||
for (path, body) in [
|
||||
("config/iam/empty.json", Vec::new()),
|
||||
("config/iam/users/ignored/extra.json", b"not JSON".to_vec()),
|
||||
] {
|
||||
env.put_object_bytes(LEGACY_META_BUCKET, path, body).await;
|
||||
}
|
||||
try_migrate_iam_config(
|
||||
env.ecstore.clone(),
|
||||
Some(std::sync::Arc::new(|_| panic!("unsupported IAM records must not be decrypted"))),
|
||||
)
|
||||
.await
|
||||
.expect("unsupported IAM records, including empty objects, must be skipped");
|
||||
|
||||
let format_path = "config/iam/format.json";
|
||||
for body in [Vec::new(), b"invalid IAM format".to_vec()] {
|
||||
env.put_object_bytes(LEGACY_META_BUCKET, format_path, body).await;
|
||||
let error = try_migrate_iam_config(env.ecstore.clone(), None)
|
||||
.await
|
||||
.expect_err("empty or incompatible supported IAM metadata must prevent startup readiness");
|
||||
let io_error = std::io::Error::from(error);
|
||||
let detail = io_error
|
||||
.get_ref()
|
||||
.and_then(|context| context.source())
|
||||
.expect("failure must retain the supported record in its source");
|
||||
assert!(detail.to_string().contains(format_path), "failure must identify the supported record");
|
||||
}
|
||||
seed_legacy_iam_object(&env, format_path, &json!({"version": 1})).await;
|
||||
|
||||
let regular_source = json!({
|
||||
"version": 1,
|
||||
"credentials": {
|
||||
@@ -155,7 +183,12 @@ async fn minio_permanent_identities_survive_migration_and_repeated_iam_loads() {
|
||||
)
|
||||
.await;
|
||||
|
||||
try_migrate_iam_config(env.ecstore.clone(), None).await;
|
||||
try_migrate_iam_config(env.ecstore.clone(), None)
|
||||
.await
|
||||
.expect("legacy IAM migration completes after source repair");
|
||||
try_migrate_iam_config(env.ecstore.clone(), None)
|
||||
.await
|
||||
.expect("completed legacy IAM migration is idempotent");
|
||||
|
||||
let store = ObjectStore::new(env.ecstore);
|
||||
assert_identity_survives(
|
||||
|
||||
@@ -1321,6 +1321,38 @@ mod tests {
|
||||
.expect("token auth must map to a source");
|
||||
}
|
||||
|
||||
/// A pending acquisition cannot take the returned-error fallback: the
|
||||
/// outer login policy must cut it off before publishing a client.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_stalled_initial_login_is_bounded_by_the_attempt_timeout() {
|
||||
let state = Arc::new(ScriptedState::default());
|
||||
let source = ScriptedSource {
|
||||
state: state.clone(),
|
||||
ttl: Duration::ZERO,
|
||||
renewable: false,
|
||||
login_delay: Duration::from_secs(60),
|
||||
};
|
||||
let policy = test_policy(Duration::from_secs(10), Duration::from_secs(5));
|
||||
let attempt_timeout = policy.retry.attempt_timeout;
|
||||
let started = Instant::now();
|
||||
let result = VaultCredentialProvider::new(test_settings(), Box::new(source), policy).await;
|
||||
assert!(
|
||||
matches!(&result, Err(KmsError::OperationTimedOut { message }) if message.starts_with("vault_login attempt 1 timed out")),
|
||||
"a stalled login must return its typed timeout without publishing a client"
|
||||
);
|
||||
assert_eq!(
|
||||
started.elapsed(),
|
||||
attempt_timeout,
|
||||
"login must consume exactly one virtual attempt budget"
|
||||
);
|
||||
assert_eq!(state.login_calls.load(Ordering::SeqCst), 0, "the acquisition must not complete");
|
||||
assert_eq!(
|
||||
state.renew_calls.load(Ordering::SeqCst),
|
||||
0,
|
||||
"failed initialization must not start renewal"
|
||||
);
|
||||
}
|
||||
|
||||
/// backlog#2369 P3: `vault token create` defaults to a 768-hour TTL, so
|
||||
/// hard-coding "no lease" for token auth left the renewal task unstarted
|
||||
/// and turned a healthy cluster into one that answers 403 a month later.
|
||||
|
||||
@@ -14,8 +14,7 @@
|
||||
|
||||
//! Fault-injection matrix for the Vault backend operation policy.
|
||||
//!
|
||||
//! Offline cases run against locally injected transport faults (a listener
|
||||
//! that never responds) — deterministic, no external
|
||||
//! Offline cases run against locally injected HTTP and transport faults — no external
|
||||
//! dependencies. Real-Vault cases are `#[ignore]`d and need a dev Vault
|
||||
//! (default `http://127.0.0.1:8200`, override with `RUSTFS_KMS_VAULT_ADDR`).
|
||||
//!
|
||||
@@ -38,9 +37,100 @@ use rustfs_kms::backends::vault::VaultKmsBackend;
|
||||
use rustfs_kms::{
|
||||
BackendConfig, DescribeKeyRequest, KmsBackend as KmsBackendKind, KmsConfig, KmsError, VaultAuthMethod, VaultConfig,
|
||||
};
|
||||
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::task::JoinHandle;
|
||||
|
||||
const OPERATIONS_TOTAL: &str = "rustfs_kms_backend_operations_total";
|
||||
const ATTEMPT_FAILURES_TOTAL: &str = "rustfs_kms_backend_attempt_failures_total";
|
||||
const LOGIN: &str = "vault_login";
|
||||
const READ_KEY: &str = "vault_kv2_read_key";
|
||||
const LOOKUP_REQUEST: &str = "GET /v1/auth/token/lookup-self HTTP/1.1";
|
||||
|
||||
/// Unlike the unit-test scripted Vault, this fixture records the credential
|
||||
/// probe too and can fail it independently of the subsequent key request.
|
||||
/// `None` parks a connection without responding; extra requests receive 599.
|
||||
struct FaultVault {
|
||||
address: String,
|
||||
requests: mpsc::UnboundedReceiver<String>,
|
||||
task: JoinHandle<()>,
|
||||
}
|
||||
|
||||
impl FaultVault {
|
||||
async fn serve(responses: Vec<Option<(u16, serde_json::Value)>>) -> Self {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind fault-injection Vault");
|
||||
let address = format!("http://{}", listener.local_addr().expect("fault-injection Vault address"));
|
||||
let (recorded, requests) = mpsc::unbounded_channel();
|
||||
let task = tokio::spawn(async move {
|
||||
let mut responses = responses.into_iter();
|
||||
let mut parked = Vec::new();
|
||||
loop {
|
||||
let (stream, _) = listener.accept().await.expect("accept Vault request");
|
||||
let mut stream = BufReader::new(stream);
|
||||
let mut line = String::new();
|
||||
assert_ne!(stream.read_line(&mut line).await.expect("read request line"), 0);
|
||||
recorded.send(line.trim_end().to_string()).expect("record Vault request");
|
||||
loop {
|
||||
line.clear();
|
||||
assert_ne!(stream.read_line(&mut line).await.expect("read request header"), 0);
|
||||
if line == "\r\n" {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let mut stream = stream.into_inner();
|
||||
let response = responses
|
||||
.next()
|
||||
.unwrap_or_else(|| Some((599, serde_json::json!({"errors": ["unexpected Vault request"]}))));
|
||||
if let Some((status, body)) = response {
|
||||
let body = body.to_string();
|
||||
let response = format!(
|
||||
"HTTP/1.1 {status} Scripted\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
|
||||
body.len()
|
||||
);
|
||||
stream.write_all(response.as_bytes()).await.expect("write Vault response");
|
||||
stream.shutdown().await.expect("close Vault response");
|
||||
} else {
|
||||
parked.push(stream);
|
||||
}
|
||||
}
|
||||
});
|
||||
Self { address, requests, task }
|
||||
}
|
||||
|
||||
async fn finish(&mut self) {
|
||||
assert!(self.requests.try_recv().is_err(), "no unexpected requests may remain");
|
||||
self.task.abort();
|
||||
let error = (&mut self.task).await.expect_err("fault server runs until aborted");
|
||||
assert!(error.is_cancelled(), "fault server must not panic: {error}");
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for FaultVault {
|
||||
fn drop(&mut self) {
|
||||
self.task.abort();
|
||||
}
|
||||
}
|
||||
|
||||
fn healthy_token_lookup() -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"data": {
|
||||
"accessor": "fault-injection-accessor",
|
||||
"creation_time": 1_700_000_000u64,
|
||||
"creation_ttl": 0,
|
||||
"display_name": "token",
|
||||
"entity_id": "",
|
||||
"explicit_max_ttl": 0,
|
||||
"id": "unused",
|
||||
"num_uses": 0,
|
||||
"orphan": true,
|
||||
"path": "auth/token/create",
|
||||
"policies": ["default"],
|
||||
"renewable": false,
|
||||
"ttl": 0
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn vault_config(address: &str, token: &str) -> VaultConfig {
|
||||
VaultConfig {
|
||||
@@ -125,39 +215,112 @@ fn counter_value(snapshot: &[MetricEntry], name: &str, labels: &[(&str, &str)])
|
||||
fn stalled_connection_is_cut_off_by_the_attempt_timeout() {
|
||||
let snapshot = record_metrics(|| {
|
||||
Box::pin(async move {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
let mut vault = FaultVault::serve(vec![Some((200, healthy_token_lookup())), None]).await;
|
||||
let attempt_timeout = Duration::from_millis(250);
|
||||
let client = VaultKmsBackend::new(kms_config(vault_config(&vault.address, "unused"), attempt_timeout, 1))
|
||||
.await
|
||||
.expect("bind stall listener");
|
||||
let address = format!("http://{}", listener.local_addr().expect("stall listener addr"));
|
||||
// Accept and park every connection without ever responding.
|
||||
tokio::spawn(async move {
|
||||
let mut parked = Vec::new();
|
||||
loop {
|
||||
let Ok((socket, _)) = listener.accept().await else { return };
|
||||
parked.push(socket);
|
||||
}
|
||||
});
|
||||
.expect("the token lookup must succeed before injecting the stalled key read");
|
||||
assert_eq!(vault.requests.try_recv().as_deref(), Ok(LOOKUP_REQUEST));
|
||||
|
||||
let client = VaultKmsBackend::new(kms_config(vault_config(&address, "unused"), Duration::from_millis(250), 1))
|
||||
.await
|
||||
.expect("client construction performs no network calls");
|
||||
let error = KmsBackendTrait::describe_key(&client, describe_key_request("fault-injection-stalled"))
|
||||
let read = KmsBackendTrait::describe_key(&client, describe_key_request("fault-injection-stalled"));
|
||||
tokio::pin!(read);
|
||||
tokio::select! {
|
||||
request = vault.requests.recv() => assert_eq!(
|
||||
request.as_deref(),
|
||||
Some("GET /v1/secret/data/rustfs/kms/fault-injection/fault-injection-stalled? HTTP/1.1")
|
||||
),
|
||||
result = &mut read => panic!("the key request must reach the stall listener: {result:?}"),
|
||||
}
|
||||
// Pause only after real loopback I/O reaches the intended request;
|
||||
// otherwise auto-advancing time could expire the login instead.
|
||||
tokio::time::pause();
|
||||
let stalled_at = tokio::time::Instant::now();
|
||||
// Tokio rounds timer deadlines up to the next millisecond.
|
||||
let virtual_step = attempt_timeout + Duration::from_millis(1);
|
||||
tokio::time::advance(virtual_step).await;
|
||||
let error = tokio::time::timeout(Duration::from_secs(1), read)
|
||||
.await
|
||||
.expect("the attempt timer must resolve without further network activity")
|
||||
.expect_err("a stalled request must be cut off by the attempt timeout");
|
||||
assert_eq!(
|
||||
stalled_at.elapsed(),
|
||||
virtual_step,
|
||||
"the read must resolve within the attempt budget plus one timer tick"
|
||||
);
|
||||
assert!(
|
||||
matches!(error, KmsError::OperationTimedOut { .. } | KmsError::BackendError { .. }),
|
||||
"got {error:?}"
|
||||
);
|
||||
vault.finish().await;
|
||||
})
|
||||
});
|
||||
|
||||
// The policy timer reports attempt_timeout; the client-level HTTP timeout
|
||||
// surfaces as a connection-class failure. Either way it is exactly one
|
||||
// attempt that was cut off.
|
||||
let cut_off = counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("error_class", "attempt_timeout")])
|
||||
+ counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("error_class", "retryable_conn")]);
|
||||
let cut_off = counter_value(
|
||||
&snapshot,
|
||||
ATTEMPT_FAILURES_TOTAL,
|
||||
&[("operation", READ_KEY), ("error_class", "attempt_timeout")],
|
||||
) + counter_value(
|
||||
&snapshot,
|
||||
ATTEMPT_FAILURES_TOTAL,
|
||||
&[("operation", READ_KEY), ("error_class", "retryable_conn")],
|
||||
);
|
||||
assert_eq!(cut_off, 1, "the single budgeted attempt must be cut off by a timeout");
|
||||
assert_eq!(counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "budget_exhausted")]), 1);
|
||||
assert_eq!(counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("operation", LOGIN)]), 0);
|
||||
assert_eq!(
|
||||
counter_value(&snapshot, OPERATIONS_TOTAL, &[("operation", LOGIN), ("outcome", "success")]),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
counter_value(&snapshot, OPERATIONS_TOTAL, &[("operation", READ_KEY), ("outcome", "budget_exhausted")]),
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
/// A returned lookup error degrades lease discovery, not Vault authorization:
|
||||
/// the subsequent forbidden key read must still fail once, without retrying.
|
||||
#[test]
|
||||
fn token_lookup_errors_do_not_bypass_key_authorization() {
|
||||
for lookup_status in [403, 503] {
|
||||
let snapshot = record_metrics(|| {
|
||||
Box::pin(async move {
|
||||
let mut vault = FaultVault::serve(vec![
|
||||
Some((lookup_status, serde_json::json!({"errors": ["token lookup unavailable"]}))),
|
||||
Some((403, serde_json::json!({"errors": ["permission denied"]}))),
|
||||
])
|
||||
.await;
|
||||
let client = VaultKmsBackend::new(kms_config(vault_config(&vault.address, "unused"), Duration::from_secs(5), 3))
|
||||
.await
|
||||
.expect("a returned token lookup error must preserve static-token fallback");
|
||||
assert_eq!(vault.requests.try_recv().as_deref(), Ok(LOOKUP_REQUEST));
|
||||
let error = KmsBackendTrait::describe_key(&client, describe_key_request("fault-injection-forbidden"))
|
||||
.await
|
||||
.expect_err("lease discovery fallback must not authorize a forbidden key read");
|
||||
assert!(matches!(error, KmsError::BackendError { .. }), "got {error:?}");
|
||||
assert_eq!(
|
||||
vault.requests.try_recv().as_deref(),
|
||||
Ok("GET /v1/secret/data/rustfs/kms/fault-injection/fault-injection-forbidden? HTTP/1.1")
|
||||
);
|
||||
vault.finish().await;
|
||||
})
|
||||
});
|
||||
assert_eq!(counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("operation", LOGIN)]), 0);
|
||||
assert_eq!(
|
||||
counter_value(&snapshot, OPERATIONS_TOTAL, &[("operation", LOGIN), ("outcome", "success")]),
|
||||
1
|
||||
);
|
||||
assert_eq!(counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("operation", READ_KEY)]), 1);
|
||||
assert_eq!(
|
||||
counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("operation", READ_KEY), ("error_class", "fatal")]),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
counter_value(&snapshot, OPERATIONS_TOTAL, &[("operation", READ_KEY), ("outcome", "fatal")]),
|
||||
1
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn real_vault_address() -> String {
|
||||
@@ -174,7 +337,7 @@ fn real_vault_invalid_token_is_fatal_and_never_retried() {
|
||||
let config = vault_config(&real_vault_address(), "fault-injection-invalid-token");
|
||||
let client = VaultKmsBackend::new(kms_config(config, Duration::from_secs(5), 3))
|
||||
.await
|
||||
.expect("client construction performs no network calls");
|
||||
.expect("a returned token lookup error must preserve static-token fallback");
|
||||
let error = KmsBackendTrait::describe_key(&client, describe_key_request("fault-injection-forbidden"))
|
||||
.await
|
||||
.expect_err("an invalid token must be rejected");
|
||||
@@ -183,13 +346,20 @@ fn real_vault_invalid_token_is_fatal_and_never_retried() {
|
||||
});
|
||||
|
||||
assert_eq!(
|
||||
counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("error_class", "fatal")]),
|
||||
counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("operation", READ_KEY), ("error_class", "fatal")]),
|
||||
1,
|
||||
"a 403 must be observed by exactly one attempt"
|
||||
);
|
||||
assert_eq!(counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "fatal")]), 1);
|
||||
assert_eq!(
|
||||
counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("error_class", "retryable_status")]),
|
||||
counter_value(&snapshot, OPERATIONS_TOTAL, &[("operation", READ_KEY), ("outcome", "fatal")]),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
counter_value(
|
||||
&snapshot,
|
||||
ATTEMPT_FAILURES_TOTAL,
|
||||
&[("operation", READ_KEY), ("error_class", "retryable_status")]
|
||||
),
|
||||
0,
|
||||
"an auth failure must never be classified as retryable"
|
||||
);
|
||||
@@ -207,7 +377,7 @@ fn real_vault_missing_key_is_resolved_in_one_attempt() {
|
||||
let config = vault_config(&real_vault_address(), &token);
|
||||
let client = VaultKmsBackend::new(kms_config(config, Duration::from_secs(5), 3))
|
||||
.await
|
||||
.expect("client construction performs no network calls");
|
||||
.expect("static-token initialization must complete against the running Vault");
|
||||
let error = KmsBackendTrait::describe_key(&client, describe_key_request("fault-injection-definitely-missing"))
|
||||
.await
|
||||
.expect_err("a missing key must resolve to key-not-found");
|
||||
@@ -216,9 +386,12 @@ fn real_vault_missing_key_is_resolved_in_one_attempt() {
|
||||
});
|
||||
|
||||
assert_eq!(
|
||||
counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("error_class", "fatal")]),
|
||||
counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("operation", READ_KEY), ("error_class", "fatal")]),
|
||||
1,
|
||||
"a 404 must be observed by exactly one attempt"
|
||||
);
|
||||
assert_eq!(counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "fatal")]), 1);
|
||||
assert_eq!(
|
||||
counter_value(&snapshot, OPERATIONS_TOTAL, &[("operation", READ_KEY), ("outcome", "fatal")]),
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
@@ -730,12 +730,16 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
|
||||
}
|
||||
}
|
||||
|
||||
// Highest priority first, like MinIO's `FilterActionableRules`. The
|
||||
// tie-breakers make this a total order: a comparator that only
|
||||
// orders same-destination pairs is not transitive, and the standard
|
||||
// library sort panics on such inputs past its insertion-sort
|
||||
// threshold (backlog#2367 C-1).
|
||||
rules.sort_by(|a, b| {
|
||||
if a.destination == b.destination {
|
||||
b.priority.cmp(&a.priority)
|
||||
} else {
|
||||
std::cmp::Ordering::Equal
|
||||
}
|
||||
b.priority
|
||||
.cmp(&a.priority)
|
||||
.then_with(|| a.destination.bucket.cmp(&b.destination.bucket))
|
||||
.then_with(|| a.id.cmp(&b.id))
|
||||
});
|
||||
|
||||
rules
|
||||
@@ -813,24 +817,19 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
|
||||
return vec![role.to_string()];
|
||||
}
|
||||
|
||||
let mut arns = Vec::new();
|
||||
let mut targets_map: HashSet<String> = HashSet::new();
|
||||
let rules = self.filter_actionable_rules(obj);
|
||||
|
||||
for rule in rules {
|
||||
// Rule order (priority descending) is the ARN order: callers that
|
||||
// iterate targets see the highest-priority destination first.
|
||||
let mut arns: Vec<String> = Vec::new();
|
||||
for rule in self.filter_actionable_rules(obj) {
|
||||
if rule.status == ReplicationRuleStatus::from_static(ReplicationRuleStatus::DISABLED) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let arn = rule.destination.bucket.trim();
|
||||
if !arn.is_empty() && !targets_map.contains(arn) {
|
||||
targets_map.insert(arn.to_string());
|
||||
if !arn.is_empty() && !arns.iter().any(|seen| seen == arn) {
|
||||
arns.push(arn.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
for arn in targets_map {
|
||||
arns.push(arn);
|
||||
}
|
||||
arns
|
||||
}
|
||||
|
||||
@@ -1908,6 +1907,84 @@ mod tests {
|
||||
assert_eq!(decisions, vec![(target_a.to_string(), false), (target_b.to_string(), true)]);
|
||||
}
|
||||
|
||||
// backlog#2367 C-1: the actionable-rule sort must be a total order. A
|
||||
// comparator that answers `Equal` for different destinations but orders
|
||||
// same-destination rules by priority is not transitive, and the standard
|
||||
// library sort panics on such inputs once the slice is past the
|
||||
// insertion-sort threshold (> 20 rules).
|
||||
#[test]
|
||||
fn actionable_rule_sort_is_a_total_order_across_destinations() {
|
||||
let targets = ["arn:target:a", "arn:target:b", "arn:target:c"];
|
||||
let mut seed: u64 = 0x2367;
|
||||
for _ in 0..200 {
|
||||
let rule_count = 21 + (seed % 200) as usize;
|
||||
let rules = (0..rule_count)
|
||||
.map(|index| {
|
||||
seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
|
||||
let target = targets[(seed >> 33) as usize % targets.len()];
|
||||
delete_marker_rule(&format!("r{index}"), target, "", index as i32, true)
|
||||
})
|
||||
.collect();
|
||||
let config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules,
|
||||
};
|
||||
let ordered = config.filter_actionable_rules(&ObjectOpts {
|
||||
name: "logs/app.log".to_string(),
|
||||
op_type: ReplicationType::Object,
|
||||
..Default::default()
|
||||
});
|
||||
assert_eq!(ordered.len(), rule_count);
|
||||
assert!(
|
||||
ordered.windows(2).all(|pair| pair[0].priority >= pair[1].priority),
|
||||
"actionable rules must be ordered by descending priority"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// backlog#2367 C-2: a V1 rule carries its prefix at the top level (no
|
||||
// <Filter>). Ignoring it made `<Prefix>logs/</Prefix>` match every object.
|
||||
#[test]
|
||||
fn top_level_rule_prefix_scopes_matching_without_a_filter() {
|
||||
let arn = "arn:target:a";
|
||||
let config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![delete_marker_rule("v1-prefix", arn, "logs/", 1, true)],
|
||||
};
|
||||
assert_eq!(config.rules[0].prefix(), "logs/");
|
||||
|
||||
let matching = config.filter_actionable_rules(&ObjectOpts {
|
||||
name: "logs/app.log".to_string(),
|
||||
op_type: ReplicationType::Object,
|
||||
..Default::default()
|
||||
});
|
||||
assert_eq!(matching.len(), 1);
|
||||
|
||||
let outside = config.filter_actionable_rules(&ObjectOpts {
|
||||
name: "data/app.log".to_string(),
|
||||
op_type: ReplicationType::Object,
|
||||
..Default::default()
|
||||
});
|
||||
assert!(outside.is_empty(), "an object outside the V1 prefix must not match: {outside:?}");
|
||||
assert!(
|
||||
config
|
||||
.filter_target_arns(&ObjectOpts {
|
||||
name: "data/app.log".to_string(),
|
||||
op_type: ReplicationType::Object,
|
||||
..Default::default()
|
||||
})
|
||||
.is_empty()
|
||||
);
|
||||
|
||||
// A <Filter> still wins over the deprecated top-level element.
|
||||
let mut filtered = delete_marker_rule("filtered", arn, "logs/", 1, true);
|
||||
filtered.filter = Some(s3s::dto::ReplicationRuleFilter {
|
||||
prefix: Some("photos/".to_string()),
|
||||
..Default::default()
|
||||
});
|
||||
assert_eq!(filtered.prefix(), "photos/");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn force_delete_targets_use_overlapping_rules_and_highest_priority_switch() {
|
||||
let target_a = "arn:target:a";
|
||||
|
||||
@@ -22,6 +22,10 @@ pub trait ReplicationRuleExt {
|
||||
}
|
||||
|
||||
impl ReplicationRuleExt for ReplicationRule {
|
||||
/// The rule's key prefix: `Filter.Prefix`, else `Filter.And.Prefix`, else
|
||||
/// the deprecated top-level `Prefix` of a V1 rule written without a
|
||||
/// `<Filter>` (backlog#2367 C-2). A rule that carries both keeps AWS's
|
||||
/// precedence: the `<Filter>` is authoritative.
|
||||
fn prefix(&self) -> &str {
|
||||
if let Some(filter) = &self.filter {
|
||||
if let Some(prefix) = &filter.prefix {
|
||||
@@ -32,7 +36,7 @@ impl ReplicationRuleExt for ReplicationRule {
|
||||
""
|
||||
}
|
||||
} else {
|
||||
""
|
||||
self.prefix.as_deref().unwrap_or("")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -96,14 +96,14 @@ pub use scanner::{
|
||||
scanner_topology_digest,
|
||||
};
|
||||
pub use scanner_io::{
|
||||
ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageClearObserver, ScannerDirtyUsageSnapshot,
|
||||
ScannerDirtyUsageState, ScannerDurableDirtyUsageReplayEntry, ScannerDurableDirtyUsageReplayError,
|
||||
ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageClearObserver, ScannerDirtyUsageMutationObserver,
|
||||
ScannerDirtyUsageSnapshot, ScannerDirtyUsageState, ScannerDurableDirtyUsageReplayEntry, ScannerDurableDirtyUsageReplayError,
|
||||
ScannerDurableDirtyUsageReplayRecord, ScannerDurableDirtyUsageReplayScope, acknowledge_dirty_usage_generation,
|
||||
acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, encode_durable_dirty_usage_producer_replay_record,
|
||||
record_dirty_usage_bucket, record_dirty_usage_bucket_from_producer, record_dirty_usage_bucket_from_producers,
|
||||
record_dirty_usage_object, record_dirty_usage_object_from_producer, record_scanner_maintenance_change,
|
||||
replay_durable_dirty_usage_producer_record, scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
|
||||
scanner_maintenance_generation, set_scanner_dirty_usage_clear_observer,
|
||||
scanner_maintenance_generation, set_scanner_dirty_usage_clear_observer, set_scanner_dirty_usage_mutation_observer,
|
||||
};
|
||||
pub use segment_invalidation::SegmentInvalidationProducerIdentity;
|
||||
pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER};
|
||||
|
||||
@@ -105,8 +105,14 @@ pub(super) fn remote_dirty_usage_acknowledgement_loss_reconciled(
|
||||
if !acknowledged_hosts.insert(acknowledgement.host.as_str()) {
|
||||
return false;
|
||||
}
|
||||
scanner_activity_dirty_usage_state_for_host(&activity_after_error, &acknowledgement.host)
|
||||
.is_some_and(|(instance_id, _generation, pending)| instance_id == acknowledgement.instance_id && !pending)
|
||||
let Some(expected_generation) = acknowledgement.expected_dirty_usage_generation() else {
|
||||
return false;
|
||||
};
|
||||
scanner_activity_dirty_usage_state_for_host(&activity_after_error, &acknowledgement.host).is_some_and(
|
||||
|(instance_id, generation, pending)| {
|
||||
instance_id == acknowledgement.instance_id && generation >= expected_generation && !pending
|
||||
},
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -509,6 +515,15 @@ pub(crate) enum ScannerDirtyUsageAcknowledgementKind {
|
||||
},
|
||||
}
|
||||
|
||||
impl ScannerDirtyUsageAcknowledgement {
|
||||
fn expected_dirty_usage_generation(&self) -> Option<u64> {
|
||||
match &self.kind {
|
||||
ScannerDirtyUsageAcknowledgementKind::Generation(generation) => Some(*generation),
|
||||
ScannerDirtyUsageAcknowledgementKind::Scoped { entries, .. } => entries.iter().map(|entry| entry.generation).max(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<ScannerDirtyUsageAcknowledgement> for crate::storage_api::EcstoreScannerDirtyUsageAcknowledgement {
|
||||
fn from(acknowledgement: ScannerDirtyUsageAcknowledgement) -> Self {
|
||||
match acknowledgement.kind {
|
||||
|
||||
@@ -1798,6 +1798,8 @@ pub(super) fn scanner_pause_backlog_now() -> u64 {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const NATIVE_RETIREMENT_DRIVES_PER_SET: usize = 2;
|
||||
|
||||
fn run_native_retirement_test<C, F>(case: C)
|
||||
where
|
||||
C: FnOnce() -> F + Send + 'static,
|
||||
@@ -1825,10 +1827,29 @@ mod tests {
|
||||
async fn native_retirement_store() -> (tempfile::TempDir, Arc<ECStore>) {
|
||||
register_scanner_pause_backlog_retirement();
|
||||
let root = tempfile::tempdir().expect("native retirement fixture directory");
|
||||
let store = super::super::tests::setup_scanner_cycle_store_at_path_with_sets(root.path(), false, 3, 2).await;
|
||||
let store = super::super::tests::setup_scanner_cycle_store_at_path_with_layout_and_disk_preinit(
|
||||
root.path(),
|
||||
false,
|
||||
3,
|
||||
2,
|
||||
NATIVE_RETIREMENT_DRIVES_PER_SET,
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
(root, store)
|
||||
}
|
||||
|
||||
async fn shutdown_native_retirement_store(store: Arc<ECStore>) {
|
||||
if let Some(token) = store.background_cancel_token() {
|
||||
token.cancel();
|
||||
}
|
||||
drop(store);
|
||||
for _ in 0..8 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
|
||||
async fn native_replica_bytes(set: &SetDisks) -> (Vec<u8>, String) {
|
||||
let mut reader = set
|
||||
.get_object_reader(
|
||||
@@ -1871,7 +1892,15 @@ mod tests {
|
||||
|
||||
register_scanner_pause_backlog_retirement();
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let old = super::super::tests::setup_scanner_cycle_store_at_path_with_sets(root.path(), false, old_pool_count, 2).await;
|
||||
let old = super::super::tests::setup_scanner_cycle_store_at_path_with_layout_and_disk_preinit(
|
||||
root.path(),
|
||||
false,
|
||||
old_pool_count,
|
||||
2,
|
||||
NATIVE_RETIREMENT_DRIVES_PER_SET,
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
let fault = unstable_source
|
||||
.then(|| NativeScannerPauseBacklogWriteFault::fail_before_write(Arc::clone(&old.pools[0].disk_set[0]), "publish", 2));
|
||||
let now = unix_now();
|
||||
@@ -1889,10 +1918,14 @@ mod tests {
|
||||
} else {
|
||||
controller.observe(observation(now + 1, true, 4)).await;
|
||||
controller.observe(observation(now + 2, false, 4)).await;
|
||||
assert!(matches!(
|
||||
controller.begin_attempt(now + 2).await,
|
||||
ScannerPauseBacklogAttemptDecision::Tracked(_)
|
||||
));
|
||||
let decision = controller.begin_attempt(now + 2).await;
|
||||
assert!(
|
||||
matches!(decision, ScannerPauseBacklogAttemptDecision::Tracked(_)),
|
||||
"expected tracked native expansion setup attempt, got {decision:?}; ledger={:?}; persistence_disabled={}; runtime_error={:?}",
|
||||
controller.loaded.ledger,
|
||||
controller.persistence_disabled,
|
||||
runtime_error()
|
||||
);
|
||||
assert!(controller.loaded.ledger.has_unfinished_attempt());
|
||||
}
|
||||
let original = controller.loaded.ledger.clone();
|
||||
@@ -1902,9 +1935,16 @@ mod tests {
|
||||
old.pool_meta_write_status()
|
||||
.await
|
||||
.expect("healthy pool metadata keeps the background recovery loop read-only");
|
||||
old.background_cancel_token().expect("old store shutdown token").cancel();
|
||||
drop(old);
|
||||
let expanded = super::super::tests::setup_scanner_cycle_store_at_path_with_sets(root.path(), false, 3, 2).await;
|
||||
shutdown_native_retirement_store(old).await;
|
||||
let expanded = super::super::tests::setup_scanner_cycle_store_at_path_with_layout_and_disk_preinit(
|
||||
root.path(),
|
||||
false,
|
||||
3,
|
||||
2,
|
||||
NATIVE_RETIREMENT_DRIVES_PER_SET,
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
for pool in expanded.pools.iter().skip(old_pool_count) {
|
||||
for set in &pool.disk_set {
|
||||
let replica = read_scanner_pause_backlog_replica(Arc::clone(set)).await;
|
||||
@@ -1999,12 +2039,16 @@ mod tests {
|
||||
}
|
||||
|
||||
store.pool_meta_write_status().await.expect("healthy metadata before restart");
|
||||
store
|
||||
.background_cancel_token()
|
||||
.expect("expanded store shutdown token")
|
||||
.cancel();
|
||||
drop(store);
|
||||
let restarted = super::super::tests::setup_scanner_cycle_store_at_path_with_sets(root.path(), false, 3, 2).await;
|
||||
shutdown_native_retirement_store(store).await;
|
||||
let restarted = super::super::tests::setup_scanner_cycle_store_at_path_with_layout_and_disk_preinit(
|
||||
root.path(),
|
||||
false,
|
||||
3,
|
||||
2,
|
||||
NATIVE_RETIREMENT_DRIVES_PER_SET,
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
let reloaded = load_scanner_pause_backlog(Arc::clone(&restarted))
|
||||
.await
|
||||
.expect("a new store must recover from disk without the failed controller");
|
||||
@@ -2021,6 +2065,8 @@ mod tests {
|
||||
assert_eq!(retry_ledger.current_attempt_serial, original.current_attempt_serial);
|
||||
assert_eq!(retry_ledger.last_finished_attempt_serial, original.current_attempt_serial);
|
||||
assert_eq!(retry_ledger.consecutive_failures, original.consecutive_failures + 1);
|
||||
drop(retried);
|
||||
shutdown_native_retirement_store(restarted).await;
|
||||
}
|
||||
|
||||
async fn assert_current_native_writer_ledger(store: &Arc<ECStore>, expected: &ScannerPauseBacklogLedger) {
|
||||
@@ -2124,6 +2170,8 @@ mod tests {
|
||||
.await
|
||||
.expect("retry must seed, commit and stabilize the stale member");
|
||||
assert_current_native_writer_ledger(&store, &expected).await;
|
||||
drop(target);
|
||||
shutdown_native_retirement_store(store).await;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -2175,6 +2223,8 @@ mod tests {
|
||||
.await
|
||||
.expect("a fresh caller may finish the seeded membership transition");
|
||||
assert_current_native_writer_ledger(&store, &expected).await;
|
||||
drop(pool_meta_lock);
|
||||
shutdown_native_retirement_store(store).await;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -2235,6 +2285,7 @@ mod tests {
|
||||
.await
|
||||
.expect("retry must seed the newly visible members before committing");
|
||||
assert_current_native_writer_ledger(&store, &expected).await;
|
||||
shutdown_native_retirement_store(store).await;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -2292,6 +2343,7 @@ mod tests {
|
||||
assert_current_native_ledger(&store, &original).await;
|
||||
}
|
||||
assert!(original.has_unfinished_attempt());
|
||||
shutdown_native_retirement_store(store).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -2337,14 +2389,22 @@ mod tests {
|
||||
.pool_meta_write_status()
|
||||
.await
|
||||
.expect("healthy pool metadata keeps the background recovery loop read-only");
|
||||
store.background_cancel_token().expect("old store shutdown token").cancel();
|
||||
drop(store);
|
||||
let restarted = super::super::tests::setup_scanner_cycle_store_at_path_with_sets(root.path(), false, 3, 2).await;
|
||||
shutdown_native_retirement_store(store).await;
|
||||
let restarted = super::super::tests::setup_scanner_cycle_store_at_path_with_layout_and_disk_preinit(
|
||||
root.path(),
|
||||
false,
|
||||
3,
|
||||
2,
|
||||
NATIVE_RETIREMENT_DRIVES_PER_SET,
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
for set_index in 0..2 {
|
||||
restarted.retire_scanner_pause_backlog_for_test(0, set_index).await.unwrap();
|
||||
assert_native_source_missing(&restarted, set_index).await;
|
||||
}
|
||||
assert_current_native_ledger(&restarted, &original).await;
|
||||
shutdown_native_retirement_store(restarted).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -2384,9 +2444,16 @@ mod tests {
|
||||
.pool_meta_write_status()
|
||||
.await
|
||||
.expect("healthy pool metadata keeps the background recovery loop read-only");
|
||||
store.background_cancel_token().expect("old store shutdown token").cancel();
|
||||
drop(store);
|
||||
let restarted = super::super::tests::setup_scanner_cycle_store_at_path_with_sets(root.path(), false, 3, 2).await;
|
||||
shutdown_native_retirement_store(store).await;
|
||||
let restarted = super::super::tests::setup_scanner_cycle_store_at_path_with_layout_and_disk_preinit(
|
||||
root.path(),
|
||||
false,
|
||||
3,
|
||||
2,
|
||||
NATIVE_RETIREMENT_DRIVES_PER_SET,
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
for set_index in 0..2 {
|
||||
restarted.retire_scanner_pause_backlog_for_test(0, set_index).await.unwrap();
|
||||
assert_native_source_missing(&restarted, set_index).await;
|
||||
@@ -2396,6 +2463,7 @@ mod tests {
|
||||
assert_eq!(bootstrapped.ledger.generation, 1);
|
||||
assert!(bootstrapped.ledger.last_updated_at_unix_secs < future_now);
|
||||
assert_current_native_ledger(&restarted, &bootstrapped.ledger).await;
|
||||
shutdown_native_retirement_store(restarted).await;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -2441,6 +2509,8 @@ mod tests {
|
||||
assert_current_native_ledger(&store, &original).await;
|
||||
store.retire_scanner_pause_backlog_for_test(0, 1).await.unwrap();
|
||||
assert_native_source_missing(&store, 1).await;
|
||||
drop(pool_meta_lock);
|
||||
shutdown_native_retirement_store(store).await;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -2536,6 +2606,9 @@ mod tests {
|
||||
"retirement never copied a stale source record"
|
||||
);
|
||||
}
|
||||
drop(pool_meta_lock);
|
||||
drop(barrier);
|
||||
shutdown_native_retirement_store(store).await;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -2669,11 +2742,12 @@ mod tests {
|
||||
.retire_scanner_pause_backlog_for_test(0, 1)
|
||||
.await
|
||||
.expect("the remaining source set follows the same native proof");
|
||||
let after = load_scanner_pause_backlog(store)
|
||||
let after = load_scanner_pause_backlog(Arc::clone(&store))
|
||||
.await
|
||||
.expect("native restart selection after handoff");
|
||||
assert_eq!(after.ledger, new_ledger);
|
||||
assert!(after.durable && after.stable_matches_ledger);
|
||||
shutdown_native_retirement_store(store).await;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -2745,6 +2819,7 @@ mod tests {
|
||||
.expect("retained target intent snapshot")
|
||||
};
|
||||
assert_eq!(after, before, "native cleanup never clears or estimates a target mutation intent");
|
||||
shutdown_native_retirement_store(store).await;
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -69,13 +69,42 @@ pub(super) async fn setup_scanner_cycle_store_at_path_with_sets(
|
||||
seed_usage_baseline: bool,
|
||||
pool_count: usize,
|
||||
sets_per_pool: usize,
|
||||
) -> Arc<ECStore> {
|
||||
setup_scanner_cycle_store_at_path_with_layout(root, seed_usage_baseline, pool_count, sets_per_pool, 4).await
|
||||
}
|
||||
|
||||
pub(super) async fn setup_scanner_cycle_store_at_path_with_layout(
|
||||
root: &Path,
|
||||
seed_usage_baseline: bool,
|
||||
pool_count: usize,
|
||||
sets_per_pool: usize,
|
||||
drives_per_set: usize,
|
||||
) -> Arc<ECStore> {
|
||||
setup_scanner_cycle_store_at_path_with_layout_and_disk_preinit(
|
||||
root,
|
||||
seed_usage_baseline,
|
||||
pool_count,
|
||||
sets_per_pool,
|
||||
drives_per_set,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn setup_scanner_cycle_store_at_path_with_layout_and_disk_preinit(
|
||||
root: &Path,
|
||||
seed_usage_baseline: bool,
|
||||
pool_count: usize,
|
||||
sets_per_pool: usize,
|
||||
drives_per_set: usize,
|
||||
preinitialize_disks: bool,
|
||||
) -> Arc<ECStore> {
|
||||
init_ecstore_config_for_scanner_tests();
|
||||
let mut pools = Vec::with_capacity(pool_count);
|
||||
for pool_index in 0..pool_count {
|
||||
let mut endpoints = Vec::new();
|
||||
for set_index in 0..sets_per_pool {
|
||||
for disk_index in 0..4 {
|
||||
for disk_index in 0..drives_per_set {
|
||||
let disk_path = if sets_per_pool == 1 {
|
||||
root.join(format!("pool{pool_index}/disk{disk_index}"))
|
||||
} else {
|
||||
@@ -95,7 +124,7 @@ pub(super) async fn setup_scanner_cycle_store_at_path_with_sets(
|
||||
pools.push(PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: sets_per_pool,
|
||||
drives_per_set: 4,
|
||||
drives_per_set,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: if pool_count == 1 && sets_per_pool == 1 {
|
||||
"scanner-cycle-metrics".to_string()
|
||||
@@ -108,9 +137,11 @@ pub(super) async fn setup_scanner_cycle_store_at_path_with_sets(
|
||||
let endpoint_pools = EndpointServerPools::from(pools);
|
||||
let instance_ctx = Arc::new(InstanceContext::new());
|
||||
instance_ctx.set_endpoints(endpoint_pools.clone());
|
||||
init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
|
||||
.await
|
||||
.expect("scanner cycle test disks should initialize");
|
||||
if preinitialize_disks {
|
||||
init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
|
||||
.await
|
||||
.expect("scanner cycle test disks should initialize");
|
||||
}
|
||||
let store = ECStore::new_with_instance_ctx(
|
||||
"127.0.0.1:0".parse().expect("test address should parse"),
|
||||
endpoint_pools,
|
||||
@@ -7649,6 +7680,25 @@ async fn scanner_cycle_confirms_lost_remote_ack_from_activity_snapshot() {
|
||||
"a new peer instance cannot confirm whether the old ACK reached durable dirty state"
|
||||
);
|
||||
|
||||
let mut stale_activity = scanner_node_activity("epoch-a", 7, 3);
|
||||
stale_activity.dirty_usage_generation = 4;
|
||||
let stale_clean_activity = BTreeMap::from([("node-2".to_string(), stale_activity)]);
|
||||
let stale_clean = remote_dirty_usage_acknowledgement_pending(
|
||||
8,
|
||||
1,
|
||||
std::slice::from_ref(&acknowledgement),
|
||||
std::future::ready(Err::<bool, _>(std::io::Error::other(
|
||||
"response lost before newer generation was observed",
|
||||
))),
|
||||
|| async { Ok(stale_clean_activity) },
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, stale_clean),
|
||||
ScannerCycleOutcome::CompletedWithPendingMaintenance,
|
||||
"a clean peer snapshot from before the acknowledged generation cannot prove the ACK reached durable dirty state"
|
||||
);
|
||||
|
||||
let mut written_activity = scanner_node_activity("epoch-a", 7, 3);
|
||||
written_activity.dirty_usage_generation = 6;
|
||||
written_activity.dirty_usage_pending = true;
|
||||
@@ -7722,6 +7772,47 @@ async fn scanner_cycle_confirms_lost_scoped_ack_only_after_same_instance_clean_a
|
||||
"a restarted peer cannot prove the scoped ACK reached the old scanner instance"
|
||||
);
|
||||
|
||||
let mut stale_activity = scanner_node_activity("epoch-a", 7, 3);
|
||||
stale_activity.dirty_usage_generation = 4;
|
||||
let stale_clean_activity = BTreeMap::from([("node-2".to_string(), stale_activity)]);
|
||||
let stale_clean = remote_dirty_usage_acknowledgement_pending(
|
||||
8,
|
||||
1,
|
||||
std::slice::from_ref(&acknowledgement),
|
||||
std::future::ready(Err::<bool, _>(std::io::Error::other(
|
||||
"scoped ACK transport failed before the requested generation was observed",
|
||||
))),
|
||||
|| async { Ok(stale_clean_activity) },
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, stale_clean),
|
||||
ScannerCycleOutcome::CompletedWithPendingMaintenance,
|
||||
"a clean peer snapshot from before the scoped ACK generation cannot prove the ACK reached durable dirty state"
|
||||
);
|
||||
|
||||
let empty_scoped_ack = ScannerDirtyUsageAcknowledgement {
|
||||
host: "node-2".to_string(),
|
||||
instance_id: "epoch-a".to_string(),
|
||||
kind: ScannerDirtyUsageAcknowledgementKind::Scoped {
|
||||
owner_id: Uuid::from_u128(0x11111111111111111111111111111111).to_string(),
|
||||
entries: Vec::new(),
|
||||
},
|
||||
};
|
||||
let empty_scoped_clean = remote_dirty_usage_acknowledgement_pending(
|
||||
8,
|
||||
1,
|
||||
&[empty_scoped_ack],
|
||||
std::future::ready(Err::<bool, _>(std::io::Error::other("empty scoped ACK failed before peer delivery"))),
|
||||
|| async { Ok(BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))])) },
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, empty_scoped_clean),
|
||||
ScannerCycleOutcome::CompletedWithPendingMaintenance,
|
||||
"an empty scoped ACK has no durable generation to reconcile after response loss"
|
||||
);
|
||||
|
||||
let mut written_activity = scanner_node_activity("epoch-a", 7, 3);
|
||||
written_activity.dirty_usage_generation = 6;
|
||||
written_activity.dirty_usage_pending = true;
|
||||
|
||||
@@ -432,7 +432,12 @@ async fn scoped_ack_publication_stale_baseline_cannot_prove_a_replaced_root() {
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scoped_ack_publication_rejects_builder_mutation_after_real_root_publish() {
|
||||
for mutation in ["remote_ack_target", "publication_epoch", "remote_lease_targets"] {
|
||||
for mutation in [
|
||||
"remote_ack_target",
|
||||
"remote_scoped_ack_target",
|
||||
"publication_epoch",
|
||||
"remote_lease_targets",
|
||||
] {
|
||||
let (_directory, store) = candidate_store().await;
|
||||
let (scan, candidate) = complete_candidate(&store, PROOF_CYCLE).await;
|
||||
let baseline = read_data_usage_persist_baseline(store.clone())
|
||||
@@ -465,6 +470,18 @@ async fn scoped_ack_publication_rejects_builder_mutation_after_real_root_publish
|
||||
instance_id: crate::scanner_activity_epoch().to_string(),
|
||||
kind: crate::scanner::ScannerDirtyUsageAcknowledgementKind::Generation(changed_generation),
|
||||
}]),
|
||||
"remote_scoped_ack_target" => scan.with_remote_dirty_usage_acknowledgements(vec![ScannerDirtyUsageAcknowledgement {
|
||||
host: "proof-peer:9000".to_string(),
|
||||
instance_id: crate::scanner_activity_epoch().to_string(),
|
||||
kind: crate::scanner::ScannerDirtyUsageAcknowledgementKind::Scoped {
|
||||
owner_id: crate::scanner_activity_epoch().to_string(),
|
||||
entries: vec![crate::storage_api::EcstoreScannerScopedDirtyUsageAckEntry {
|
||||
bucket: PROOF_BUCKET.to_string(),
|
||||
bucket_incarnation: uuid::Uuid::from_u128(0x11111111111111111111111111111111),
|
||||
generation: changed_generation,
|
||||
}],
|
||||
},
|
||||
}]),
|
||||
"publication_epoch" => scan.with_publication_epoch(Some(changed_epoch)),
|
||||
"remote_lease_targets" => scan.with_remote_publication_lease_targets(vec![(
|
||||
"proof-peer:9000".to_string(),
|
||||
|
||||
@@ -306,7 +306,7 @@ fn scanner_scoped_dirty_usage_ack_exceeds_cost_threshold(
|
||||
|
||||
fn resolve_remote_dirty_usage_scope(
|
||||
requested_scope: ScannerBucketScanScope,
|
||||
mut dirty_buckets: HashSet<String>,
|
||||
dirty_usage_snapshot: &DirtyUsageSnapshot,
|
||||
remote_dirty_usage: VerifiedRemoteDirtyUsage,
|
||||
all_buckets: &[BucketInfo],
|
||||
baseline_proof: ScannerCacheBaselineProof<'_>,
|
||||
@@ -319,12 +319,21 @@ fn resolve_remote_dirty_usage_scope(
|
||||
|
||||
let peer_count = remote_dirty_usage.peer_count;
|
||||
let dirty_peer_count = remote_dirty_usage.dirty_peer_count;
|
||||
let remote_dirty_buckets = remote_dirty_usage.dirty_buckets.clone();
|
||||
let mut dirty_buckets = dirty_usage_snapshot.buckets.keys().cloned().collect::<HashSet<_>>();
|
||||
dirty_buckets.extend(remote_dirty_usage.dirty_buckets);
|
||||
// Peer snapshots contribute bucket names only; the local prefix scopes
|
||||
// would narrow a bucket a peer dirtied elsewhere, so the merged scope
|
||||
// stays at bucket granularity (same rule as the local fallthrough).
|
||||
let scope =
|
||||
scoped_scan_scope_from_dirty_buckets(requested_scope, dirty_buckets, None, true, false, all_buckets, baseline_proof);
|
||||
let scope = scoped_scan_scope_from_dirty_buckets(
|
||||
requested_scope.clone(),
|
||||
dirty_buckets.clone(),
|
||||
None,
|
||||
true,
|
||||
false,
|
||||
all_buckets,
|
||||
baseline_proof,
|
||||
);
|
||||
if scope.is_default() {
|
||||
return default_result(scope);
|
||||
}
|
||||
@@ -358,19 +367,45 @@ fn resolve_remote_dirty_usage_scope(
|
||||
}
|
||||
let has_scoped_acknowledgements = !scoped_acknowledgements.is_empty();
|
||||
|
||||
let distributed_segment_invalidation_evidence =
|
||||
(dirty_peer_count > 0 && has_scoped_acknowledgements).then_some(DistributedSegmentInvalidationEvidence {
|
||||
invalidation_domain: crate::segment_invalidation::SegmentInvalidationDomain::DistributedEc,
|
||||
distributed_ec_invalidation: true,
|
||||
peer_count,
|
||||
dirty_peer_count,
|
||||
same_window_remote_proof: true,
|
||||
all_peers_bound_to_generation_window: true,
|
||||
});
|
||||
let segment_reuse_activation_preflight = scanner_segment_reuse_activation_preflight_for_baseline_with_evidence(
|
||||
dirty_usage_snapshot,
|
||||
true,
|
||||
baseline_proof,
|
||||
distributed_segment_invalidation_evidence,
|
||||
);
|
||||
let scope = if segment_reuse_activation_preflight.scanner_segment_reuse_activated {
|
||||
let local_only_scopes = dirty_usage_snapshot
|
||||
.scopes
|
||||
.iter()
|
||||
.filter(|(bucket, _)| !remote_dirty_buckets.contains(bucket.as_str()))
|
||||
.map(|(bucket, scope)| (bucket.clone(), scope.clone()))
|
||||
.collect::<DirtyUsageBucketScopes>();
|
||||
scoped_scan_scope_from_dirty_buckets(
|
||||
requested_scope,
|
||||
dirty_buckets,
|
||||
(!local_only_scopes.is_empty()).then_some(&local_only_scopes),
|
||||
true,
|
||||
true,
|
||||
all_buckets,
|
||||
baseline_proof,
|
||||
)
|
||||
} else {
|
||||
scope
|
||||
};
|
||||
|
||||
ScannerBucketScopeResolutionResult {
|
||||
scope,
|
||||
remote_dirty_usage_acknowledgements: scoped_acknowledgements,
|
||||
distributed_segment_invalidation_evidence: (dirty_peer_count > 0 && has_scoped_acknowledgements).then_some(
|
||||
DistributedSegmentInvalidationEvidence {
|
||||
invalidation_domain: crate::segment_invalidation::SegmentInvalidationDomain::DistributedEc,
|
||||
distributed_ec_invalidation: true,
|
||||
peer_count,
|
||||
dirty_peer_count,
|
||||
same_window_remote_proof: true,
|
||||
all_peers_bound_to_generation_window: true,
|
||||
},
|
||||
),
|
||||
distributed_segment_invalidation_evidence,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -596,7 +631,7 @@ fn scanner_segment_reuse_activation_preflight_for_cycle(
|
||||
production_activation: true,
|
||||
producer_identity_coverage_complete: dirty_usage_producer_evidence.producer_identity_coverage_complete,
|
||||
durable_producer_identity: dirty_usage_producer_evidence.durable_producer_identity,
|
||||
durable_dirty_producer_journal: dirty_usage_producer_evidence.durable_producer_identity,
|
||||
durable_dirty_producer_journal: dirty_usage_producer_evidence.durable_dirty_producer_journal,
|
||||
restart_gap_absent: dirty_usage_producer_evidence.restart_gap_absent,
|
||||
generation_window_bound: dirty_usage_snapshot.covers_all_pending
|
||||
&& dirty_usage_snapshot.generation != 0
|
||||
@@ -616,6 +651,15 @@ fn scanner_segment_reuse_activation_preflight_for_baseline(
|
||||
dirty_usage_snapshot: &DirtyUsageSnapshot,
|
||||
distributed: bool,
|
||||
baseline_proof: ScannerCacheBaselineProof<'_>,
|
||||
) -> ScannerSegmentReuseActivationPreflight {
|
||||
scanner_segment_reuse_activation_preflight_for_baseline_with_evidence(dirty_usage_snapshot, distributed, baseline_proof, None)
|
||||
}
|
||||
|
||||
fn scanner_segment_reuse_activation_preflight_for_baseline_with_evidence(
|
||||
dirty_usage_snapshot: &DirtyUsageSnapshot,
|
||||
distributed: bool,
|
||||
baseline_proof: ScannerCacheBaselineProof<'_>,
|
||||
distributed_segment_invalidation_evidence: Option<DistributedSegmentInvalidationEvidence>,
|
||||
) -> ScannerSegmentReuseActivationPreflight {
|
||||
let (dirty_usage_producer_evidence, cold_zero_walk_oracle) =
|
||||
scanner_segment_reuse_baseline_producer_evidence(dirty_usage_snapshot, baseline_proof);
|
||||
@@ -623,7 +667,7 @@ fn scanner_segment_reuse_activation_preflight_for_baseline(
|
||||
dirty_usage_snapshot,
|
||||
dirty_usage_producer_evidence,
|
||||
distributed,
|
||||
None,
|
||||
distributed_segment_invalidation_evidence,
|
||||
cold_zero_walk_oracle,
|
||||
)
|
||||
}
|
||||
@@ -1568,14 +1612,14 @@ pub(crate) use cache::{
|
||||
current_cache_root_or_prepare_with_generation,
|
||||
};
|
||||
pub use dirty_usage::{
|
||||
ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageClearObserver, ScannerDirtyUsageSnapshot,
|
||||
ScannerDirtyUsageState, ScannerDurableDirtyUsageReplayEntry, ScannerDurableDirtyUsageReplayError,
|
||||
ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageClearObserver, ScannerDirtyUsageMutationObserver,
|
||||
ScannerDirtyUsageSnapshot, ScannerDirtyUsageState, ScannerDurableDirtyUsageReplayEntry, ScannerDurableDirtyUsageReplayError,
|
||||
ScannerDurableDirtyUsageReplayRecord, ScannerDurableDirtyUsageReplayScope, acknowledge_dirty_usage_generation,
|
||||
acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, encode_durable_dirty_usage_producer_replay_record,
|
||||
record_dirty_usage_bucket, record_dirty_usage_bucket_from_producer, record_dirty_usage_bucket_from_producers,
|
||||
record_dirty_usage_object, record_dirty_usage_object_from_producer, record_scanner_maintenance_change,
|
||||
replay_durable_dirty_usage_producer_record, scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
|
||||
scanner_maintenance_generation, set_scanner_dirty_usage_clear_observer,
|
||||
scanner_maintenance_generation, set_scanner_dirty_usage_clear_observer, set_scanner_dirty_usage_mutation_observer,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use dirty_usage::{clear_dirty_usage_buckets_for_tests, dirty_usage_buckets_for_tests};
|
||||
|
||||
@@ -30,6 +30,8 @@ pub(super) static DIRTY_USAGE_PRODUCER_IDENTITIES: LazyLock<StdMutex<DirtyUsageP
|
||||
LazyLock::new(|| StdMutex::new(BTreeMap::new()));
|
||||
static DIRTY_USAGE_CLEAR_OBSERVER: LazyLock<StdRwLock<Option<ScannerDirtyUsageClearObserver>>> =
|
||||
LazyLock::new(|| StdRwLock::new(None));
|
||||
static DIRTY_USAGE_MUTATION_OBSERVER: LazyLock<StdRwLock<Option<ScannerDirtyUsageMutationObserver>>> =
|
||||
LazyLock::new(|| StdRwLock::new(None));
|
||||
pub(super) static DIRTY_USAGE_PRODUCER_COVERAGE: AtomicU64 = AtomicU64::new(0);
|
||||
pub(super) static DIRTY_USAGE_BUCKET_NOTIFY: LazyLock<Notify> = LazyLock::new(Notify::new);
|
||||
pub(super) static SCANNER_ACTIVITY_EPOCH: LazyLock<String> = LazyLock::new(|| format!("{:032x}", rand::random::<u128>()));
|
||||
@@ -53,6 +55,8 @@ pub struct ScannerDirtyUsageBucket {
|
||||
}
|
||||
|
||||
pub type ScannerDirtyUsageClearObserver = Arc<dyn Fn(Vec<ScannerDirtyUsageBucket>) + Send + Sync + 'static>;
|
||||
pub type ScannerDirtyUsageMutationObserver =
|
||||
Arc<dyn Fn(&str, &str, crate::segment_invalidation::SegmentInvalidationProducerIdentity) + Send + Sync + 'static>;
|
||||
|
||||
/// A non-durable optimization hint for a dirty bucket.
|
||||
///
|
||||
@@ -83,6 +87,7 @@ pub(super) type DirtyUsageProducerIdentities = BTreeMap<String, DirtyUsageProduc
|
||||
pub(super) struct DirtyUsageProducerEvidence {
|
||||
pub(super) producer_identity_coverage_complete: bool,
|
||||
pub(super) durable_producer_identity: bool,
|
||||
pub(super) durable_dirty_producer_journal: bool,
|
||||
pub(super) restart_gap_absent: bool,
|
||||
pub(super) generation_window_bound: bool,
|
||||
pub(super) generation_start: u64,
|
||||
@@ -112,6 +117,15 @@ pub fn set_scanner_dirty_usage_clear_observer(
|
||||
std::mem::replace(&mut *slot, observer)
|
||||
}
|
||||
|
||||
pub fn set_scanner_dirty_usage_mutation_observer(
|
||||
observer: Option<ScannerDirtyUsageMutationObserver>,
|
||||
) -> Option<ScannerDirtyUsageMutationObserver> {
|
||||
let mut slot = DIRTY_USAGE_MUTATION_OBSERVER
|
||||
.write()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
std::mem::replace(&mut *slot, observer)
|
||||
}
|
||||
|
||||
fn notify_dirty_usage_clear(cleared: Vec<ScannerDirtyUsageBucket>) {
|
||||
if cleared.is_empty() {
|
||||
return;
|
||||
@@ -125,6 +139,30 @@ fn notify_dirty_usage_clear(cleared: Vec<ScannerDirtyUsageBucket>) {
|
||||
}
|
||||
}
|
||||
|
||||
fn notify_dirty_usage_mutation(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
producers: &[crate::segment_invalidation::SegmentInvalidationProducerIdentity],
|
||||
) {
|
||||
if bucket.is_empty() {
|
||||
return;
|
||||
}
|
||||
let observer = DIRTY_USAGE_MUTATION_OBSERVER
|
||||
.read()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.clone();
|
||||
let Some(observer) = observer else {
|
||||
return;
|
||||
};
|
||||
if producers.is_empty() {
|
||||
observer(bucket, object, crate::segment_invalidation::SegmentInvalidationProducerIdentity::Unknown);
|
||||
} else {
|
||||
for producer in producers {
|
||||
observer(bucket, object, *producer);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A point-in-time view of the local dirty bucket generations.
|
||||
///
|
||||
/// `complete == false` is an all-or-nothing overflow signal: `buckets` is
|
||||
@@ -566,6 +604,7 @@ mod scoped_dirty_usage_tests {
|
||||
let evidence = dirty_usage_producer_evidence(&snapshot);
|
||||
assert!(evidence.producer_identity_coverage_complete);
|
||||
assert!(evidence.durable_producer_identity);
|
||||
assert!(evidence.durable_dirty_producer_journal);
|
||||
assert!(evidence.restart_gap_absent);
|
||||
assert_eq!(evidence.generation_start, 7);
|
||||
assert_eq!(evidence.generation_end, 7);
|
||||
@@ -583,10 +622,45 @@ mod scoped_dirty_usage_tests {
|
||||
);
|
||||
let changed_evidence = dirty_usage_producer_evidence(&changed);
|
||||
assert!(!changed_evidence.durable_producer_identity);
|
||||
assert!(!changed_evidence.durable_dirty_producer_journal);
|
||||
assert!(!changed_evidence.restart_gap_absent);
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn dirty_usage_mutation_observer_tracks_typed_and_conservative_events() {
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
let observed = Arc::new(Mutex::new(Vec::new()));
|
||||
let observed_clone = observed.clone();
|
||||
let previous = set_scanner_dirty_usage_mutation_observer(Some(Arc::new(move |bucket, object, producer| {
|
||||
observed_clone.lock().expect("observer lock should not be poisoned").push((
|
||||
bucket.to_string(),
|
||||
object.to_string(),
|
||||
producer,
|
||||
));
|
||||
})));
|
||||
|
||||
record_dirty_usage_object_from_producer("photos", "2026/image.jpg", SegmentInvalidationProducerIdentity::PutObject);
|
||||
record_dirty_usage_bucket("archive");
|
||||
set_scanner_dirty_usage_mutation_observer(previous);
|
||||
|
||||
assert_eq!(
|
||||
*observed.lock().expect("observer lock should not be poisoned"),
|
||||
vec![
|
||||
(
|
||||
"photos".to_string(),
|
||||
"2026/image.jpg".to_string(),
|
||||
SegmentInvalidationProducerIdentity::PutObject,
|
||||
),
|
||||
("archive".to_string(), String::new(), SegmentInvalidationProducerIdentity::Unknown,),
|
||||
]
|
||||
);
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn durable_dirty_usage_replay_rejects_invalid_records_without_partial_state() {
|
||||
@@ -745,6 +819,7 @@ fn record_dirty_usage_bucket_inner<I>(bucket: &str, producers: I)
|
||||
where
|
||||
I: IntoIterator<Item = crate::segment_invalidation::SegmentInvalidationProducerIdentity>,
|
||||
{
|
||||
let producers = producers.into_iter().collect::<Vec<_>>();
|
||||
let pending_buckets = {
|
||||
let mut dirty_buckets = dirty_usage_buckets();
|
||||
let mut dirty_scopes = dirty_usage_bucket_scopes();
|
||||
@@ -752,7 +827,12 @@ where
|
||||
let generation = advance_generation(&DIRTY_USAGE_BUCKET_GENERATION);
|
||||
dirty_buckets.insert(bucket.to_string(), generation);
|
||||
dirty_scopes.insert(bucket.to_string(), DirtyUsageBucketScope::WholeBucket);
|
||||
record_segment_invalidation_producer_identities_for_generation(&mut producer_identities, bucket, generation, producers);
|
||||
record_segment_invalidation_producer_identities_for_generation(
|
||||
&mut producer_identities,
|
||||
bucket,
|
||||
generation,
|
||||
producers.iter().copied(),
|
||||
);
|
||||
dirty_buckets.len()
|
||||
};
|
||||
global_metrics().record_scanner_dirty_usage_pending(usize_to_u64_saturated(pending_buckets));
|
||||
@@ -760,6 +840,7 @@ where
|
||||
// admin/console consumers never ride the full TTL after a change
|
||||
// (rustfs/backlog#1872).
|
||||
crate::prefix_usage::invalidate_prefix_usage_cache(bucket);
|
||||
notify_dirty_usage_mutation(bucket, "", &producers);
|
||||
DIRTY_USAGE_BUCKET_NOTIFY.notify_one();
|
||||
}
|
||||
|
||||
@@ -817,11 +898,17 @@ where
|
||||
if overflowed {
|
||||
*scope = DirtyUsageBucketScope::WholeBucket;
|
||||
}
|
||||
record_segment_invalidation_producer_identities_for_generation(&mut producer_identities, bucket, generation, producers);
|
||||
record_segment_invalidation_producer_identities_for_generation(
|
||||
&mut producer_identities,
|
||||
bucket,
|
||||
generation,
|
||||
producers.iter().copied(),
|
||||
);
|
||||
dirty_buckets.len()
|
||||
};
|
||||
global_metrics().record_scanner_dirty_usage_pending(usize_to_u64_saturated(pending_buckets));
|
||||
crate::prefix_usage::invalidate_prefix_usage_cache(bucket);
|
||||
notify_dirty_usage_mutation(bucket, object, &producers);
|
||||
DIRTY_USAGE_BUCKET_NOTIFY.notify_one();
|
||||
}
|
||||
|
||||
@@ -1193,7 +1280,7 @@ pub(super) fn dirty_usage_producer_evidence(snapshot: &DirtyUsageSnapshot) -> Di
|
||||
&& DIRTY_USAGE_PRODUCER_COVERAGE.load(Ordering::Acquire)
|
||||
& crate::segment_invalidation::SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION_COVERAGE_MASK
|
||||
== crate::segment_invalidation::SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION_COVERAGE_MASK;
|
||||
let durable_producer_identity = producer_identity_coverage_complete
|
||||
let durable_dirty_producer_journal = producer_identity_coverage_complete
|
||||
&& snapshot
|
||||
.buckets
|
||||
.keys()
|
||||
@@ -1205,8 +1292,9 @@ pub(super) fn dirty_usage_producer_evidence(snapshot: &DirtyUsageSnapshot) -> Di
|
||||
|
||||
DirtyUsageProducerEvidence {
|
||||
producer_identity_coverage_complete,
|
||||
durable_producer_identity,
|
||||
restart_gap_absent: durable_producer_identity,
|
||||
durable_producer_identity: durable_dirty_producer_journal,
|
||||
durable_dirty_producer_journal,
|
||||
restart_gap_absent: durable_dirty_producer_journal,
|
||||
generation_window_bound,
|
||||
generation_start: if generation_window_bound { generation_start } else { 0 },
|
||||
generation_end: if generation_window_bound { generation_end } else { 0 },
|
||||
|
||||
@@ -190,7 +190,7 @@ where
|
||||
};
|
||||
let remote_resolution = resolve_remote_dirty_usage_scope(
|
||||
resolution.requested_scope,
|
||||
dirty_buckets,
|
||||
resolution.dirty_usage_snapshot,
|
||||
remote_dirty_usage,
|
||||
resolution.all_buckets,
|
||||
resolution.baseline_proof,
|
||||
|
||||
@@ -37,7 +37,7 @@ use rustfs_concurrency::{
|
||||
};
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use serial_test::serial;
|
||||
use std::collections::BTreeMap;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::sync::Arc;
|
||||
use temp_env::with_var;
|
||||
use time::OffsetDateTime;
|
||||
@@ -285,6 +285,7 @@ fn scanner_durable_segment_invalidation_evidence_requires_matching_complete_set_
|
||||
|
||||
assert!(durable_evidence.producer_identity_coverage_complete);
|
||||
assert!(durable_evidence.durable_producer_identity);
|
||||
assert!(!durable_evidence.durable_dirty_producer_journal);
|
||||
assert!(durable_evidence.restart_gap_absent);
|
||||
|
||||
let mut stale_epoch = results.clone();
|
||||
@@ -297,12 +298,14 @@ fn scanner_durable_segment_invalidation_evidence_requires_matching_complete_set_
|
||||
let stale_evidence = scanner_durable_segment_invalidation_evidence(&dirty_usage_snapshot, &stale_epoch, &expected_sources);
|
||||
assert!(stale_evidence.producer_identity_coverage_complete);
|
||||
assert!(!stale_evidence.durable_producer_identity);
|
||||
assert!(!stale_evidence.durable_dirty_producer_journal);
|
||||
assert!(!stale_evidence.restart_gap_absent);
|
||||
|
||||
record_dirty_usage_bucket("videos");
|
||||
let changed_evidence = scanner_durable_segment_invalidation_evidence(&dirty_usage_snapshot, &results, &expected_sources);
|
||||
assert!(!changed_evidence.producer_identity_coverage_complete);
|
||||
assert!(!changed_evidence.durable_producer_identity);
|
||||
assert!(!changed_evidence.durable_dirty_producer_journal);
|
||||
assert!(!changed_evidence.restart_gap_absent);
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
@@ -316,6 +319,19 @@ fn scanner_segment_reuse_activation_replays_cold_durable_baseline() {
|
||||
for producer in SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION {
|
||||
record_dirty_usage_object_from_producer("photos", "2026/object", producer);
|
||||
}
|
||||
let replay_generation = dirty_usage_generation();
|
||||
replay_durable_dirty_usage_producer_record(
|
||||
&encode_durable_dirty_usage_producer_replay_record(vec![ScannerDurableDirtyUsageReplayEntry {
|
||||
bucket: "photos".to_string(),
|
||||
generation: replay_generation,
|
||||
scope: ScannerDurableDirtyUsageReplayScope::TopLevelEntries {
|
||||
entries: BTreeSet::from(["2026".to_string()]),
|
||||
},
|
||||
producers: SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION.into_iter().collect(),
|
||||
}])
|
||||
.expect("durable producer replay should encode"),
|
||||
)
|
||||
.expect("durable producer replay should restore restart authority");
|
||||
let dirty_usage_snapshot =
|
||||
snapshot_dirty_usage_buckets(&[bucket_info("photos"), bucket_info("archive")], dirty_usage_generation());
|
||||
let mut segment_proof = dirty_usage_producer_evidence(&dirty_usage_snapshot)
|
||||
@@ -413,8 +429,11 @@ fn scanner_segment_reuse_activation_replays_cold_durable_baseline() {
|
||||
scan_plan_digest,
|
||||
},
|
||||
);
|
||||
assert!(preflight.scanner_segment_reuse_activated);
|
||||
assert_eq!(preflight.fail_closed_blockers().collect::<Vec<_>>(), Vec::<&str>::new());
|
||||
assert!(!preflight.scanner_segment_reuse_activated);
|
||||
assert_eq!(
|
||||
preflight.fail_closed_blockers().collect::<Vec<_>>(),
|
||||
vec!["missing_durable_journal_replay"]
|
||||
);
|
||||
|
||||
record_dirty_usage_bucket("photos");
|
||||
let unidentified_snapshot =
|
||||
@@ -473,6 +492,7 @@ fn complete_process_local_producer_evidence() -> DirtyUsageProducerEvidence {
|
||||
DirtyUsageProducerEvidence {
|
||||
producer_identity_coverage_complete: true,
|
||||
durable_producer_identity: false,
|
||||
durable_dirty_producer_journal: false,
|
||||
restart_gap_absent: false,
|
||||
generation_window_bound: true,
|
||||
generation_start: 7,
|
||||
@@ -1468,6 +1488,7 @@ fn dirty_usage_producer_evidence_tracks_process_local_coverage_without_durable_r
|
||||
assert!(evidence.generation_window_bound);
|
||||
assert!(evidence.producer_identity_coverage_complete);
|
||||
assert!(!evidence.durable_producer_identity);
|
||||
assert!(!evidence.durable_dirty_producer_journal);
|
||||
assert!(!evidence.restart_gap_absent);
|
||||
assert_eq!(evidence.generation_start, snapshot.buckets["photos"]);
|
||||
assert_eq!(evidence.generation_end, snapshot.buckets["photos"]);
|
||||
@@ -1858,6 +1879,44 @@ fn complete_usage_baseline(
|
||||
bytes::Bytes::from(serde_json::to_vec(&baseline).expect("test baseline should encode"))
|
||||
}
|
||||
|
||||
fn complete_segment_reuse_baseline(
|
||||
source: DataUsageCacheSource,
|
||||
scan_plan_digest: DataUsageScanPlanDigest,
|
||||
scanner_cycle: u64,
|
||||
scanner_epoch: u64,
|
||||
evidence: DirtyUsageProducerEvidence,
|
||||
buckets: &[&str],
|
||||
) -> bytes::Bytes {
|
||||
let mut proof = evidence
|
||||
.segment_invalidation_proof()
|
||||
.expect("durable producer evidence should produce segment proof");
|
||||
proof.cold_zero_walk_oracle = true;
|
||||
let baseline = DataUsageInfo {
|
||||
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(10)),
|
||||
scanner_cycle: Some(scanner_cycle),
|
||||
scanner_epoch: Some(scanner_epoch),
|
||||
buckets_count: u64::try_from(buckets.len()).expect("test bucket count should fit"),
|
||||
buckets_usage: buckets
|
||||
.iter()
|
||||
.map(|bucket| ((*bucket).to_string(), Default::default()))
|
||||
.collect(),
|
||||
usage_snapshot_complete: true,
|
||||
usage_snapshot_converged: Some(true),
|
||||
usage_snapshot_set_states: vec![DataUsageSnapshotSetState {
|
||||
pool_index: u64::try_from(source.pool_index).expect("test pool index should fit"),
|
||||
set_index: u64::try_from(source.set_index).expect("test set index should fit"),
|
||||
scanner_cycle: Some(scanner_cycle),
|
||||
scanner_epoch: Some(scanner_epoch),
|
||||
scan_plan_digest: Some(scan_plan_digest.0),
|
||||
complete: true,
|
||||
tombstone: false,
|
||||
segment_invalidation_proof: Some(proof),
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
bytes::Bytes::from(serde_json::to_vec(&baseline).expect("test baseline should encode"))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scoped_scan_requires_a_converged_complete_baseline_with_exact_set_provenance() {
|
||||
let source = DataUsageCacheSource::new(1, 2);
|
||||
@@ -2183,6 +2242,12 @@ fn remote_dirty_usage_invalidates_local_prefix_hints_until_distributed_proof_exi
|
||||
"photos".to_string(),
|
||||
DirtyUsageBucketScope::TopLevelEntries(HashSet::from(["2026".to_string()])),
|
||||
)]);
|
||||
let dirty_usage_snapshot = DirtyUsageSnapshot {
|
||||
buckets: Arc::new(HashMap::from([("photos".to_string(), 7)])),
|
||||
scopes: Arc::new(dirty_scopes.clone()),
|
||||
generation: 7,
|
||||
covers_all_pending: true,
|
||||
};
|
||||
let locally_scoped = scoped_scan_scope_from_dirty_buckets(
|
||||
ScannerBucketScanScope::default(),
|
||||
HashSet::from(["photos".to_string()]),
|
||||
@@ -2206,7 +2271,7 @@ fn remote_dirty_usage_invalidates_local_prefix_hints_until_distributed_proof_exi
|
||||
|
||||
let distributed = resolve_remote_dirty_usage_scope(
|
||||
ScannerBucketScanScope::default(),
|
||||
HashSet::from(["photos".to_string()]),
|
||||
&dirty_usage_snapshot,
|
||||
remote_dirty_usage,
|
||||
&[bucket_info("photos")],
|
||||
ScannerCacheBaselineProof {
|
||||
@@ -2246,6 +2311,82 @@ fn remote_dirty_usage_invalidates_local_prefix_hints_until_distributed_proof_exi
|
||||
assert!(evidence.all_peers_bound_to_generation_window);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn distributed_segment_reuse_activation_keeps_remote_dirty_buckets_at_bucket_scope() {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
let source = DataUsageCacheSource::new(1, 2);
|
||||
let expected_sources = HashSet::from([source]);
|
||||
let scan_plan_digest = DataUsageScanPlanDigest([9; 32]);
|
||||
let entries = BTreeSet::from(["2026".to_string()]);
|
||||
let bytes = encode_durable_dirty_usage_producer_replay_record(vec![ScannerDurableDirtyUsageReplayEntry {
|
||||
bucket: "photos".to_string(),
|
||||
generation: 7,
|
||||
scope: ScannerDurableDirtyUsageReplayScope::TopLevelEntries { entries },
|
||||
producers: crate::segment_invalidation::SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION
|
||||
.iter()
|
||||
.copied()
|
||||
.collect(),
|
||||
}])
|
||||
.expect("durable dirty usage replay record should encode");
|
||||
replay_durable_dirty_usage_producer_record(&bytes).expect("durable dirty usage replay should restore producer proof");
|
||||
let dirty_usage_snapshot =
|
||||
snapshot_dirty_usage_buckets(&[bucket_info("photos"), bucket_info("archive")], dirty_usage_generation());
|
||||
let evidence = dirty_usage_producer_evidence(&dirty_usage_snapshot);
|
||||
let baseline = complete_segment_reuse_baseline(source, scan_plan_digest, 7, 11, evidence, &["photos", "archive"]);
|
||||
let expected_peers = HashMap::from([(
|
||||
"node-a:9000".to_string(),
|
||||
ScannerPeerDirtyUsageExpectation {
|
||||
instance_id: "instance-a".to_string(),
|
||||
generation: 3,
|
||||
pending: true,
|
||||
},
|
||||
)]);
|
||||
let remote_dirty_usage = verified_remote_dirty_usage(
|
||||
&expected_peers,
|
||||
vec![(
|
||||
"node-a:9000".to_string(),
|
||||
peer_dirty_usage_snapshot("instance-a", 3, true, &[("archive", 3)]),
|
||||
)],
|
||||
)
|
||||
.expect("fixture remote dirty usage should verify at bucket granularity");
|
||||
|
||||
let result = resolve_remote_dirty_usage_scope(
|
||||
ScannerBucketScanScope::default(),
|
||||
&dirty_usage_snapshot,
|
||||
remote_dirty_usage,
|
||||
&[bucket_info("photos"), bucket_info("archive")],
|
||||
ScannerCacheBaselineProof {
|
||||
authoritative_data: Some(&baseline),
|
||||
observed_candidate_data: None,
|
||||
expected_sources: &expected_sources,
|
||||
leader_epoch: 11,
|
||||
want_cycle: 8,
|
||||
scan_plan_digest,
|
||||
},
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
result
|
||||
.scope
|
||||
.selected_buckets
|
||||
.as_deref()
|
||||
.expect("distributed reuse still selects both dirty buckets"),
|
||||
&HashSet::from(["photos".to_string(), "archive".to_string()])
|
||||
);
|
||||
assert!(
|
||||
result.scope.prefix_scope_for("photos").is_some(),
|
||||
"durable local producer proof may activate local segment reuse after distributed ACK capability evidence"
|
||||
);
|
||||
assert!(
|
||||
result.scope.prefix_scope_for("archive").is_none(),
|
||||
"remote dirty usage has only bucket-granularity evidence and must not be narrowed to a local prefix"
|
||||
);
|
||||
assert_eq!(result.remote_dirty_usage_acknowledgements.len(), 1);
|
||||
assert!(result.distributed_segment_invalidation_evidence.is_some());
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
fn peer_dirty_usage_snapshot(
|
||||
instance_id: &str,
|
||||
generation: u64,
|
||||
@@ -2419,7 +2560,12 @@ fn remote_dirty_usage_scope_resolution_falls_back_when_ack_batch_exceeds_thresho
|
||||
|
||||
let result = resolve_remote_dirty_usage_scope(
|
||||
ScannerBucketScanScope::default(),
|
||||
HashSet::new(),
|
||||
&DirtyUsageSnapshot {
|
||||
buckets: Arc::new(HashMap::new()),
|
||||
scopes: Arc::new(HashMap::new()),
|
||||
generation: 7,
|
||||
covers_all_pending: true,
|
||||
},
|
||||
remote_dirty_usage,
|
||||
&all_buckets,
|
||||
ScannerCacheBaselineProof {
|
||||
|
||||
@@ -226,6 +226,34 @@ fn record_segment_dirty_usage(bucket: &str) {
|
||||
}
|
||||
}
|
||||
|
||||
fn replay_segment_dirty_usage(bucket: &str) {
|
||||
replay_dirty_usage(
|
||||
bucket,
|
||||
ScannerDurableDirtyUsageReplayScope::TopLevelEntries {
|
||||
entries: BTreeSet::from(["hot-segment".to_string()]),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
fn replay_whole_bucket_dirty_usage(bucket: &str) {
|
||||
replay_dirty_usage(bucket, ScannerDurableDirtyUsageReplayScope::WholeBucket);
|
||||
}
|
||||
|
||||
fn replay_dirty_usage(bucket: &str, scope: ScannerDurableDirtyUsageReplayScope) {
|
||||
replay_durable_dirty_usage_producer_record(
|
||||
&encode_durable_dirty_usage_producer_replay_record(vec![ScannerDurableDirtyUsageReplayEntry {
|
||||
bucket: bucket.to_string(),
|
||||
generation: dirty_usage_generation(),
|
||||
scope,
|
||||
producers: crate::segment_invalidation::SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION
|
||||
.into_iter()
|
||||
.collect(),
|
||||
}])
|
||||
.expect("durable segment replay should encode"),
|
||||
)
|
||||
.expect("durable segment replay should restore producer authority");
|
||||
}
|
||||
|
||||
// The scoped fallback fixture keeps two EC pools and several scan futures live
|
||||
// at once. Run the async cases on a dedicated stack so Linux libtest defaults
|
||||
// exercise the assertions instead of aborting before the oracle finishes.
|
||||
@@ -267,6 +295,7 @@ async fn scoped_entry_fallback_distinguishes_planned_scope_from_real_cold_walks_
|
||||
create_bucket(&store, &hot).await;
|
||||
create_bucket(&store, &cold).await;
|
||||
record_segment_dirty_usage(&hot);
|
||||
replay_segment_dirty_usage(&hot);
|
||||
let baseline = run_entry(&store, 1, None, true, false, false).await;
|
||||
persist_baseline(&store, &baseline).await;
|
||||
|
||||
@@ -283,6 +312,7 @@ async fn scoped_entry_fallback_distinguishes_planned_scope_from_real_cold_walks_
|
||||
"hot-segment/object",
|
||||
crate::segment_invalidation::SegmentInvalidationProducerIdentity::PutObject,
|
||||
);
|
||||
replay_segment_dirty_usage(&hot);
|
||||
let usage = run_entry(&store, 3, Some(&hot), true, true, true).await;
|
||||
assert_eq!(usage.buckets_usage[&hot].objects_count, 2);
|
||||
assert_eq!(usage.buckets_usage[&cold].objects_count, 1);
|
||||
@@ -298,6 +328,7 @@ async fn scoped_entry_fallback_distinguishes_planned_scope_from_real_cold_walks_
|
||||
crate::segment_invalidation::SegmentInvalidationProducerIdentity::PutObject,
|
||||
);
|
||||
}
|
||||
replay_whole_bucket_dirty_usage(&hot);
|
||||
let usage = run_entry(&store, 4, Some(&hot), true, true, false).await;
|
||||
assert_eq!(usage.objects_total_count, 3);
|
||||
|
||||
|
||||
@@ -364,6 +364,29 @@ scripts/python_bin.sh scripts/check_test_wiring.py \
|
||||
--check-scanner-heal-release-bundle /path/to/release-evidence.json
|
||||
```
|
||||
|
||||
For a single Linux handoff checklist that keeps the measured runners in a
|
||||
stable order, generate the Scanner/Heal Linux evidence plan:
|
||||
|
||||
```bash
|
||||
scripts/python_bin.sh scripts/run_scanner_heal_linux_evidence_plan.py \
|
||||
--write-plan --out-dir /path/to/plan-dir
|
||||
```
|
||||
|
||||
The plan is only an execution manifest. Its `evidence_type` is `plan_only`, and
|
||||
it cannot satisfy any Gxx/Wxx/Rxx gate. Use `--run-preflight` only for the
|
||||
lightweight registry and runner self-tests before starting a long Linux run.
|
||||
After or during a Linux run, check which planned artifacts are still missing
|
||||
without approving the release bundle:
|
||||
|
||||
```bash
|
||||
scripts/python_bin.sh scripts/run_scanner_heal_linux_evidence_plan.py \
|
||||
--status-root /path/to/run-root --format json
|
||||
```
|
||||
|
||||
The status command exits nonzero while evidence is missing or malformed and
|
||||
keeps `release_approved: false`; use its `pending_gates` and `next_step` fields
|
||||
for issue writeback and failure triage.
|
||||
|
||||
Lane descriptors can be assembled into that bundle with:
|
||||
|
||||
```bash
|
||||
@@ -416,12 +439,14 @@ sample, or save-frequency cost counters. Missing, synthetic, stale, tampered,
|
||||
undersized, cross-run, or topology-mismatched evidence returns a compact blocked
|
||||
or invalid JSON result and a nonzero exit.
|
||||
|
||||
The status-and-outcome descriptor producer consumes three measured raw JSON
|
||||
artifacts for G05, G06, and R-D. Those inputs must all carry schema 1, measured
|
||||
evidence, matching `source_revision`, a shared `run_id`, a shared
|
||||
The status-and-outcome raw collector normalizes live observations into the three
|
||||
measured raw JSON artifacts for G05, G06, and R-D. The descriptor producer then
|
||||
consumes those artifacts. The inputs must all carry schema 1, measured evidence,
|
||||
matching `source_revision`, a shared `run_id`, a shared
|
||||
`measurement_window_id`, matching `started_at`/`finished_at` timestamps, and
|
||||
non-empty command provenance. The producer rejects command-line run/window/time
|
||||
overrides that would relabel raw artifacts from another status-and-outcome run.
|
||||
non-empty command provenance. The collector and producer reject synthetic input,
|
||||
missing required cases, and command-line run/window/time overrides that would
|
||||
relabel raw artifacts from another status-and-outcome run.
|
||||
|
||||
The scheduler-pressure lane must also carry the numbers needed to close W09,
|
||||
W10, and W11: bounded deferred item/byte/age limits, zero duplicate tasks,
|
||||
|
||||
@@ -77,7 +77,7 @@
|
||||
|
||||
rustfs = rustPlatform.buildRustPackage {
|
||||
pname = "rustfs";
|
||||
version = "1.0.0-rc.5";
|
||||
version = "1.0.0-rc.6";
|
||||
|
||||
src = ./.;
|
||||
|
||||
|
||||
@@ -2,8 +2,8 @@ apiVersion: v2
|
||||
name: rustfs
|
||||
description: RustFS helm chart to deploy RustFS on kubernetes cluster.
|
||||
type: application
|
||||
version: "1.0.0-rc.5"
|
||||
appVersion: "1.0.0-rc.5"
|
||||
version: "1.0.0-rc.6"
|
||||
appVersion: "1.0.0-rc.6"
|
||||
home: https://rustfs.com
|
||||
icon: https://media.sys.truenas.net/apps/rustfs/icons/icon.svg
|
||||
maintainers:
|
||||
|
||||
+5
-2
@@ -1,9 +1,9 @@
|
||||
%global _enable_debug_packages 0
|
||||
%global _empty_manifest_terminate_build 0
|
||||
%global prerelease rc.5
|
||||
%global prerelease rc.6
|
||||
Name: rustfs
|
||||
Version: 1.0.0
|
||||
Release: rc.5
|
||||
Release: rc.6
|
||||
Summary: High-performance distributed object storage for MinIO alternative
|
||||
|
||||
License: Apache-2.0
|
||||
@@ -58,6 +58,9 @@ install %_builddir/%{name}-%{version}-%{prerelease}/target/%_arch/%_arch-unknown
|
||||
%_bindir/rustfs
|
||||
|
||||
%changelog
|
||||
* Thu Sep 10 2026 overtrue <anzhengchao@gmail.com>
|
||||
- Update RPM package to RustFS 1.0.0-rc.6
|
||||
|
||||
* Mon Aug 31 2026 overtrue <anzhengchao@gmail.com>
|
||||
- Update RPM package to RustFS 1.0.0-rc.5
|
||||
|
||||
|
||||
+168
-21
@@ -22,9 +22,9 @@ use crate::server::rate_limit::{
|
||||
apply_throttle_headers, client_ip,
|
||||
};
|
||||
use crate::server::{
|
||||
APPLE_TOUCH_ICON_PATH, APPLE_TOUCH_ICON_PRECOMPOSED_PATH, CONSOLE_PREFIX, FAVICON_PATH, HEALTH_PREFIX, HEALTH_READY_PATH,
|
||||
HeaderMapCarrier, HealthProbe, LICENSE, RUSTFS_ADMIN_PREFIX, RequestContextLayer, VERSION, build_health_response_parts,
|
||||
collect_probe_readiness,
|
||||
APPLE_TOUCH_ICON_PATH, APPLE_TOUCH_ICON_PRECOMPOSED_PATH, FAVICON_PATH, HEALTH_PREFIX, HEALTH_READY_PATH, HeaderMapCarrier,
|
||||
HealthProbe, LICENSE, RUSTFS_ADMIN_PREFIX, RequestContextLayer, VERSION, build_health_response_parts,
|
||||
collect_probe_readiness, console_prefix,
|
||||
};
|
||||
use crate::version::{self, build};
|
||||
use axum::{
|
||||
@@ -83,7 +83,7 @@ async fn static_handler(uri: Uri) -> impl IntoResponse {
|
||||
return Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header("Content-Type", mime_type.to_string())
|
||||
.body(Body::from(file.data))
|
||||
.body(Body::from(rewrite_console_asset(path, file.data, crate::server::console_prefix())))
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
@@ -95,7 +95,7 @@ async fn static_handler(uri: Uri) -> impl IntoResponse {
|
||||
return Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header("Content-Type", mime_type.to_string())
|
||||
.body(Body::from(file.data))
|
||||
.body(Body::from(rewrite_console_asset(&index_path, file.data, crate::server::console_prefix())))
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
@@ -106,7 +106,11 @@ async fn static_handler(uri: Uri) -> impl IntoResponse {
|
||||
Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header("Content-Type", mime_type.to_string())
|
||||
.body(Body::from(file.data))
|
||||
.body(Body::from(rewrite_console_asset(
|
||||
"index.html",
|
||||
file.data,
|
||||
crate::server::console_prefix(),
|
||||
)))
|
||||
.unwrap()
|
||||
} else {
|
||||
Response::builder()
|
||||
@@ -116,6 +120,57 @@ async fn static_handler(uri: Uri) -> impl IntoResponse {
|
||||
}
|
||||
}
|
||||
|
||||
// Next exports bake the base path into HTML, chunk loaders, and RSC text payloads.
|
||||
// Rewrite only path references, preserving external URLs such as the source repository.
|
||||
fn rewrite_console_asset<'a>(path: &str, data: std::borrow::Cow<'a, [u8]>, prefix: &str) -> std::borrow::Cow<'a, [u8]> {
|
||||
use std::borrow::Cow;
|
||||
|
||||
if prefix == crate::server::CONSOLE_PREFIX
|
||||
|| !matches!(
|
||||
path.rsplit('.').next(),
|
||||
Some("html" | "js" | "css" | "json" | "txt" | "webmanifest" | "svg")
|
||||
)
|
||||
{
|
||||
return data;
|
||||
}
|
||||
let Ok(text) = std::str::from_utf8(&data) else {
|
||||
return data;
|
||||
};
|
||||
let mut rewritten = Cow::Borrowed(text);
|
||||
let escaped_default = crate::server::CONSOLE_PREFIX.replace('/', "\\/");
|
||||
let escaped_prefix = prefix.replace('/', "\\/");
|
||||
for (source, target) in [
|
||||
(crate::server::CONSOLE_PREFIX, prefix),
|
||||
(escaped_default.as_str(), escaped_prefix.as_str()),
|
||||
] {
|
||||
let mut output = String::new();
|
||||
let mut copied = 0;
|
||||
for (offset, _) in rewritten.match_indices(source) {
|
||||
let end = offset + source.len();
|
||||
let before = rewritten.as_bytes().get(offset.wrapping_sub(1)).copied();
|
||||
let after = rewritten.as_bytes().get(end).copied();
|
||||
let starts_path = before
|
||||
.is_none_or(|byte| byte.is_ascii_whitespace() || matches!(byte, b'"' | b'\'' | b'`' | b'(' | b'=' | b'}' | b'>'));
|
||||
let ends_prefix = after.is_none_or(|byte| {
|
||||
byte.is_ascii_whitespace() || matches!(byte, b'/' | b'\\' | b'"' | b'\'' | b'`' | b'?' | b'#' | b')' | b'<')
|
||||
});
|
||||
if starts_path && ends_prefix {
|
||||
output.push_str(&rewritten[copied..offset]);
|
||||
output.push_str(target);
|
||||
copied = end;
|
||||
}
|
||||
}
|
||||
if copied != 0 {
|
||||
output.push_str(&rewritten[copied..]);
|
||||
rewritten = Cow::Owned(output);
|
||||
}
|
||||
}
|
||||
match rewritten {
|
||||
Cow::Borrowed(_) => data,
|
||||
Cow::Owned(text) => Cow::Owned(text.into_bytes()),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Clone)]
|
||||
pub(crate) struct Config {
|
||||
#[serde(skip)]
|
||||
@@ -468,7 +523,7 @@ fn get_console_config_from_env() -> (bool, u32, u64, String) {
|
||||
/// - `true` if the path is for console access, `false` otherwise.
|
||||
pub fn is_console_path(path: &str) -> bool {
|
||||
matches!(path, FAVICON_PATH | APPLE_TOUCH_ICON_PATH | APPLE_TOUCH_ICON_PRECOMPOSED_PATH)
|
||||
|| has_path_prefix(path, CONSOLE_PREFIX)
|
||||
|| has_path_prefix(path, console_prefix())
|
||||
}
|
||||
|
||||
/// Setup comprehensive middleware stack with tower-http features
|
||||
@@ -487,34 +542,35 @@ fn setup_console_middleware_stack(
|
||||
rate_limit_rpm: u32,
|
||||
auth_timeout: u64,
|
||||
) -> Router {
|
||||
let console_prefix = console_prefix();
|
||||
let mut app = Router::new()
|
||||
.route(FAVICON_PATH, get(static_handler))
|
||||
.route(&format!("{CONSOLE_PREFIX}{LICENSE}"), get(license_handler))
|
||||
.route(&format!("{CONSOLE_PREFIX}{VERSION}"), get(version_handler))
|
||||
.nest(CONSOLE_PREFIX, Router::new().fallback_service(get(static_handler)))
|
||||
.route(&format!("{console_prefix}{LICENSE}"), get(license_handler))
|
||||
.route(&format!("{console_prefix}{VERSION}"), get(version_handler))
|
||||
.nest(console_prefix, Router::new().fallback_service(get(static_handler)))
|
||||
.fallback_service(get(static_handler));
|
||||
|
||||
if rustfs_utils::get_env_bool(rustfs_config::ENV_HEALTH_ENDPOINT_ENABLE, rustfs_config::DEFAULT_HEALTH_ENDPOINT_ENABLE) {
|
||||
app = app
|
||||
.route(&format!("{CONSOLE_PREFIX}{HEALTH_PREFIX}"), get(health_check).head(health_check))
|
||||
.route(&format!("{console_prefix}{HEALTH_PREFIX}"), get(health_check).head(health_check))
|
||||
.route(
|
||||
&format!("{CONSOLE_PREFIX}{}", crate::server::HEALTH_COMPAT_LIVE_PATH),
|
||||
&format!("{console_prefix}{}", crate::server::HEALTH_COMPAT_LIVE_PATH),
|
||||
get(health_check).head(health_check),
|
||||
)
|
||||
.route(&format!("{CONSOLE_PREFIX}{HEALTH_READY_PATH}"), get(health_check).head(health_check));
|
||||
.route(&format!("{console_prefix}{HEALTH_READY_PATH}"), get(health_check).head(health_check));
|
||||
} else {
|
||||
// Keep disabled health probes from falling through to the SPA fallback.
|
||||
app = app
|
||||
.route(
|
||||
&format!("{CONSOLE_PREFIX}{HEALTH_PREFIX}"),
|
||||
&format!("{console_prefix}{HEALTH_PREFIX}"),
|
||||
get(health_route_disabled).head(health_route_disabled),
|
||||
)
|
||||
.route(
|
||||
&format!("{CONSOLE_PREFIX}{}", crate::server::HEALTH_COMPAT_LIVE_PATH),
|
||||
&format!("{console_prefix}{}", crate::server::HEALTH_COMPAT_LIVE_PATH),
|
||||
get(health_route_disabled).head(health_route_disabled),
|
||||
)
|
||||
.route(
|
||||
&format!("{CONSOLE_PREFIX}{HEALTH_READY_PATH}"),
|
||||
&format!("{console_prefix}{HEALTH_READY_PATH}"),
|
||||
get(health_route_disabled).head(health_route_disabled),
|
||||
);
|
||||
}
|
||||
@@ -624,7 +680,7 @@ async fn health_check(
|
||||
uri: Uri,
|
||||
server_ctx: Option<Extension<Arc<crate::runtime_sources::ServerContextSlot>>>,
|
||||
) -> Response {
|
||||
let probe = if uri.path().strip_prefix(CONSOLE_PREFIX) == Some(HEALTH_READY_PATH) {
|
||||
let probe = if uri.path().strip_prefix(console_prefix()) == Some(HEALTH_READY_PATH) {
|
||||
HealthProbe::Readiness
|
||||
} else {
|
||||
HealthProbe::Liveness
|
||||
@@ -786,6 +842,7 @@ pub(crate) fn make_console_server() -> Router {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::server::CONSOLE_PREFIX;
|
||||
use axum::body::Body;
|
||||
use axum::routing::get;
|
||||
use http::{Request, StatusCode};
|
||||
@@ -867,9 +924,14 @@ mod tests {
|
||||
async fn console_config_handler_serializes_admin_discovery_paths() {
|
||||
init_console_cfg(IpAddr::V4(Ipv4Addr::LOCALHOST), 9001);
|
||||
|
||||
let response = config_handler(Uri::from_static("http://127.0.0.1:9001/rustfs/console/api/v1/config"), HeaderMap::new())
|
||||
.await
|
||||
.into_response();
|
||||
let response = config_handler(
|
||||
format!("http://127.0.0.1:9001{CONSOLE_PREFIX}/api/v1/config")
|
||||
.parse()
|
||||
.expect("console URI"),
|
||||
HeaderMap::new(),
|
||||
)
|
||||
.await
|
||||
.into_response();
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = response.into_body();
|
||||
@@ -889,7 +951,8 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn external_admin_paths_are_not_console_paths() {
|
||||
assert!(is_console_path("/rustfs/console/"));
|
||||
assert!(is_console_path(&format!("{CONSOLE_PREFIX}/")));
|
||||
assert!(!is_console_path(&format!("{CONSOLE_PREFIX}-other/index.html")));
|
||||
assert!(is_console_path("/apple-touch-icon.png"));
|
||||
assert!(is_console_path("/apple-touch-icon-precomposed.png"));
|
||||
assert!(!is_console_path("/minio/admin/v3/info"));
|
||||
@@ -1273,3 +1336,87 @@ mod tests {
|
||||
assert!(value.get("expired").is_none());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod console_asset_prefix_tests {
|
||||
use super::rewrite_console_asset;
|
||||
use crate::server::CONSOLE_PREFIX;
|
||||
use std::borrow::Cow;
|
||||
|
||||
#[test]
|
||||
fn rewrites_exported_assets_and_client_routes() {
|
||||
let fixtures = [
|
||||
(
|
||||
"index.html",
|
||||
r#"<script src="/rustfs/console/_next/app.js"></script>"#,
|
||||
r#"<script src="/console/_next/app.js"></script>"#,
|
||||
),
|
||||
(
|
||||
"app.js",
|
||||
r#"let base="/rustfs/console";fetch(`${host}/rustfs/console/version`)"#,
|
||||
r#"let base="/console";fetch(`${host}/console/version`)"#,
|
||||
),
|
||||
("app.css", "url(/rustfs/console/logo.svg)", "url(/console/logo.svg)"),
|
||||
(
|
||||
"route.txt",
|
||||
r#"2:I[1,["/rustfs/console/_next/app.js"],"default"]"#,
|
||||
r#"2:I[1,["/console/_next/app.js"],"default"]"#,
|
||||
),
|
||||
("config.json", r#"{"url":"\/rustfs\/console\/login"}"#, r#"{"url":"\/console\/login"}"#),
|
||||
("site.webmanifest", r#"{"start_url":"/rustfs/console/"}"#, r#"{"start_url":"/console/"}"#),
|
||||
(
|
||||
"logo.svg",
|
||||
r#"<image href="/rustfs/console/logo.png"/>"#,
|
||||
r#"<image href="/console/logo.png"/>"#,
|
||||
),
|
||||
];
|
||||
for (path, input, expected) in fixtures {
|
||||
let input = input
|
||||
.replace("/rustfs/console", CONSOLE_PREFIX)
|
||||
.replace("\\/rustfs\\/console", &CONSOLE_PREFIX.replace('/', "\\/"));
|
||||
assert_eq!(
|
||||
rewrite_console_asset(path, Cow::Borrowed(input.as_bytes()), "/console").as_ref(),
|
||||
expected.as_bytes(),
|
||||
"{path}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restores_the_standard_path_from_an_oem_build() {
|
||||
let input = format!(r#"<script src="{CONSOLE_PREFIX}/_next/app.js"></script>"#);
|
||||
let output = rewrite_console_asset("index.html", Cow::Borrowed(input.as_bytes()), rustfs_config::DEFAULT_CONSOLE_PREFIX);
|
||||
assert_eq!(output.as_ref(), br#"<script src="/rustfs/console/_next/app.js"></script>"#);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preserves_unrelated_urls_paths_and_default_bytes() {
|
||||
let input = format!(
|
||||
r#"["https://github.com/rustfs/console","/other{CONSOLE_PREFIX}","{CONSOLE_PREFIX}-extra","{CONSOLE_PREFIX}/index.html"]"#
|
||||
);
|
||||
let expected = format!(
|
||||
r#"["https://github.com/rustfs/console","/other{CONSOLE_PREFIX}","{CONSOLE_PREFIX}-extra","/console/index.html"]"#
|
||||
);
|
||||
assert_eq!(
|
||||
rewrite_console_asset("app.js", Cow::Borrowed(input.as_bytes()), "/console").as_ref(),
|
||||
expected.as_bytes()
|
||||
);
|
||||
let unchanged = rewrite_console_asset("app.js", Cow::Borrowed(input.as_bytes()), CONSOLE_PREFIX);
|
||||
assert!(matches!(unchanged, Cow::Borrowed(_)));
|
||||
assert_eq!(unchanged.as_ref(), input.as_bytes());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preserves_binary_invalid_utf8_and_text_without_paths() {
|
||||
let invalid_utf8 = [b"\xff".as_slice(), CONSOLE_PREFIX.as_bytes()].concat();
|
||||
for (path, bytes) in [
|
||||
("image.png", CONSOLE_PREFIX.as_bytes()),
|
||||
("app.js", invalid_utf8.as_slice()),
|
||||
("app.js", b"https://github.com/rustfs/console".as_slice()),
|
||||
] {
|
||||
let output = rewrite_console_asset(path, Cow::Borrowed(bytes), "/console");
|
||||
assert!(matches!(output, Cow::Borrowed(_)), "{path}");
|
||||
assert_eq!(output.as_ref(), bytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -38,7 +38,7 @@ use super::supervise_admin_mutation;
|
||||
use crate::admin::auth::validate_admin_request;
|
||||
use crate::admin::router::{AdminOperation, Operation, S3Router};
|
||||
use crate::admin::runtime_sources::{current_action_credentials, current_ready_iam_handle, object_store_from_req};
|
||||
use crate::admin::service::caller_identity::CallerIdentity;
|
||||
use crate::admin::service::caller_identity::{CallerIdentity, oidc_profile_fields};
|
||||
use crate::admin::storage_api::s3::{self, Body, S3ErrorCode, S3Request, S3Response, S3Result};
|
||||
use crate::admin::utils::read_compatible_admin_body;
|
||||
use crate::auth::constant_time_eq;
|
||||
@@ -73,6 +73,16 @@ pub fn register_account_route(r: &mut S3Router<AdminOperation>) -> std::io::Resu
|
||||
/// `GET /rustfs/admin/v3/account/info`
|
||||
pub struct SelfAccountInfoHandler {}
|
||||
|
||||
#[derive(Debug, serde::Serialize)]
|
||||
struct SelfAccountInfoResponse {
|
||||
#[serde(flatten)]
|
||||
account: SelfAccountInfo,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
username: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
email: Option<String>,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for SelfAccountInfoHandler {
|
||||
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
@@ -124,17 +134,22 @@ impl Operation for SelfAccountInfoHandler {
|
||||
None => return Err(s3::error(S3ErrorCode::ServiceUnavailable, "the object store is not ready")),
|
||||
};
|
||||
|
||||
let info = SelfAccountInfo {
|
||||
access_key: caller.access_key.clone(),
|
||||
identity_type: caller.identity_type,
|
||||
session_access_key: caller.session_access_key.clone(),
|
||||
is_admin: caller.is_owner,
|
||||
status,
|
||||
member_of,
|
||||
policies,
|
||||
credentials_source: caller.credentials_source,
|
||||
mutable: caller.mutability(),
|
||||
mfa,
|
||||
let (username, email) = oidc_profile_fields(&caller.credentials);
|
||||
let info = SelfAccountInfoResponse {
|
||||
account: SelfAccountInfo {
|
||||
access_key: caller.access_key.clone(),
|
||||
identity_type: caller.identity_type,
|
||||
session_access_key: caller.session_access_key.clone(),
|
||||
is_admin: caller.is_owner,
|
||||
status,
|
||||
member_of,
|
||||
policies,
|
||||
credentials_source: caller.credentials_source,
|
||||
mutable: caller.mutability(),
|
||||
mfa,
|
||||
},
|
||||
username,
|
||||
email,
|
||||
};
|
||||
|
||||
admin_json_response(req.uri.path(), &caller.credentials.secret_key, StatusCode::OK, &info)
|
||||
@@ -548,6 +563,38 @@ fn validate_new_secret_key(request: &ChangePasswordRequest) -> S3Result<()> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::server::ADMIN_PREFIX;
|
||||
use rustfs_madmin::account::{AccountMutability, CredentialsSource};
|
||||
|
||||
#[test]
|
||||
fn self_account_info_response_adds_oidc_display_fields_without_changing_base_type() {
|
||||
let mut response = SelfAccountInfoResponse {
|
||||
account: SelfAccountInfo {
|
||||
access_key: "virtual-parent".to_string(),
|
||||
identity_type: IdentityType::Sts,
|
||||
session_access_key: Some("temporary-key".to_string()),
|
||||
is_admin: false,
|
||||
status: "enabled".to_string(),
|
||||
member_of: Vec::new(),
|
||||
policies: Vec::new(),
|
||||
credentials_source: CredentialsSource::Iam,
|
||||
mutable: AccountMutability::default(),
|
||||
mfa: AccountMfaSummary::default(),
|
||||
},
|
||||
username: Some("oidc-user".to_string()),
|
||||
email: Some("oidc-user@example.test".to_string()),
|
||||
};
|
||||
|
||||
let value = serde_json::to_value(&response).expect("serialize account response");
|
||||
assert_eq!(value["access_key"], "virtual-parent");
|
||||
assert_eq!(value["username"], "oidc-user");
|
||||
assert_eq!(value["email"], "oidc-user@example.test");
|
||||
|
||||
response.username = None;
|
||||
response.email = None;
|
||||
let legacy_shape = serde_json::to_value(&response).expect("serialize account response without OIDC fields");
|
||||
assert!(!legacy_shape.as_object().unwrap().contains_key("username"));
|
||||
assert!(!legacy_shape.as_object().unwrap().contains_key("email"));
|
||||
}
|
||||
|
||||
fn change_request(current: &str, new: &str) -> ChangePasswordRequest {
|
||||
ChangePasswordRequest {
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
use crate::admin::auth::authenticate_request;
|
||||
use crate::admin::router::{AdminOperation, Operation, S3Router};
|
||||
use crate::admin::runtime_sources::{current_action_credentials, object_store_from_req};
|
||||
use crate::admin::service::caller_identity::oidc_profile_fields;
|
||||
use crate::admin::storage_api::bucket::versioning_sys::BucketVersioningSys;
|
||||
use crate::admin::storage_api::contract::admin::StorageAdminApi;
|
||||
use crate::admin::storage_api::contract::bucket::{BucketOperations, BucketOptions};
|
||||
@@ -52,6 +53,16 @@ pub struct AccountInfo {
|
||||
|
||||
pub struct AccountInfoHandler {}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct AccountInfoResponse {
|
||||
#[serde(flatten)]
|
||||
account: rustfs_madmin::AccountInfo,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
username: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
email: Option<String>,
|
||||
}
|
||||
|
||||
pub fn register_account_info_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
|
||||
r.insert(
|
||||
Method::GET,
|
||||
@@ -242,6 +253,7 @@ impl Operation for AccountInfoHandler {
|
||||
let policy_str = serde_json::to_string(&effective_policy)
|
||||
.map_err(|_e| S3Error::with_message(S3ErrorCode::InternalError, "parse policy failed"))?;
|
||||
|
||||
let (username, email) = oidc_profile_fields(&cred);
|
||||
let mut account_info = rustfs_madmin::AccountInfo {
|
||||
account_name,
|
||||
server: StorageAdminApi::backend_info(store.as_ref()).await,
|
||||
@@ -288,8 +300,12 @@ impl Operation for AccountInfoHandler {
|
||||
}
|
||||
}
|
||||
|
||||
let data = serde_json::to_vec(&account_info)
|
||||
.map_err(|_e| S3Error::with_message(S3ErrorCode::InternalError, "parse accountInfo failed"))?;
|
||||
let data = serde_json::to_vec(&AccountInfoResponse {
|
||||
account: account_info,
|
||||
username,
|
||||
email,
|
||||
})
|
||||
.map_err(|_e| S3Error::with_message(S3ErrorCode::InternalError, "parse accountInfo failed"))?;
|
||||
|
||||
let mut header = HeaderMap::new();
|
||||
header.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||
@@ -305,6 +321,25 @@ mod tests {
|
||||
use rustfs_policy::policy::BucketPolicy;
|
||||
use s3s::dto::{Destination, ReplicationRule};
|
||||
|
||||
#[test]
|
||||
fn accountinfo_response_adds_optional_oidc_display_fields() {
|
||||
let mut response = AccountInfoResponse {
|
||||
account: rustfs_madmin::AccountInfo::default(),
|
||||
username: Some("oidc-user".to_string()),
|
||||
email: Some("oidc-user@example.test".to_string()),
|
||||
};
|
||||
|
||||
let value = serde_json::to_value(&response).expect("serialize accountinfo response");
|
||||
assert_eq!(value["username"], "oidc-user");
|
||||
assert_eq!(value["email"], "oidc-user@example.test");
|
||||
|
||||
response.username = None;
|
||||
response.email = None;
|
||||
let legacy_shape = serde_json::to_value(&response).expect("serialize accountinfo response without OIDC fields");
|
||||
assert!(!legacy_shape.as_object().unwrap().contains_key("username"));
|
||||
assert!(!legacy_shape.as_object().unwrap().contains_key("email"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_account_info_structure() {
|
||||
// Test AccountInfo struct creation and serialization
|
||||
|
||||
@@ -24,7 +24,7 @@ use crate::admin::storage_api::config::{
|
||||
read_admin_config_without_migrate, read_admin_server_config_snapshot, save_admin_server_config_snapshot,
|
||||
};
|
||||
use crate::admin::utils::json_response;
|
||||
use crate::server::{ADMIN_PREFIX, CONSOLE_PREFIX, MINIO_ADMIN_PREFIX};
|
||||
use crate::server::{ADMIN_PREFIX, MINIO_ADMIN_PREFIX, console_prefix};
|
||||
use http::StatusCode;
|
||||
use hyper::Method;
|
||||
use matchit::Params;
|
||||
@@ -824,7 +824,8 @@ fn build_console_redirect(
|
||||
let fragment =
|
||||
build_console_callback_fragment(access_key, secret_key, session_token, expiration, redirect_after, logout_token);
|
||||
|
||||
let callback_path = format!("{CONSOLE_PREFIX}{CONSOLE_OIDC_CALLBACK_SUFFIX}");
|
||||
let console_prefix = console_prefix();
|
||||
let callback_path = format!("{console_prefix}{CONSOLE_OIDC_CALLBACK_SUFFIX}");
|
||||
if let Some(base_url) = browser_redirect_url(&callback_path)? {
|
||||
return Ok(format!("{base_url}#{fragment}"));
|
||||
}
|
||||
@@ -836,7 +837,8 @@ fn build_console_redirect(
|
||||
}
|
||||
|
||||
fn build_console_login_redirect(req: &S3Request<Body>) -> S3Result<String> {
|
||||
let login_path = format!("{CONSOLE_PREFIX}{CONSOLE_LOGIN_SUFFIX}");
|
||||
let console_prefix = console_prefix();
|
||||
let login_path = format!("{console_prefix}{CONSOLE_LOGIN_SUFFIX}");
|
||||
if let Some(url) = browser_redirect_url(&login_path)? {
|
||||
return Ok(url);
|
||||
}
|
||||
@@ -1672,6 +1674,26 @@ mod tests {
|
||||
assert_eq!(callback, "https://internal:9000/rustfs/admin/v3/oidc/callback/default");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn console_prefix_process_case_oidc() {
|
||||
if std::env::var_os("RUSTFS_TEST_CONSOLE_PREFIX_PROCESS").is_none() {
|
||||
return;
|
||||
}
|
||||
crate::server::init_console_prefix().expect("initialize console prefix");
|
||||
let prefix = console_prefix();
|
||||
let req = build_oidc_request("http://internal/rustfs/admin/v3/oidc/callback/default", Some("internal:9000"), None);
|
||||
assert_eq!(
|
||||
build_console_login_redirect(&req).expect("login URL"),
|
||||
format!("https://console.example.com{prefix}/auth/login")
|
||||
);
|
||||
let redirect = build_console_redirect(&req, "access", "secret", "token", None, None, None).expect("console callback URL");
|
||||
assert!(redirect.starts_with(&format!("https://console.example.com{prefix}/auth/oidc-callback/#")));
|
||||
assert_eq!(
|
||||
derive_callback_uri(&req, "default").expect("admin callback URL"),
|
||||
"https://console.example.com/rustfs/admin/v3/oidc/callback/default"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_console_redirect_uses_browser_redirect_url() {
|
||||
let req = build_oidc_request("http://internal/rustfs/admin/v3/oidc/callback/default", Some("internal:9000"), None);
|
||||
@@ -1681,7 +1703,7 @@ mod tests {
|
||||
.expect("console redirect should use browser redirect URL")
|
||||
});
|
||||
|
||||
assert!(redirect.starts_with("https://console.example.com/rustfs/console/auth/oidc-callback/#"));
|
||||
assert!(redirect.starts_with(&format!("https://console.example.com{}/auth/oidc-callback/#", console_prefix())));
|
||||
assert!(redirect.contains("redirect=%2Fbuckets"));
|
||||
assert!(redirect.contains("logoutToken=logout-token"));
|
||||
}
|
||||
@@ -1694,7 +1716,7 @@ mod tests {
|
||||
build_console_login_redirect(&req).expect("login redirect should use browser redirect URL")
|
||||
});
|
||||
|
||||
assert_eq!(redirect, "https://console.example.com/rustfs/console/auth/login");
|
||||
assert_eq!(redirect, format!("https://console.example.com{}/auth/login", console_prefix()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -44,7 +44,8 @@ use crate::admin::utils::{empty_response, json_response, read_compatible_admin_b
|
||||
use crate::error::ApiError;
|
||||
use crate::server::ADMIN_PREFIX;
|
||||
use crate::site_replication::identity::{
|
||||
canonical_endpoint, is_https_endpoint, mark_unknown_peer_sync_enabled, same_identity_endpoint, site_identity_key,
|
||||
canonical_endpoint, deployment_id_for_endpoint, is_https_endpoint, mark_unknown_peer_sync_enabled, same_identity_endpoint,
|
||||
site_identity_key,
|
||||
};
|
||||
use crate::storage::storage_api::{lock_bucket_targets_metadata, with_config_object_write_lock};
|
||||
use base64_simd::URL_SAFE_NO_PAD;
|
||||
@@ -312,6 +313,8 @@ struct SRPeerJoinResponse {
|
||||
peer: PeerInfo,
|
||||
#[serde(rename = "initialSyncErrorMessage", default, skip_serializing_if = "String::is_empty")]
|
||||
initial_sync_error_message: String,
|
||||
#[serde(rename = "initialSyncDeferred", default, skip_serializing_if = "std::ops::Not::not")]
|
||||
initial_sync_deferred: bool,
|
||||
/// Whether the receiving site actually applied this join.
|
||||
///
|
||||
/// Three-valued on purpose. `None` means the peer did not report — MinIO
|
||||
@@ -330,6 +333,8 @@ struct SRPeerJoinEnvelope {
|
||||
request: SRPeerJoinReq,
|
||||
#[serde(rename = "deferSyncStateEnable", default, skip_serializing_if = "std::ops::Not::not")]
|
||||
defer_sync_state_enable: bool,
|
||||
#[serde(rename = "deferInitialSync", default, skip_serializing_if = "std::ops::Not::not")]
|
||||
defer_initial_sync: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
@@ -2792,6 +2797,19 @@ fn prune_in_sync_status_details(status: &mut SRStatusInfo, opts: &SRStatusOption
|
||||
}
|
||||
}
|
||||
|
||||
fn peer_states_from_infos(
|
||||
site_infos: BTreeMap<String, SRInfo>,
|
||||
reachable_peers: &HashSet<String>,
|
||||
) -> BTreeMap<String, SRStateInfo> {
|
||||
// Failed metainfo fetches leave default entries in site_infos for comparison;
|
||||
// they must not become fabricated peer state. PeerErrors describes the failure.
|
||||
site_infos
|
||||
.into_iter()
|
||||
.filter(|(deployment_id, _)| reachable_peers.contains(deployment_id))
|
||||
.map(|(deployment_id, info)| (deployment_id, info.state))
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn build_status_info(state: &SiteReplicationState, local_peer: &PeerInfo, uri: &Uri) -> S3Result<SRStatusInfo> {
|
||||
let opts = sr_status_options(uri);
|
||||
let mut local_info = Some(filter_sr_info(build_sr_info(state, local_peer).await?, &opts));
|
||||
@@ -2920,17 +2938,7 @@ async fn build_status_info(state: &SiteReplicationState, local_peer: &PeerInfo,
|
||||
}
|
||||
|
||||
if opts.peer_state {
|
||||
for (deployment_id, peer) in &state.peers {
|
||||
status.peer_states.insert(
|
||||
deployment_id.clone(),
|
||||
SRStateInfo {
|
||||
name: peer.name.clone(),
|
||||
peers: state.peers.clone(),
|
||||
updated_at: state.updated_at,
|
||||
api_version: Some(SITE_REPL_API_VERSION.to_string()),
|
||||
},
|
||||
);
|
||||
}
|
||||
status.peer_states = peer_states_from_infos(site_infos, &reachable_peers);
|
||||
}
|
||||
|
||||
Ok(status)
|
||||
@@ -2940,6 +2948,7 @@ fn merge_add_sites(
|
||||
mut state: SiteReplicationState,
|
||||
local_peer: PeerInfo,
|
||||
sites: Vec<PeerSite>,
|
||||
preflight_infos: &[SiteReplicationAddPreflightInfo],
|
||||
service_account_access_key: String,
|
||||
service_account_parent: String,
|
||||
replicate_ilm_expiry: bool,
|
||||
@@ -2949,11 +2958,36 @@ fn merge_add_sites(
|
||||
state.service_account_parent = service_account_parent;
|
||||
state.updated_at = Some(OffsetDateTime::now_utc());
|
||||
state.peers = build_join_peers(&state, &local_peer, sites, replicate_ilm_expiry);
|
||||
// Every join must carry the verified identities, including peers that
|
||||
// have not joined yet. Fixing only the coordinator after each reply
|
||||
// leaves the other sites holding endpoint-derived placeholders.
|
||||
for info in preflight_infos {
|
||||
if let Some(mut peer) = existing_peer_for_endpoint(&state, &info.endpoint) {
|
||||
peer.deployment_id = info.deployment_id.clone();
|
||||
state = reconcile_peer_with_actual_identity(state, peer);
|
||||
}
|
||||
}
|
||||
state
|
||||
}
|
||||
|
||||
fn update_peer(mut state: SiteReplicationState, incoming: PeerInfo, ilm_expiry_override: Option<bool>) -> SiteReplicationState {
|
||||
let mut peer = normalize_peer_info(incoming);
|
||||
// An older sender may still hold a placeholder after this site has
|
||||
// learned the real ID. Do not let that delivery downgrade the identity.
|
||||
if peer.deployment_id == deployment_id_for_endpoint(&peer.endpoint)
|
||||
&& let Some(existing) = state.peers.values().find(|existing| {
|
||||
same_identity_endpoint(&existing.endpoint, &peer.endpoint)
|
||||
&& existing.deployment_id != deployment_id_for_endpoint(&existing.endpoint)
|
||||
})
|
||||
{
|
||||
peer.deployment_id = existing.deployment_id.clone();
|
||||
}
|
||||
// Remove the placeholder before persistence normalizes duplicate
|
||||
// endpoints; otherwise map ordering can discard the real identity.
|
||||
state.peers.retain(|_, existing| {
|
||||
!same_identity_endpoint(&existing.endpoint, &peer.endpoint)
|
||||
|| existing.deployment_id != deployment_id_for_endpoint(&existing.endpoint)
|
||||
});
|
||||
if let Some(enabled) = ilm_expiry_override {
|
||||
peer.replicate_ilm_expiry = enabled;
|
||||
}
|
||||
@@ -3539,6 +3573,13 @@ fn align_peer_edit_deployment_id(state: &SiteReplicationState, incoming: &mut Pe
|
||||
return;
|
||||
};
|
||||
if matches.next().is_none() {
|
||||
if same_identity_endpoint(&peer.endpoint, &incoming.endpoint)
|
||||
&& peer.deployment_id == deployment_id_for_endpoint(&peer.endpoint)
|
||||
&& !incoming.deployment_id.is_empty()
|
||||
&& incoming.deployment_id != deployment_id_for_endpoint(&incoming.endpoint)
|
||||
{
|
||||
return;
|
||||
}
|
||||
incoming.deployment_id = peer.deployment_id.clone();
|
||||
}
|
||||
}
|
||||
@@ -3853,6 +3894,30 @@ fn pending_remote_peer_ids(peers: &BTreeMap<String, PeerInfo>, local_peer: &Peer
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The peers a pending remove / rotation still has to notify: every remote
|
||||
/// peer that has not acked, with the local site excluded by the same
|
||||
/// deployment-id-or-endpoint identity [`pending_remote_peer_ids`] finalizes
|
||||
/// on. The tick-driven `local_peer` carries the node's own listen address
|
||||
/// rather than the registered site endpoint (and a handler's carries the
|
||||
/// request `Host`, which behind a load balancer differs too), so an
|
||||
/// endpoint-only check dialed the site itself, timed out against the
|
||||
/// lifecycle lock this very request holds, and reported the operation as
|
||||
/// `Partial` (backlog#2367 A-4).
|
||||
fn pending_peers_awaiting_notification<'a>(
|
||||
peers: &'a BTreeMap<String, PeerInfo>,
|
||||
local_peer: &PeerInfo,
|
||||
acked_deployment_ids: &BTreeSet<String>,
|
||||
) -> Vec<&'a PeerInfo> {
|
||||
peers
|
||||
.values()
|
||||
.filter(|peer| {
|
||||
peer.deployment_id != local_peer.deployment_id
|
||||
&& !same_identity_endpoint(&peer.endpoint, &local_peer.endpoint)
|
||||
&& !acked_deployment_ids.contains(&peer.deployment_id)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn pending_all_remote_peers_acked(
|
||||
peers: &BTreeMap<String, PeerInfo>,
|
||||
local_peer: &PeerInfo,
|
||||
@@ -4068,12 +4133,7 @@ async fn drive_pending_rotation(pending: &PendingRotation, local_peer: &PeerInfo
|
||||
};
|
||||
|
||||
let mut peer_errors = Vec::new();
|
||||
for peer in pending.peers.values() {
|
||||
if same_identity_endpoint(&peer.endpoint, &local_peer.endpoint)
|
||||
|| pending.acked_deployment_ids.contains(&peer.deployment_id)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
for peer in pending_peers_awaiting_notification(&pending.peers, local_peer, &pending.acked_deployment_ids) {
|
||||
// A superseded join returns BEFORE `apply_iam`, so a no-op answer
|
||||
// means the peer never installed the new secret. Acking it would
|
||||
// finalize a rotation half the mesh cannot authenticate against
|
||||
@@ -4376,12 +4436,9 @@ async fn drive_pending_remove(pending_remove: &PendingRemove, local_peer: &PeerI
|
||||
if secret_candidates.is_empty() {
|
||||
peer_errors.push("site replication service account secret unavailable".to_string());
|
||||
} else {
|
||||
for peer in pending_remove.original_peers.values() {
|
||||
if same_identity_endpoint(&peer.endpoint, &local_peer.endpoint)
|
||||
|| pending_remove.acked_deployment_ids.contains(&peer.deployment_id)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
for peer in
|
||||
pending_peers_awaiting_notification(&pending_remove.original_peers, local_peer, &pending_remove.acked_deployment_ids)
|
||||
{
|
||||
if let Err(err) = PeerAdminRequest::put(
|
||||
&runtime_peer_connection(peer)?,
|
||||
SITE_REPLICATION_PEER_REMOVE_PATH,
|
||||
@@ -4931,7 +4988,7 @@ async fn ensure_site_replication_bucket_targets(bucket: &str) -> S3Result<()> {
|
||||
return Ok(());
|
||||
};
|
||||
let config = bucket_replication_config_for_target_refresh(bucket).await?;
|
||||
ensure_site_replication_bucket_targets_with_runtime(
|
||||
let written = ensure_site_replication_bucket_targets_with_runtime(
|
||||
bucket,
|
||||
&runtime.state,
|
||||
&runtime.local_peer,
|
||||
@@ -4939,7 +4996,11 @@ async fn ensure_site_replication_bucket_targets(bucket: &str) -> S3Result<()> {
|
||||
&runtime.service_account_secret_key,
|
||||
expected_incarnation_id,
|
||||
)
|
||||
.await
|
||||
.await?;
|
||||
if written {
|
||||
reload_bucket_metadata_on_peers(bucket, "site_replication_bucket_targets", false).await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn ensure_site_replication_bucket_setup(bucket: &str) -> S3Result<bool> {
|
||||
@@ -5009,6 +5070,9 @@ async fn cleanup_removed_site_replication_bucket(bucket: &str, removed_deploymen
|
||||
Err(err) => return Err(ApiError::from(err).into()),
|
||||
}
|
||||
|
||||
if removed > 0 {
|
||||
reload_bucket_metadata_on_peers(bucket, "site_replication_bucket_cleanup", true).await;
|
||||
}
|
||||
Ok(removed)
|
||||
}
|
||||
|
||||
@@ -5325,7 +5389,7 @@ async fn refresh_bucket_targets_after_endpoint_edit(pending_id: &str, service_ac
|
||||
let local_peer = current_local_runtime_peer(&target_state);
|
||||
let _targets_guard = lock_bucket_targets_metadata(&bucket.name).await;
|
||||
let replication_config = bucket_replication_config_for_target_refresh(&bucket.name).await?;
|
||||
ensure_site_replication_bucket_targets_with_runtime(
|
||||
let written = ensure_site_replication_bucket_targets_with_runtime(
|
||||
&bucket.name,
|
||||
&target_state,
|
||||
&local_peer,
|
||||
@@ -5334,6 +5398,9 @@ async fn refresh_bucket_targets_after_endpoint_edit(pending_id: &str, service_ac
|
||||
expected_incarnation_id,
|
||||
)
|
||||
.await?;
|
||||
if written {
|
||||
reload_bucket_metadata_on_peers(&bucket.name, "site_replication_endpoint_refresh", false).await;
|
||||
}
|
||||
|
||||
rewritten.push(bucket.name.clone());
|
||||
|
||||
@@ -5377,16 +5444,12 @@ async fn site_bucket_resync_manifest_entry(bucket: &str, peer: &PeerInfo, now: O
|
||||
..Default::default()
|
||||
};
|
||||
let _targets_guard = lock_bucket_targets_metadata(bucket).await;
|
||||
let (config, _) = match metadata_sys::get_replication_config(bucket).await {
|
||||
Ok(config) => config,
|
||||
Err(err) => {
|
||||
entry.status = "failed".to_string();
|
||||
entry.err_detail = summarize_peer_error_detail(&err.to_string());
|
||||
return entry;
|
||||
}
|
||||
};
|
||||
let targets = match metadata_sys::list_bucket_targets(bucket).await {
|
||||
Ok(targets) => targets,
|
||||
// Read what is persisted, not this node's cache: the wiring may have
|
||||
// been written by another node moments ago (`start_site_bucket_resync`
|
||||
// already reads its targets from disk), and an operator resync must see
|
||||
// the same records the drive will use.
|
||||
let (config, targets) = match site_bucket_resync_persisted_wiring(bucket).await {
|
||||
Ok(wiring) => wiring,
|
||||
Err(err) => {
|
||||
entry.status = "failed".to_string();
|
||||
entry.err_detail = summarize_peer_error_detail(&err.to_string());
|
||||
@@ -5417,6 +5480,15 @@ async fn site_bucket_resync_manifest_entry(bucket: &str, peer: &PeerInfo, now: O
|
||||
entry
|
||||
}
|
||||
|
||||
/// The persisted replication configuration and bucket targets, bypassing the
|
||||
/// node-local metadata cache. `ConfigNotFound` surfaces for a bucket without
|
||||
/// a replication configuration, matching the cached read's error.
|
||||
async fn site_bucket_resync_persisted_wiring(bucket: &str) -> Result<(ReplicationConfiguration, BucketTargets), StorageError> {
|
||||
let metadata = metadata_sys::get_config_from_disk(bucket).await?;
|
||||
let config = metadata.replication_config.ok_or(StorageError::ConfigNotFound)?;
|
||||
Ok((config, metadata.bucket_target_config.unwrap_or_default()))
|
||||
}
|
||||
|
||||
async fn start_site_bucket_resync(bucket: &str, target_arn: &str, resync_id: &str) -> ResyncBucketStatus {
|
||||
let mut bucket_status = ResyncBucketStatus {
|
||||
bucket: bucket.to_string(),
|
||||
@@ -5439,17 +5511,8 @@ async fn start_site_bucket_resync(bucket: &str, target_arn: &str, resync_id: &st
|
||||
}
|
||||
};
|
||||
|
||||
let (config, _) = match metadata_sys::get_replication_config(bucket).await {
|
||||
Ok(config) => config,
|
||||
Err(err) => {
|
||||
bucket_status.status = "failed".to_string();
|
||||
bucket_status.err_detail = err.to_string();
|
||||
return bucket_status;
|
||||
}
|
||||
};
|
||||
|
||||
let targets = match metadata_sys::list_bucket_targets_from_disk(bucket).await {
|
||||
Ok(targets) => targets,
|
||||
let (config, targets) = match site_bucket_resync_persisted_wiring(bucket).await {
|
||||
Ok(wiring) => wiring,
|
||||
Err(err) => {
|
||||
bucket_status.status = "failed".to_string();
|
||||
bucket_status.err_detail = err.to_string();
|
||||
@@ -6046,6 +6109,10 @@ async fn apply_bucket_meta_item(item: SRBucketMeta) -> S3Result<()> {
|
||||
drop(lifecycle_guard);
|
||||
drop(targets_guard);
|
||||
|
||||
if !skip_config_write {
|
||||
reload_bucket_metadata_on_peers(&item.bucket, "site_replication_bucket_meta", item.r#type == "lc-config").await;
|
||||
}
|
||||
|
||||
if item.r#type == "replication-config" {
|
||||
// Rebuild the local outbound rules too: a site that joined an already-replicated
|
||||
// bucket receives this item before it has any `site-repl-*` rule of its own.
|
||||
@@ -6684,6 +6751,7 @@ fn parse_peer_join_response(body: &[u8], fallback_peer: PeerInfo) -> Result<SRPe
|
||||
return Ok(SRPeerJoinResponse {
|
||||
peer: fallback_peer,
|
||||
initial_sync_error_message: String::new(),
|
||||
initial_sync_deferred: false,
|
||||
applied: None,
|
||||
});
|
||||
}
|
||||
@@ -6775,6 +6843,7 @@ impl Operation for SiteReplicationAddHandler {
|
||||
current_state,
|
||||
local_peer.clone(),
|
||||
sites.clone(),
|
||||
&preflight_infos,
|
||||
service_account_access_key.clone(),
|
||||
admin_access_key,
|
||||
replicate_ilm_expiry,
|
||||
@@ -6789,66 +6858,86 @@ impl Operation for SiteReplicationAddHandler {
|
||||
updated_at: state.updated_at,
|
||||
},
|
||||
defer_sync_state_enable: true,
|
||||
defer_initial_sync: true,
|
||||
};
|
||||
let peer_join_path = with_site_replication_bootstrap_token(
|
||||
SITE_REPLICATION_PEER_JOIN_PATH,
|
||||
&add_in_progress_guard.token.to_string(),
|
||||
);
|
||||
|
||||
let mut joined_endpoints = HashSet::new();
|
||||
// Install every site's service account before any receiver probes or
|
||||
// backfills to a peer that may not have joined yet. Reuse the join
|
||||
// snapshot for the second pass without repeating IAM/topology writes.
|
||||
// Only peers acknowledging deferral get a second request; older
|
||||
// receivers retain their one-pass behavior and reported errors.
|
||||
let initial_sync_path = format!("{peer_join_path}&initial-sync=true");
|
||||
let mut initial_sync_errors = SiteReplicationErrorSummary::default();
|
||||
for (site, preflight) in sites.iter().zip(preflight_infos.iter()) {
|
||||
if same_identity_endpoint(&site.endpoint, &local_peer.endpoint)
|
||||
|| !joined_endpoints.insert(site_identity_key(&site.endpoint))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let mut deferred_endpoints = HashSet::new();
|
||||
for (path, defer_initial_sync) in [(&peer_join_path, true), (&initial_sync_path, false)] {
|
||||
let mut joined_endpoints = HashSet::new();
|
||||
for (site, preflight) in sites.iter().zip(preflight_infos.iter()) {
|
||||
if same_identity_endpoint(&site.endpoint, &local_peer.endpoint)
|
||||
|| (!defer_initial_sync && !deferred_endpoints.contains(&site_identity_key(&site.endpoint)))
|
||||
|| !joined_endpoints.insert(site_identity_key(&site.endpoint))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut peer_join_req = join_req.clone();
|
||||
peer_join_req.request.svc_acct_parent = site.access_key.clone();
|
||||
let connection = PeerConnection::try_from(site)?;
|
||||
let body = PeerAdminRequest::put(&connection, &peer_join_path, &site.access_key)
|
||||
.send(&site.secret_key, &peer_join_req)
|
||||
.await?;
|
||||
let mut peer_join_req = join_req.clone();
|
||||
peer_join_req.defer_initial_sync = defer_initial_sync;
|
||||
peer_join_req.request.svc_acct_parent = site.access_key.clone();
|
||||
let connection = PeerConnection::try_from(site)?;
|
||||
let body = PeerAdminRequest::put(&connection, path, &site.access_key)
|
||||
.send(&site.secret_key, &peer_join_req)
|
||||
.await?;
|
||||
|
||||
let mut fallback_peer = existing_peer_for_endpoint(&state, &site.endpoint)
|
||||
.unwrap_or_else(|| normalize_peer_site(site.clone(), replicate_ilm_expiry));
|
||||
fallback_peer.deployment_id = preflight.deployment_id.clone();
|
||||
let join_response = parse_peer_join_response(&body, fallback_peer).map_err(|e| {
|
||||
S3Error::with_message(
|
||||
S3ErrorCode::InternalError,
|
||||
format!("parse peer join response from {} failed: {e}", site.endpoint),
|
||||
)
|
||||
})?;
|
||||
if !join_response.initial_sync_error_message.is_empty() {
|
||||
initial_sync_errors.push(format!("{}: {}", site.endpoint, join_response.initial_sync_error_message));
|
||||
let mut fallback_peer = existing_peer_for_endpoint(&state, &site.endpoint)
|
||||
.unwrap_or_else(|| normalize_peer_site(site.clone(), replicate_ilm_expiry));
|
||||
fallback_peer.deployment_id = preflight.deployment_id.clone();
|
||||
let join_response = parse_peer_join_response(&body, fallback_peer).map_err(|e| {
|
||||
S3Error::with_message(
|
||||
S3ErrorCode::InternalError,
|
||||
format!("parse peer join response from {} failed: {e}", site.endpoint),
|
||||
)
|
||||
})?;
|
||||
if join_response.initial_sync_deferred {
|
||||
if defer_initial_sync {
|
||||
deferred_endpoints.insert(site_identity_key(&site.endpoint));
|
||||
} else {
|
||||
initial_sync_errors.push(format!("{}: peer did not complete initial sync", site.endpoint));
|
||||
}
|
||||
}
|
||||
if !join_response.initial_sync_error_message.is_empty() {
|
||||
initial_sync_errors.push(format!("{}: {}", site.endpoint, join_response.initial_sync_error_message));
|
||||
}
|
||||
// An explicit no-op join. The peer answered 200 but wrote nothing —
|
||||
// its persisted state is already newer than the snapshot it was
|
||||
// sent — so the add is only PARTIALLY configured and saying
|
||||
// "configured successfully" would be a lie (rustfs/rustfs#5963).
|
||||
// `None` (a MinIO peer, or one older than the field) is not a
|
||||
// no-op signal and is deliberately not reported.
|
||||
if join_response.applied == Some(false) {
|
||||
let phase = if defer_initial_sync { "join" } else { "initial sync" };
|
||||
initial_sync_errors.push(format!(
|
||||
"{}: peer did not apply the {phase} (its site replication state is newer than the snapshot it was sent); \
|
||||
the site is not configured against this peer",
|
||||
site.endpoint
|
||||
));
|
||||
}
|
||||
state = reconcile_peer_with_actual_identity(state, join_response.peer);
|
||||
let reconciled_peer = existing_peer_for_endpoint(&state, &site.endpoint).ok_or_else(|| {
|
||||
S3Error::with_message(
|
||||
S3ErrorCode::InternalError,
|
||||
format!("peer join response from {} did not identify the requested site", site.endpoint),
|
||||
)
|
||||
})?;
|
||||
validate_proposed_peer(&reconciled_peer).map_err(|err| {
|
||||
S3Error::with_message(
|
||||
S3ErrorCode::InvalidRequest,
|
||||
format!("invalid peer join response from {}: {err}", site.endpoint),
|
||||
)
|
||||
})?;
|
||||
}
|
||||
// An explicit no-op join. The peer answered 200 but wrote nothing —
|
||||
// its persisted state is already newer than the snapshot it was
|
||||
// sent — so the add is only PARTIALLY configured and saying
|
||||
// "configured successfully" would be a lie (rustfs/rustfs#5963).
|
||||
// `None` (a MinIO peer, or one older than the field) is not a
|
||||
// no-op signal and is deliberately not reported.
|
||||
if join_response.applied == Some(false) {
|
||||
initial_sync_errors.push(format!(
|
||||
"{}: peer did not apply the join (its site replication state is newer than the snapshot it was sent); \
|
||||
the site is not configured against this peer",
|
||||
site.endpoint
|
||||
));
|
||||
}
|
||||
state = reconcile_peer_with_actual_identity(state, join_response.peer);
|
||||
let reconciled_peer = existing_peer_for_endpoint(&state, &site.endpoint).ok_or_else(|| {
|
||||
S3Error::with_message(
|
||||
S3ErrorCode::InternalError,
|
||||
format!("peer join response from {} did not identify the requested site", site.endpoint),
|
||||
)
|
||||
})?;
|
||||
validate_proposed_peer(&reconciled_peer).map_err(|err| {
|
||||
S3Error::with_message(
|
||||
S3ErrorCode::InvalidRequest,
|
||||
format!("invalid peer join response from {}: {err}", site.endpoint),
|
||||
)
|
||||
})?;
|
||||
}
|
||||
|
||||
mark_unknown_peer_sync_enabled(&mut state.peers);
|
||||
@@ -7122,6 +7211,24 @@ impl Operation for SiteReplicationNetPerfHandler {
|
||||
|
||||
pub struct SRPeerJoinHandler {}
|
||||
|
||||
fn ensure_initial_sync_join_current(state: &SiteReplicationState, join_req: &SRPeerJoinReq) -> S3Result<()> {
|
||||
if !state.enabled()
|
||||
|| join_req.updated_at.is_none()
|
||||
|| state.updated_at != join_req.updated_at
|
||||
|| state.service_account_access_key.is_empty()
|
||||
|| state.service_account_access_key != join_req.svc_acct_access_key
|
||||
|| state.pending_remove.is_some()
|
||||
|| state.pending_rotation.is_some()
|
||||
|| pending_endpoint_refresh(state).is_some()
|
||||
{
|
||||
return Err(s3_error!(
|
||||
InvalidRequest,
|
||||
"site replication changed before initial sync; re-run replicate add"
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// What the join admission decided about an incoming peer join. The verdict —
|
||||
/// and the committed state the back-fill afterwards needs — travel out of
|
||||
/// [`admit_peer_join`] instead of being answered where they are decided.
|
||||
@@ -7284,6 +7391,7 @@ fn superseded_join_response(peer: PeerInfo) -> SRPeerJoinResponse {
|
||||
SRPeerJoinResponse {
|
||||
peer,
|
||||
initial_sync_error_message: String::new(),
|
||||
initial_sync_deferred: false,
|
||||
applied: Some(false),
|
||||
}
|
||||
}
|
||||
@@ -7293,6 +7401,7 @@ fn applied_join_response(peer: PeerInfo, initial_sync_error_message: String) ->
|
||||
SRPeerJoinResponse {
|
||||
peer,
|
||||
initial_sync_error_message,
|
||||
initial_sync_deferred: false,
|
||||
applied: Some(true),
|
||||
}
|
||||
}
|
||||
@@ -7302,17 +7411,30 @@ impl Operation for SRPeerJoinHandler {
|
||||
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
let cred = validate_site_replication_admin_request(&req, AdminAction::SiteReplicationAddAction).await?;
|
||||
let bootstrap_token = site_replication_bootstrap_token(&req.uri);
|
||||
let initial_sync_only = query_pairs(&req.uri).get("initial-sync").is_some_and(|value| value == "true");
|
||||
let local_endpoint = site_replication_local_endpoint(&req.uri, &req.headers);
|
||||
// The body is fully read before the admission takes the lifecycle
|
||||
// guard: a sender that stalls mid-body must not block this node's
|
||||
// add/remove/rotate/reconciler.
|
||||
let join_envelope: SRPeerJoinEnvelope = read_site_replication_json(req, &cred.secret_key, true).await?;
|
||||
let defer_sync_state_enable = join_envelope.defer_sync_state_enable;
|
||||
let defer_initial_sync = join_envelope.defer_initial_sync;
|
||||
let join_req = join_envelope.request;
|
||||
validate_join_peer_snapshot(&join_req.peers)?;
|
||||
|
||||
let committed =
|
||||
admit_peer_join(local_endpoint, join_req, defer_sync_state_enable, apply_peer_join_service_account).await?;
|
||||
let _initial_sync_guard = if initial_sync_only {
|
||||
Some(SiteReplicationLifecycleGuard::acquire().await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let committed = if initial_sync_only {
|
||||
let state = load_site_replication_state().await?;
|
||||
ensure_initial_sync_join_current(&state, &join_req)?;
|
||||
let local_peer = local_peer_at_endpoint(local_endpoint, &state);
|
||||
PeerJoinOutcome::Applied(Box::new(state), local_peer)
|
||||
} else {
|
||||
admit_peer_join(local_endpoint, join_req, defer_sync_state_enable, apply_peer_join_service_account).await?
|
||||
};
|
||||
// Committed; the reverse-reachability probe and the bucket back-fill
|
||||
// run outside the transaction — their transport helpers' retry-event
|
||||
// bookkeeping re-enters it (P1-15).
|
||||
@@ -7329,6 +7451,12 @@ impl Operation for SRPeerJoinHandler {
|
||||
return json_response(StatusCode::OK, &superseded_join_response(peer));
|
||||
}
|
||||
};
|
||||
if defer_initial_sync && !initial_sync_only {
|
||||
let mut response =
|
||||
applied_join_response(state.peers.get(&local_peer.deployment_id).cloned().unwrap_or(local_peer), String::new());
|
||||
response.initial_sync_deferred = true;
|
||||
return json_response(StatusCode::OK, &response);
|
||||
}
|
||||
// Fix 1 (receiving side): ensure the joining peer also sets up replication for any
|
||||
// buckets it already owns so the reverse direction works from the start. Per-bucket
|
||||
// failures are logged (BUG2) so a reverse-direction back-fill gap is observable.
|
||||
@@ -7432,6 +7560,7 @@ impl Operation for SRPeerBucketOpsHandler {
|
||||
)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
reload_bucket_metadata_on_peers(&bucket, "site_replication_make_bucket", false).await;
|
||||
}
|
||||
"configure-replication" => {
|
||||
store
|
||||
@@ -8455,9 +8584,92 @@ impl Operation for SRRotateServiceAccountHandler {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::site_replication::identity::deployment_id_for_endpoint;
|
||||
use rustfs_madmin::SRSessionPolicy;
|
||||
|
||||
#[test]
|
||||
fn peer_states_preserve_each_sites_actual_membership_and_metadata() {
|
||||
let local = SRStateInfo {
|
||||
name: "local".to_string(),
|
||||
peers: BTreeMap::from([(
|
||||
"actual-remote".to_string(),
|
||||
PeerInfo {
|
||||
deployment_id: "actual-remote".to_string(),
|
||||
endpoint: "http://remote:9000".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
)]),
|
||||
updated_at: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
api_version: Some("1".to_string()),
|
||||
};
|
||||
let remote = SRStateInfo {
|
||||
name: "remote-reported-name".to_string(),
|
||||
peers: BTreeMap::from([(
|
||||
"legacy-placeholder".to_string(),
|
||||
PeerInfo {
|
||||
deployment_id: "legacy-placeholder".to_string(),
|
||||
endpoint: "http://local:9000".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
)]),
|
||||
updated_at: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(10)),
|
||||
api_version: None,
|
||||
};
|
||||
let infos = BTreeMap::from([
|
||||
(
|
||||
"local".to_string(),
|
||||
SRInfo {
|
||||
state: local.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
(
|
||||
"remote".to_string(),
|
||||
SRInfo {
|
||||
state: remote.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
]);
|
||||
let states = peer_states_from_infos(infos, &HashSet::from(["local".to_string(), "remote".to_string()]));
|
||||
assert_eq!(states.len(), 2);
|
||||
assert_eq!(serde_json::to_value(&states["local"]).unwrap(), serde_json::to_value(local).unwrap());
|
||||
assert_eq!(serde_json::to_value(&states["remote"]).unwrap(), serde_json::to_value(remote).unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_states_omit_unreachable_peers_instead_of_defaulting_them() {
|
||||
let infos = BTreeMap::from([
|
||||
("local".to_string(), SRInfo::default()),
|
||||
("offline".to_string(), SRInfo::default()),
|
||||
]);
|
||||
let states = peer_states_from_infos(infos, &HashSet::from(["local".to_string()]));
|
||||
assert_eq!(states.len(), 1);
|
||||
assert!(states.contains_key("local"));
|
||||
assert!(!states.contains_key("offline"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_states_preserve_a_reachable_peers_empty_membership() {
|
||||
let states = peer_states_from_infos(
|
||||
BTreeMap::from([(
|
||||
"remote".to_string(),
|
||||
SRInfo {
|
||||
enabled: false,
|
||||
state: SRStateInfo {
|
||||
name: "remote".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
},
|
||||
)]),
|
||||
&HashSet::from(["remote".to_string()]),
|
||||
);
|
||||
assert_eq!(states["remote"].name, "remote");
|
||||
assert!(states["remote"].peers.is_empty());
|
||||
assert!(states["remote"].updated_at.is_none());
|
||||
assert!(states["remote"].api_version.is_none());
|
||||
}
|
||||
|
||||
/// A peer the status probe could not reach must render as offline.
|
||||
///
|
||||
/// Regression: `build_metrics_summary` used to hardcode `online: true` and
|
||||
@@ -10889,6 +11101,7 @@ mod tests {
|
||||
secret_key: "remote-sk".to_string(),
|
||||
..PeerSite::default()
|
||||
}],
|
||||
&[],
|
||||
"svc-ak".to_string(),
|
||||
"root".to_string(),
|
||||
true,
|
||||
@@ -10922,6 +11135,7 @@ mod tests {
|
||||
..PeerSite::default()
|
||||
},
|
||||
],
|
||||
&[],
|
||||
"svc-ak".to_string(),
|
||||
"root".to_string(),
|
||||
true,
|
||||
@@ -12128,6 +12342,173 @@ mod tests {
|
||||
assert!(normalized.contains_key("hash-remote"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_peer_identity_join_snapshot_uses_verified_ids() {
|
||||
let actual = ["site-a", "site-b", "site-c"].map(|name| PeerInfo {
|
||||
deployment_id: format!("{name}-deployment"),
|
||||
..peer(name, &format!("https://{name}.example.com:9000"))
|
||||
});
|
||||
let preflight = actual
|
||||
.iter()
|
||||
.map(|peer| preflight_site("reported-name", &peer.endpoint, &peer.deployment_id, 0))
|
||||
.collect::<Vec<_>>();
|
||||
let sites = actual
|
||||
.iter()
|
||||
.map(|peer| PeerSite {
|
||||
name: peer.name.clone(),
|
||||
endpoint: peer.endpoint.clone(),
|
||||
skip_tls_verify: true,
|
||||
ca_cert_pem: "requested-ca".to_string(),
|
||||
..Default::default()
|
||||
})
|
||||
.collect();
|
||||
let state = merge_add_sites(
|
||||
SiteReplicationState::default(),
|
||||
actual[0].clone(),
|
||||
sites,
|
||||
&preflight,
|
||||
"svc-ak".to_string(),
|
||||
"root".to_string(),
|
||||
false,
|
||||
);
|
||||
assert_eq!(state.peers.len(), actual.len());
|
||||
for expected in &actual {
|
||||
let stored = state
|
||||
.peers
|
||||
.get(&expected.deployment_id)
|
||||
.expect("verified ID in initial join map");
|
||||
assert_eq!(stored.name, expected.name);
|
||||
assert_eq!(stored.endpoint, expected.endpoint);
|
||||
assert!(stored.skip_tls_verify);
|
||||
assert_eq!(stored.ca_cert_pem, "requested-ca");
|
||||
}
|
||||
for local in &actual[1..] {
|
||||
let mut joined = SiteReplicationState::default();
|
||||
apply_peer_join(
|
||||
&mut joined,
|
||||
local,
|
||||
SRPeerJoinReq {
|
||||
peers: state.peers.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
true,
|
||||
);
|
||||
assert_eq!(joined.peers.keys().collect::<Vec<_>>(), state.peers.keys().collect::<Vec<_>>());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_peer_identity_legacy_edit_does_not_restore_placeholder() {
|
||||
let actual = PeerInfo {
|
||||
deployment_id: "actual-remote".to_string(),
|
||||
..peer("remote", "http://remote.example.com:9000")
|
||||
};
|
||||
for name in ["remote", ""] {
|
||||
let state = SiteReplicationState {
|
||||
peers: BTreeMap::from([(actual.deployment_id.clone(), actual.clone())]),
|
||||
..Default::default()
|
||||
};
|
||||
let mut incoming = PeerInfo {
|
||||
deployment_id: deployment_id_for_endpoint("https://REMOTE.example.com:9000/"),
|
||||
sync_state: SyncStatus::Enable,
|
||||
..peer(name, "https://REMOTE.example.com:9000/")
|
||||
};
|
||||
align_peer_edit_deployment_id(&state, &mut incoming);
|
||||
let state = update_peer(state, incoming, None);
|
||||
assert_eq!(state.peers.len(), 1);
|
||||
assert_eq!(state.peers[&actual.deployment_id].sync_state, SyncStatus::Enable);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_peer_identity_finalization_repairs_legacy_three_site_join() {
|
||||
let actual = ["site-a", "site-b", "site-c"].map(|name| PeerInfo {
|
||||
deployment_id: format!("{name}-deployment"),
|
||||
..peer(name, &format!("http://{name}.example.com:9000"))
|
||||
});
|
||||
let sites = actual
|
||||
.iter()
|
||||
.map(|peer| PeerSite {
|
||||
name: peer.name.clone(),
|
||||
endpoint: peer.endpoint.clone(),
|
||||
..Default::default()
|
||||
})
|
||||
.collect();
|
||||
let mut coordinator = merge_add_sites(
|
||||
SiteReplicationState::default(),
|
||||
actual[0].clone(),
|
||||
sites,
|
||||
&[],
|
||||
"svc-ak".to_string(),
|
||||
"root".to_string(),
|
||||
true,
|
||||
);
|
||||
let join = SRPeerJoinReq {
|
||||
peers: coordinator.peers.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
for remote in &actual[1..] {
|
||||
coordinator = reconcile_peer_with_actual_identity(coordinator, remote.clone());
|
||||
}
|
||||
mark_unknown_peer_sync_enabled(&mut coordinator.peers);
|
||||
|
||||
for local in &actual[1..] {
|
||||
let mut state = SiteReplicationState::default();
|
||||
apply_peer_join(&mut state, local, join.clone(), true);
|
||||
for mut incoming in coordinator.peers.values().cloned() {
|
||||
align_peer_edit_deployment_id(&state, &mut incoming);
|
||||
state = apply_internal_peer_edit(state, local, incoming, None).expect("finalize peer identity");
|
||||
}
|
||||
assert_eq!(state.peers.len(), actual.len(), "finalization must not retain placeholder peers");
|
||||
for expected in &actual {
|
||||
let stored = existing_peer_for_endpoint(&state, &expected.endpoint).expect("peer remains configured");
|
||||
assert_eq!(stored.deployment_id, expected.deployment_id, "observer: {}", local.name);
|
||||
assert_eq!(stored.sync_state, SyncStatus::Enable);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_peer_identity_edit_replaces_placeholder_for_canonical_endpoint() {
|
||||
let local = PeerInfo {
|
||||
deployment_id: "local-deployment".to_string(),
|
||||
..peer("local", "https://local.example.com:9000")
|
||||
};
|
||||
let endpoint = "http://remote.example.com:9000";
|
||||
let placeholder = PeerInfo {
|
||||
deployment_id: deployment_id_for_endpoint(endpoint),
|
||||
..peer("remote", endpoint)
|
||||
};
|
||||
for deployment_id in ["00000000-0000-4000-8000-000000000001", "ffffffff-ffff-4fff-bfff-ffffffffffff"] {
|
||||
for name in ["remote", ""] {
|
||||
for already_present in [false, true] {
|
||||
let mut incoming = PeerInfo {
|
||||
deployment_id: deployment_id.to_string(),
|
||||
sync_state: SyncStatus::Enable,
|
||||
..peer(name, "https://REMOTE.example.com:9000/")
|
||||
};
|
||||
let mut state = SiteReplicationState {
|
||||
peers: BTreeMap::from([
|
||||
(local.deployment_id.clone(), local.clone()),
|
||||
(placeholder.deployment_id.clone(), placeholder.clone()),
|
||||
]),
|
||||
..Default::default()
|
||||
};
|
||||
if already_present {
|
||||
state.peers.insert(incoming.deployment_id.clone(), incoming.clone());
|
||||
}
|
||||
align_peer_edit_deployment_id(&state, &mut incoming);
|
||||
let state = apply_internal_peer_edit(state, &local, incoming, None).expect("repair peer identity");
|
||||
assert_eq!(state.peers.len(), 2, "repair must replace, not duplicate, the placeholder");
|
||||
assert!(!state.peers.contains_key(&placeholder.deployment_id));
|
||||
assert!(state.peers.contains_key(deployment_id));
|
||||
let normalized = normalize_peer_map_by_identity(state.peers);
|
||||
assert!(normalized.contains_key(deployment_id), "normalization must retain the actual ID");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reconcile_peer_with_actual_identity_replaces_endpoint_hash_key() {
|
||||
let mut state = SiteReplicationState::default();
|
||||
@@ -13450,6 +13831,7 @@ mod tests {
|
||||
assert_eq!(response.peer.deployment_id, "remote-deployment");
|
||||
assert_eq!(response.peer.endpoint, "https://remote.example.com");
|
||||
assert!(response.initial_sync_error_message.is_empty());
|
||||
assert!(!response.initial_sync_deferred);
|
||||
assert_eq!(
|
||||
response.applied, None,
|
||||
"a MinIO empty-body success reports nothing; it must not read as a no-op join"
|
||||
@@ -13459,6 +13841,7 @@ mod tests {
|
||||
let json = serde_json::to_vec(&SRPeerJoinResponse {
|
||||
peer: peer("actual", "https://actual.example.com"),
|
||||
initial_sync_error_message: "sync failed".to_string(),
|
||||
initial_sync_deferred: false,
|
||||
applied: Some(true),
|
||||
})
|
||||
.expect("serialize join response");
|
||||
@@ -14311,6 +14694,69 @@ mod tests {
|
||||
assert_eq!(value.get("deferSyncStateEnable"), Some(&Value::Bool(true)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_initial_sync_join_requires_the_committed_snapshot() {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let mut state = SiteReplicationState {
|
||||
updated_at: Some(now),
|
||||
service_account_access_key: "replicator".to_string(),
|
||||
peers: BTreeMap::from([
|
||||
("a".to_string(), peer("a", "https://a.example.com")),
|
||||
("b".to_string(), peer("b", "https://b.example.com")),
|
||||
]),
|
||||
..Default::default()
|
||||
};
|
||||
let mut request = SRPeerJoinReq {
|
||||
updated_at: Some(now),
|
||||
svc_acct_access_key: "replicator".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
ensure_initial_sync_join_current(&state, &request).expect("same committed join");
|
||||
for timestamp in [None, Some(now - time::Duration::SECOND), Some(now + time::Duration::SECOND)] {
|
||||
request.updated_at = timestamp;
|
||||
assert!(ensure_initial_sync_join_current(&state, &request).is_err());
|
||||
}
|
||||
request.updated_at = Some(now);
|
||||
request.svc_acct_access_key = "another-replicator".to_string();
|
||||
assert!(ensure_initial_sync_join_current(&state, &request).is_err());
|
||||
request.svc_acct_access_key.clone_from(&state.service_account_access_key);
|
||||
state.pending_remove = Some(PendingRemove::default());
|
||||
assert!(ensure_initial_sync_join_current(&state, &request).is_err());
|
||||
state.pending_remove = None;
|
||||
state.pending_rotation = Some(PendingRotation::default());
|
||||
assert!(ensure_initial_sync_join_current(&state, &request).is_err());
|
||||
state.pending_rotation = None;
|
||||
state.pending_endpoint_refresh = Some(PendingEndpointRefresh::default());
|
||||
assert!(ensure_initial_sync_join_current(&state, &request).is_err());
|
||||
state.pending_endpoint_refresh = None;
|
||||
state.service_account_access_key.clear();
|
||||
request.svc_acct_access_key.clear();
|
||||
assert!(ensure_initial_sync_join_current(&state, &request).is_err());
|
||||
state.service_account_access_key = "replicator".to_string();
|
||||
request.svc_acct_access_key.clone_from(&state.service_account_access_key);
|
||||
state.peers.clear();
|
||||
assert!(ensure_initial_sync_join_current(&state, &request).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_join_initial_sync_deferral_preserves_legacy_requests() {
|
||||
let legacy: SRPeerJoinEnvelope = serde_json::from_str("{}").expect("legacy join");
|
||||
assert!(!legacy.defer_initial_sync);
|
||||
assert!(serde_json::to_value(&legacy).unwrap().get("deferInitialSync").is_none());
|
||||
let deferred: SRPeerJoinEnvelope = serde_json::from_str(r#"{"deferInitialSync":true}"#).expect("deferred join");
|
||||
assert!(deferred.defer_initial_sync);
|
||||
assert_eq!(serde_json::to_value(deferred).unwrap()["deferInitialSync"], true);
|
||||
let mut response = applied_join_response(peer("b", "https://b.example.com"), String::new());
|
||||
assert!(serde_json::to_value(&response).unwrap().get("initialSyncDeferred").is_none());
|
||||
response.initial_sync_deferred = true;
|
||||
let wire = serde_json::to_vec(&response).unwrap();
|
||||
assert!(
|
||||
parse_peer_join_response(&wire, PeerInfo::default())
|
||||
.unwrap()
|
||||
.initial_sync_deferred
|
||||
);
|
||||
}
|
||||
|
||||
// BUG2: pre-existing-bucket back-fill failures must be surfaced in the add response's
|
||||
// initial_sync_error_message, not swallowed behind an unqualified success.
|
||||
#[test]
|
||||
@@ -14363,6 +14809,7 @@ mod tests {
|
||||
let value = serde_json::to_value(SRPeerJoinResponse {
|
||||
peer: peer("remote", "https://remote.example.com"),
|
||||
initial_sync_error_message: "bucket setup failed".to_string(),
|
||||
initial_sync_deferred: false,
|
||||
applied: Some(true),
|
||||
})
|
||||
.expect("serialize peer join response");
|
||||
@@ -14374,6 +14821,7 @@ mod tests {
|
||||
let value = serde_json::to_value(SRPeerJoinResponse {
|
||||
peer: peer("remote", "https://remote.example.com"),
|
||||
initial_sync_error_message: String::new(),
|
||||
initial_sync_deferred: false,
|
||||
applied: None,
|
||||
})
|
||||
.expect("serialize peer join response");
|
||||
@@ -15434,4 +15882,54 @@ mod tests {
|
||||
"no replicated config write may bypass the source stamp"
|
||||
);
|
||||
}
|
||||
|
||||
/// backlog#2367 A-4: `remove --all` notified "the peer" at the site's own
|
||||
/// registered endpoint. The tick-driven local peer carries the node's
|
||||
/// listen address, so an endpoint-only self check let the loop dial the
|
||||
/// site itself and report `Partial: failed to notify 1 peer(s)`.
|
||||
#[test]
|
||||
fn pending_notifications_skip_the_local_site_by_deployment_id() {
|
||||
let local_registered = PeerInfo {
|
||||
deployment_id: "site-b".to_string(),
|
||||
..peer("site-b", "http://site-b.example.com:9000")
|
||||
};
|
||||
let remote = PeerInfo {
|
||||
deployment_id: "site-a".to_string(),
|
||||
..peer("site-a", "http://site-a.example.com:9000")
|
||||
};
|
||||
let acked = PeerInfo {
|
||||
deployment_id: "site-c".to_string(),
|
||||
..peer("site-c", "http://site-c.example.com:9000")
|
||||
};
|
||||
let peers = BTreeMap::from([
|
||||
(local_registered.deployment_id.clone(), local_registered.clone()),
|
||||
(remote.deployment_id.clone(), remote),
|
||||
(acked.deployment_id.clone(), acked.clone()),
|
||||
]);
|
||||
let acked_ids = BTreeSet::from([acked.deployment_id]);
|
||||
|
||||
// The tick resolves the local peer from its own listen address.
|
||||
let local_from_tick = PeerInfo {
|
||||
deployment_id: "site-b".to_string(),
|
||||
..peer("site-b", "http://127.0.0.1:9000")
|
||||
};
|
||||
let to_notify: Vec<&str> = pending_peers_awaiting_notification(&peers, &local_from_tick, &acked_ids)
|
||||
.iter()
|
||||
.map(|peer| peer.deployment_id.as_str())
|
||||
.collect();
|
||||
assert_eq!(to_notify, vec!["site-a"], "the local site and the acked peer are never dialed");
|
||||
|
||||
// Identity stays consistent with what finalization waits for.
|
||||
assert_eq!(
|
||||
pending_remote_peer_ids(&peers, &local_from_tick),
|
||||
BTreeSet::from(["site-a".to_string(), "site-c".to_string()])
|
||||
);
|
||||
|
||||
// A handler-resolved local peer (registered endpoint) agrees.
|
||||
let to_notify: Vec<&str> = pending_peers_awaiting_notification(&peers, &local_registered, &acked_ids)
|
||||
.iter()
|
||||
.map(|peer| peer.deployment_id.as_str())
|
||||
.collect();
|
||||
assert_eq!(to_notify, vec!["site-a"]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -73,6 +73,7 @@ const SITE_REPLICATION_RESYNC_ROUTE: &str = "/rustfs/admin/v3/site-replication/r
|
||||
const SITE_REPLICATION_REPAIR_ROUTE: &str = "/rustfs/admin/v3/site-replication/repair";
|
||||
const SITE_REPLICATION_REPAIR_STATUS_ROUTE: &str = "/rustfs/admin/v3/site-replication/repair/status";
|
||||
const IAM_POLICY_ATTACH_ROUTE: &str = "/rustfs/admin/v3/idp/builtin/policy/attach";
|
||||
const DATA_USAGE_INFO_ROUTE: &str = "/rustfs/admin/v3/datausageinfo";
|
||||
const IAM_POLICY_DETACH_ROUTE: &str = "/rustfs/admin/v3/idp/builtin/policy/detach";
|
||||
const IAM_POLICY_ENTITIES_ROUTE: &str = "/rustfs/admin/v3/idp/builtin/policy-entities";
|
||||
const IAM_ACCESS_KEYS_BULK_ROUTE: &str = "/rustfs/admin/v3/list-access-keys-bulk";
|
||||
@@ -1077,12 +1078,25 @@ fn advertised_admin_capabilities() -> Vec<AdvertisedAdminCapability> {
|
||||
("admin.account.mfa", HttpMethod::Get, ACCOUNT_MFA_ROUTE),
|
||||
("admin.mfa.challenge", HttpMethod::Get, MFA_CHALLENGE_ROUTE),
|
||||
("admin.user.mfa", HttpMethod::Get, USER_MFA_ROUTE),
|
||||
// `rc du` is gated on this name. Before it was advertised the client
|
||||
// inferred it from a `1.0.0-rc.` version prefix, which no longer
|
||||
// matches once the server reports `1.0.0` (backlog#2367 E-2).
|
||||
("admin.data-usage", HttpMethod::Get, DATA_USAGE_INFO_ROUTE),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|(name, method, route)| AdvertisedAdminCapability {
|
||||
name,
|
||||
status: admin_route_capability(method, route),
|
||||
})
|
||||
.chain(std::iter::once(AdvertisedAdminCapability {
|
||||
// `rc watch` streams `GET /{bucket}?events=`, a misc extension route
|
||||
// dispatched by `admin::router` rather than an admin policy route,
|
||||
// so its status is not an inventory lookup (same version-prefix
|
||||
// inference on the client as `admin.data-usage`).
|
||||
name: "listen_notification",
|
||||
status: CapabilityStatus::supported()
|
||||
.with_reason("bucket listen notification (?events=) is dispatched by the admin router"),
|
||||
}))
|
||||
.collect()
|
||||
}
|
||||
|
||||
@@ -1258,6 +1272,9 @@ mod tests {
|
||||
"admin.iam.access-keys-bulk",
|
||||
"admin.iam.access-keys-bulk.ldap",
|
||||
"admin.iam.access-keys-bulk.openid",
|
||||
// rc pinned these two by version prefix until 1.0.0 (backlog#2367 E-2).
|
||||
"admin.data-usage",
|
||||
"listen_notification",
|
||||
];
|
||||
for name in expected_supported {
|
||||
let entry = response
|
||||
|
||||
@@ -33,6 +33,7 @@ use rustfs_credentials::Credentials;
|
||||
use rustfs_iam::federation::OIDC_VIRTUAL_PARENT_CLAIM;
|
||||
use rustfs_iam::sys::is_rustfs_oidc_claims;
|
||||
use rustfs_madmin::account::{AccountMutability, CredentialsSource, IdentityType};
|
||||
use serde_json::Value;
|
||||
|
||||
/// Claim written by the Keystone middleware onto its synthesized credentials.
|
||||
const KEYSTONE_ROLES_CLAIM: &str = "keystone_roles";
|
||||
@@ -54,6 +55,23 @@ pub(crate) fn session_parent_identity(credentials: &Credentials) -> Option<&str>
|
||||
.and_then(|value| value.as_str())
|
||||
}
|
||||
|
||||
/// Human-readable OIDC identity metadata for self-service responses. These
|
||||
/// values never replace the issuer-scoped virtual parent used for authorization.
|
||||
pub(crate) fn oidc_profile_fields(credentials: &Credentials) -> (Option<String>, Option<String>) {
|
||||
let Some(claims) = credentials.claims.as_ref().filter(|claims| is_rustfs_oidc_claims(claims)) else {
|
||||
return (None, None);
|
||||
};
|
||||
let string_claim = |name| {
|
||||
claims
|
||||
.get(name)
|
||||
.and_then(Value::as_str)
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
.map(ToOwned::to_owned)
|
||||
};
|
||||
|
||||
(string_claim("preferred_username"), string_claim("email"))
|
||||
}
|
||||
|
||||
/// Why a credential may not change its own authentication material.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum CredentialMutationDenial {
|
||||
@@ -398,6 +416,48 @@ mod tests {
|
||||
assert!(!caller.mutability().password);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oidc_profile_fields_return_normalized_display_claims() {
|
||||
let mut credentials = sts_session("TEMPKEY", "oidc-parent");
|
||||
credentials.claims = Some(HashMap::from([
|
||||
("iss".to_string(), Value::String("rustfs-oidc".to_string())),
|
||||
("oidc_provider".to_string(), Value::String("entraid".to_string())),
|
||||
("sub".to_string(), Value::String("subject-123".to_string())),
|
||||
("preferred_username".to_string(), Value::String("j.bruijns@pay.nl".to_string())),
|
||||
("email".to_string(), Value::String("fallback@pay.nl".to_string())),
|
||||
]));
|
||||
|
||||
assert_eq!(
|
||||
oidc_profile_fields(&credentials),
|
||||
(Some("j.bruijns@pay.nl".to_string()), Some("fallback@pay.nl".to_string()))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oidc_profile_fields_omit_missing_blank_and_non_string_values() {
|
||||
let mut credentials = sts_session("TEMPKEY", "oidc-parent");
|
||||
credentials.claims = Some(HashMap::from([
|
||||
("iss".to_string(), Value::String("rustfs-oidc".to_string())),
|
||||
("oidc_provider".to_string(), Value::String("keycloak".to_string())),
|
||||
("sub".to_string(), Value::String("subject-123".to_string())),
|
||||
("preferred_username".to_string(), Value::String(" ".to_string())),
|
||||
("email".to_string(), Value::Array(vec![Value::String("user@example.test".to_string())])),
|
||||
]));
|
||||
|
||||
assert_eq!(oidc_profile_fields(&credentials), (None, None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oidc_profile_fields_ignore_non_oidc_claim_shapes() {
|
||||
let mut credentials = sts_session("TEMPKEY", "ordinary-parent");
|
||||
credentials.claims = Some(HashMap::from([
|
||||
("preferred_username".to_string(), Value::String("attacker".to_string())),
|
||||
("email".to_string(), Value::String("attacker@example.test".to_string())),
|
||||
]));
|
||||
|
||||
assert_eq!(oidc_profile_fields(&credentials), (None, None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keystone_session_is_reported_as_federated() {
|
||||
let mut credentials = sts_session("TEMPKEY", "keystone-parent");
|
||||
|
||||
@@ -69,7 +69,7 @@ use super::storage_api::multipart_usecase::{
|
||||
};
|
||||
use crate::app::object::{
|
||||
ConcurrencyManager, ForegroundWriteAdmission, get_concurrency_manager, guard_put_object_body_read_timeout,
|
||||
put_object_body_read_timeout,
|
||||
put_object_body_read_timeout, reject_oversize_single_upload,
|
||||
};
|
||||
use crate::app::object_data_cache::{
|
||||
ObjectDataCacheAdapter, invalidate_object_data_cache_after_complete_multipart_success,
|
||||
@@ -1169,6 +1169,9 @@ impl DefaultMultipartUsecase {
|
||||
validate_table_catalog_object_mutation(&bucket, &key).await?;
|
||||
|
||||
let mut size = resolve_upload_part_size(&req.headers, content_length)?;
|
||||
if let Some(size) = size {
|
||||
reject_oversize_single_upload(size)?;
|
||||
}
|
||||
let mut body_stream = body.ok_or_else(|| s3_error!(IncompleteBody))?;
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
@@ -3209,6 +3212,36 @@ mod tests {
|
||||
assert_eq!(err.code(), &S3ErrorCode::IncompleteBody);
|
||||
}
|
||||
|
||||
/// issue #7596: a part whose declared length exceeds the 5 GiB
|
||||
/// single-request ceiling is rejected before the body is polled or the
|
||||
/// store is consulted. Exact-cap and zero-length parts pass admission.
|
||||
#[tokio::test]
|
||||
async fn execute_upload_part_rejects_oversize_declared_part_before_reading_the_body() {
|
||||
let ceiling = i64::try_from(rustfs_config::MAX_SINGLE_PUT_OBJECT_SIZE).expect("ceiling fits i64");
|
||||
|
||||
for (declared, expect_too_large) in [(ceiling + 1, true), (ceiling, false), (0, false)] {
|
||||
let (body, polls) = crate::app::object::PollCountingBody::streaming_blob();
|
||||
let input = UploadPartInput::builder()
|
||||
.bucket("bucket".to_string())
|
||||
.key("object".to_string())
|
||||
.upload_id("upload-id".to_string())
|
||||
.part_number(1)
|
||||
.body(Some(body))
|
||||
.content_length(Some(declared))
|
||||
.build()
|
||||
.unwrap();
|
||||
let req = build_request(input, Method::PUT);
|
||||
|
||||
let err = make_usecase().execute_upload_part(req).await.unwrap_err();
|
||||
if expect_too_large {
|
||||
assert_eq!(err.code(), &S3ErrorCode::EntityTooLarge, "declared {declared}");
|
||||
assert_eq!(polls.load(std::sync::atomic::Ordering::SeqCst), 0, "body must not be polled");
|
||||
} else {
|
||||
assert_ne!(err.code(), &S3ErrorCode::EntityTooLarge, "declared {declared} must pass admission");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_upload_part_rejects_invalid_part_number_before_body_lookup() {
|
||||
for part_number in [-1, 0, 10001] {
|
||||
|
||||
+201
-42
@@ -16,6 +16,67 @@
|
||||
|
||||
use super::*;
|
||||
|
||||
async fn authorize_recursive_delete<T>(
|
||||
req: &mut S3Request<T>,
|
||||
store: &Arc<ECStore>,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
versioned: bool,
|
||||
replica: bool,
|
||||
) -> S3Result<()> {
|
||||
let original_info = req_info_ref(req)?.clone();
|
||||
let descendant_prefix = if prefix.ends_with('/') {
|
||||
prefix.to_owned()
|
||||
} else {
|
||||
format!("{prefix}/")
|
||||
};
|
||||
let result = async {
|
||||
let mut marker = None;
|
||||
let mut version_marker = None;
|
||||
loop {
|
||||
let page = store
|
||||
.clone()
|
||||
.list_object_versions(bucket, prefix, marker.clone(), version_marker.clone(), None, 1000)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
for object in page.objects {
|
||||
// Disk prefix deletion follows path boundaries, while S3's
|
||||
// string-prefix listing can also return unrelated siblings.
|
||||
if object.name != prefix && !object.name.starts_with(&descendant_prefix) {
|
||||
continue;
|
||||
}
|
||||
let object_version = object.version_id.filter(|version| !version.is_nil());
|
||||
let version_id = (versioned || object_version.is_some())
|
||||
.then(|| object_version.map_or_else(|| "null".to_owned(), |id| id.to_string()));
|
||||
let info = req_info_mut(req)?;
|
||||
info.object = Some(object.name.clone());
|
||||
info.version_id = version_id.clone();
|
||||
let action = if replica {
|
||||
Action::S3Action(S3Action::ReplicateDeleteAction)
|
||||
} else {
|
||||
delete_object_authorize_action(version_id.as_deref())
|
||||
};
|
||||
authorize_request(req, action).await?;
|
||||
if has_bypass_governance_header(&req.headers) {
|
||||
authorize_request(req, Action::S3Action(S3Action::BypassGovernanceRetentionAction)).await?;
|
||||
}
|
||||
validate_table_catalog_object_mutation(bucket, &object.name).await?;
|
||||
}
|
||||
if !page.is_truncated {
|
||||
return Ok(());
|
||||
}
|
||||
if page.next_marker.is_none() || (marker == page.next_marker && version_marker == page.next_version_idmarker) {
|
||||
return Err(s3_error!(InternalError, "Recursive delete listing did not advance"));
|
||||
}
|
||||
marker = page.next_marker;
|
||||
version_marker = page.next_version_idmarker;
|
||||
}
|
||||
}
|
||||
.await;
|
||||
req.extensions.insert(original_info);
|
||||
result
|
||||
}
|
||||
|
||||
fn successful_delete_audit_objects(
|
||||
delete: &s3s::dto::Delete,
|
||||
successful_results: impl IntoIterator<Item = bool>,
|
||||
@@ -411,14 +472,6 @@ impl DefaultObjectUsecase {
|
||||
));
|
||||
}
|
||||
|
||||
let is_owner = req_info_ref(&req).map(|info| info.is_owner).unwrap_or(false);
|
||||
if !recursive_force_delete_is_authorized(&req.headers, is_owner, false) {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::AccessDenied,
|
||||
"Recursive force-delete is restricted to administrative requests",
|
||||
));
|
||||
}
|
||||
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
@@ -474,7 +527,7 @@ impl DefaultObjectUsecase {
|
||||
req_info.version_id = version_id.clone();
|
||||
}
|
||||
|
||||
let auth_res = authorize_request(&mut req, Action::S3Action(S3Action::DeleteObjectAction)).await;
|
||||
let auth_res = authorize_request(&mut req, delete_object_authorize_action(version_id.as_deref())).await;
|
||||
if auth_res.is_err() {
|
||||
if !bulk_denial_logged {
|
||||
bulk_denial_logged = true;
|
||||
@@ -882,11 +935,11 @@ impl DefaultObjectUsecase {
|
||||
authorize_request(&mut req, Action::S3Action(S3Action::ReplicateDeleteAction)).await?;
|
||||
}
|
||||
|
||||
let is_owner = req_info_ref(&req).map(|info| info.is_owner).unwrap_or(false);
|
||||
if !recursive_force_delete_is_authorized(&req.headers, is_owner, replica) {
|
||||
let authenticated = req_info_ref(&req).is_ok_and(|info| info.is_owner || info.cred.is_some());
|
||||
if !recursive_force_delete_has_authenticated_caller(&req.headers, authenticated, replica) {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::AccessDenied,
|
||||
"Recursive force-delete is restricted to internal or administrative requests",
|
||||
"Recursive force-delete requires an authenticated caller",
|
||||
));
|
||||
}
|
||||
validate_table_catalog_object_mutation(&bucket, &key).await?;
|
||||
@@ -901,6 +954,22 @@ impl DefaultObjectUsecase {
|
||||
};
|
||||
validate_bucket_exists(&store, &bucket).await?;
|
||||
|
||||
// Lock order is bucket lifecycle, then object/commit locks in storage.
|
||||
// Keep this guard alive through the physical delete: a preflight without
|
||||
// writer exclusion could authorize one subtree and delete a newer one.
|
||||
let recursive_delete_guard = if rustfs_utils::http::get_header(&req.headers, rustfs_utils::http::SUFFIX_FORCE_DELETE)
|
||||
.is_some_and(|value| value == "true")
|
||||
{
|
||||
Some(
|
||||
store
|
||||
.lock_bucket_for_recursive_delete(&bucket)
|
||||
.await
|
||||
.map_err(ApiError::from)?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let metadata = extract_metadata(&req.headers);
|
||||
// Clone version_id before it's moved
|
||||
let version_id_clone = version_id.clone();
|
||||
@@ -924,6 +993,12 @@ impl DefaultObjectUsecase {
|
||||
apply_bucket_generation_guard(&req, &bucket, &mut opts)?;
|
||||
let force_delete = opts.delete_prefix;
|
||||
|
||||
if let Some(guard) = &recursive_delete_guard {
|
||||
opts.add_bucket_lifecycle_lock_guard(guard);
|
||||
authorize_recursive_delete(&mut req, &store, &bucket, &key, opts.versioned || opts.version_suspended, replica)
|
||||
.await?;
|
||||
}
|
||||
|
||||
// let mut vid = opts.version_id.clone();
|
||||
|
||||
if replica {
|
||||
@@ -1026,6 +1101,7 @@ impl DefaultObjectUsecase {
|
||||
}
|
||||
}
|
||||
};
|
||||
drop(recursive_delete_guard);
|
||||
|
||||
if force_delete {
|
||||
let _ = invalidate_object_data_cache_prefix_after_delete(&cache_adapter, &bucket, &key).await;
|
||||
@@ -2212,14 +2288,122 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recursive_force_delete_requires_administrative_or_replica_context() {
|
||||
#[serial_test::serial]
|
||||
fn recursive_delete_holds_writer_exclusion_after_authorization() {
|
||||
crate::app::gating_test_env::run_large_stack_test("recursive-delete-writer-exclusion", || async {
|
||||
use crate::app::storage_api::test::contract::bucket::{
|
||||
BucketOperations as _, DeleteBucketOptions, MakeBucketOptions,
|
||||
};
|
||||
use std::time::Duration;
|
||||
|
||||
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
|
||||
if current_app_context().is_none() {
|
||||
crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await;
|
||||
}
|
||||
let context = current_app_context().expect("recursive delete test requires an AppContext");
|
||||
let bucket = format!("recursive-delete-writer-{}", Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create test bucket");
|
||||
let mut reader = PutObjReader::from_vec(b"old".to_vec());
|
||||
store
|
||||
.put_object(&bucket, "folder/old", &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("seed old object");
|
||||
|
||||
let policy_json = format!(
|
||||
r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"Allow","Principal":{{"AWS":"*"}},"Action":["s3:DeleteObject","s3:DeleteObjectVersion"],"Resource":["arn:aws:s3:::{bucket}/*"]}}]}}"#
|
||||
);
|
||||
let mut metadata = (*crate::storage::get_bucket_metadata(&bucket)
|
||||
.await
|
||||
.expect("load test metadata"))
|
||||
.clone();
|
||||
metadata.policy_config = Some(serde_json::from_str(&policy_json).expect("parse test policy"));
|
||||
metadata.policy_config_json = policy_json.into_bytes();
|
||||
crate::storage::storage_api::set_bucket_metadata(bucket.clone(), metadata)
|
||||
.await
|
||||
.expect("publish test policy");
|
||||
|
||||
let input = DeleteObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key("folder/".to_owned())
|
||||
.build()
|
||||
.expect("build force delete");
|
||||
let mut req = build_request(input, Method::DELETE);
|
||||
req.headers.insert("x-rustfs-force-delete", HeaderValue::from_static("true"));
|
||||
req.extensions.insert(crate::storage::access::ReqInfo {
|
||||
is_owner: true,
|
||||
bucket: Some(bucket.clone()),
|
||||
object: Some("folder/".to_owned()),
|
||||
..Default::default()
|
||||
});
|
||||
let loaded = Arc::new(tokio::sync::Barrier::new(2));
|
||||
let resume = Arc::new(tokio::sync::Barrier::new(2));
|
||||
install_delete_source_test_hook(bucket.clone(), Arc::clone(&loaded), Arc::clone(&resume));
|
||||
let usecase = DefaultObjectUsecase::with_context(Some(context));
|
||||
let delete = tokio::spawn(async move { usecase.execute_delete_object(req).await });
|
||||
tokio::time::timeout(Duration::from_secs(30), loaded.wait())
|
||||
.await
|
||||
.expect("force delete reaches authorized pre-commit pause");
|
||||
|
||||
let writer_store = Arc::clone(&store);
|
||||
let writer_bucket = bucket.clone();
|
||||
let mut writer = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(b"new".to_vec());
|
||||
writer_store
|
||||
.put_object(&writer_bucket, "folder/new", &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
let before_delete = tokio::time::timeout(Duration::from_secs(1), &mut writer).await;
|
||||
resume.wait().await;
|
||||
tokio::time::timeout(Duration::from_secs(30), delete)
|
||||
.await
|
||||
.expect("force delete completes without lock recursion")
|
||||
.expect("delete task joins")
|
||||
.expect("force delete succeeds");
|
||||
assert!(
|
||||
before_delete.is_err(),
|
||||
"a writer must not enter the authorized subtree before deletion commits"
|
||||
);
|
||||
tokio::time::timeout(Duration::from_secs(30), writer)
|
||||
.await
|
||||
.expect("writer resumes after deletion")
|
||||
.expect("writer task joins")
|
||||
.expect("writer succeeds");
|
||||
store
|
||||
.get_object_info(&bucket, "folder/new", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("post-delete writer's object survives");
|
||||
assert!(
|
||||
store
|
||||
.get_object_info(&bucket, "folder/old", &ObjectOptions::default())
|
||||
.await
|
||||
.is_err(),
|
||||
"authorized old object is removed"
|
||||
);
|
||||
store
|
||||
.delete_bucket(
|
||||
&bucket,
|
||||
&DeleteBucketOptions {
|
||||
force: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("remove test bucket");
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recursive_force_delete_requires_authenticated_or_replica_context() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("x-rustfs-force-delete", HeaderValue::from_static("true"));
|
||||
|
||||
assert!(!recursive_force_delete_is_authorized(&headers, false, false));
|
||||
assert!(recursive_force_delete_is_authorized(&headers, true, false));
|
||||
assert!(recursive_force_delete_is_authorized(&headers, false, true));
|
||||
assert!(recursive_force_delete_is_authorized(&HeaderMap::new(), false, false));
|
||||
assert!(!recursive_force_delete_has_authenticated_caller(&headers, false, false));
|
||||
assert!(recursive_force_delete_has_authenticated_caller(&headers, true, false));
|
||||
assert!(recursive_force_delete_has_authenticated_caller(&headers, false, true));
|
||||
assert!(recursive_force_delete_has_authenticated_caller(&HeaderMap::new(), false, false));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -2240,31 +2424,6 @@ mod tests {
|
||||
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_delete_objects_rejects_untrusted_force_delete_before_store_access() {
|
||||
let input = DeleteObjectsInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.delete(Delete {
|
||||
objects: vec![ObjectIdentifier {
|
||||
key: "prefix/object".to_string(),
|
||||
version_id: None,
|
||||
..Default::default()
|
||||
}],
|
||||
quiet: None,
|
||||
})
|
||||
.build()
|
||||
.unwrap();
|
||||
let mut req = build_request(input, Method::POST);
|
||||
req.headers.insert("x-rustfs-force-delete", HeaderValue::from_static("true"));
|
||||
req.extensions.insert(crate::storage::access::ReqInfo::default());
|
||||
|
||||
let err = DefaultObjectUsecase::without_context()
|
||||
.execute_delete_objects(req)
|
||||
.await
|
||||
.expect_err("untrusted force-delete must be rejected before storage lookup");
|
||||
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
|
||||
}
|
||||
|
||||
// backlog#929 (HP-8): the pre-delete stat may only be skipped when every
|
||||
// consumer of its result is provably idle. Each guard flips one condition
|
||||
// to prove the skip is fenced on all four data dependencies.
|
||||
|
||||
@@ -21,8 +21,9 @@ use crate::storage_api::table::get_bucket_metadata;
|
||||
|
||||
use super::storage_api::object_usecase::access::{
|
||||
PostObjectRequestMarker, apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard, authorize_request,
|
||||
has_bypass_governance_header, load_bucket_generation_from_store, odm_read_generation, prepare_odm_read_generation,
|
||||
recursive_force_delete_is_authorized, replication_request_authorized, req_info_mut, req_info_ref,
|
||||
delete_object_authorize_action, has_bypass_governance_header, load_bucket_generation_from_store, odm_read_generation,
|
||||
prepare_odm_read_generation, recursive_force_delete_has_authenticated_caller, replication_request_authorized, req_info_mut,
|
||||
req_info_ref,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use super::storage_api::object_usecase::bucket::quota::BucketQuota;
|
||||
@@ -64,7 +65,7 @@ pub(crate) use super::storage_api::object_usecase::concurrency::{
|
||||
#[cfg(test)]
|
||||
use super::storage_api::object_usecase::contract::http::HTTPPreconditions;
|
||||
use super::storage_api::object_usecase::contract::namespace::NamespaceLocking;
|
||||
use super::storage_api::object_usecase::contract::object::{ObjectIO as _, ObjectOperations as _};
|
||||
use super::storage_api::object_usecase::contract::object::{ListOperations as _, ObjectIO as _, ObjectOperations as _};
|
||||
use super::storage_api::object_usecase::contract::range::HTTPRangeSpec;
|
||||
use super::storage_api::object_usecase::data_usage::{
|
||||
quota_object_size, record_bucket_delete_marker_memory, record_bucket_object_delete_memory,
|
||||
@@ -223,8 +224,10 @@ pub(crate) use self::extract::*;
|
||||
pub(crate) use self::get::*;
|
||||
pub(crate) use self::internal_put::*;
|
||||
pub(crate) use self::on_demand_migration_put::*;
|
||||
#[cfg(test)]
|
||||
pub(crate) use self::put::PollCountingBody;
|
||||
use self::put::*;
|
||||
pub(crate) use self::put::{guard_put_object_body_read_timeout, put_object_body_read_timeout};
|
||||
pub(crate) use self::put::{guard_put_object_body_read_timeout, put_object_body_read_timeout, reject_oversize_single_upload};
|
||||
#[cfg(test)]
|
||||
pub(crate) use self::restore::RestoreStatusCommitBarrier;
|
||||
pub(crate) use self::shared::*;
|
||||
|
||||
@@ -109,6 +109,21 @@ fn resolve_put_object_authoritative_size(headers: &HeaderMap, content_length: Op
|
||||
Ok(size)
|
||||
}
|
||||
|
||||
/// Reject a declared upload length above the single-request ceiling
|
||||
/// ([`rustfs_config::MAX_SINGLE_PUT_OBJECT_SIZE`]) with `EntityTooLarge`.
|
||||
///
|
||||
/// Applies to `PutObject` and `UploadPart`. A negative or unknown length is
|
||||
/// left to the caller's existing validation.
|
||||
pub(crate) fn reject_oversize_single_upload(size: i64) -> S3Result<()> {
|
||||
if u64::try_from(size).is_ok_and(|size| size > rustfs_config::MAX_SINGLE_PUT_OBJECT_SIZE) {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::EntityTooLarge,
|
||||
ApiError::error_code_to_message(&S3ErrorCode::EntityTooLarge),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Resolve the S3 request-body inter-chunk read timeout from the environment.
|
||||
///
|
||||
/// Returns `Duration::ZERO` when disabled (`RUSTFS_HTTP_REQUEST_BODY_READ_TIMEOUT=0`),
|
||||
@@ -1287,6 +1302,12 @@ impl DefaultObjectUsecase {
|
||||
// Resolve the authoritative decoded/plain object length (rejecting negative/unknown) before anything else consumes it.
|
||||
let size = resolve_put_object_authoritative_size(&req.headers, content_length)?;
|
||||
|
||||
// The streaming-body limit (s3s `put_object_max_size`) only fires once the
|
||||
// client has already streamed 5 GiB. The declared length is authoritative,
|
||||
// so reject an oversize single PUT here, before any body byte is read
|
||||
// (issue #7596).
|
||||
reject_oversize_single_upload(size)?;
|
||||
|
||||
if let Some(limit) = max_content_length
|
||||
&& u64::try_from(size).is_ok_and(|size| size > limit)
|
||||
{
|
||||
@@ -3318,6 +3339,77 @@ mod tests {
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidStorageClass);
|
||||
}
|
||||
|
||||
/// issue #7596: a single PUT whose declared length exceeds the 5 GiB
|
||||
/// ceiling must be rejected from the headers, before any body byte is
|
||||
/// requested.
|
||||
#[tokio::test]
|
||||
async fn execute_put_object_rejects_oversize_content_length_before_reading_the_body() {
|
||||
let ceiling = i64::try_from(rustfs_config::MAX_SINGLE_PUT_OBJECT_SIZE).expect("ceiling fits i64");
|
||||
let (body, polls) = PollCountingBody::streaming_blob();
|
||||
let input = PutObjectInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("huge.bin".to_string())
|
||||
.body(Some(body))
|
||||
.content_length(Some(ceiling + 1))
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let req = build_request(input, Method::PUT);
|
||||
let usecase = DefaultObjectUsecase::without_context();
|
||||
let fs = FS::new();
|
||||
|
||||
let err = Box::pin(usecase.execute_put_object(&fs, req)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::EntityTooLarge);
|
||||
assert_eq!(polls.load(std::sync::atomic::Ordering::SeqCst), 0, "body must not be polled");
|
||||
}
|
||||
|
||||
/// Admission uses the logical object size, not the wire length: a signed
|
||||
/// aws-chunked request whose framed `Content-Length` exceeds the cap but
|
||||
/// whose decoded length is within it must not be rejected as oversize,
|
||||
/// while a decoded length above the cap must be.
|
||||
#[tokio::test]
|
||||
async fn execute_put_object_oversize_admission_uses_decoded_length_for_aws_chunked() {
|
||||
let ceiling = i64::try_from(rustfs_config::MAX_SINGLE_PUT_OBJECT_SIZE).expect("ceiling fits i64");
|
||||
let framing_overhead = 1_000_000;
|
||||
|
||||
for (decoded, expect_too_large) in [(ceiling, false), (ceiling + 1, true)] {
|
||||
let (body, polls) = PollCountingBody::streaming_blob();
|
||||
let input = PutObjectInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("huge.bin".to_string())
|
||||
.body(Some(body))
|
||||
.content_length(Some(decoded + framing_overhead))
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let mut req = build_request(input, Method::PUT);
|
||||
req.headers
|
||||
.insert(http::header::CONTENT_ENCODING, HeaderValue::from_static("aws-chunked"));
|
||||
req.headers.insert(
|
||||
HeaderName::from_static("x-amz-content-sha256"),
|
||||
HeaderValue::from_static("STREAMING-AWS4-HMAC-SHA256-PAYLOAD"),
|
||||
);
|
||||
req.headers.insert(
|
||||
HeaderName::from_static("x-amz-decoded-content-length"),
|
||||
HeaderValue::from_str(&decoded.to_string()).unwrap(),
|
||||
);
|
||||
let usecase = DefaultObjectUsecase::without_context();
|
||||
let fs = FS::new();
|
||||
|
||||
let err = Box::pin(usecase.execute_put_object(&fs, req)).await.unwrap_err();
|
||||
if expect_too_large {
|
||||
assert_eq!(err.code(), &S3ErrorCode::EntityTooLarge, "decoded {decoded}");
|
||||
assert_eq!(polls.load(std::sync::atomic::Ordering::SeqCst), 0, "body must not be polled");
|
||||
} else {
|
||||
assert_ne!(
|
||||
err.code(),
|
||||
&S3ErrorCode::EntityTooLarge,
|
||||
"framed wire length above the cap must not reject a decoded length at the cap"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_put_object_rejects_post_object_sse_kms_from_headers() {
|
||||
let input = PutObjectInput::builder()
|
||||
@@ -4184,3 +4276,55 @@ mod tests {
|
||||
assert!(is_err_object_not_found(&lookup_err), "{lookup_err}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Test-only request body that records how often it is polled, so admission
|
||||
/// tests can prove a rejection happened before any body byte was requested.
|
||||
#[cfg(test)]
|
||||
pub(crate) struct PollCountingBody {
|
||||
pub(crate) polls: std::sync::Arc<std::sync::atomic::AtomicUsize>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl PollCountingBody {
|
||||
pub(crate) fn streaming_blob() -> (StreamingBlob, std::sync::Arc<std::sync::atomic::AtomicUsize>) {
|
||||
let polls = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let body = StreamingBlob::new(Self {
|
||||
polls: std::sync::Arc::clone(&polls),
|
||||
});
|
||||
(body, polls)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Stream for PollCountingBody {
|
||||
type Item = Result<Bytes, StdError>;
|
||||
|
||||
fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||
self.polls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
Poll::Ready(Some(Ok(Bytes::from_static(b"x"))))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl ByteStream for PollCountingBody {}
|
||||
|
||||
#[cfg(test)]
|
||||
mod oversize_single_upload_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn reject_oversize_single_upload_enforces_the_single_request_ceiling() {
|
||||
let ceiling = i64::try_from(rustfs_config::MAX_SINGLE_PUT_OBJECT_SIZE).expect("ceiling fits i64");
|
||||
|
||||
assert!(reject_oversize_single_upload(0).is_ok());
|
||||
assert!(reject_oversize_single_upload(ceiling).is_ok(), "exact ceiling is allowed");
|
||||
assert!(reject_oversize_single_upload(-1).is_ok(), "unknown length is left to later validation");
|
||||
|
||||
let err = reject_oversize_single_upload(ceiling + 1).expect_err("one byte over must be rejected");
|
||||
assert_eq!(*err.code(), S3ErrorCode::EntityTooLarge);
|
||||
assert_eq!(
|
||||
err.message(),
|
||||
Some(ApiError::error_code_to_message(&S3ErrorCode::EntityTooLarge).as_str())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -266,10 +266,10 @@ pub(crate) mod access {
|
||||
pub(crate) use crate::storage::storage_api::access_consumer::ReqInfo;
|
||||
pub(crate) use crate::storage::storage_api::access_consumer::{
|
||||
PostObjectRequestMarker, apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard, authorize_request,
|
||||
bucket_config_mutation_incarnation, has_bypass_governance_header, load_bucket_generation_from_store,
|
||||
log_list_buckets_iam_implicit_deny, odm_read_generation, prepare_list_buckets_iam_authorization,
|
||||
prepare_odm_read_generation, recursive_force_delete_is_authorized, replication_request_authorized, req_info_mut,
|
||||
req_info_ref,
|
||||
bucket_config_mutation_incarnation, delete_object_authorize_action, has_bypass_governance_header,
|
||||
load_bucket_generation_from_store, log_list_buckets_iam_implicit_deny, odm_read_generation,
|
||||
prepare_list_buckets_iam_authorization, prepare_odm_read_generation, recursive_force_delete_has_authenticated_caller,
|
||||
replication_request_authorized, req_info_mut, req_info_ref,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1196,7 +1196,7 @@ pub(crate) mod object_usecase {
|
||||
}
|
||||
|
||||
pub(crate) mod object {
|
||||
pub(crate) use super::super::super::storage_contracts::{ObjectIO, ObjectOperations};
|
||||
pub(crate) use super::super::super::storage_contracts::{ListOperations, ObjectIO, ObjectOperations};
|
||||
}
|
||||
|
||||
pub(crate) mod range {
|
||||
|
||||
@@ -478,6 +478,57 @@ fn error_chain_s3s_body_stream_error(err: &(dyn std::error::Error + 'static)) ->
|
||||
None
|
||||
}
|
||||
|
||||
/// Walk an error chain (including `io::Error` custom payloads) and return
|
||||
/// whether any link satisfies `pred`.
|
||||
fn error_chain_any(err: &(dyn std::error::Error + 'static), pred: &dyn Fn(&(dyn std::error::Error + 'static)) -> bool) -> bool {
|
||||
if pred(err) {
|
||||
return true;
|
||||
}
|
||||
if let Some(io_err) = err.downcast_ref::<std::io::Error>()
|
||||
&& let Some(inner) = io_err.get_ref()
|
||||
&& error_chain_any(inner, pred)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
let mut current = err.source();
|
||||
while let Some(err) = current {
|
||||
if error_chain_any(err, pred) {
|
||||
return true;
|
||||
}
|
||||
current = err.source();
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// s3s raises `BodySizeLimitExceeded` when the streaming-body budget
|
||||
/// (`put_object_max_size`) runs out mid-stream. The type lives in s3s's
|
||||
/// private `http` module, so it is recognised by its `Display` form
|
||||
/// (`body size {size} exceeds limit {limit}`), like the other s3s body-stream
|
||||
/// errors above. Switch to a typed downcast once s3s re-exports the type.
|
||||
fn is_body_size_limit_exceeded_display(err: &(dyn std::error::Error + 'static)) -> bool {
|
||||
let text = err.to_string();
|
||||
text.starts_with("body size ") && text.contains(" exceeds limit ")
|
||||
}
|
||||
|
||||
fn error_chain_has_body_size_limit_exceeded(err: &(dyn std::error::Error + 'static)) -> bool {
|
||||
error_chain_any(err, &is_body_size_limit_exceeded_display)
|
||||
}
|
||||
|
||||
/// hyper reports a request body whose connection hit EOF before
|
||||
/// `Content-Length` bytes arrived as a `Kind::Body` error carrying an
|
||||
/// `UnexpectedEof` `io::Error` (its `IncompleteBody` marker is private).
|
||||
/// That is a client-side short body, not a server fault.
|
||||
fn is_hyper_body_eof(err: &(dyn std::error::Error + 'static)) -> bool {
|
||||
err.downcast_ref::<hyper::Error>()
|
||||
.and_then(|hyper_err| std::error::Error::source(hyper_err))
|
||||
.and_then(|cause| cause.downcast_ref::<std::io::Error>())
|
||||
.is_some_and(|io_err| io_err.kind() == std::io::ErrorKind::UnexpectedEof)
|
||||
}
|
||||
|
||||
fn error_chain_has_hyper_body_eof(err: &(dyn std::error::Error + 'static)) -> bool {
|
||||
error_chain_any(err, &is_hyper_body_eof)
|
||||
}
|
||||
|
||||
impl From<ApiError> for S3Error {
|
||||
fn from(err: ApiError) -> Self {
|
||||
let status = custom_error_status(&err.code);
|
||||
@@ -535,6 +586,22 @@ impl From<StorageError> for ApiError {
|
||||
};
|
||||
}
|
||||
|
||||
if error_chain_has_body_size_limit_exceeded(inner) {
|
||||
return ApiError {
|
||||
code: S3ErrorCode::EntityTooLarge,
|
||||
message: ApiError::error_code_to_message(&S3ErrorCode::EntityTooLarge),
|
||||
source: Some(Box::new(err)),
|
||||
};
|
||||
}
|
||||
|
||||
if error_chain_has_hyper_body_eof(inner) {
|
||||
return ApiError {
|
||||
code: S3ErrorCode::IncompleteBody,
|
||||
message: ApiError::error_code_to_message(&S3ErrorCode::IncompleteBody),
|
||||
source: Some(Box::new(err)),
|
||||
};
|
||||
}
|
||||
|
||||
if matches!(s3s_body_stream_error, Some(S3sBodyStreamError::IncompleteBody)) {
|
||||
return ApiError {
|
||||
code: S3ErrorCode::IncompleteBody,
|
||||
@@ -678,6 +745,22 @@ impl From<std::io::Error> for ApiError {
|
||||
source: Some(Box::new(err)),
|
||||
};
|
||||
}
|
||||
if error_chain_has_body_size_limit_exceeded(inner) {
|
||||
return ApiError {
|
||||
code: S3ErrorCode::EntityTooLarge,
|
||||
message: ApiError::error_code_to_message(&S3ErrorCode::EntityTooLarge),
|
||||
source: Some(Box::new(err)),
|
||||
};
|
||||
}
|
||||
|
||||
if error_chain_has_hyper_body_eof(inner) {
|
||||
return ApiError {
|
||||
code: S3ErrorCode::IncompleteBody,
|
||||
message: ApiError::error_code_to_message(&S3ErrorCode::IncompleteBody),
|
||||
source: Some(Box::new(err)),
|
||||
};
|
||||
}
|
||||
|
||||
if matches!(s3s_body_stream_error, Some(S3sBodyStreamError::IncompleteBody)) {
|
||||
return ApiError {
|
||||
code: S3ErrorCode::IncompleteBody,
|
||||
@@ -950,6 +1033,117 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn body_size_limit_exceeded_maps_to_entity_too_large_across_io_boundaries() {
|
||||
// Shape observed in production (issue #7596):
|
||||
// Custom { UnexpectedEof, Custom { Other, BodySizeLimitExceeded { size, limit } } }
|
||||
let nested = || {
|
||||
IoError::new(
|
||||
ErrorKind::UnexpectedEof,
|
||||
IoError::other(MockS3sBodyStreamError("body size 16384 exceeds limit 6389")),
|
||||
)
|
||||
};
|
||||
|
||||
let direct: ApiError = nested().into();
|
||||
assert_eq!(direct.code, S3ErrorCode::EntityTooLarge);
|
||||
assert_eq!(direct.message, ApiError::error_code_to_message(&S3ErrorCode::EntityTooLarge));
|
||||
|
||||
let storage: ApiError = StorageError::Io(nested()).into();
|
||||
assert_eq!(storage.code, S3ErrorCode::EntityTooLarge);
|
||||
assert!(storage.source.is_some());
|
||||
|
||||
// An unrelated message that merely mentions a limit stays internal.
|
||||
let other: ApiError = IoError::other(MockS3sBodyStreamError("limit exceeded for something else")).into();
|
||||
assert_eq!(other.code, S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
/// Trip s3s's real streaming-body budget with a tiny limit so the
|
||||
/// display-based matcher is checked against the pinned dependency's
|
||||
/// actual error, not only the mocked string.
|
||||
#[tokio::test]
|
||||
async fn real_s3s_body_size_limit_error_maps_to_entity_too_large() {
|
||||
use futures::StreamExt;
|
||||
|
||||
let real_error = || async {
|
||||
let mut body = s3s::Body::from(bytes::Bytes::from_static(b"hello"));
|
||||
body.set_limit(Some(4));
|
||||
body.next()
|
||||
.await
|
||||
.expect("one frame")
|
||||
.expect_err("five bytes must exceed a four-byte budget")
|
||||
};
|
||||
|
||||
let err = real_error().await;
|
||||
assert!(is_body_size_limit_exceeded_display(err.as_ref()), "unexpected display: {err}");
|
||||
|
||||
let err = real_error().await;
|
||||
let storage: ApiError = StorageError::Io(IoError::new(ErrorKind::UnexpectedEof, IoError::other(err))).into();
|
||||
assert_eq!(storage.code, S3ErrorCode::EntityTooLarge);
|
||||
assert_eq!(storage.message, ApiError::error_code_to_message(&S3ErrorCode::EntityTooLarge));
|
||||
|
||||
let err = real_error().await;
|
||||
let direct: ApiError = IoError::other(err).into();
|
||||
assert_eq!(direct.code, S3ErrorCode::EntityTooLarge);
|
||||
}
|
||||
|
||||
/// Drive a real hyper HTTP/1 server so the test sees hyper's own body EOF
|
||||
/// error (`hyper::Error(Body, UnexpectedEof, IncompleteBody)`), which has no
|
||||
/// public constructor.
|
||||
async fn capture_hyper_body_eof_error() -> hyper::Error {
|
||||
use http_body_util::BodyExt;
|
||||
use hyper::service::service_fn;
|
||||
use hyper_util::rt::TokioIo;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("bind");
|
||||
let addr = listener.local_addr().expect("local addr");
|
||||
let captured: Arc<Mutex<Option<hyper::Error>>> = Arc::new(Mutex::new(None));
|
||||
let server_slot = Arc::clone(&captured);
|
||||
let server = tokio::spawn(async move {
|
||||
let (stream, _) = listener.accept().await.expect("accept");
|
||||
let slot = server_slot;
|
||||
let service = service_fn(move |req: hyper::Request<hyper::body::Incoming>| {
|
||||
let slot = Arc::clone(&slot);
|
||||
async move {
|
||||
let err = req.into_body().collect().await.expect_err("short body must fail");
|
||||
*slot.lock().expect("slot") = Some(err);
|
||||
Ok::<_, std::convert::Infallible>(hyper::Response::new(String::new()))
|
||||
}
|
||||
});
|
||||
let _ = hyper::server::conn::http1::Builder::new()
|
||||
.serve_connection(TokioIo::new(stream), service)
|
||||
.await;
|
||||
});
|
||||
|
||||
let mut client = tokio::net::TcpStream::connect(addr).await.expect("connect");
|
||||
client
|
||||
.write_all(b"PUT /bucket/key HTTP/1.1\r\nHost: localhost\r\nContent-Length: 100\r\n\r\nabc")
|
||||
.await
|
||||
.expect("write partial body");
|
||||
client.shutdown().await.expect("shutdown write side");
|
||||
let _ = tokio::time::timeout(std::time::Duration::from_secs(10), server).await;
|
||||
let captured = captured.lock().expect("slot").take();
|
||||
captured.expect("hyper body error captured")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn hyper_body_eof_maps_to_incomplete_body_across_io_boundaries() {
|
||||
let hyper_err = capture_hyper_body_eof_error().await;
|
||||
assert!(is_hyper_body_eof(&hyper_err), "unexpected hyper error shape: {hyper_err:?}");
|
||||
|
||||
// Shape observed in production (issue #7596):
|
||||
// Custom { UnexpectedEof, Custom { Other, hyper::Error(Body, UnexpectedEof, IncompleteBody) } }
|
||||
let nested = IoError::new(ErrorKind::UnexpectedEof, IoError::other(hyper_err));
|
||||
let storage: ApiError = StorageError::Io(nested).into();
|
||||
assert_eq!(storage.code, S3ErrorCode::IncompleteBody);
|
||||
assert_eq!(storage.message, ApiError::error_code_to_message(&S3ErrorCode::IncompleteBody));
|
||||
|
||||
let hyper_err = capture_hyper_body_eof_error().await;
|
||||
let direct: ApiError = IoError::other(hyper_err).into();
|
||||
assert_eq!(direct.code, S3ErrorCode::IncompleteBody);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn server_side_source_read_error_maps_to_service_unavailable_before_incomplete_body() {
|
||||
let short_source = IoError::new(ErrorKind::UnexpectedEof, rustfs_rio::IncompleteBody { remaining: 17 });
|
||||
|
||||
@@ -418,7 +418,7 @@ impl PathCategory {
|
||||
PathCategory::InternodeRpc
|
||||
} else if path.starts_with("/rustfs/admin/") || path.starts_with("/minio/admin/") {
|
||||
PathCategory::AdminApi
|
||||
} else if path.starts_with("/rustfs/console") {
|
||||
} else if crate::server::has_path_prefix(path, crate::server::console_prefix()) {
|
||||
PathCategory::Console
|
||||
} else if path == "/health"
|
||||
|| path.starts_with("/health/")
|
||||
@@ -766,8 +766,10 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_path_category_classify_console() {
|
||||
assert_eq!(PathCategory::classify("/rustfs/console/index.html"), PathCategory::Console);
|
||||
assert_eq!(PathCategory::classify("/rustfs/console"), PathCategory::Console);
|
||||
let prefix = crate::server::console_prefix();
|
||||
assert_eq!(PathCategory::classify(&format!("{prefix}/index.html")), PathCategory::Console);
|
||||
assert_eq!(PathCategory::classify(prefix), PathCategory::Console);
|
||||
assert_eq!(PathCategory::classify(&format!("{prefix}-other/index.html")), PathCategory::S3DataPlane);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -158,13 +158,11 @@ static HTTP_STATUS_CLASS_METRICS: std::sync::LazyLock<[HttpStatusClassMetrics; 6
|
||||
static HTTP_TRANSPORT_FAILURES_COUNTER: std::sync::LazyLock<metrics::Counter> =
|
||||
std::sync::LazyLock::new(|| counter!(METRIC_HTTP_SERVER_FAILURES_TOTAL, LABEL_HTTP_STATUS_CLASS => "transport"));
|
||||
|
||||
const RUSTFS_S3_PUT_OBJECT_MAX_SIZE: u64 = 5 * 1024 * 1024 * 1024;
|
||||
|
||||
fn rustfs_s3_config() -> S3Config {
|
||||
let mut s3_config = S3Config::default();
|
||||
s3_config.normalize_forward_slash_path = true;
|
||||
s3_config.enable_sig_v2 = true;
|
||||
s3_config.put_object_max_size = Some(RUSTFS_S3_PUT_OBJECT_MAX_SIZE);
|
||||
s3_config.put_object_max_size = Some(rustfs_config::MAX_SINGLE_PUT_OBJECT_SIZE);
|
||||
s3_config.sig_v4_allowed_services.push("s3tables".to_string());
|
||||
s3_config
|
||||
}
|
||||
@@ -966,6 +964,7 @@ pub async fn start_http_server(
|
||||
readiness: Arc<GlobalReadiness>,
|
||||
server_ctx: Arc<ServerContextSlot>,
|
||||
) -> Result<(ShutdownHandle, SocketAddr)> {
|
||||
crate::server::init_console_prefix()?;
|
||||
let server_addr = parse_and_resolve_address(config.address.as_str()).map_err(Error::other)?;
|
||||
|
||||
// The listening address and port are obtained from the parameters
|
||||
@@ -1213,6 +1212,7 @@ pub async fn start_http_server(
|
||||
let now_time = jiff::Zoned::now().strftime("%Y-%m-%d %H:%M:%S").to_string();
|
||||
if config.console_enable {
|
||||
admin::console::init_console_cfg(local_ip, local_port);
|
||||
let console_prefix = crate::server::console_prefix();
|
||||
|
||||
info!(
|
||||
target: "rustfs::console::startup",
|
||||
@@ -1220,7 +1220,7 @@ pub async fn start_http_server(
|
||||
component = LOG_COMPONENT_SERVER,
|
||||
subsystem = LOG_SUBSYSTEM_STARTUP,
|
||||
service = "console",
|
||||
endpoint = %format!("{protocol}://{local_ip_str}:{local_port}/rustfs/console/index.html"),
|
||||
endpoint = %format!("{protocol}://{local_ip_str}:{local_port}{console_prefix}/index.html"),
|
||||
"Startup endpoint available"
|
||||
);
|
||||
info!(
|
||||
@@ -1229,7 +1229,7 @@ pub async fn start_http_server(
|
||||
component = LOG_COMPONENT_SERVER,
|
||||
subsystem = LOG_SUBSYSTEM_STARTUP,
|
||||
service = "console_localhost",
|
||||
endpoint = %format!("{protocol}://127.0.0.1:{local_port}/rustfs/console/index.html"),
|
||||
endpoint = %format!("{protocol}://127.0.0.1:{local_port}{console_prefix}/index.html"),
|
||||
"Startup endpoint available"
|
||||
);
|
||||
} else {
|
||||
@@ -3049,7 +3049,7 @@ mod tests {
|
||||
assert!(s3_config.normalize_forward_slash_path);
|
||||
assert!(s3_config.normalize_content_length);
|
||||
assert!(s3_config.enable_sig_v2);
|
||||
assert_eq!(s3_config.put_object_max_size, Some(RUSTFS_S3_PUT_OBJECT_MAX_SIZE));
|
||||
assert_eq!(s3_config.put_object_max_size, Some(rustfs_config::MAX_SINGLE_PUT_OBJECT_SIZE));
|
||||
assert!(s3_config.sig_v4_allowed_services.iter().any(|service| service == "s3"));
|
||||
assert!(s3_config.sig_v4_allowed_services.iter().any(|service| service == "sts"));
|
||||
assert!(s3_config.sig_v4_allowed_services.iter().any(|service| service == "s3tables"));
|
||||
|
||||
@@ -20,10 +20,10 @@ use crate::server::RemoteAddr;
|
||||
use crate::server::cors;
|
||||
use crate::server::hybrid::{HybridBody, is_grpc_request};
|
||||
use crate::server::{
|
||||
ADMIN_PREFIX, CONSOLE_PREFIX, HEALTH_COMPAT_LIVE_PATH, HEALTH_PREFIX, HEALTH_READY_PATH, HealthProbe, MINIO_ADMIN_PREFIX,
|
||||
ADMIN_PREFIX, HEALTH_COMPAT_LIVE_PATH, HEALTH_PREFIX, HEALTH_READY_PATH, HealthProbe, MINIO_ADMIN_PREFIX,
|
||||
MINIO_ADMIN_V3_PREFIX, MINIO_HEALTH_CLUSTER_PATH, MINIO_HEALTH_CLUSTER_READ_PATH, MINIO_HEALTH_LIVE_PATH,
|
||||
MINIO_HEALTH_READY_PATH, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH, RPC_PREFIX, RUSTFS_ADMIN_PREFIX, active_http_requests,
|
||||
build_health_response_parts, collect_probe_readiness, has_path_prefix, is_admin_path, is_table_catalog_path,
|
||||
build_health_response_parts, collect_probe_readiness, console_prefix, has_path_prefix, is_admin_path, is_table_catalog_path,
|
||||
kms_probe_staleness_limit, kms_ready_from_probe,
|
||||
};
|
||||
use crate::shared_types::ReadinessDegradedReason;
|
||||
@@ -625,7 +625,7 @@ where
|
||||
// Create redirect response
|
||||
let redirect_response = Response::builder()
|
||||
.status(StatusCode::FOUND)
|
||||
.header(http::header::LOCATION, "/rustfs/console/")
|
||||
.header(http::header::LOCATION, format!("{}/", console_prefix()))
|
||||
.body(HybridBody::Rest {
|
||||
rest_body: RestBody::default(),
|
||||
})
|
||||
@@ -1861,7 +1861,7 @@ fn is_object_attributes_request<B>(req: &HttpRequest<B>) -> bool {
|
||||
|| has_path_prefix(path, RUSTFS_ADMIN_PREFIX)
|
||||
|| has_path_prefix(path, MINIO_ADMIN_V3_PREFIX)
|
||||
|| is_table_catalog_path(path)
|
||||
|| has_path_prefix(path, CONSOLE_PREFIX)
|
||||
|| has_path_prefix(path, console_prefix())
|
||||
|| has_path_prefix(path, RPC_PREFIX)
|
||||
{
|
||||
return false;
|
||||
@@ -2242,7 +2242,74 @@ fn rewrite_double_slash_root(uri: &Uri) -> Option<Uri> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[tokio::test]
|
||||
async fn console_prefix_process_case_browser_redirect() {
|
||||
if std::env::var_os("RUSTFS_TEST_CONSOLE_PREFIX_PROCESS").is_none() {
|
||||
return;
|
||||
}
|
||||
crate::server::init_console_prefix().expect("initialize console prefix");
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("redirect listener");
|
||||
let addr = listener.local_addr().expect("redirect listener address");
|
||||
let server = tokio::spawn(async move {
|
||||
let (stream, _) = listener.accept().await.expect("redirect client");
|
||||
let inner = tower::service_fn(|_request: Request<Incoming>| async {
|
||||
Ok::<_, Infallible>(Response::new(HybridBody::<Empty<Bytes>, Empty<Bytes>>::Rest { rest_body: Empty::new() }))
|
||||
});
|
||||
let service = RedirectLayer.layer(inner);
|
||||
hyper::server::conn::http1::Builder::new()
|
||||
.serve_connection(
|
||||
hyper_util::rt::TokioIo::new(stream),
|
||||
hyper_util::service::TowerToHyperService::new(service),
|
||||
)
|
||||
.await
|
||||
.expect("redirect connection");
|
||||
});
|
||||
let client = reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.http1_only()
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.timeout(Duration::from_secs(5))
|
||||
.build()
|
||||
.expect("redirect client");
|
||||
let response = client
|
||||
.get(format!("http://{addr}/"))
|
||||
.header(http::header::USER_AGENT, "Mozilla/5.0")
|
||||
.header(http::header::CONNECTION, "close")
|
||||
.send()
|
||||
.await
|
||||
.expect("browser response");
|
||||
assert_eq!(response.status(), StatusCode::FOUND);
|
||||
assert_eq!(response.headers()[http::header::LOCATION], format!("{}/", console_prefix()));
|
||||
response.bytes().await.expect("redirect body");
|
||||
tokio::time::timeout(Duration::from_secs(5), server)
|
||||
.await
|
||||
.expect("bounded redirect server shutdown")
|
||||
.expect("redirect task");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn console_prefix_process_case_classification() {
|
||||
if std::env::var_os("RUSTFS_TEST_CONSOLE_PREFIX_PROCESS").is_none() {
|
||||
return;
|
||||
}
|
||||
crate::server::init_console_prefix().expect("initialize console prefix");
|
||||
let prefix = crate::server::console_prefix();
|
||||
let console_uri = format!("{prefix}/index.html").parse().expect("console URI");
|
||||
assert!(is_empty_body_console_path(&Method::GET, &console_uri));
|
||||
let request = HttpRequest::builder()
|
||||
.uri(format!("{prefix}/index.html?attributes"))
|
||||
.body(())
|
||||
.expect("console attributes request");
|
||||
assert!(!is_object_attributes_request(&request));
|
||||
let s3_request = HttpRequest::builder()
|
||||
.uri("/bucket/object?attributes")
|
||||
.body(())
|
||||
.expect("S3 attributes request");
|
||||
assert!(is_object_attributes_request(&s3_request));
|
||||
}
|
||||
|
||||
use super::*;
|
||||
use crate::server::CONSOLE_PREFIX;
|
||||
use crate::server::compress::{HttpCompressionConfig, PathAwareHttpCompressionPredicate, PathCategoryInjectionLayer};
|
||||
use crate::server::{FAVICON_PATH, LICENSE, RemoteAddr, VERSION};
|
||||
use futures::future::{Ready, ready};
|
||||
@@ -2333,7 +2400,7 @@ mod tests {
|
||||
for path in [
|
||||
"/rustfs/admin/v3/metrics",
|
||||
"/minio/admin/v3/storageinfo",
|
||||
"/rustfs/console/",
|
||||
CONSOLE_PREFIX,
|
||||
"/rustfs/rpc/test",
|
||||
"/health/ready",
|
||||
"/_iceberg/v1/config",
|
||||
@@ -2624,7 +2691,7 @@ mod tests {
|
||||
for path in [
|
||||
"/rustfs/admin/v3/info",
|
||||
"/minio/admin/v3/info",
|
||||
"/rustfs/console/",
|
||||
CONSOLE_PREFIX,
|
||||
HEALTH_PREFIX,
|
||||
"/iceberg/v1/config",
|
||||
"/rustfs/rpc/v1/read-file",
|
||||
@@ -3983,7 +4050,7 @@ mod tests {
|
||||
"/minio/admin/v3/pools/cancel?versionId=unused",
|
||||
"/rustfs/admin/v3/pools/cancel?versionId=unused",
|
||||
"/rustfs/rpc/read_file_stream?versionId=unused",
|
||||
"/rustfs/console/index.html?versionId=unused",
|
||||
&format!("{CONSOLE_PREFIX}/index.html?versionId=unused"),
|
||||
"/health?versionId=unused",
|
||||
"/health/ready?versionId=unused",
|
||||
"/profile/cpu?versionId=unused",
|
||||
|
||||
@@ -72,7 +72,7 @@ pub(crate) use prefix::{
|
||||
HEALTH_COMPAT_LIVE_PATH, HEALTH_PREFIX, HEALTH_READY_PATH, LICENSE, MINIO_ADMIN_PREFIX, MINIO_ADMIN_V3_PREFIX,
|
||||
MINIO_HEALTH_CLUSTER_PATH, MINIO_HEALTH_CLUSTER_READ_PATH, MINIO_HEALTH_LIVE_PATH, MINIO_HEALTH_READY_PATH, PROFILE_CPU_PATH,
|
||||
PROFILE_MEMORY_PATH, RPC_PREFIX, RUSTFS_ADMIN_PREFIX, TABLE_CATALOG_COMPAT_PREFIX, TABLE_CATALOG_PREFIX, TONIC_PREFIX,
|
||||
VERSION, has_path_prefix, is_admin_path, is_table_catalog_path,
|
||||
VERSION, console_prefix, has_path_prefix, init_console_prefix, is_admin_path, is_table_catalog_path,
|
||||
};
|
||||
pub(crate) use readiness::ReadinessDegradedReason;
|
||||
pub(crate) use readiness::ReadinessGateLayer;
|
||||
|
||||
+219
-4
@@ -83,10 +83,81 @@ pub(crate) const RUSTFS_ADMIN_PREFIX: &str = "/rustfs/admin/v3";
|
||||
/// MinIO-compatible admin API prefix accepted by RustFS.
|
||||
pub(crate) const MINIO_ADMIN_V3_PREFIX: &str = "/minio/admin/v3";
|
||||
|
||||
/// Predefined console prefix for RustFS server routes.
|
||||
/// This prefix is used for endpoints that handle console-related tasks
|
||||
/// such as user interface and management.
|
||||
pub(crate) const CONSOLE_PREFIX: &str = "/rustfs/console";
|
||||
/// Console asset base path embedded at build time and used as the startup default.
|
||||
/// It must match NEXT_PUBLIC_BASE_PATH when building the bundled frontend.
|
||||
pub(crate) const CONSOLE_PREFIX: &str = match option_env!("RUSTFS_CONSOLE_BASE_PATH") {
|
||||
Some(path) if !path.is_empty() => path,
|
||||
_ => rustfs_config::DEFAULT_CONSOLE_PREFIX,
|
||||
};
|
||||
|
||||
static CONFIGURED_CONSOLE_PREFIX: std::sync::OnceLock<String> = std::sync::OnceLock::new();
|
||||
|
||||
/// The prefix is fixed before listeners start; request handling never reads the environment.
|
||||
pub(crate) fn console_prefix() -> &'static str {
|
||||
CONFIGURED_CONSOLE_PREFIX.get().map(String::as_str).unwrap_or(CONSOLE_PREFIX)
|
||||
}
|
||||
|
||||
pub(crate) fn init_console_prefix() -> std::io::Result<()> {
|
||||
let raw = match std::env::var(rustfs_config::ENV_RUSTFS_CONSOLE_PREFIX) {
|
||||
Ok(value) => value,
|
||||
Err(std::env::VarError::NotPresent) => CONSOLE_PREFIX.to_string(),
|
||||
Err(err) => return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, err)),
|
||||
};
|
||||
let prefix = validate_console_prefix(&raw)?;
|
||||
if CONFIGURED_CONSOLE_PREFIX.get_or_init(|| prefix.clone()) != &prefix {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
"RUSTFS_CONSOLE_PREFIX cannot change after server initialization",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_console_prefix(raw: &str) -> std::io::Result<String> {
|
||||
let prefix = raw.strip_suffix('/').unwrap_or(raw);
|
||||
// Keep the value safe in HTTP headers, Axum routes, and embedded HTML/JS.
|
||||
if !prefix.starts_with('/')
|
||||
|| prefix.len() > 256
|
||||
|| prefix[1..].split('/').any(|segment| {
|
||||
segment.is_empty()
|
||||
|| matches!(segment, "." | "..")
|
||||
|| !segment
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~'))
|
||||
})
|
||||
{
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
"RUSTFS_CONSOLE_PREFIX must be a non-root absolute path of at most 256 bytes with nonempty URL-safe segments",
|
||||
));
|
||||
}
|
||||
let reserved = [
|
||||
ADMIN_PREFIX,
|
||||
MINIO_ADMIN_PREFIX,
|
||||
TABLE_CATALOG_PREFIX,
|
||||
TABLE_CATALOG_COMPAT_PREFIX,
|
||||
RPC_PREFIX,
|
||||
TONIC_PREFIX,
|
||||
"/rustfs/peer",
|
||||
HEALTH_PREFIX,
|
||||
"/minio/health",
|
||||
"/profile",
|
||||
"/index.html",
|
||||
FAVICON_PATH,
|
||||
APPLE_TOUCH_ICON_PATH,
|
||||
APPLE_TOUCH_ICON_PRECOMPOSED_PATH,
|
||||
];
|
||||
if reserved
|
||||
.iter()
|
||||
.any(|path| has_path_prefix(prefix, path) || has_path_prefix(path, prefix))
|
||||
{
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
"RUSTFS_CONSOLE_PREFIX overlaps a reserved server route",
|
||||
));
|
||||
}
|
||||
Ok(prefix.to_string())
|
||||
}
|
||||
|
||||
/// Predefined RPC prefix for RustFS server routes.
|
||||
/// This prefix is used for endpoints that handle remote procedure calls (RPC).
|
||||
@@ -111,3 +182,147 @@ pub const LOGO: &str = r#"
|
||||
░▀░▀░▀▀▀░▀▀▀░░▀░░▀░░░▀▀▀
|
||||
|
||||
"#;
|
||||
|
||||
#[cfg(test)]
|
||||
mod console_prefix_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn console_prefix_validation() {
|
||||
for (raw, expected) in [
|
||||
(CONSOLE_PREFIX, CONSOLE_PREFIX),
|
||||
("/console", "/console"),
|
||||
("/management/console/", "/management/console"),
|
||||
("/health-dashboard", "/health-dashboard"),
|
||||
] {
|
||||
assert_eq!(validate_console_prefix(raw).expect("valid console prefix"), expected);
|
||||
}
|
||||
for raw in [
|
||||
"",
|
||||
"/",
|
||||
"console",
|
||||
"//console",
|
||||
"/console//",
|
||||
"/a//b",
|
||||
"/a/../b",
|
||||
"/a/./b",
|
||||
"/%2e%2e",
|
||||
"/console?x=1",
|
||||
"/console#x",
|
||||
"/console\\x",
|
||||
"/a\n",
|
||||
"/{param}",
|
||||
"/<script>",
|
||||
"/控制台",
|
||||
"/rustfs",
|
||||
"/rustfs/admin",
|
||||
"/rustfs/admin/v3/ui",
|
||||
"/minio",
|
||||
"/minio/admin",
|
||||
"/health",
|
||||
"/health/ui",
|
||||
"/iceberg",
|
||||
"/_iceberg/v1",
|
||||
"/rustfs/rpc",
|
||||
"/rustfs/peer",
|
||||
"/node_service.NodeService",
|
||||
"/profile",
|
||||
"/index.html",
|
||||
"/favicon.ico",
|
||||
"/index.html",
|
||||
] {
|
||||
assert_eq!(validate_console_prefix(raw).expect_err(raw).kind(), std::io::ErrorKind::InvalidInput);
|
||||
}
|
||||
assert!(validate_console_prefix(&format!("/{}", "a".repeat(255))).is_ok());
|
||||
assert!(validate_console_prefix(&format!("/{}", "a".repeat(256))).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn configured_console_prefix_subprocesses() {
|
||||
// Startup configuration is process-wide; isolate each value from other unit tests.
|
||||
for prefix in [None, Some("/console"), Some("/management/console/")] {
|
||||
let mut command = std::process::Command::new(std::env::current_exe().expect("test executable"));
|
||||
command
|
||||
.args(["console_prefix_process_case", "--test-threads=1"])
|
||||
.env("RUSTFS_TEST_CONSOLE_PREFIX_PROCESS", "1")
|
||||
.env("RUSTFS_CONSOLE_BASE_PATH", "/runtime-ignored/console")
|
||||
.env("RUSTFS_BROWSER_REDIRECT_URL", "https://console.example.com")
|
||||
.env("RUSTFS_HEALTH_ENDPOINT_ENABLE", "true")
|
||||
.env("RUSTFS_CONSOLE_RATE_LIMIT_ENABLE", "false");
|
||||
if let Some(prefix) = prefix {
|
||||
command.env(rustfs_config::ENV_RUSTFS_CONSOLE_PREFIX, prefix);
|
||||
} else {
|
||||
command.env_remove(rustfs_config::ENV_RUSTFS_CONSOLE_PREFIX);
|
||||
}
|
||||
let output = command.output().expect("run isolated console tests");
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"prefix {prefix:?}: {}{}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn console_prefix_process_case_routes() {
|
||||
if std::env::var_os("RUSTFS_TEST_CONSOLE_PREFIX_PROCESS").is_none() {
|
||||
return;
|
||||
}
|
||||
use axum::body::Body;
|
||||
use http::{Request, StatusCode};
|
||||
use tower::ServiceExt;
|
||||
init_console_prefix().expect("initialize configured console prefix");
|
||||
let expected = std::env::var(rustfs_config::ENV_RUSTFS_CONSOLE_PREFIX).unwrap_or_else(|_| CONSOLE_PREFIX.to_string());
|
||||
let prefix = expected.trim_end_matches('/');
|
||||
assert_eq!(console_prefix(), prefix);
|
||||
assert!(crate::admin::console::is_console_path(&format!("{prefix}/index.html")));
|
||||
assert!(!crate::admin::console::is_console_path(&format!("{prefix}-other/index.html")));
|
||||
for path in ["/rustfs/admin/v3/info", "/minio/admin/v3/info", "/health", "/bucket/object"] {
|
||||
assert!(!crate::admin::console::is_console_path(path), "reserved or S3 path {path}");
|
||||
}
|
||||
assert_eq!(
|
||||
crate::server::compress::PathCategory::classify(&format!("{prefix}/asset.js")),
|
||||
crate::server::compress::PathCategory::Console
|
||||
);
|
||||
if prefix != CONSOLE_PREFIX {
|
||||
assert!(!crate::admin::console::is_console_path(CONSOLE_PREFIX));
|
||||
}
|
||||
crate::admin::console::init_console_cfg(std::net::Ipv4Addr::LOCALHOST.into(), 9001);
|
||||
let router = crate::admin::console::make_console_server();
|
||||
for (suffix, expected_status, expected_ready) in [
|
||||
("/health", StatusCode::OK, None),
|
||||
("/health/live", StatusCode::OK, None),
|
||||
("/health/ready", StatusCode::SERVICE_UNAVAILABLE, Some(false)),
|
||||
] {
|
||||
let response = router
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri(format!("{prefix}{suffix}"))
|
||||
.body(Body::empty())
|
||||
.expect("health request"),
|
||||
)
|
||||
.await
|
||||
.expect("health response");
|
||||
assert_eq!(response.status(), expected_status, "{suffix}");
|
||||
let body = axum::body::to_bytes(response.into_body(), 65536).await.expect("health body");
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("health JSON");
|
||||
assert_eq!(payload.get("ready").and_then(serde_json::Value::as_bool), expected_ready, "{suffix}");
|
||||
}
|
||||
for suffix in ["/version", "/license"] {
|
||||
let response = router
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri(format!("{prefix}{suffix}"))
|
||||
.body(Body::empty())
|
||||
.expect("console request"),
|
||||
)
|
||||
.await
|
||||
.expect("console response");
|
||||
assert_eq!(response.status(), StatusCode::OK, "{suffix}");
|
||||
assert_eq!(response.headers()[http::header::CONTENT_TYPE], "application/json");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,9 +54,10 @@
|
||||
//! dimension, whose key space (bucket names) is attacker-chosen.
|
||||
|
||||
use crate::server::{
|
||||
CONSOLE_PREFIX, FAVICON_PATH, HEALTH_COMPAT_LIVE_PATH, HEALTH_PREFIX, HEALTH_READY_PATH, MINIO_HEALTH_CLUSTER_PATH,
|
||||
FAVICON_PATH, HEALTH_COMPAT_LIVE_PATH, HEALTH_PREFIX, HEALTH_READY_PATH, MINIO_HEALTH_CLUSTER_PATH,
|
||||
MINIO_HEALTH_CLUSTER_READ_PATH, MINIO_HEALTH_LIVE_PATH, MINIO_HEALTH_READY_PATH, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH,
|
||||
RPC_PREFIX, RemoteAddr, TONIC_PREFIX, has_path_prefix, is_admin_path, is_table_catalog_path, strip_valid_port_suffix,
|
||||
RPC_PREFIX, RemoteAddr, TONIC_PREFIX, console_prefix, has_path_prefix, is_admin_path, is_table_catalog_path,
|
||||
strip_valid_port_suffix,
|
||||
};
|
||||
use crate::storage_api::server::layer::request_context::RequestContext;
|
||||
use bytes::Bytes;
|
||||
@@ -405,7 +406,7 @@ fn is_rate_limit_exempt_path(path: &str) -> bool {
|
||||
| FAVICON_PATH
|
||||
) || has_path_prefix(path, RPC_PREFIX)
|
||||
|| has_path_prefix(path, TONIC_PREFIX)
|
||||
|| has_path_prefix(path, CONSOLE_PREFIX)
|
||||
|| has_path_prefix(path, console_prefix())
|
||||
}
|
||||
|
||||
/// Apply the standard throttling headers shared by every rate-limited scope.
|
||||
@@ -630,6 +631,18 @@ where
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn console_prefix_process_case_classification() {
|
||||
if std::env::var_os("RUSTFS_TEST_CONSOLE_PREFIX_PROCESS").is_none() {
|
||||
return;
|
||||
}
|
||||
crate::server::init_console_prefix().expect("initialize console prefix");
|
||||
let prefix = crate::server::console_prefix();
|
||||
assert!(is_rate_limit_exempt_path(&format!("{prefix}/version")));
|
||||
assert!(!is_rate_limit_exempt_path(&format!("{prefix}-other/version")));
|
||||
assert!(!is_rate_limit_exempt_path("/bucket/object"));
|
||||
}
|
||||
|
||||
use super::*;
|
||||
use http_body_util::BodyExt;
|
||||
use serial_test::serial;
|
||||
@@ -779,7 +792,7 @@ mod tests {
|
||||
"/favicon.ico",
|
||||
"/rustfs/rpc/anything",
|
||||
"/node_service.NodeService/Ping",
|
||||
"/rustfs/console/index.html",
|
||||
&format!("{}/index.html", console_prefix()),
|
||||
] {
|
||||
assert!(is_rate_limit_exempt_path(path), "{path} must be exempt");
|
||||
}
|
||||
|
||||
@@ -137,7 +137,7 @@ fn is_probe_path(path: &str) -> bool {
|
||||
let is_prefix_probe = has_path_prefix(path, crate::server::RUSTFS_ADMIN_PREFIX)
|
||||
|| has_path_prefix(path, crate::server::MINIO_ADMIN_V3_PREFIX)
|
||||
|| is_table_catalog_path(path)
|
||||
|| has_path_prefix(path, crate::server::CONSOLE_PREFIX)
|
||||
|| has_path_prefix(path, crate::server::console_prefix())
|
||||
|| has_path_prefix(path, crate::server::RPC_PREFIX)
|
||||
|| has_path_prefix(path, crate::server::ADMIN_PREFIX)
|
||||
|| has_path_prefix(path, crate::server::MINIO_ADMIN_PREFIX)
|
||||
@@ -1158,6 +1158,18 @@ where
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn console_prefix_process_case_classification() {
|
||||
if std::env::var_os("RUSTFS_TEST_CONSOLE_PREFIX_PROCESS").is_none() {
|
||||
return;
|
||||
}
|
||||
crate::server::init_console_prefix().expect("initialize console prefix");
|
||||
let prefix = crate::server::console_prefix();
|
||||
assert!(is_probe_path(&format!("{prefix}/index.html")));
|
||||
assert!(!is_probe_path(&format!("{prefix}-other/index.html")));
|
||||
assert!(!is_probe_path("/bucket/object"));
|
||||
}
|
||||
|
||||
use super::*;
|
||||
use crate::storage_api::server::readiness::{DiskOption, new_disk};
|
||||
use rustfs_madmin::{BackendInfo, Disk};
|
||||
@@ -1624,7 +1636,7 @@ mod tests {
|
||||
assert!(is_probe_path("/rustfs/admin/v3/info"));
|
||||
assert!(is_probe_path(&format!("{}/config", crate::server::TABLE_CATALOG_PREFIX)));
|
||||
assert!(is_probe_path("/_iceberg/v1/config"));
|
||||
assert!(is_probe_path("/rustfs/console/"));
|
||||
assert!(is_probe_path(&format!("{}/", crate::server::console_prefix())));
|
||||
assert!(!is_probe_path("/minio/adminx/object"));
|
||||
assert!(!is_probe_path("/rustfs/adminx/object"));
|
||||
assert!(!is_probe_path("/bucket/object"));
|
||||
|
||||
@@ -1630,6 +1630,44 @@ pub(crate) fn build_site_replication_config(
|
||||
}
|
||||
}
|
||||
|
||||
/// Reload `bucket`'s metadata on every other node of this site after a
|
||||
/// site-replication write. Every S3 bucket-config write does this
|
||||
/// (`app::bucket_usecase::notify_bucket_metadata_reload`); the
|
||||
/// site-replication writers did not, so on a multi-node site a node other
|
||||
/// than the one that applied the write served the previous targets and
|
||||
/// rules for up to the 15-minute refresh — a `resync start` routed to such a
|
||||
/// node reported every freshly wired bucket as `Config not found` or
|
||||
/// `recorded remote target no longer exists` (backlog#2367 A-5, backlog#2195
|
||||
/// item 2). Best effort like the S3 path: the write is durable and the
|
||||
/// refresh loop is the fallback, so an unreachable node must not fail the
|
||||
/// operation that already committed.
|
||||
pub(crate) async fn reload_bucket_metadata_on_peers(bucket: &str, operation: &'static str, scanner_maintenance_change: bool) {
|
||||
if scanner_maintenance_change {
|
||||
rustfs_scanner::record_scanner_maintenance_change(bucket);
|
||||
}
|
||||
let Some(notification_sys) = crate::admin::runtime_sources::current_notification_system() else {
|
||||
return;
|
||||
};
|
||||
let result = if scanner_maintenance_change {
|
||||
notification_sys.load_bucket_metadata_for_scanner_maintenance(bucket).await
|
||||
} else {
|
||||
notification_sys.load_bucket_metadata(bucket).await
|
||||
};
|
||||
if let Err(err) = result {
|
||||
warn!(
|
||||
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
|
||||
bucket = %bucket,
|
||||
operation,
|
||||
result = "peer_metadata_reload_failed",
|
||||
error = %err,
|
||||
"admin site replication state"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns whether the bucket targets were rewritten.
|
||||
pub(crate) async fn ensure_site_replication_bucket_targets_with_runtime(
|
||||
bucket: &str,
|
||||
state: &SiteReplicationState,
|
||||
@@ -1637,7 +1675,7 @@ pub(crate) async fn ensure_site_replication_bucket_targets_with_runtime(
|
||||
config: Option<&ReplicationConfiguration>,
|
||||
service_account_secret_key: &str,
|
||||
expected_incarnation_id: Uuid,
|
||||
) -> S3Result<()> {
|
||||
) -> S3Result<bool> {
|
||||
let existing = match metadata_sys::list_bucket_targets(bucket).await {
|
||||
Ok(targets) => targets,
|
||||
Err(StorageError::ConfigNotFound) => BucketTargets::default(),
|
||||
@@ -1649,7 +1687,7 @@ pub(crate) async fn ensure_site_replication_bucket_targets_with_runtime(
|
||||
let updated =
|
||||
reconcile_site_replication_bucket_targets(existing, bucket, state, local_peer, config, service_account_secret_key)?;
|
||||
if updated.targets.is_empty() {
|
||||
return Ok(());
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let json_targets = serde_json::to_vec(&updated)
|
||||
@@ -1658,12 +1696,12 @@ pub(crate) async fn ensure_site_replication_bucket_targets_with_runtime(
|
||||
// client — noticeable now that startup reconciles all buckets, not just the one bucket
|
||||
// an operation touched.
|
||||
if json_targets == existing_json {
|
||||
return Ok(());
|
||||
return Ok(false);
|
||||
}
|
||||
metadata_sys::update_if_incarnation(bucket, BUCKET_TARGETS_FILE, json_targets, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
Ok(())
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
pub(crate) async fn bucket_replication_config_for_target_refresh(bucket: &str) -> S3Result<Option<ReplicationConfiguration>> {
|
||||
@@ -1674,13 +1712,14 @@ pub(crate) async fn bucket_replication_config_for_target_refresh(bucket: &str) -
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns whether the replication configuration was rewritten.
|
||||
pub(crate) async fn ensure_site_replication_bucket_replication_config_with_runtime(
|
||||
bucket: &str,
|
||||
state: &SiteReplicationState,
|
||||
local_peer: &PeerInfo,
|
||||
service_account_secret_key: &str,
|
||||
expected_incarnation_id: Uuid,
|
||||
) -> S3Result<()> {
|
||||
) -> S3Result<bool> {
|
||||
let existing = match metadata_sys::get_replication_config(bucket).await {
|
||||
Ok((existing, _)) => Some(existing),
|
||||
Err(StorageError::ConfigNotFound) => None,
|
||||
@@ -1689,7 +1728,7 @@ pub(crate) async fn ensure_site_replication_bucket_replication_config_with_runti
|
||||
|
||||
let Some(desired) = build_site_replication_config(bucket, state, local_peer, service_account_secret_key, existing.as_ref())?
|
||||
else {
|
||||
return Ok(());
|
||||
return Ok(false);
|
||||
};
|
||||
|
||||
// Derived rules are state owned by this site: rebuild them from the current peer
|
||||
@@ -1721,7 +1760,7 @@ pub(crate) async fn ensure_site_replication_bucket_replication_config_with_runti
|
||||
};
|
||||
|
||||
if rules == existing_rules && role == existing_role {
|
||||
return Ok(());
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let data = serialize(&ReplicationConfiguration { role, rules })
|
||||
@@ -1730,7 +1769,7 @@ pub(crate) async fn ensure_site_replication_bucket_replication_config_with_runti
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
Ok(())
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
pub(crate) async fn ensure_site_replication_bucket_setup_with_runtime(
|
||||
@@ -1748,9 +1787,9 @@ pub(crate) async fn ensure_site_replication_bucket_setup_with_runtime_for_incarn
|
||||
runtime: &SiteReplicationRuntime,
|
||||
expected_incarnation_id: Uuid,
|
||||
) -> S3Result<()> {
|
||||
let _targets_guard = lock_bucket_targets_metadata(bucket).await;
|
||||
let targets_guard = lock_bucket_targets_metadata(bucket).await;
|
||||
let config = bucket_replication_config_for_target_refresh(bucket).await?;
|
||||
ensure_site_replication_bucket_targets_with_runtime(
|
||||
let targets_written = ensure_site_replication_bucket_targets_with_runtime(
|
||||
bucket,
|
||||
&runtime.state,
|
||||
&runtime.local_peer,
|
||||
@@ -1759,7 +1798,7 @@ pub(crate) async fn ensure_site_replication_bucket_setup_with_runtime_for_incarn
|
||||
expected_incarnation_id,
|
||||
)
|
||||
.await?;
|
||||
ensure_site_replication_bucket_replication_config_with_runtime(
|
||||
let config_written = ensure_site_replication_bucket_replication_config_with_runtime(
|
||||
bucket,
|
||||
&runtime.state,
|
||||
&runtime.local_peer,
|
||||
@@ -1767,6 +1806,10 @@ pub(crate) async fn ensure_site_replication_bucket_setup_with_runtime_for_incarn
|
||||
expected_incarnation_id,
|
||||
)
|
||||
.await?;
|
||||
drop(targets_guard);
|
||||
if targets_written || config_written {
|
||||
reload_bucket_metadata_on_peers(bucket, "site_replication_bucket_setup", config_written).await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1791,6 +1834,7 @@ pub(crate) async fn ensure_site_replication_bucket_versioning(bucket: &str) -> S
|
||||
metadata_sys::update_if_incarnation(bucket, BUCKET_VERSIONING_CONFIG, bucket_versioning_xml()?, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
reload_bucket_metadata_on_peers(bucket, "site_replication_bucket_versioning", false).await;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -52,7 +52,11 @@ pub(crate) struct SiteReplicationRetryEvent {
|
||||
/// deletion body (if it was a deletion) recorded in
|
||||
/// [`SiteReplicationState::iam_deletion_replays`]. Only then may a
|
||||
/// successful deletion replay plus a stable snapshot resend settle the
|
||||
/// entry; a legacy entry (or one degraded by record overflow) keeps the
|
||||
/// entry. Every entry this binary creates starts recorded: the IAM
|
||||
/// change hook records deletion bodies, and the other creators (the add
|
||||
/// bootstrap's snapshot send, the drain's own replay) never carry a
|
||||
/// deletion. A legacy entry persisted by a binary that predates recording
|
||||
/// (serde default `false`), or one degraded by record overflow, keeps the
|
||||
/// escalation semantics because an unrecorded deletion may hide in it.
|
||||
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
|
||||
pub(crate) deletions_recorded: bool,
|
||||
@@ -308,7 +312,12 @@ fn push_site_replication_retry_event(
|
||||
updated_at: Some(OffsetDateTime::now_utc()),
|
||||
edit_generation: generation,
|
||||
peer_unreachable,
|
||||
deletions_recorded: false,
|
||||
// See the field doc: only a row persisted by an older binary is
|
||||
// unrecorded. Stamping at creation is what lets an entry first
|
||||
// created by the bootstrap snapshot send settle after a later
|
||||
// deletion is replayed, instead of escalating forever
|
||||
// (backlog#2367 A-3).
|
||||
deletions_recorded: true,
|
||||
});
|
||||
Ok(evicted)
|
||||
}
|
||||
@@ -598,22 +607,7 @@ pub(crate) fn record_failed_iam_delivery(
|
||||
item: &SRIAMItem,
|
||||
error: &str,
|
||||
) -> S3Result<()> {
|
||||
let existed = state
|
||||
.retry_queue
|
||||
.iter()
|
||||
.any(|event| retry_event_matches(event, peer, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH));
|
||||
upsert_site_replication_retry_event(&mut state.retry_queue, peer, SITE_REPLICATION_PEER_IAM_ITEM_WIRE_PATH, error, None)?;
|
||||
if !existed
|
||||
&& let Some(event) = state
|
||||
.retry_queue
|
||||
.iter_mut()
|
||||
.find(|event| retry_event_matches(event, peer, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH))
|
||||
{
|
||||
// Fresh entry: every failure it will ever collapse goes through this
|
||||
// recording path, so a deletion replay plus a stable snapshot resend
|
||||
// can later settle it instead of escalating.
|
||||
event.deletions_recorded = true;
|
||||
}
|
||||
|
||||
let Some(entity) = iam_item_deletion_entity(item) else {
|
||||
return Ok(());
|
||||
@@ -1418,6 +1412,33 @@ pub(crate) fn site_replication_retry_backoff_elapsed(event: &SiteReplicationRetr
|
||||
now.unix_timestamp().saturating_sub(updated_at.unix_timestamp()) >= delay
|
||||
}
|
||||
|
||||
/// Backoff evaluation time for the heavyweight tick: halfway to the next
|
||||
/// tick. Backoffs are multiples of the tick interval, so an entry stamped δ
|
||||
/// seconds after a tick is `600 − δ` old at the next one and slipped a whole
|
||||
/// extra interval for every δ > 0 — a first replay landed at T+1200 rather
|
||||
/// than T+600 (backlog#2367 A-1). Evaluating at the midpoint bounds the slip
|
||||
/// to half an interval either way; timestamps written back stay real time.
|
||||
pub(crate) fn heavyweight_retry_drain_horizon(now: OffsetDateTime) -> OffsetDateTime {
|
||||
let half_interval = crate::site_replication_reconcile::RECONCILE_INTERVAL / 2;
|
||||
now + time::Duration::seconds(i64::try_from(half_interval.as_secs()).unwrap_or(i64::MAX))
|
||||
}
|
||||
|
||||
/// What the lightweight 30-second pass may act on. It replays bounded bucket
|
||||
/// ops only, but probes every backed-off class: promotion is a state flip
|
||||
/// the heavyweight tick then replays, so an IAM or bucket-metadata snapshot
|
||||
/// owed to a peer that came back is resent at the next tick instead of
|
||||
/// after its own backoff has fully elapsed (backlog#2367 A-1).
|
||||
pub(crate) fn lightweight_retry_drain_partition(
|
||||
state: &SiteReplicationState,
|
||||
now: OffsetDateTime,
|
||||
) -> (Vec<SiteReplicationRetryEvent>, Vec<SiteReplicationRetryEvent>) {
|
||||
let mut actionable = actionable_site_replication_retry_events(state, now);
|
||||
actionable.retain(|event| {
|
||||
classify_site_replication_retry_event(event).is_some_and(|action| is_lightweight_retry_drain_action(&action))
|
||||
});
|
||||
(actionable, deferred_site_replication_retry_events(state, now))
|
||||
}
|
||||
|
||||
/// The subset of the retry queue the background drain is allowed to touch.
|
||||
pub(crate) fn actionable_site_replication_retry_events(
|
||||
state: &SiteReplicationState,
|
||||
@@ -1666,14 +1687,7 @@ async fn drain_site_replication_retry_queue_lightweight_inner() -> S3Result<()>
|
||||
return Ok(());
|
||||
}
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let mut actionable = actionable_site_replication_retry_events(&runtime.state, now);
|
||||
let mut deferred = deferred_site_replication_retry_events(&runtime.state, now);
|
||||
actionable.retain(|event| {
|
||||
classify_site_replication_retry_event(event).is_some_and(|action| is_lightweight_retry_drain_action(&action))
|
||||
});
|
||||
deferred.retain(|event| {
|
||||
classify_site_replication_retry_event(event).is_some_and(|action| is_lightweight_retry_drain_action(&action))
|
||||
});
|
||||
let (actionable, deferred) = lightweight_retry_drain_partition(&runtime.state, now);
|
||||
if actionable.is_empty() && deferred.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
@@ -1697,10 +1711,7 @@ async fn drain_site_replication_retry_queue_lightweight_inner() -> S3Result<()>
|
||||
return Ok(());
|
||||
}
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let mut actionable = actionable_site_replication_retry_events(&runtime.state, now);
|
||||
actionable.retain(|event| {
|
||||
classify_site_replication_retry_event(event).is_some_and(|action| is_lightweight_retry_drain_action(&action))
|
||||
});
|
||||
let (actionable, _) = lightweight_retry_drain_partition(&runtime.state, now);
|
||||
if actionable.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
@@ -1717,9 +1728,9 @@ pub(crate) async fn drain_site_replication_retry_queue_inner() -> S3Result<()> {
|
||||
// The alert must fire even when nothing is drainable this tick —
|
||||
// escalated markers are exactly the entries the drain skips.
|
||||
log_site_replication_retry_liabilities(&runtime.state);
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let actionable = actionable_site_replication_retry_events(&runtime.state, now);
|
||||
let deferred = deferred_site_replication_retry_events(&runtime.state, now);
|
||||
let horizon = heavyweight_retry_drain_horizon(OffsetDateTime::now_utc());
|
||||
let actionable = actionable_site_replication_retry_events(&runtime.state, horizon);
|
||||
let deferred = deferred_site_replication_retry_events(&runtime.state, horizon);
|
||||
if actionable.is_empty() && deferred.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
@@ -1764,8 +1775,8 @@ pub(crate) async fn drain_site_replication_retry_queue_inner() -> S3Result<()> {
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let actionable = actionable_site_replication_retry_events(&runtime.state, now);
|
||||
let horizon = heavyweight_retry_drain_horizon(OffsetDateTime::now_utc());
|
||||
let actionable = actionable_site_replication_retry_events(&runtime.state, horizon);
|
||||
if actionable.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -98,11 +98,46 @@ async fn spawn_test_tls_server_with_response(response: &'static [u8]) -> (String
|
||||
break;
|
||||
}
|
||||
}
|
||||
stream.write_all(response).await.is_ok()
|
||||
// Flush buffered TLS records and send close_notify before dropping the socket.
|
||||
stream.write_all(response).await.is_ok() && stream.shutdown().await.is_ok()
|
||||
});
|
||||
(endpoint, ca_pem, task)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tls_test_server_delivers_response_and_closes_cleanly() {
|
||||
use rustls_pki_types::pem::PemObject;
|
||||
|
||||
let (endpoint, ca_pem, server) = spawn_test_tls_server().await;
|
||||
let mut roots = rustls::RootCertStore::empty();
|
||||
roots
|
||||
.add(rustls_pki_types::CertificateDer::from_pem_slice(ca_pem.as_bytes()).expect("parse test CA"))
|
||||
.expect("trust test CA");
|
||||
let config = rustls::ClientConfig::builder()
|
||||
.with_root_certificates(roots)
|
||||
.with_no_client_auth();
|
||||
let connector = tokio_rustls::TlsConnector::from(Arc::new(config));
|
||||
let socket = tokio::net::TcpStream::connect(endpoint.strip_prefix("https://").expect("TLS endpoint"))
|
||||
.await
|
||||
.expect("connect to TLS test server");
|
||||
let mut stream = connector
|
||||
.connect(rustls_pki_types::ServerName::try_from("127.0.0.1").expect("test server name"), socket)
|
||||
.await
|
||||
.expect("trust TLS test server");
|
||||
stream
|
||||
.write_all(b"GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n")
|
||||
.await
|
||||
.expect("write test request");
|
||||
stream.flush().await.expect("flush test request");
|
||||
let mut response = Vec::new();
|
||||
tokio::time::timeout(Duration::from_secs(5), stream.read_to_end(&mut response))
|
||||
.await
|
||||
.expect("TLS response must finish")
|
||||
.expect("TLS test server must send close_notify before closing");
|
||||
assert!(response.ends_with(b"\r\n\r\nok"));
|
||||
assert!(server.await.expect("TLS test server task"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_connection_validation_accepts_supported_combinations() {
|
||||
let ca = valid_test_ca_pem("peer.example.com");
|
||||
@@ -845,7 +880,8 @@ fn test_record_failed_iam_delivery_records_deletions_and_flags_entry() {
|
||||
record_failed_iam_delivery(&mut state, &target, &policy_delete_item("readonly"), "peer offline").expect("record failure");
|
||||
assert_eq!(state.iam_deletion_replays.len(), 2);
|
||||
|
||||
// A legacy entry (created without recording) is never stamped.
|
||||
// A legacy entry (persisted by a binary that predates recording, so it
|
||||
// deserialized with the `false` default) is never stamped.
|
||||
let legacy = PeerInfo {
|
||||
deployment_id: "legacy-dep".to_string(),
|
||||
..peer("legacy", "https://legacy.example.com")
|
||||
@@ -859,6 +895,12 @@ fn test_record_failed_iam_delivery_records_deletions_and_flags_entry() {
|
||||
None,
|
||||
)
|
||||
.expect("upsert retry event");
|
||||
state
|
||||
.retry_queue
|
||||
.iter_mut()
|
||||
.find(|event| event.peer_deployment_id == legacy.deployment_id)
|
||||
.expect("legacy entry")
|
||||
.deletions_recorded = false;
|
||||
record_failed_iam_delivery(&mut state, &legacy, &user_delete_item("bob"), "peer offline").expect("record failure");
|
||||
let legacy_event = state
|
||||
.retry_queue
|
||||
@@ -871,6 +913,43 @@ fn test_record_failed_iam_delivery_records_deletions_and_flags_entry() {
|
||||
);
|
||||
}
|
||||
|
||||
/// backlog#2367 A-3: an entry first created by a non-deletion failure — the
|
||||
/// add bootstrap's snapshot send, or the drain's own replay — hides no
|
||||
/// unrecorded deletion, so a deletion recorded later plus a stable snapshot
|
||||
/// resend must settle it instead of escalating it to the permanent marker
|
||||
/// that only `replicate repair` clears.
|
||||
#[test]
|
||||
fn test_bootstrap_created_iam_entry_settles_after_deletion_replay() {
|
||||
let target = PeerInfo {
|
||||
deployment_id: "remote-dep".to_string(),
|
||||
..peer("remote", "https://remote.example.com")
|
||||
};
|
||||
let mut state = deletion_replay_state(&target);
|
||||
upsert_site_replication_retry_event(
|
||||
&mut state.retry_queue,
|
||||
&target,
|
||||
SITE_REPLICATION_PEER_IAM_ITEM_WIRE_PATH,
|
||||
"peer request to https://remote.example.com failed (connect): connection refused",
|
||||
None,
|
||||
)
|
||||
.expect("bootstrap send failure");
|
||||
assert!(state.retry_queue[0].deletions_recorded, "a fresh entry carries no unrecorded deletion");
|
||||
|
||||
record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "peer offline").expect("record failure");
|
||||
assert_eq!(state.retry_queue.len(), 1, "the hook failure collapses into the bootstrap entry");
|
||||
assert!(state.retry_queue[0].deletions_recorded);
|
||||
assert_eq!(state.iam_deletion_replays.len(), 1);
|
||||
|
||||
let observed = state.retry_queue[0].clone();
|
||||
let replayed: Vec<String> = state.iam_deletion_replays.iter().map(|record| record.id.clone()).collect();
|
||||
assert!(
|
||||
settle_replayed_iam_retry_events(&mut state, &target, &observed, &replayed),
|
||||
"the replayed deletion plus the snapshot resend settle the entry"
|
||||
);
|
||||
assert!(state.retry_queue.is_empty(), "no escalation marker may remain: {:?}", state.retry_queue);
|
||||
assert!(state.iam_deletion_replays.is_empty());
|
||||
}
|
||||
|
||||
/// Overflowing the per-peer record cap degrades the entry back to the
|
||||
/// escalation semantics: the record set is no longer complete, so a replay
|
||||
/// can no longer prove the peer converged.
|
||||
@@ -1626,6 +1705,74 @@ fn test_deferred_retry_events_do_not_probe_fresh_application_failures() {
|
||||
assert!(actionable_site_replication_retry_events(&state, now).is_empty());
|
||||
}
|
||||
|
||||
/// backlog#2367 A-1: the lightweight pass replays bucket ops only, but
|
||||
/// probes every backed-off class so a recovered peer's IAM snapshot is
|
||||
/// promoted within 30 seconds instead of waiting for the heavyweight tick
|
||||
/// to notice it.
|
||||
#[test]
|
||||
fn test_lightweight_partition_probes_snapshot_entries_but_replays_bucket_ops_only() {
|
||||
let now = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp");
|
||||
let mut state = SiteReplicationState::default();
|
||||
state
|
||||
.peers
|
||||
.insert("remote".to_string(), peer("remote", "https://remote.example.com"));
|
||||
|
||||
let bucket_make = "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning";
|
||||
let mut iam_unreachable = drain_event(
|
||||
"remote",
|
||||
SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH,
|
||||
3,
|
||||
Some(now - time::Duration::seconds(30)),
|
||||
);
|
||||
iam_unreachable.peer_unreachable = true;
|
||||
let mut bucket_unreachable = drain_event("remote", bucket_make, 3, Some(now - time::Duration::seconds(30)));
|
||||
bucket_unreachable.peer_unreachable = true;
|
||||
state.retry_queue = vec![
|
||||
iam_unreachable,
|
||||
bucket_unreachable,
|
||||
// Already promoted (or never stamped): due now.
|
||||
drain_event("remote", SITE_REPLICATION_RETRY_BUCKET_METADATA_SNAPSHOT_PATH, 1, None),
|
||||
drain_event("remote", bucket_make, 1, None),
|
||||
];
|
||||
|
||||
let (actionable, deferred) = lightweight_retry_drain_partition(&state, now);
|
||||
let deferred_paths: Vec<&str> = deferred.iter().map(|event| event.path.as_str()).collect();
|
||||
assert!(
|
||||
deferred_paths.contains(&SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH),
|
||||
"the backed-off IAM snapshot must be probed by the lightweight pass: {deferred_paths:?}"
|
||||
);
|
||||
assert!(deferred_paths.contains(&bucket_make));
|
||||
assert_eq!(
|
||||
actionable.iter().map(|event| event.path.as_str()).collect::<Vec<_>>(),
|
||||
vec![bucket_make],
|
||||
"only the bounded bucket op is replayed by the lightweight pass"
|
||||
);
|
||||
}
|
||||
|
||||
/// backlog#2367 A-1: the heavyweight tick evaluates backoff halfway to its
|
||||
/// next tick. A first failure stamped one second after a tick is 599 s old
|
||||
/// at the next tick; without the horizon it slipped to the tick after.
|
||||
#[test]
|
||||
fn test_heavyweight_horizon_absorbs_tick_phase() {
|
||||
let now = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp");
|
||||
let horizon = heavyweight_retry_drain_horizon(now);
|
||||
assert_eq!(horizon - now, time::Duration::seconds(300));
|
||||
|
||||
let elapsed_at_horizon = |secs_ago: i64| {
|
||||
site_replication_retry_backoff_elapsed(
|
||||
&drain_event("remote", "/p", 1, Some(now - time::Duration::seconds(secs_ago))),
|
||||
horizon,
|
||||
)
|
||||
};
|
||||
// Stamped just after the previous tick: due at this tick, not the next.
|
||||
assert!(elapsed_at_horizon(599));
|
||||
// Due before the next tick's midpoint: drained now rather than a whole
|
||||
// interval late.
|
||||
assert!(elapsed_at_horizon(301));
|
||||
// Due after the midpoint: waits for the next tick.
|
||||
assert!(!elapsed_at_horizon(299));
|
||||
}
|
||||
|
||||
/// The drain settles a peer-edit success under a freshly allocated
|
||||
/// generation; legacy queue entries carry `edit_generation: None` and
|
||||
/// must be cleared by that generation-scoped settlement (`(Some, None)`
|
||||
|
||||
@@ -32,7 +32,7 @@ use tokio::time::Instant;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::warn;
|
||||
|
||||
const RECONCILE_INTERVAL: Duration = Duration::from_secs(600);
|
||||
pub(crate) const RECONCILE_INTERVAL: Duration = Duration::from_secs(600);
|
||||
pub(crate) const RETRY_DRAIN_INTERVAL: Duration = Duration::from_secs(30);
|
||||
|
||||
/// A reconciler reports its own failures; the outcome carries no value because neither
|
||||
|
||||
@@ -62,10 +62,10 @@ pub(crate) async fn init_embedded_bucket_metadata_runtime(store: Arc<ECStore>, c
|
||||
|
||||
let buckets: Vec<String> = buckets_list.into_iter().map(|v| v.name).collect();
|
||||
|
||||
try_migrate_bucket_metadata(store.clone()).await;
|
||||
try_migrate_bucket_metadata(store.clone()).await?;
|
||||
init_on_demand_migration_runtime();
|
||||
init_bucket_metadata_sys(store.clone(), buckets.clone()).await;
|
||||
try_migrate_iam_config(store).await;
|
||||
try_migrate_iam_config(store).await?;
|
||||
spawn_bucket_resync_startup_reconcile(buckets.clone(), ctx.clone(), false);
|
||||
|
||||
Ok(buckets)
|
||||
@@ -82,9 +82,9 @@ pub(crate) async fn init_bucket_metadata_runtime(store: Arc<ECStore>, ctx: Cance
|
||||
|
||||
let buckets: Vec<String> = buckets_list.into_iter().map(|v| v.name).collect();
|
||||
|
||||
try_migrate_bucket_metadata(store.clone()).await;
|
||||
try_migrate_bucket_metadata(store.clone()).await?;
|
||||
|
||||
try_migrate_iam_config(store.clone()).await;
|
||||
try_migrate_iam_config(store.clone()).await?;
|
||||
init_on_demand_migration_runtime();
|
||||
init_bucket_metadata_sys(store, buckets.clone()).await;
|
||||
spawn_bucket_resync_startup_reconcile(buckets.clone(), ctx, true);
|
||||
|
||||
@@ -81,6 +81,7 @@ pub(crate) async fn init_startup_listen_context(
|
||||
config: &Config,
|
||||
instance_ctx: &Arc<InstanceContext>,
|
||||
) -> Result<StartupListenContext> {
|
||||
crate::server::init_console_prefix()?;
|
||||
log_sanitized_server_config(config);
|
||||
let readiness = Arc::new(GlobalReadiness::new());
|
||||
|
||||
|
||||
+78
-206
@@ -56,6 +56,10 @@ use std::sync::Arc;
|
||||
use std::sync::OnceLock;
|
||||
use url::{Url, form_urlencoded};
|
||||
|
||||
const EVENT_OBJECT_TAG_AUTHORIZATION: &str = "object_tag_authorization";
|
||||
const LOG_COMPONENT_ACCESS: &str = "storage_access";
|
||||
const LOG_SUBSYSTEM_AUTHORIZATION: &str = "authorization";
|
||||
|
||||
#[derive(Default, Clone, Debug)]
|
||||
pub(crate) struct ReqInfo {
|
||||
pub cred: Option<rustfs_credentials::Credentials>,
|
||||
@@ -102,9 +106,13 @@ async fn authorize_replication_only_put_headers<T>(req: &mut S3Request<T>) -> S3
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn recursive_force_delete_is_authorized(headers: &HeaderMap, is_owner: bool, replica_request: bool) -> bool {
|
||||
pub(crate) fn recursive_force_delete_has_authenticated_caller(
|
||||
headers: &HeaderMap,
|
||||
authenticated: bool,
|
||||
replica_request: bool,
|
||||
) -> bool {
|
||||
!get_header(headers, SUFFIX_FORCE_DELETE).is_some_and(|value| value.eq_ignore_ascii_case("true"))
|
||||
|| is_owner
|
||||
|| authenticated
|
||||
|| replica_request
|
||||
}
|
||||
|
||||
@@ -833,15 +841,12 @@ pub(crate) fn log_list_buckets_iam_implicit_deny<T>(req: &S3Request<T>) -> S3Res
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Extra action that may be evaluated in the same authorization flow and can
|
||||
/// independently require `ExistingObjectTag` conditions.
|
||||
fn secondary_tag_hint_action(action: Action, version_id: Option<&str>) -> Option<Action> {
|
||||
match action {
|
||||
Action::S3Action(S3Action::DeleteObjectAction) if version_id.is_some() => {
|
||||
Some(Action::S3Action(S3Action::DeleteObjectVersionAction))
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
pub(crate) fn delete_object_authorize_action(version_id: Option<&str>) -> Action {
|
||||
Action::S3Action(if version_id.is_some() {
|
||||
S3Action::DeleteObjectVersionAction
|
||||
} else {
|
||||
S3Action::DeleteObjectAction
|
||||
})
|
||||
}
|
||||
|
||||
/// GHSA-3ppv: select the IAM action for an object read by whether the request
|
||||
@@ -1015,14 +1020,14 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
|
||||
deny_only: false,
|
||||
};
|
||||
let prepared = iam_store.prepare_auth(&action_args).await;
|
||||
let mut needs_tag_from_iam = prepared.needs_existing_object_tag;
|
||||
let needs_tag_from_iam = prepared.needs_existing_object_tag;
|
||||
|
||||
let bucket_tag_hint = if !bucket.is_empty() && !object.is_empty() {
|
||||
Some(load_bucket_policy_existing_object_tag_hint(store.as_ref(), bucket.as_str(), action).await)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut needs_tag_from_bucket = if let Some(hint) = bucket_tag_hint.as_ref() {
|
||||
let needs_tag_from_bucket = if let Some(hint) = bucket_tag_hint.as_ref() {
|
||||
let bucket_args = BucketPolicyArgs {
|
||||
bucket: bucket.as_str(),
|
||||
action,
|
||||
@@ -1037,44 +1042,18 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
|
||||
false
|
||||
};
|
||||
|
||||
let secondary_action = secondary_tag_hint_action(action, version_id.as_deref());
|
||||
if let Some(extra_action) = secondary_action {
|
||||
let extra_args = Args {
|
||||
account: &cred.access_key,
|
||||
groups: &cred.groups,
|
||||
action: extra_action,
|
||||
bucket: bucket.as_str(),
|
||||
conditions: &conditions,
|
||||
is_owner,
|
||||
object: object.as_str(),
|
||||
claims,
|
||||
deny_only: false,
|
||||
};
|
||||
needs_tag_from_iam |= prepared.needs_existing_object_tag_for_args(&extra_args).await;
|
||||
|
||||
if let Some(hint) = bucket_tag_hint.as_ref() {
|
||||
let extra_bucket_args = BucketPolicyArgs {
|
||||
bucket: bucket.as_str(),
|
||||
action: extra_action,
|
||||
is_owner,
|
||||
account: cred.access_key.as_str(),
|
||||
groups: &cred.groups,
|
||||
conditions: &conditions,
|
||||
object: object.as_str(),
|
||||
};
|
||||
needs_tag_from_bucket |= bucket_policy_needs_existing_object_tag_from_hint(hint, &extra_bucket_args).await;
|
||||
}
|
||||
}
|
||||
|
||||
let needs_tag = needs_tag_from_iam || needs_tag_from_bucket;
|
||||
if needs_tag {
|
||||
tracing::debug!(
|
||||
event = EVENT_OBJECT_TAG_AUTHORIZATION,
|
||||
component = LOG_COMPONENT_ACCESS,
|
||||
subsystem = LOG_SUBSYSTEM_AUTHORIZATION,
|
||||
anonymous = false,
|
||||
bucket = %bucket,
|
||||
?action,
|
||||
?secondary_action,
|
||||
needs_tag_from_iam,
|
||||
needs_tag_from_bucket,
|
||||
"authorize_request ExistingObjectTag hint requires tag conditions"
|
||||
"Object tag authorization conditions required"
|
||||
);
|
||||
}
|
||||
maybe_merge_object_tag_conditions(
|
||||
@@ -1116,41 +1095,8 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
|
||||
return Err(denial.deny("bucket_policy_explicit_deny", action));
|
||||
}
|
||||
|
||||
if action == Action::S3Action(S3Action::DeleteObjectAction) && version_id.is_some() {
|
||||
let delete_version_args = Args {
|
||||
account: &cred.access_key,
|
||||
groups: &cred.groups,
|
||||
action: Action::S3Action(S3Action::DeleteObjectVersionAction),
|
||||
bucket: bucket.as_str(),
|
||||
conditions: &conditions,
|
||||
is_owner,
|
||||
object: object.as_str(),
|
||||
claims,
|
||||
deny_only: false,
|
||||
};
|
||||
let delete_version_allowed = iam_store.eval_prepared(&prepared, &delete_version_args).await;
|
||||
if !delete_version_allowed
|
||||
&& !PolicySys::try_is_allowed_for_store(
|
||||
store.as_ref(),
|
||||
&BucketPolicyArgs {
|
||||
bucket: bucket.as_str(),
|
||||
action: Action::S3Action(S3Action::DeleteObjectVersionAction),
|
||||
is_owner,
|
||||
account: &cred.access_key,
|
||||
groups: &cred.groups,
|
||||
conditions: &conditions,
|
||||
object: object.as_str(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(ApiError::from)?
|
||||
{
|
||||
return Err(denial.deny("delete_object_version_denied", Action::S3Action(S3Action::DeleteObjectVersionAction)));
|
||||
}
|
||||
}
|
||||
|
||||
let iam_allowed = {
|
||||
let final_args = Args {
|
||||
let mut final_args = Args {
|
||||
account: &cred.access_key,
|
||||
groups: &cred.groups,
|
||||
action,
|
||||
@@ -1161,7 +1107,28 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
|
||||
claims,
|
||||
deny_only: false,
|
||||
};
|
||||
iam_store.eval_prepared(&prepared, &final_args).await
|
||||
let allowed = iam_store.eval_prepared(&prepared, &final_args).await;
|
||||
if !allowed
|
||||
&& matches!(
|
||||
action,
|
||||
Action::S3Action(
|
||||
S3Action::DeleteObjectAction
|
||||
| S3Action::DeleteObjectVersionAction
|
||||
| S3Action::ListBucketVersionsAction
|
||||
| S3Action::BypassGovernanceRetentionAction
|
||||
| S3Action::ReplicateDeleteAction
|
||||
)
|
||||
)
|
||||
&& prepared.combined_policy_for_view().is_some()
|
||||
{
|
||||
// Bucket policy Allow may supplement an implicit IAM denial,
|
||||
// but must not override an explicit deletion-policy Deny.
|
||||
final_args.deny_only = true;
|
||||
if !iam_store.eval_prepared(&prepared, &final_args).await {
|
||||
return Err(denial.deny("iam_explicit_deny", action));
|
||||
}
|
||||
}
|
||||
allowed
|
||||
};
|
||||
|
||||
if iam_allowed {
|
||||
@@ -1199,42 +1166,6 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if action == Action::S3Action(S3Action::ListBucketVersionsAction) {
|
||||
let list_bucket_args = Args {
|
||||
account: &cred.access_key,
|
||||
groups: &cred.groups,
|
||||
action: Action::S3Action(S3Action::ListBucketAction),
|
||||
bucket: bucket.as_str(),
|
||||
conditions: &conditions,
|
||||
is_owner,
|
||||
object: object.as_str(),
|
||||
claims,
|
||||
deny_only: false,
|
||||
};
|
||||
let list_bucket_allowed = iam_store.eval_prepared(&prepared, &list_bucket_args).await;
|
||||
if list_bucket_allowed {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if PolicySys::try_is_allowed_for_store(
|
||||
store.as_ref(),
|
||||
&BucketPolicyArgs {
|
||||
bucket: bucket.as_str(),
|
||||
action: Action::S3Action(S3Action::ListBucketAction),
|
||||
is_owner,
|
||||
account: &cred.access_key,
|
||||
groups: &cred.groups,
|
||||
conditions: &conditions,
|
||||
object: object.as_str(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(ApiError::from)?
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let default_cred = rustfs_credentials::Credentials::default();
|
||||
let client_info = req.extensions.get::<ClientInfo>();
|
||||
@@ -1254,7 +1185,7 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut needs_tag_from_bucket = if let Some(hint) = bucket_tag_hint.as_ref() {
|
||||
let needs_tag_from_bucket = if let Some(hint) = bucket_tag_hint.as_ref() {
|
||||
let bucket_args = BucketPolicyArgs {
|
||||
bucket: bucket.as_str(),
|
||||
action,
|
||||
@@ -1268,27 +1199,15 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
|
||||
} else {
|
||||
false
|
||||
};
|
||||
let secondary_action = secondary_tag_hint_action(action, version_id.as_deref());
|
||||
if let Some(extra_action) = secondary_action
|
||||
&& let Some(hint) = bucket_tag_hint.as_ref()
|
||||
{
|
||||
let extra_bucket_args = BucketPolicyArgs {
|
||||
bucket: bucket.as_str(),
|
||||
action: extra_action,
|
||||
is_owner: false,
|
||||
account: "",
|
||||
groups: &no_groups,
|
||||
conditions: &conditions,
|
||||
object: object.as_str(),
|
||||
};
|
||||
needs_tag_from_bucket |= bucket_policy_needs_existing_object_tag_from_hint(hint, &extra_bucket_args).await;
|
||||
}
|
||||
if needs_tag_from_bucket {
|
||||
tracing::debug!(
|
||||
event = EVENT_OBJECT_TAG_AUTHORIZATION,
|
||||
component = LOG_COMPONENT_ACCESS,
|
||||
subsystem = LOG_SUBSYSTEM_AUTHORIZATION,
|
||||
anonymous = true,
|
||||
bucket = %bucket,
|
||||
?action,
|
||||
?secondary_action,
|
||||
"anonymous authorize_request ExistingObjectTag hint requires tag conditions"
|
||||
"Object tag authorization conditions required"
|
||||
);
|
||||
}
|
||||
maybe_merge_object_tag_conditions(
|
||||
@@ -1324,28 +1243,6 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
|
||||
}
|
||||
|
||||
if action != Action::S3Action(S3Action::ListAllMyBucketsAction) {
|
||||
if action == Action::S3Action(S3Action::DeleteObjectAction) && version_id.is_some() {
|
||||
let delete_version_allowed = PolicySys::try_is_allowed_for_store(
|
||||
store.as_ref(),
|
||||
&BucketPolicyArgs {
|
||||
bucket: bucket.as_str(),
|
||||
action: Action::S3Action(S3Action::DeleteObjectVersionAction),
|
||||
is_owner: false,
|
||||
account: "",
|
||||
groups: &None,
|
||||
conditions: &conditions,
|
||||
object: object.as_str(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
if !delete_version_allowed {
|
||||
return Err(
|
||||
denial.deny("delete_object_version_denied", Action::S3Action(S3Action::DeleteObjectVersionAction))
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let policy_allowed = PolicySys::try_is_allowed_for_store(
|
||||
store.as_ref(),
|
||||
&BucketPolicyArgs {
|
||||
@@ -1361,27 +1258,6 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
// A bucket policy granting s3:ListBucket also covers listing versions. This
|
||||
// fallback has to feed the same post-authorization gates as the direct grant
|
||||
// below, otherwise a public bucket keeps serving anonymous
|
||||
// ListObjectVersions after RestrictPublicBuckets is turned on.
|
||||
let policy_allowed = policy_allowed
|
||||
|| (action == Action::S3Action(S3Action::ListBucketVersionsAction)
|
||||
&& PolicySys::try_is_allowed_for_store(
|
||||
store.as_ref(),
|
||||
&BucketPolicyArgs {
|
||||
bucket: bucket.as_str(),
|
||||
action: Action::S3Action(S3Action::ListBucketAction),
|
||||
is_owner: false,
|
||||
account: "",
|
||||
groups: &None,
|
||||
conditions: &conditions,
|
||||
object: "",
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(ApiError::from)?);
|
||||
|
||||
if policy_allowed {
|
||||
deny_anonymous_table_data_plane_if_needed(req, action, bucket.as_str(), object.as_str()).await?;
|
||||
// RestrictPublicBuckets: when true, deny public access even if bucket policy allows it.
|
||||
@@ -2173,20 +2049,18 @@ impl S3Access for FS {
|
||||
req_info.bucket = Some(req.input.bucket.clone());
|
||||
req_info.object = Some(req.input.key.clone());
|
||||
req_info.version_id = req.input.version_id.clone();
|
||||
let is_owner = req_info.is_owner;
|
||||
let authenticated = req_info.is_owner || req_info.cred.is_some();
|
||||
let action = delete_object_authorize_action(req_info.version_id.as_deref());
|
||||
|
||||
authorize_request(req, Action::S3Action(S3Action::DeleteObjectAction)).await?;
|
||||
authorize_request(req, action).await?;
|
||||
|
||||
let replica_request = req
|
||||
.headers
|
||||
.get(AMZ_BUCKET_REPLICATION_STATUS)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.is_some_and(|value| value == ReplicationStatusType::Replica.as_str());
|
||||
if !recursive_force_delete_is_authorized(&req.headers, is_owner, replica_request) {
|
||||
return Err(s3_error!(
|
||||
AccessDenied,
|
||||
"Recursive force-delete is restricted to internal or administrative requests"
|
||||
));
|
||||
if !recursive_force_delete_has_authenticated_caller(&req.headers, authenticated, replica_request) {
|
||||
return Err(s3_error!(AccessDenied, "Recursive force-delete requires an authenticated caller"));
|
||||
}
|
||||
|
||||
// S3 Standard: When bypass_governance header is set, must have s3:BypassGovernanceRetention permission
|
||||
@@ -3108,12 +2982,12 @@ mod tests {
|
||||
PostObjectRequestMarker, ReqInfo, S3Access, StorageError, TableDataPlanePublicationGuards, apply_bucket_generation_guard,
|
||||
apply_copy_source_bucket_generation_guard, authorization_conditions, bucket_policy_needs_existing_object_tag_from_hint,
|
||||
bucket_website_config_authorize_action, classify_bucket_policy_raw_load_error,
|
||||
complete_multipart_upload_authorize_action, get_bucket_policy_authorize_action, has_write_offset_bytes_header,
|
||||
install_restore_authorization_test_hook, legal_hold_write_requested, list_parts_authorize_action,
|
||||
load_bucket_policy_existing_object_tag_hint, maybe_merge_object_tag_conditions, merge_list_bucket_query_conditions,
|
||||
merge_request_object_tag_conditions, owner_can_bypass_policy_deny, post_object_authorize_action,
|
||||
put_bucket_policy_authorize_action, request_context_from_req, request_object_store, retention_write_requested,
|
||||
secondary_tag_hint_action, table_data_plane_admin_action, table_data_plane_content_mutation,
|
||||
complete_multipart_upload_authorize_action, delete_object_authorize_action, get_bucket_policy_authorize_action,
|
||||
has_write_offset_bytes_header, install_restore_authorization_test_hook, legal_hold_write_requested,
|
||||
list_parts_authorize_action, load_bucket_policy_existing_object_tag_hint, maybe_merge_object_tag_conditions,
|
||||
merge_list_bucket_query_conditions, merge_request_object_tag_conditions, owner_can_bypass_policy_deny,
|
||||
post_object_authorize_action, put_bucket_policy_authorize_action, request_context_from_req, request_object_store,
|
||||
retention_write_requested, table_data_plane_admin_action, table_data_plane_content_mutation,
|
||||
table_data_plane_resource_for_request, table_publication_guard_error, validate_post_object_success_controls,
|
||||
versioned_read_action,
|
||||
};
|
||||
@@ -3946,20 +3820,18 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_secondary_tag_hint_action_for_delete_object_version() {
|
||||
assert_eq!(
|
||||
secondary_tag_hint_action(Action::S3Action(S3Action::DeleteObjectAction), Some("v1")),
|
||||
Some(Action::S3Action(S3Action::DeleteObjectVersionAction))
|
||||
);
|
||||
assert_eq!(secondary_tag_hint_action(Action::S3Action(S3Action::DeleteObjectAction), None), None);
|
||||
assert_eq!(
|
||||
secondary_tag_hint_action(Action::S3Action(S3Action::ListBucketVersionsAction), None),
|
||||
None
|
||||
);
|
||||
fn delete_authorization_selects_the_addressed_version() {
|
||||
assert_eq!(delete_object_authorize_action(None), Action::S3Action(S3Action::DeleteObjectAction));
|
||||
for version_id in ["null", "8f418ad0-f9f4-4458-83b2-cc72bc6f1b70"] {
|
||||
assert_eq!(
|
||||
delete_object_authorize_action(Some(version_id)),
|
||||
Action::S3Action(S3Action::DeleteObjectVersionAction)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_anonymous_delete_object_with_version_requires_secondary_policy_and_tag_hint() {
|
||||
async fn test_anonymous_version_delete_uses_version_policy_and_tag_hint() {
|
||||
let policy: BucketPolicy = serde_json::from_str(
|
||||
r#"{
|
||||
"Version":"2012-10-17",
|
||||
@@ -4013,16 +3885,16 @@ mod tests {
|
||||
"DeleteObjectVersion should still be denied without matching ExistingObjectTag conditions"
|
||||
);
|
||||
|
||||
let needs_tag_main = bucket_policy_needs_existing_object_tag_from_hint(&hint, &args_delete).await;
|
||||
let needs_tag_secondary = bucket_policy_needs_existing_object_tag_from_hint(&hint, &args_delete_version).await;
|
||||
assert!(!needs_tag_main, "DeleteObject statement itself does not require ExistingObjectTag");
|
||||
let needs_tag_current = bucket_policy_needs_existing_object_tag_from_hint(&hint, &args_delete).await;
|
||||
let needs_tag_version = bucket_policy_needs_existing_object_tag_from_hint(&hint, &args_delete_version).await;
|
||||
assert!(!needs_tag_current, "DeleteObject statement itself does not require ExistingObjectTag");
|
||||
assert!(
|
||||
needs_tag_secondary,
|
||||
needs_tag_version,
|
||||
"DeleteObjectVersion statement requires ExistingObjectTag when version delete is evaluated"
|
||||
);
|
||||
assert!(
|
||||
needs_tag_main || needs_tag_secondary,
|
||||
"combined primary+secondary check must require tag fetch for DeleteObject(versionId)"
|
||||
needs_tag_version,
|
||||
"the selected version action must require tag fetch for DeleteObject(versionId)"
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -5939,8 +5939,11 @@ mod tests {
|
||||
})
|
||||
.await
|
||||
.expect_err("mismatched kms context should fail");
|
||||
assert_eq!(err.code, S3ErrorCode::InternalError);
|
||||
assert_eq!(err.message, ApiError::error_code_to_message(&S3ErrorCode::InternalError));
|
||||
assert_eq!(err.code, S3ErrorCode::InvalidRequest);
|
||||
assert_eq!(
|
||||
err.message,
|
||||
"Encryption context mismatch: Context mismatch for key 'tenant': expected 'alpha', got 'beta'"
|
||||
);
|
||||
assert_eq!(super::kms_data_plane_error_class(&err), "context_mismatch");
|
||||
|
||||
manager.stop().await.expect("kms service should stop cleanly");
|
||||
|
||||
@@ -119,9 +119,9 @@ pub(crate) use super::sse::{
|
||||
pub(crate) mod access_consumer {
|
||||
pub(crate) use super::super::access::{
|
||||
PostObjectRequestMarker, ReqInfo, apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard,
|
||||
authorize_internal_object_request, authorize_request, bucket_config_mutation_incarnation, has_bypass_governance_header,
|
||||
load_bucket_generation_from_store, log_list_buckets_iam_implicit_deny, odm_read_generation,
|
||||
prepare_list_buckets_iam_authorization, prepare_odm_read_generation, recursive_force_delete_is_authorized,
|
||||
authorize_internal_object_request, authorize_request, bucket_config_mutation_incarnation, delete_object_authorize_action,
|
||||
has_bypass_governance_header, load_bucket_generation_from_store, log_list_buckets_iam_implicit_deny, odm_read_generation,
|
||||
prepare_list_buckets_iam_authorization, prepare_odm_read_generation, recursive_force_delete_has_authenticated_caller,
|
||||
replication_request_authorized, req_info_mut, req_info_ref,
|
||||
};
|
||||
}
|
||||
@@ -955,7 +955,10 @@ pub(crate) async fn get_local_server_property() -> rustfs_madmin::ServerProperti
|
||||
|
||||
pub(crate) async fn init_background_replication(store: Arc<ECStore>) {
|
||||
let durable_dirty_usage_journal = super::scanner_dirty_journal::start_durable_dirty_usage_journal(store.clone()).await;
|
||||
let journal_writer = durable_dirty_usage_journal.clone();
|
||||
let mutation_journal = durable_dirty_usage_journal.clone();
|
||||
rustfs_scanner::set_scanner_dirty_usage_mutation_observer(Some(Arc::new(move |bucket, object, producer| {
|
||||
mutation_journal.record_committed_mutation(bucket, object, producer);
|
||||
})));
|
||||
ecstore_bucket::replication::set_scanner_dirty_usage_mutation_observer(Some(Arc::new(move |bucket, object, source| {
|
||||
let producer = match source {
|
||||
ecstore_bucket::replication::ScannerDirtyUsageMutationSource::Replication => {
|
||||
@@ -966,7 +969,6 @@ pub(crate) async fn init_background_replication(store: Arc<ECStore>) {
|
||||
}
|
||||
};
|
||||
rustfs_scanner::record_dirty_usage_object_from_producer(bucket, object, producer);
|
||||
journal_writer.record_committed_mutation(bucket, object, producer);
|
||||
})));
|
||||
rustfs_scanner::set_scanner_dirty_usage_clear_observer(Some(Arc::new(move |cleared| {
|
||||
durable_dirty_usage_journal.clear_confirmed_buckets(cleared);
|
||||
@@ -1183,17 +1185,21 @@ pub(crate) fn get_global_transition_state() -> Arc<TransitionState> {
|
||||
ecstore_bucket::lifecycle::bucket_lifecycle_ops::get_global_transition_state()
|
||||
}
|
||||
|
||||
pub(crate) async fn try_migrate_bucket_metadata(store: Arc<ECStore>) {
|
||||
ecstore_bucket::migration::try_migrate_bucket_metadata(store).await;
|
||||
pub(crate) async fn try_migrate_bucket_metadata(store: Arc<ECStore>) -> std::io::Result<()> {
|
||||
ecstore_bucket::migration::try_migrate_bucket_metadata(store)
|
||||
.await
|
||||
.map_err(ecstore_bucket::migration::migration_startup_error)
|
||||
}
|
||||
|
||||
pub(crate) async fn try_migrate_iam_config(store: Arc<ECStore>) {
|
||||
pub(crate) async fn try_migrate_iam_config(store: Arc<ECStore>) -> std::io::Result<()> {
|
||||
// MinIO encrypts IAM identity/service-account files at rest with a key derived
|
||||
// from the root credentials. Inject the IAM crate's decryption so those blobs
|
||||
// are decrypted before normalization instead of being skipped as "incompatible".
|
||||
let decrypt_fn: ecstore_bucket::migration::LegacyBlobDecryptFn =
|
||||
Arc::new(|data: &[u8]| rustfs_iam::try_decrypt_iam_blob(data));
|
||||
ecstore_bucket::migration::try_migrate_iam_config(store, Some(decrypt_fn)).await;
|
||||
ecstore_bucket::migration::try_migrate_iam_config(store, Some(decrypt_fn))
|
||||
.await
|
||||
.map_err(ecstore_bucket::migration::migration_startup_error)
|
||||
}
|
||||
|
||||
pub(crate) fn init_ecstore_config() {
|
||||
|
||||
@@ -0,0 +1,326 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#![recursion_limit = "256"]
|
||||
|
||||
use reqwest::StatusCode;
|
||||
use rustfs::embedded::{RustFSServerBuilder, find_available_port};
|
||||
use rustfs_ecstore::api::config::com::{delete_config, read_config};
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Stdio;
|
||||
use std::time::Duration;
|
||||
use tokio::process::Command;
|
||||
|
||||
mod common;
|
||||
|
||||
const TEST_NAME: &str = "native_migration_failure_blocks_server_startup_and_repair_preserves_records";
|
||||
const STAGE_ENV: &str = "RUSTFS_NATIVE_MIGRATION_TEST_STAGE";
|
||||
const ROOT_ENV: &str = "RUSTFS_NATIVE_MIGRATION_TEST_ROOT";
|
||||
const ADDRESS_ENV: &str = "RUSTFS_NATIVE_MIGRATION_TEST_ADDRESS";
|
||||
const FAILURE_ENV: &str = "RUSTFS_NATIVE_MIGRATION_TEST_FAILURE";
|
||||
const STOP_ENV: &str = "RUSTFS_NATIVE_MIGRATION_TEST_STOP";
|
||||
const ACCESS_KEY: &str = "native-migration-root";
|
||||
const SECRET_KEY: &str = "native-migration-root-secret";
|
||||
const LEGACY_BUCKET: &str = ".minio.sys";
|
||||
const TARGET_BUCKET: &str = ".rustfs.sys";
|
||||
const BUCKET_METADATA: &str = "buckets/interop/.metadata.bin";
|
||||
const IAM_RECORD: &str = "config/iam/groups/migration-group/members.json";
|
||||
const IAM_FORMAT: &str = "config/iam/format.json";
|
||||
const EXISTING_FORMAT: &[u8] = br#"{"version":1}"#;
|
||||
const STARTUP_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
enum StartupMode {
|
||||
Server,
|
||||
Embedded,
|
||||
}
|
||||
|
||||
fn volumes(root: &Path) -> Vec<PathBuf> {
|
||||
(1..=4).map(|index| root.join(format!("disk{index}"))).collect()
|
||||
}
|
||||
|
||||
fn minio_bucket_metadata() -> Vec<u8> {
|
||||
let hex: String = include_str!("../../crates/ecstore/tests/fixtures/minio/bucket_metadata.blob.hex")
|
||||
.chars()
|
||||
.filter(|ch| !ch.is_whitespace())
|
||||
.collect();
|
||||
hex.as_bytes()
|
||||
.as_chunks::<2>()
|
||||
.0
|
||||
.iter()
|
||||
.map(|pair| {
|
||||
u8::from_str_radix(std::str::from_utf8(pair).expect("fixture hex is UTF-8"), 16).expect("valid MinIO fixture hex")
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn prepare_or_verify_fixture(root: &Path, seed: bool) {
|
||||
let env = rustfs_test_utils::TestECStoreEnv::builder()
|
||||
.base_dir(root)
|
||||
.disk_count(4)
|
||||
.build()
|
||||
.await;
|
||||
if seed {
|
||||
env.make_bucket("interop", false).await;
|
||||
env.make_bucket(LEGACY_BUCKET, false).await;
|
||||
env.put_object_bytes(LEGACY_BUCKET, BUCKET_METADATA, minio_bucket_metadata())
|
||||
.await;
|
||||
env.put_object_bytes(
|
||||
LEGACY_BUCKET,
|
||||
IAM_RECORD,
|
||||
br#"{"version":1,"status":"enabled","members":[],"updatedAt":"2026-09-10T00:00:00Z"}"#.to_vec(),
|
||||
)
|
||||
.await;
|
||||
env.put_object_bytes(TARGET_BUCKET, IAM_FORMAT, EXISTING_FORMAT.to_vec())
|
||||
.await;
|
||||
// A completed record must be skipped before reading even a broken old copy.
|
||||
env.put_object_bytes(LEGACY_BUCKET, IAM_FORMAT, b"do not overwrite the existing target".to_vec())
|
||||
.await;
|
||||
delete_config(env.ecstore.clone(), BUCKET_METADATA)
|
||||
.await
|
||||
.expect("leave bucket metadata pending migration");
|
||||
} else {
|
||||
assert_eq!(
|
||||
read_config(env.ecstore.clone(), BUCKET_METADATA)
|
||||
.await
|
||||
.expect("migrated bucket metadata"),
|
||||
minio_bucket_metadata(),
|
||||
"migration must preserve the MinIO bucket settings"
|
||||
);
|
||||
let group: serde_json::Value = serde_json::from_slice(
|
||||
&read_config(env.ecstore.clone(), IAM_RECORD)
|
||||
.await
|
||||
.expect("migrated IAM group"),
|
||||
)
|
||||
.expect("valid migrated IAM JSON");
|
||||
assert_eq!(group["status"], "enabled");
|
||||
assert_eq!(group["members"], serde_json::json!([]));
|
||||
}
|
||||
assert_eq!(
|
||||
read_config(env.ecstore.clone(), IAM_FORMAT)
|
||||
.await
|
||||
.expect("existing IAM format"),
|
||||
EXISTING_FORMAT,
|
||||
"retry must not overwrite records already migrated"
|
||||
);
|
||||
}
|
||||
|
||||
async fn run_embedded_child(root: &Path) {
|
||||
let address = std::env::var(ADDRESS_ENV).expect("embedded child address");
|
||||
let result = RustFSServerBuilder::new()
|
||||
.address(address)
|
||||
.access_key(ACCESS_KEY)
|
||||
.secret_key(SECRET_KEY)
|
||||
.volumes(volumes(root).iter().map(|path| path.to_string_lossy().into_owned()).collect())
|
||||
.build()
|
||||
.await;
|
||||
match result {
|
||||
Ok(server) => {
|
||||
let stop = PathBuf::from(std::env::var_os(STOP_ENV).expect("embedded stop path"));
|
||||
while !stop.exists() {
|
||||
tokio::time::sleep(Duration::from_millis(25)).await;
|
||||
}
|
||||
server.shutdown().await;
|
||||
}
|
||||
Err(error) => {
|
||||
fs::write(std::env::var_os(FAILURE_ENV).expect("embedded failure path"), error.to_string())
|
||||
.expect("record the actual embedded startup error");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn child_command(root: &Path, stage: &str, log: &Path) -> Command {
|
||||
let mut command = Command::new(std::env::current_exe().expect("integration test executable"));
|
||||
command
|
||||
.args(["--exact", TEST_NAME, "--nocapture"])
|
||||
.env(STAGE_ENV, stage)
|
||||
.env(ROOT_ENV, root);
|
||||
configure_process(&mut command, log);
|
||||
command
|
||||
}
|
||||
|
||||
fn configure_process(command: &mut Command, log: &Path) {
|
||||
let output = fs::File::create(log).expect("create isolated process log");
|
||||
command
|
||||
// These disposable erasure volumes intentionally share the test runner's disk.
|
||||
.env("RUSTFS_UNSAFE_BYPASS_DISK_CHECK", "true")
|
||||
.env("RUSTFS_CONSOLE_ENABLE", "false")
|
||||
.env("NO_PROXY", "localhost,127.0.0.1,::1")
|
||||
.env("no_proxy", "localhost,127.0.0.1,::1")
|
||||
.env("RUST_LOG", "warn")
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::from(output.try_clone().expect("clone process log")))
|
||||
.stderr(Stdio::from(output))
|
||||
.kill_on_drop(true);
|
||||
}
|
||||
|
||||
async fn fixture_process(root: &Path, stage: &str) {
|
||||
let log = root.join(format!("{stage}.log"));
|
||||
let status = tokio::time::timeout(STARTUP_TIMEOUT, child_command(root, stage, &log).status())
|
||||
.await
|
||||
.expect("fixture process must finish")
|
||||
.expect("run fixture process");
|
||||
assert!(status.success(), "{stage} failed: {}", fs::read_to_string(log).expect("fixture log"));
|
||||
}
|
||||
|
||||
async fn check_startup(root: &Path, mode: StartupMode, failure_record: Option<&str>, label: &str) {
|
||||
let ready = failure_record.is_none();
|
||||
let address = format!("127.0.0.1:{}", find_available_port().expect("free startup probe port"));
|
||||
let log = root.join(format!("{label}.log"));
|
||||
let failure = root.join(format!("{label}.failure"));
|
||||
let stop = root.join(format!("{label}.stop"));
|
||||
let mut command = match mode {
|
||||
StartupMode::Server => {
|
||||
let mut command = Command::new(env!("CARGO_BIN_EXE_rustfs"));
|
||||
command
|
||||
.args(["--address", &address, "--access-key", ACCESS_KEY, "--secret-key", SECRET_KEY])
|
||||
.args(volumes(root));
|
||||
configure_process(&mut command, &log);
|
||||
command
|
||||
}
|
||||
StartupMode::Embedded => {
|
||||
let mut command = child_command(root, "embedded", &log);
|
||||
command
|
||||
.env(ADDRESS_ENV, &address)
|
||||
.env(FAILURE_ENV, &failure)
|
||||
.env(STOP_ENV, &stop);
|
||||
command
|
||||
}
|
||||
};
|
||||
let mut child = command.spawn().expect("start isolated server process");
|
||||
let http = reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.timeout(Duration::from_millis(500))
|
||||
.build()
|
||||
.expect("local readiness client");
|
||||
let result = tokio::time::timeout(STARTUP_TIMEOUT, async {
|
||||
loop {
|
||||
if let Ok(response) = http.get(format!("http://{address}/health/ready")).send().await
|
||||
&& response.status() == StatusCode::OK
|
||||
{
|
||||
assert!(ready, "{mode:?} published Ready after a migration I/O failure");
|
||||
return;
|
||||
}
|
||||
if let Some(status) = child.try_wait().expect("poll server process") {
|
||||
let details = fs::read_to_string(&log).expect("startup log");
|
||||
assert!(!ready, "{mode:?} exited before Ready ({status}): {details}");
|
||||
let record = failure_record.expect("failed startup has an obstructed record");
|
||||
match mode {
|
||||
StartupMode::Server => {
|
||||
assert_eq!(status.code(), Some(1), "startup must fail: {details}");
|
||||
assert_migration_io_error(&details, record);
|
||||
}
|
||||
StartupMode::Embedded => {
|
||||
assert!(status.success(), "embedded test process failed unexpectedly: {details}");
|
||||
let error = fs::read_to_string(&failure).expect("embedded startup returned an error");
|
||||
assert_migration_io_error(&error, record);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(25)).await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
result.is_ok(),
|
||||
"{mode:?} did not reach the expected startup outcome: {}",
|
||||
fs::read_to_string(&log).expect("startup diagnostics")
|
||||
);
|
||||
if ready {
|
||||
match mode {
|
||||
StartupMode::Embedded => {
|
||||
fs::write(stop, b"stop").expect("request embedded shutdown");
|
||||
assert!(
|
||||
tokio::time::timeout(STARTUP_TIMEOUT, child.wait())
|
||||
.await
|
||||
.expect("embedded shutdown completes")
|
||||
.expect("wait for embedded shutdown")
|
||||
.success()
|
||||
);
|
||||
}
|
||||
StartupMode::Server => child.kill().await.expect("stop the isolated server"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_migration_io_error(error: &str, record: &str) {
|
||||
let lower = error.to_ascii_lowercase();
|
||||
assert!(
|
||||
(lower.contains("access denied")
|
||||
|| lower.contains("access is denied")
|
||||
|| lower.contains("not a directory")
|
||||
|| lower.contains("not regular"))
|
||||
&& error.contains(&format!("{TARGET_BUCKET}/{record}")),
|
||||
"startup must fail because of the obstructed metadata record, not an unrelated initialization error: {error}"
|
||||
);
|
||||
}
|
||||
|
||||
async fn run_startup_cases(mode: StartupMode) {
|
||||
let ordinary = tempfile::TempDir::with_prefix("rustfs-no-legacy-").expect("ordinary store");
|
||||
for volume in volumes(ordinary.path()) {
|
||||
fs::create_dir_all(volume).expect("ordinary volume");
|
||||
}
|
||||
check_startup(ordinary.path(), mode, None, "ordinary").await;
|
||||
|
||||
let control = tempfile::TempDir::with_prefix("rustfs-migration-control-").expect("control fixture");
|
||||
fixture_process(control.path(), "seed").await;
|
||||
check_startup(control.path(), mode, None, "control").await;
|
||||
fixture_process(control.path(), "verify").await;
|
||||
|
||||
for record in [BUCKET_METADATA, IAM_RECORD] {
|
||||
let target = tempfile::TempDir::with_prefix("rustfs-migration-failure-").expect("disposable migration target");
|
||||
fixture_process(target.path(), "seed").await;
|
||||
let blockers: Vec<_> = volumes(target.path())
|
||||
.iter()
|
||||
.map(|volume| volume.join(TARGET_BUCKET).join(record))
|
||||
.collect();
|
||||
for blocker in &blockers {
|
||||
fs::create_dir_all(blocker.parent().expect("record parent")).expect("create target parent");
|
||||
assert!(!blocker.exists(), "the record must still need migration");
|
||||
// A non-directory target causes real filesystem I/O errors even when tests run as root.
|
||||
fs::write(blocker, b"blocked migration target").expect("block only the destination record");
|
||||
}
|
||||
check_startup(target.path(), mode, Some(record), "blocked").await;
|
||||
for blocker in blockers {
|
||||
fs::remove_file(blocker).expect("repair the same partially migrated target");
|
||||
}
|
||||
check_startup(target.path(), mode, None, "repaired").await;
|
||||
fixture_process(target.path(), "verify").await;
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_migration_failure_blocks_server_startup_and_repair_preserves_records() {
|
||||
// Cold processes keep failed initialization and cached metadata out of subsequent restart attempts.
|
||||
common::run_embedded_test(|| async {
|
||||
match std::env::var(STAGE_ENV).ok().as_deref() {
|
||||
Some("seed") => {
|
||||
prepare_or_verify_fixture(&PathBuf::from(std::env::var_os(ROOT_ENV).expect("fixture root")), true).await
|
||||
}
|
||||
Some("verify") => {
|
||||
prepare_or_verify_fixture(&PathBuf::from(std::env::var_os(ROOT_ENV).expect("fixture root")), false).await
|
||||
}
|
||||
Some("embedded") => run_embedded_child(&PathBuf::from(std::env::var_os(ROOT_ENV).expect("fixture root"))).await,
|
||||
None => run_startup_cases(StartupMode::Server).await,
|
||||
Some(stage) => panic!("unknown native migration test stage: {stage}"),
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_migration_failure_blocks_embedded_startup_and_repair_preserves_records() {
|
||||
common::run_embedded_test(|| run_startup_cases(StartupMode::Embedded));
|
||||
}
|
||||
@@ -61,10 +61,12 @@ their issue closes.
|
||||
| `run_scanner_heal_checkpoint_crash_evidence.py` | dev-tool | Assembles measured Scanner/Heal G02/R-E checkpoint and restart release descriptors from scanner restart diagnostic reports | `diagnose_scanner_enumeration_restart.py`; `test_scanner_heal_checkpoint_crash_evidence.sh` |
|
||||
| `run_scanner_heal_g14_multiset_evidence.py` | dev-tool | Assembles measured Scanner/Heal G14 same-window EC8+4 multi-set/multi-pool release descriptors from e2e case directories or an operator-collected proof | `.config/scanner-heal-required-tests.json`; `test_scanner_heal_g14_multiset_evidence.sh` |
|
||||
| `run_scanner_heal_g09_upgrade_evidence.sh` | dev-tool | Runs the G09 mixed-version and rollback upgrade E2E lanes against a pinned previous release and verifies the raw evidence artifacts | `docs/testing/ci-gates.md`; `.github/workflows/e2e-upgrade.yml`; `test_scanner_heal_g09_upgrade_evidence.sh` |
|
||||
| `run_scanner_heal_linux_evidence_plan.py` | dev-tool | Writes the unified Scanner/Heal Linux release-evidence execution manifest and can run lightweight preflight checks without producing measured evidence | `docs/testing/ci-gates.md`; `test_scanner_heal_linux_evidence_plan.sh` |
|
||||
| `run_scanner_heal_scoped_ack_evidence.py` | dev-tool | Assembles measured Scanner/Heal G03 scoped ACK publication and mixed-peer fallback release descriptors | `.config/scanner-heal-required-tests.json`; `test_scanner_heal_scoped_ack_evidence.sh` |
|
||||
| `run_scanner_heal_legacy_rollback_evidence.py` | dev-tool | Assembles measured Scanner/Heal R-L legacy source-conflict, migration-gap, and source-retirement release descriptors | `.config/scanner-heal-required-tests.json`; `test_scanner_heal_legacy_rollback_evidence.sh` |
|
||||
| `run_scanner_heal_mrf_evidence.py` | dev-tool | Assembles measured Scanner/Heal G07/G08/P4 MRF release descriptors from W13 raw artifacts | `.config/scanner-heal-required-tests.json`; `test_scanner_heal_w13_mrf_evidence.sh` |
|
||||
| `run_scanner_heal_scheduler_pressure_evidence.py` | dev-tool | Assembles measured Scanner/Heal G10/P1/P3 scheduler-pressure release descriptors from a completed measured ABBA run, recovery-window proof, and profile artifacts | `docs/operations/scanner-benchmark-runbook.md`; `test_scanner_heal_scheduler_pressure_evidence.sh` |
|
||||
| `run_scanner_heal_status_outcome_probe.py` | dev-tool | Normalizes live Scanner/Heal status/outcome observations into the measured G05/G06/R-D raw artifacts consumed by the descriptor producer | `.config/scanner-heal-required-tests.json`; `test_scanner_heal_status_outcome_evidence.sh` |
|
||||
| `run_scanner_heal_status_outcome_evidence.py` | dev-tool | Assembles measured Scanner/Heal G05/G06/R-D status-and-outcome release descriptors from same-run status, compatibility, and disposition artifacts | `.config/scanner-heal-required-tests.json`; `test_scanner_heal_status_outcome_evidence.sh` |
|
||||
| `run_scanner_heal_maintenance_evidence.py` | dev-tool | Assembles measured Scanner/Heal G11/G13 maintenance-producer release descriptors from operator-collected proof JSON | `.config/scanner-heal-required-tests.json`; `test_scanner_heal_maintenance_evidence.sh` |
|
||||
| `run_scanner_heal_w13_mrf_evidence.sh` | dev-tool | Runs the W13 durable MRF replay lanes and writes G07/G08/P4 bundle-ready evidence descriptors | `docs/testing/ci-gates.md`; `test_scanner_heal_w13_mrf_evidence.sh` |
|
||||
|
||||
@@ -428,6 +428,7 @@ SCANNER_HEAL_RELEASE_SCOPED_ACK_CASES = {
|
||||
"exact-generation",
|
||||
"scanner-instance",
|
||||
"participating-peer-set",
|
||||
"cleared-count-bound",
|
||||
),
|
||||
"participating_peer_capability_snapshot": (
|
||||
"supports-scoped-ack",
|
||||
@@ -452,6 +453,7 @@ SCANNER_HEAL_RELEASE_G03_REQUIRED_TRUE_FIELDS = {
|
||||
"scanner_instance_observed",
|
||||
"participating_peer_set_observed",
|
||||
"whole_cycle_fallback_observed",
|
||||
"cleared_count_bound_observed",
|
||||
),
|
||||
"participating_peer_capability_snapshot": (
|
||||
"capability_probe_observed",
|
||||
@@ -1682,6 +1684,25 @@ def check_scanner_heal_evidence(root: Path, directory: Path, case_id: str) -> li
|
||||
expected_outage_target_required = requirement.get("outage_target_manifest_required", True)
|
||||
require(oracle.get("outage_target_manifest_required", True) is expected_outage_target_required,
|
||||
"oracle outage target-manifest contract mismatch")
|
||||
outage_write = oracle.get("outage_write")
|
||||
outage_write_deferred = oracle.get("outage_write_deferred_until_rejoin", False)
|
||||
require(type(outage_write_deferred) is bool, "invalid outage-write deferred flag")
|
||||
if not expected_outage_target_required and outage_write is not None:
|
||||
require(isinstance(outage_write, dict), "invalid optional outage-write diagnostic")
|
||||
require(outage_write.get("attempted") is True, "optional outage-write diagnostic was not attempted")
|
||||
require(outage_write.get("required") is False, "optional outage-write diagnostic required flag mismatch")
|
||||
require(outage_write.get("accepted") is True, "optional outage-write final S3 body was not accepted")
|
||||
outage_write_attempts = evidence_integer(outage_write.get("attempts"), "optional outage-write attempts", 1, 1024)
|
||||
outage_write_service_unavailable = evidence_integer(
|
||||
outage_write.get("service_unavailable"), "optional outage-write ServiceUnavailable count", 0, 1024
|
||||
)
|
||||
if outage_write_deferred:
|
||||
require(
|
||||
outage_write_service_unavailable == outage_write_attempts,
|
||||
"deferred outage write must account for every down-window attempt",
|
||||
)
|
||||
else:
|
||||
require(outage_write_deferred is False, "outage-write deferred flag requires optional outage-write diagnostic")
|
||||
if requirement.get("sets", 1) > 1 or requirement.get("pools", 1) > 1:
|
||||
require(oracle.get("distributed_ec_invalidation") is True,
|
||||
"oracle missing distributed EC invalidation proof")
|
||||
@@ -1708,8 +1729,8 @@ def check_scanner_heal_evidence(root: Path, directory: Path, case_id: str) -> li
|
||||
require(isinstance(objects, list) and requirement["min_objects"] <= len(objects) <= requirement["max_objects"],
|
||||
"incomplete/oversized object oracle")
|
||||
require(len({obj["key"] for obj in objects}) == len(objects), "duplicate object identity")
|
||||
require(sum(obj["expected_physical"] is None for obj in objects) == 1,
|
||||
"only the outage object may lack a pre-fault target manifest")
|
||||
outage_object_count = sum(obj["expected_physical"] is None for obj in objects)
|
||||
require(outage_object_count == 1, "only the outage object may lack a pre-fault target manifest")
|
||||
for obj in objects:
|
||||
require(isinstance(obj["key"], str) and 0 < len(obj["key"].encode()) <= 1024, "invalid object identity")
|
||||
require(obj["version_id"] is None, "this case only covers unversioned objects")
|
||||
@@ -1827,6 +1848,7 @@ def release_bundle_json_artifact_mirrored_fields(gate: str, field: str) -> tuple
|
||||
"raw_entry_budget",
|
||||
"max_raw_entries_per_round",
|
||||
"max_objects_processed_per_round",
|
||||
"bounded_work_quantum_observed",
|
||||
"durable_checkpoint_committed",
|
||||
"no_unbounded_tail",
|
||||
))
|
||||
@@ -2255,6 +2277,8 @@ def validate_release_bundle_domain_evidence(gate: str, field: str, evidence: dic
|
||||
f"{gate}.{field}.max_objects_processed_per_round", 1, 4096)
|
||||
require(max_raw <= budget, f"{gate}.{field} raw entries exceed fixed budget")
|
||||
require(max_objects <= budget, f"{gate}.{field} processed objects exceed fixed budget")
|
||||
release_bundle_bool_true(evidence.get("bounded_work_quantum_observed"),
|
||||
f"{gate}.{field}.bounded_work_quantum_observed")
|
||||
release_bundle_bool_true(evidence.get("durable_checkpoint_committed"),
|
||||
f"{gate}.{field}.durable_checkpoint_committed")
|
||||
release_bundle_bool_true(evidence.get("no_unbounded_tail"), f"{gate}.{field}.no_unbounded_tail")
|
||||
@@ -2772,10 +2796,10 @@ def validate_release_bundle_artifact(bundle_path: Path, source_revision: str, ga
|
||||
require(evidence.get("stale_journals_after_gc") == 0,
|
||||
f"{gate}.{field} requires zero stale journals after GC")
|
||||
if field == "segment_activation_preflight":
|
||||
require(evidence.get("production_activation") is False,
|
||||
f"{gate}.{field} must keep production activation disabled")
|
||||
require(evidence.get("scanner_segment_reuse_activated") is False,
|
||||
f"{gate}.{field} must prove the runtime activation gate is disabled")
|
||||
require(evidence.get("production_activation") is True,
|
||||
f"{gate}.{field} must prove production activation is enabled")
|
||||
require(evidence.get("scanner_segment_reuse_activated") is True,
|
||||
f"{gate}.{field} must prove the runtime activation gate is enabled")
|
||||
evidence_exact_strings(evidence.get("proof_inputs"),
|
||||
SCANNER_HEAL_SEGMENT_ACTIVATION_PROOF_INPUTS,
|
||||
f"{gate}.{field}.proof_inputs")
|
||||
@@ -3319,6 +3343,7 @@ def write_scanner_heal_release_bundle_fixture(root: Path, directory: Path) -> Pa
|
||||
"raw_entry_budget": 8,
|
||||
"max_raw_entries_per_round": 8,
|
||||
"max_objects_processed_per_round": 8,
|
||||
"bounded_work_quantum_observed": True,
|
||||
"durable_checkpoint_committed": True,
|
||||
"no_unbounded_tail": True,
|
||||
})
|
||||
@@ -3739,6 +3764,7 @@ class SelfTests(unittest.TestCase):
|
||||
"raw_entry_budget": 8,
|
||||
"max_raw_entries_per_round": 8,
|
||||
"max_objects_processed_per_round": 8,
|
||||
"bounded_work_quantum_observed": True,
|
||||
"durable_checkpoint_committed": True,
|
||||
"no_unbounded_tail": True,
|
||||
})
|
||||
@@ -3944,8 +3970,8 @@ class SelfTests(unittest.TestCase):
|
||||
},
|
||||
]
|
||||
if field == "segment_activation_preflight":
|
||||
evidence["production_activation"] = False
|
||||
evidence["scanner_segment_reuse_activated"] = False
|
||||
evidence["production_activation"] = True
|
||||
evidence["scanner_segment_reuse_activated"] = True
|
||||
evidence["proof_inputs"] = list(SCANNER_HEAL_SEGMENT_ACTIVATION_PROOF_INPUTS)
|
||||
evidence["fail_closed_checks"] = list(SCANNER_HEAL_SEGMENT_ACTIVATION_FAIL_CLOSED_CHECKS)
|
||||
if field == "cold_segment_reuse_measurement":
|
||||
@@ -4447,7 +4473,14 @@ class SelfTests(unittest.TestCase):
|
||||
"zero stale journals",
|
||||
),
|
||||
("same-window-fields", "G14", "same_window_field_evidence", lambda item: item.update({"same_window_fields": ["ec8_4_evidence", "multi_set_evidence"]}), "JSON artifact same_window_fields mismatch"),
|
||||
("activation-enabled", "G11", "segment_activation_preflight", lambda item: item.update({"production_activation": True}), "production activation disabled"),
|
||||
("activation-disabled", "G11", "segment_activation_preflight", lambda item: item.update({"production_activation": False}), "production activation is enabled"),
|
||||
(
|
||||
"activation-runtime-disabled",
|
||||
"G11",
|
||||
"segment_activation_preflight",
|
||||
lambda item: item.update({"scanner_segment_reuse_activated": False}),
|
||||
"runtime activation gate is enabled",
|
||||
),
|
||||
(
|
||||
"activation-missing-fail-closed",
|
||||
"G11",
|
||||
@@ -5156,6 +5189,35 @@ class SelfTests(unittest.TestCase):
|
||||
errors,
|
||||
)
|
||||
|
||||
def test_scanner_heal_multipool_case_accepts_deferred_optional_outage_write(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root, run_dir = self.scanner_heal_fixture(Path(tmp))
|
||||
path = run_dir / "background-target-crash-ec8-4-multi-pool.json"
|
||||
oracle = read_json(path)
|
||||
oracle["outage_write_deferred_until_rejoin"] = True
|
||||
oracle["outage_write"] = {
|
||||
"attempted": True,
|
||||
"required": False,
|
||||
"accepted": True,
|
||||
"attempts": 36,
|
||||
"service_unavailable": 36,
|
||||
}
|
||||
write_json(path, oracle)
|
||||
(run_dir / "execution.json").unlink()
|
||||
finish_scanner_heal_receipt(run_dir, 0, root)
|
||||
|
||||
self.assertEqual(check_scanner_heal_evidence(root, run_dir, "background-target-crash-ec8-4-multi-pool"), [])
|
||||
|
||||
oracle["outage_write"]["accepted"] = False
|
||||
write_json(path, oracle)
|
||||
(run_dir / "execution.json").unlink()
|
||||
finish_scanner_heal_receipt(run_dir, 0, root)
|
||||
errors = check_scanner_heal_evidence(root, run_dir, "background-target-crash-ec8-4-multi-pool")
|
||||
self.assertTrue(
|
||||
any("optional outage-write final S3 body was not accepted" in error for error in errors),
|
||||
errors,
|
||||
)
|
||||
|
||||
def test_scanner_heal_pending_gate_cannot_map_to_implemented_lane(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root, run_dir = self.scanner_heal_fixture(Path(tmp))
|
||||
|
||||
+23
-1
@@ -15,6 +15,7 @@ ROOT = Path(__file__).resolve().parent.parent
|
||||
ALWAYS_JOBS = ("classify-changes", "typos", "quick-checks")
|
||||
CODE_JOBS = (
|
||||
"test-and-lint", "test-ilm-integration-serial", "test-and-lint-rio-v2",
|
||||
"offline-enrollment-root-boundary",
|
||||
"connect-short-credential-boundary", "test-and-lint-protocols",
|
||||
"build-rustfs-debug-binary", "uring-integration", "e2e-tests",
|
||||
"s3-implemented-tests", "s3-lifecycle-behavior-tests",
|
||||
@@ -59,7 +60,7 @@ def expected_results(mode: str, event: str, ref: str) -> dict[str, str]:
|
||||
expected.update({job: "success" if mode == "full" else "skipped" for job in CODE_JOBS})
|
||||
rio = mode == "full" and event in ("schedule", "workflow_dispatch")
|
||||
expected.update({job: "success" if rio else "skipped" for job in OPTIONAL_JOBS[:2]})
|
||||
full = mode == "full" and (event in ("merge_group", "workflow_dispatch") or (event == "push" and ref == "refs/heads/main"))
|
||||
full = mode == "full" and (event in ("merge_group", "workflow_dispatch") or (event == "push" and ref in ("refs/heads/main", "refs/heads/release")))
|
||||
expected["e2e-full"] = "success" if full else "skipped"
|
||||
return expected
|
||||
|
||||
@@ -219,6 +220,27 @@ class SelfTests(unittest.TestCase):
|
||||
bad = {**good, "classify-changes": {"result": "success", "outputs": selection}}
|
||||
self.assertTrue(verify_results(bad, event, "refs/heads/main"))
|
||||
|
||||
def test_full_e2e_gate_preserves_workflow_branch_and_event_scope(self):
|
||||
for event, ref, required in (
|
||||
("push", "refs/heads/main", "success"),
|
||||
("push", "refs/heads/release", "success"),
|
||||
("push", "refs/heads/feature", "skipped"),
|
||||
("push", "refs/heads/release-candidate", "skipped"),
|
||||
("push", "refs/tags/release", "skipped"),
|
||||
("pull_request", "refs/pull/1/merge", "skipped"),
|
||||
("schedule", "refs/heads/release", "skipped"),
|
||||
("workflow_dispatch", "refs/heads/feature", "success"),
|
||||
("merge_group", "refs/heads/gh-readonly-queue/release/pr-1", "success"),
|
||||
):
|
||||
with self.subTest(event=event, ref=ref):
|
||||
expected = expected_results("full", event, ref)
|
||||
self.assertEqual(expected["e2e-full"], required)
|
||||
needs = {job: {"result": result} for job, result in expected.items()}
|
||||
needs["classify-changes"]["outputs"] = {"mode": "full"}
|
||||
for result in ("success", "skipped", "failure", "cancelled"):
|
||||
needs["e2e-full"]["result"] = result
|
||||
self.assertEqual(verify_results(needs, event, ref) == [], result == required)
|
||||
|
||||
def test_repository_wiring_and_missing_dependency_regression(self):
|
||||
self.assertEqual(check_workflow(ROOT), [])
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
|
||||
@@ -74,10 +74,31 @@ def converged(report, objects):
|
||||
|
||||
|
||||
def replays_raw_window(previous, current):
|
||||
if (previous["raw_entries"] == 0 or current["raw_entries"] == 0
|
||||
or previous["raw_first_entry"] is None or current["raw_first_entry"] is None
|
||||
or previous["raw_last_entry"] is None or current["raw_last_entry"] is None):
|
||||
return False
|
||||
raw_page_index_progressed = (
|
||||
previous["raw_page_index_parent"] == current["raw_page_index_parent"]
|
||||
and (
|
||||
current["raw_page_index_committed_entries"] > previous["raw_page_index_committed_entries"]
|
||||
or (current["raw_page_index_complete"] and not previous["raw_page_index_complete"])
|
||||
)
|
||||
)
|
||||
return (previous["raw_first_entry"] == current["raw_first_entry"]
|
||||
and previous["raw_last_entry"] == current["raw_last_entry"]
|
||||
and previous["objects_retained"] == current["objects_before"]
|
||||
and current["objects_retained"] == previous["objects_retained"])
|
||||
and current["objects_retained"] == previous["objects_retained"]
|
||||
and not raw_page_index_progressed)
|
||||
|
||||
|
||||
def fully_retained(report, objects):
|
||||
return all(report[key] == objects for key in
|
||||
("objects_retained", "versions_retained", "bytes_retained"))
|
||||
|
||||
|
||||
def final_complete_recheck_after_full_retention(previous, current, objects):
|
||||
return fully_retained(previous, objects) and converged(current, objects)
|
||||
|
||||
|
||||
def validate_recoverable_quantum(reports, *, objects, budget, require_converged):
|
||||
@@ -97,7 +118,8 @@ def validate_recoverable_quantum(reports, *, objects, budget, require_converged)
|
||||
raise ValueError("durable retained coverage did not survive process restart")
|
||||
if report["objects_retained"] < previous["objects_retained"]:
|
||||
raise ValueError("durable retained coverage regressed across restart")
|
||||
if replays_raw_window(previous, report):
|
||||
if (not final_complete_recheck_after_full_retention(previous, report, objects)
|
||||
and replays_raw_window(previous, report)):
|
||||
raise ValueError("raw enumeration window replayed without durable coverage")
|
||||
if (report["raw_page_index_parent"] == previous["raw_page_index_parent"]
|
||||
and report["raw_page_index_committed_entries"] < previous["raw_page_index_committed_entries"]
|
||||
|
||||
@@ -32,6 +32,11 @@ def positive_int(value: Any, name: str, minimum: int = 1) -> int:
|
||||
return value
|
||||
|
||||
|
||||
def reject_non_measured_markers(payload: dict[str, Any], label: str) -> None:
|
||||
for marker in ("fixture", "fixture_only", "dry_run", "synthetic"):
|
||||
require(payload.get(marker) is not True, f"{label} is {marker}")
|
||||
|
||||
|
||||
def timestamp(value: Any, name: str) -> str:
|
||||
require(isinstance(value, str) and value.endswith("Z"), f"invalid {name}")
|
||||
datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
@@ -63,6 +68,7 @@ def load_reports(directory: Path) -> list[dict[str, Any]]:
|
||||
"raw_page_index_indexed_entries",
|
||||
}
|
||||
for index, report in enumerate(reports):
|
||||
reject_non_measured_markers(report, f"round {index}")
|
||||
require(report.get("schema") == 1, f"round {index} has wrong schema")
|
||||
require(report.get("round") == index, f"round {index} order mismatch")
|
||||
for key in (
|
||||
@@ -88,8 +94,7 @@ def load_reports(directory: Path) -> list[dict[str, Any]]:
|
||||
|
||||
def require_measured_manifest(path: Path, source_revision: str) -> dict[str, Any]:
|
||||
manifest = read_json(path)
|
||||
for marker in ("fixture", "fixture_only", "dry_run", "synthetic"):
|
||||
require(manifest.get(marker) is not True, f"checkpoint/crash manifest is {marker}")
|
||||
reject_non_measured_markers(manifest, "checkpoint/crash manifest")
|
||||
require(manifest.get("schema") == 1, "unsupported manifest schema")
|
||||
require(manifest.get("evidence_type") == "measured", "manifest must be measured")
|
||||
require(manifest.get("source_revision") == source_revision, "manifest source revision mismatch")
|
||||
@@ -108,8 +113,7 @@ def derived_measured_manifest(directory: Path, source_revision: str, reports: li
|
||||
request_path = directory / "request.json"
|
||||
request = read_json(request_path)
|
||||
require(isinstance(request, dict), "diagnostic request must be a JSON object")
|
||||
for marker in ("fixture", "fixture_only", "dry_run", "synthetic"):
|
||||
require(request.get(marker) is not True, f"diagnostic request is {marker}")
|
||||
reject_non_measured_markers(request, "diagnostic request")
|
||||
objects = positive_int(request.get("objects"), "request.objects")
|
||||
raw_entry_budget = positive_int(request.get("raw_entry_budget"), "request.raw_entry_budget")
|
||||
final_round = positive_int(request.get("round"), "request.round", 0)
|
||||
@@ -251,6 +255,7 @@ def build_descriptor(args: argparse.Namespace) -> Path:
|
||||
"raw_entry_budget": summary["raw_entry_budget"],
|
||||
"max_raw_entries_per_round": summary["max_raw_entries_per_round"],
|
||||
"max_objects_processed_per_round": summary["max_objects_processed_per_round"],
|
||||
"bounded_work_quantum_observed": True,
|
||||
"durable_checkpoint_committed": True,
|
||||
"no_unbounded_tail": True,
|
||||
}),
|
||||
@@ -422,6 +427,24 @@ def run_self_test() -> None:
|
||||
else:
|
||||
raise ValueError("self-test accepted non-converged diagnostic")
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
source_revision = git_head()
|
||||
manifest, diagnostic = write_self_test_inputs(root, source_revision)
|
||||
report = read_json(diagnostic / "round-0.json")
|
||||
report["synthetic"] = True
|
||||
write_json(diagnostic / "round-0.json", report)
|
||||
try:
|
||||
build_descriptor(parse_args([
|
||||
"--manifest", str(manifest),
|
||||
"--diagnostic-dir", str(diagnostic),
|
||||
"--out-dir", str(root / "out"),
|
||||
]))
|
||||
except ValueError as err:
|
||||
require("round 0 is synthetic" in str(err), "wrong self-test failure for synthetic round report")
|
||||
else:
|
||||
raise ValueError("self-test accepted synthetic diagnostic round report")
|
||||
|
||||
|
||||
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
|
||||
@@ -28,6 +28,9 @@ The script intentionally runs a single case, not the release pseudo-case. After
|
||||
a successful case run it verifies that the release gate still remains blocked.
|
||||
Set RUSTFS_E2E_TEST_PORT_MIN and RUSTFS_E2E_TEST_PORT_RANGE to move the e2e
|
||||
port allocator when the default 20000..30000 test range is unavailable.
|
||||
Set RUSTFS_SCANNER_HEAL_SKIP_CLEAN=1 to reuse an existing cargo target directory
|
||||
while narrowing a case locally; release evidence should keep the default clean
|
||||
build.
|
||||
USAGE
|
||||
}
|
||||
|
||||
@@ -109,7 +112,7 @@ apply_runtime_profile() {
|
||||
export RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS="${RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS:-180}"
|
||||
;;
|
||||
background-ec8-4-multi-pool)
|
||||
export RUSTFS_HEAL_CHAOS_OBJECT_COUNT="${RUSTFS_HEAL_CHAOS_OBJECT_COUNT:-16}"
|
||||
export RUSTFS_HEAL_CHAOS_OBJECT_COUNT="${RUSTFS_HEAL_CHAOS_OBJECT_COUNT:-64}"
|
||||
export RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES="${RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES:-4194304}"
|
||||
export RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS="${RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS:-240}"
|
||||
;;
|
||||
@@ -255,18 +258,31 @@ if [[ -n "$(git status --porcelain --untracked-files=no)" ]]; then
|
||||
echo "commit tracked source changes before creating evidence" >&2
|
||||
exit 1
|
||||
fi
|
||||
NOFILE_SOFT="$(ulimit -Sn)"
|
||||
NOFILE_HARD="$(ulimit -Hn)"
|
||||
if [[ "$NOFILE_SOFT" =~ ^[0-9]+$ && "$NOFILE_HARD" =~ ^[0-9]+$ && "$NOFILE_SOFT" -lt 65535 ]]; then
|
||||
if [[ "$NOFILE_HARD" -ge 65535 ]]; then
|
||||
ulimit -n 65535 || true
|
||||
elif [[ "$NOFILE_HARD" -gt "$NOFILE_SOFT" ]]; then
|
||||
ulimit -n "$NOFILE_HARD" || true
|
||||
fi
|
||||
fi
|
||||
mkdir -p "$(dirname "$RUN_DIR")"
|
||||
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/rustfs-scanner-heal-evidence.XXXXXX")"
|
||||
trap 'rm -rf "$TMP_DIR"' EXIT
|
||||
|
||||
BUILD_FEATURES="${RUSTFS_BUILD_FEATURES:-}"
|
||||
cargo clean -p rustfs
|
||||
TARGET_DIR="${CARGO_TARGET_DIR:-$ROOT/target}"
|
||||
DEBUG_DIR="$TARGET_DIR/debug"
|
||||
if [[ "${RUSTFS_SCANNER_HEAL_SKIP_CLEAN:-0}" != "1" ]]; then
|
||||
cargo clean -p rustfs
|
||||
fi
|
||||
if [[ -n "$BUILD_FEATURES" ]]; then
|
||||
cargo build --locked -p rustfs --bins --features "$BUILD_FEATURES"
|
||||
else
|
||||
cargo build --locked -p rustfs --bins
|
||||
fi
|
||||
printf '%s' "$BUILD_FEATURES" >"$ROOT/target/debug/rustfs.features"
|
||||
printf '%s' "$BUILD_FEATURES" >"$DEBUG_DIR/rustfs.features"
|
||||
|
||||
LISTING_TMP="$TMP_DIR/listing.json"
|
||||
NO_PROXY="${NO_PROXY:-127.0.0.1,localhost}" \
|
||||
@@ -278,12 +294,13 @@ cargo nextest list --profile "$PROFILE" -p e2e_test -E "$TEST_FILTER" --message-
|
||||
TEST_BINARY="$(test_binary_from_listing "$LISTING_TMP" "$CASE_ID")"
|
||||
|
||||
export RUSTFS_E2E_EXPECTED_FEATURES="${RUSTFS_E2E_EXPECTED_FEATURES:-default}"
|
||||
"$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --begin-scanner-heal "$RUN_DIR" "$ROOT/target/debug/rustfs" "$TEST_BINARY"
|
||||
"$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --begin-scanner-heal "$RUN_DIR" "$DEBUG_DIR/rustfs" "$TEST_BINARY"
|
||||
cp "$LISTING_TMP" "$RUN_DIR/listing.json"
|
||||
export RUSTFS_E2E_LOG_DIR="${RUSTFS_E2E_LOG_DIR:-$RUN_DIR/e2e-logs}"
|
||||
export RUSTFS_HEAL_CHAOS_LOG_DIR="${RUSTFS_HEAL_CHAOS_LOG_DIR:-$RUSTFS_E2E_LOG_DIR}"
|
||||
mkdir -p "$RUSTFS_E2E_LOG_DIR"
|
||||
|
||||
JUNIT_PATH="$ROOT/target/nextest/$PROFILE/junit.xml"
|
||||
JUNIT_PATH="$TARGET_DIR/nextest/$PROFILE/junit.xml"
|
||||
rm -f "$JUNIT_PATH"
|
||||
set +e
|
||||
NO_PROXY="${NO_PROXY:-127.0.0.1,localhost}" \
|
||||
|
||||
@@ -48,6 +48,11 @@ def positive_int(value: Any, name: str, minimum: int = 1) -> int:
|
||||
return value
|
||||
|
||||
|
||||
def reject_non_measured_markers(payload: dict[str, Any], label: str) -> None:
|
||||
for marker in ("fixture", "fixture_only", "dry_run", "synthetic"):
|
||||
require(payload.get(marker) is not True, f"{label} is {marker}")
|
||||
|
||||
|
||||
def parse_case_dir_arg(value: str) -> tuple[str | None, Path]:
|
||||
if "=" in value:
|
||||
case_id, raw_path = value.split("=", 1)
|
||||
@@ -72,8 +77,7 @@ def proof_case_artifact_path(proof_path: Path, sample: dict[str, Any], index: in
|
||||
|
||||
def load_proof(path: Path, source_revision: str) -> dict[str, Any]:
|
||||
proof = read_json(path)
|
||||
for marker in ("fixture", "fixture_only", "dry_run", "synthetic"):
|
||||
require(proof.get(marker) is not True, f"G14 proof is {marker}")
|
||||
reject_non_measured_markers(proof, "G14 proof")
|
||||
require(proof.get("schema") == 1, "unsupported G14 proof schema")
|
||||
require(proof.get("evidence_type") == "measured", "G14 proof must be measured")
|
||||
require(proof.get("source_revision") == source_revision, "G14 proof source revision mismatch")
|
||||
@@ -110,8 +114,7 @@ def load_proof(path: Path, source_revision: str) -> dict[str, Any]:
|
||||
f"G14 case evidence {index} measurement window mismatch")
|
||||
artifact_payload = read_json(artifact)
|
||||
require(isinstance(artifact_payload, dict), f"G14 case evidence {index} artifact must be a JSON object")
|
||||
for marker in ("fixture", "fixture_only", "dry_run", "synthetic"):
|
||||
require(artifact_payload.get(marker) is not True, f"G14 case evidence {index} artifact is {marker}")
|
||||
reject_non_measured_markers(artifact_payload, f"G14 case evidence {index} artifact")
|
||||
require(artifact_payload.get("case") == sample["case"], f"G14 case evidence {index} artifact case mismatch")
|
||||
if "source_revision" in artifact_payload:
|
||||
require(artifact_payload["source_revision"] == source_revision,
|
||||
@@ -127,6 +130,10 @@ def load_case_directory(raw_value: str, source_revision: str) -> dict[str, Any]:
|
||||
require(directory.is_dir(), f"G14 case directory is missing: {directory}")
|
||||
run = read_json(directory / "run.json")
|
||||
execution = read_json(directory / "execution.json")
|
||||
require(isinstance(run, dict), "G14 case run must be a JSON object")
|
||||
require(isinstance(execution, dict), "G14 case execution must be a JSON object")
|
||||
reject_non_measured_markers(run, "G14 case run")
|
||||
reject_non_measured_markers(execution, "G14 case execution")
|
||||
require(run.get("schema") == 1, "G14 case run schema mismatch")
|
||||
require(isinstance(run.get("run_id"), str) and re.fullmatch(r"[0-9a-f]{32}", run["run_id"]),
|
||||
"invalid G14 case run id")
|
||||
@@ -156,6 +163,7 @@ def load_case_directory(raw_value: str, source_revision: str) -> dict[str, Any]:
|
||||
if expected_case is not None:
|
||||
require(oracle.get("case") == expected_case, "G14 case directory case id mismatch")
|
||||
require(oracle.get("source_revision") == source_revision, "G14 oracle source revision mismatch")
|
||||
reject_non_measured_markers(oracle, "G14 oracle")
|
||||
require(oracle.get("evidence") in {"process-restart", "process-crash-restart"}, "G14 oracle has wrong evidence type")
|
||||
topology = oracle.get("topology")
|
||||
require(isinstance(topology, dict), "G14 oracle missing topology")
|
||||
@@ -504,6 +512,27 @@ def run_self_test() -> None:
|
||||
else:
|
||||
raise ValueError("self-test accepted case evidence without multi-pool proof")
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
source_revision = git_head()
|
||||
multi_pool = write_self_test_case_dir(root, source_revision, "background-target-crash-ec8-4-multi-pool", 3, 3)
|
||||
oracle_path = multi_pool / "background-target-crash-ec8-4-multi-pool.json"
|
||||
oracle = read_json(oracle_path)
|
||||
oracle["fixture_only"] = True
|
||||
write_json(oracle_path, oracle)
|
||||
execution = read_json(multi_pool / "execution.json")
|
||||
execution["artifacts"][oracle_path.name] = digest(oracle_path)
|
||||
write_json(multi_pool / "execution.json", execution)
|
||||
try:
|
||||
build_descriptor(parse_args([
|
||||
"--case-dir", f"background-target-crash-ec8-4-multi-pool={multi_pool}",
|
||||
"--out-dir", str(root / "out"),
|
||||
]))
|
||||
except ValueError as err:
|
||||
require("G14 oracle is fixture_only" in str(err), "wrong self-test failure for fixture oracle")
|
||||
else:
|
||||
raise ValueError("self-test accepted fixture-only G14 case oracle")
|
||||
|
||||
|
||||
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
|
||||
+715
@@ -0,0 +1,715 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Plan the Scanner/Heal Linux release-evidence validation flow.
|
||||
|
||||
The planner is intentionally not a release-evidence producer. It writes a
|
||||
machine-readable execution manifest for the existing measured runners and can
|
||||
run only their lightweight preflight checks. Long-running Linux, distributed,
|
||||
mixed-version, ABBA, and profile lanes remain explicit operator actions.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
REGISTRY = ROOT / ".config" / "scanner-heal-required-tests.json"
|
||||
DEFAULT_OUTPUT_ROOT = ROOT / "target" / "scanner-heal-linux-evidence-plan"
|
||||
REQUIRED_GATES = {
|
||||
"G01",
|
||||
"G02",
|
||||
"G03",
|
||||
"G04",
|
||||
"G05",
|
||||
"G06",
|
||||
"G07",
|
||||
"G08",
|
||||
"G09",
|
||||
"G10",
|
||||
"G11",
|
||||
"G12",
|
||||
"G13",
|
||||
"G14",
|
||||
"P1",
|
||||
"P2",
|
||||
"P3",
|
||||
"P4",
|
||||
"R-D",
|
||||
"R-E",
|
||||
"R-L",
|
||||
}
|
||||
|
||||
|
||||
def command(*parts: str) -> list[str]:
|
||||
return list(parts)
|
||||
|
||||
|
||||
def expected_outputs(*parts: str) -> list[str]:
|
||||
return list(parts)
|
||||
|
||||
|
||||
def git_output(*args: str) -> str:
|
||||
return subprocess.check_output(command("git", *args), cwd=ROOT, text=True).strip()
|
||||
|
||||
|
||||
def source_revision(explicit: str | None) -> str:
|
||||
if explicit:
|
||||
if len(explicit) != 40 or any(char not in "0123456789abcdef" for char in explicit):
|
||||
raise ValueError("--source-revision must be a 40-character lowercase Git SHA")
|
||||
return explicit
|
||||
return git_output("rev-parse", "HEAD")
|
||||
|
||||
|
||||
def load_registry() -> dict[str, Any]:
|
||||
with REGISTRY.open() as stream:
|
||||
registry = json.load(stream)
|
||||
if registry.get("schema") != 2:
|
||||
raise ValueError("Scanner/Heal release registry must use schema 2")
|
||||
return registry
|
||||
|
||||
|
||||
def validate_registry(registry: dict[str, Any]) -> None:
|
||||
gates = {item["gate"] for item in registry.get("release_requirements", [])}
|
||||
missing = sorted(REQUIRED_GATES - gates)
|
||||
if missing:
|
||||
raise ValueError(f"release registry is missing gates: {', '.join(missing)}")
|
||||
lanes = registry.get("release_lanes")
|
||||
if not isinstance(lanes, dict) or not lanes:
|
||||
raise ValueError("release registry is missing release_lanes")
|
||||
lane_gates = set()
|
||||
for lane_name, lane in lanes.items():
|
||||
if not isinstance(lane, dict):
|
||||
raise ValueError(f"release lane {lane_name} must be an object")
|
||||
lane_gates.update(lane.get("gates", []))
|
||||
missing_from_lanes = sorted(gates - lane_gates)
|
||||
if missing_from_lanes:
|
||||
raise ValueError(f"release lanes do not cover gates: {', '.join(missing_from_lanes)}")
|
||||
|
||||
|
||||
def existing_script_command(*parts: str) -> dict[str, Any]:
|
||||
for part in parts:
|
||||
if part.startswith("scripts/") and "$" not in part and not (ROOT / part).is_file():
|
||||
raise ValueError(f"missing planned script: {part}")
|
||||
return {"command": command(*parts), "script": parts[0]}
|
||||
|
||||
|
||||
def preflight_steps() -> list[dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
"id": "repo-source-clean",
|
||||
"description": "Verify the checkout revision and tracked-source cleanliness before measured evidence.",
|
||||
"commands": [
|
||||
{"command": command("git", "rev-parse", "HEAD")},
|
||||
{"command": command("git", "status", "--porcelain", "--untracked-files=no")},
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "release-registry-self-test",
|
||||
"description": "Exercise the release registry, descriptor, bundle, and negative parser cases.",
|
||||
"commands": [existing_script_command("scripts/python_bin.sh", "scripts/check_test_wiring.py", "--self-test")],
|
||||
"preflight_runnable": True,
|
||||
},
|
||||
{
|
||||
"id": "runner-self-tests",
|
||||
"description": "Run lightweight CLI/schema tests for every Scanner/Heal evidence runner.",
|
||||
"commands": [
|
||||
existing_script_command("scripts/run_scanner_heal_evidence_case.sh", "--self-test"),
|
||||
existing_script_command("scripts/test_scanner_heal_authority_evidence.sh"),
|
||||
existing_script_command("scripts/test_scanner_heal_checkpoint_crash_evidence.sh"),
|
||||
existing_script_command("scripts/test_scanner_heal_status_outcome_evidence.sh"),
|
||||
existing_script_command("scripts/test_scanner_heal_scoped_ack_evidence.sh"),
|
||||
existing_script_command("scripts/test_scanner_heal_legacy_rollback_evidence.sh"),
|
||||
existing_script_command("scripts/test_scanner_heal_g14_multiset_evidence.sh"),
|
||||
existing_script_command("scripts/test_scanner_heal_g09_upgrade_evidence.sh"),
|
||||
existing_script_command("scripts/test_scanner_heal_scheduler_pressure_evidence.sh"),
|
||||
existing_script_command("scripts/test_scanner_heal_maintenance_evidence.sh"),
|
||||
existing_script_command("scripts/test_scanner_heal_w13_mrf_evidence.sh"),
|
||||
existing_script_command("scripts/test_scanner_heal_w16_recovery_evidence.sh"),
|
||||
],
|
||||
"preflight_runnable": True,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def concrete_case_steps(registry: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
cases = registry["cases"]
|
||||
ordered_cases = [
|
||||
"background-target-restart",
|
||||
"background-target-crash",
|
||||
"background-target-restart-ec8-4",
|
||||
"background-target-crash-ec8-4",
|
||||
"ec84-target-drive-restart",
|
||||
"background-target-restart-ec8-4-multi-set",
|
||||
"background-target-crash-ec8-4-multi-pool",
|
||||
]
|
||||
missing = [case for case in ordered_cases if case not in cases]
|
||||
if missing:
|
||||
raise ValueError(f"release registry is missing concrete cases: {', '.join(missing)}")
|
||||
return [
|
||||
{
|
||||
"id": f"case-{case_id}",
|
||||
"description": cases[case_id]["scope"],
|
||||
"covers": {
|
||||
"gate": cases[case_id]["gate"],
|
||||
"task": cases[case_id]["task"],
|
||||
"lane": cases[case_id]["lane"],
|
||||
"evidence": cases[case_id]["evidence"],
|
||||
},
|
||||
"commands": [
|
||||
existing_script_command(
|
||||
"scripts/run_scanner_heal_evidence_case.sh",
|
||||
"--case",
|
||||
case_id,
|
||||
"--run-dir",
|
||||
f"$RUN_ROOT/cases/{case_id}",
|
||||
)
|
||||
],
|
||||
"expected_outputs": expected_outputs(
|
||||
f"$RUN_ROOT/cases/{case_id}/run.json",
|
||||
f"$RUN_ROOT/cases/{case_id}/execution.json",
|
||||
f"$RUN_ROOT/cases/{case_id}/listing.json",
|
||||
f"$RUN_ROOT/cases/{case_id}/junit.xml",
|
||||
f"$RUN_ROOT/cases/{case_id}/{cases[case_id]['oracle']}",
|
||||
f"$RUN_ROOT/cases/{case_id}/release-status.json",
|
||||
),
|
||||
}
|
||||
for case_id in ordered_cases
|
||||
]
|
||||
|
||||
|
||||
def descriptor_steps() -> list[dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
"id": "authority-coverage",
|
||||
"description": "Assemble the G01 authority descriptor from operator-collected root and quota authority artifacts.",
|
||||
"covers": {"gates": ["G01"], "issues": ["2270"]},
|
||||
"commands": [
|
||||
existing_script_command(
|
||||
"scripts/python_bin.sh",
|
||||
"scripts/run_scanner_heal_authority_evidence.py",
|
||||
"--root-authority-json",
|
||||
"$RUN_ROOT/raw/authority/root-authority.json",
|
||||
"--quota-authority-json",
|
||||
"$RUN_ROOT/raw/authority/quota-authority.json",
|
||||
"--out-dir",
|
||||
"$RUN_ROOT/descriptors/authority",
|
||||
)
|
||||
],
|
||||
"expected_outputs": expected_outputs("$RUN_ROOT/descriptors/authority/release-bundle-authority.json"),
|
||||
},
|
||||
{
|
||||
"id": "checkpoint-and-restart",
|
||||
"description": "Assemble G02/R-E bounded checkpoint and restart descriptors from measured diagnostic reports.",
|
||||
"covers": {"gates": ["G02", "R-E"], "issues": ["2269"]},
|
||||
"commands": [
|
||||
existing_script_command(
|
||||
"scripts/python_bin.sh",
|
||||
"scripts/run_scanner_heal_checkpoint_crash_evidence.py",
|
||||
"--diagnostic-dir",
|
||||
"$RUN_ROOT/raw/checkpoint",
|
||||
"--out-dir",
|
||||
"$RUN_ROOT/descriptors/checkpoint",
|
||||
)
|
||||
],
|
||||
"expected_outputs": expected_outputs("$RUN_ROOT/descriptors/checkpoint/release-bundle-checkpoint-crash.json"),
|
||||
},
|
||||
{
|
||||
"id": "status-and-outcome",
|
||||
"description": "Collect live G05/G06/R-D raw observations, then assemble their descriptor.",
|
||||
"covers": {"gates": ["G05", "G06", "R-D"], "issues": ["2278"]},
|
||||
"commands": [
|
||||
existing_script_command(
|
||||
"scripts/python_bin.sh",
|
||||
"scripts/run_scanner_heal_status_outcome_probe.py",
|
||||
"--observations-json",
|
||||
"$RUN_ROOT/raw/status-outcome/observations.json",
|
||||
"--out-dir",
|
||||
"$RUN_ROOT/raw/status-outcome",
|
||||
),
|
||||
existing_script_command(
|
||||
"scripts/python_bin.sh",
|
||||
"scripts/run_scanner_heal_status_outcome_evidence.py",
|
||||
"--status-outcome-json",
|
||||
"$RUN_ROOT/raw/status-outcome/status-outcome.json",
|
||||
"--status-compat-json",
|
||||
"$RUN_ROOT/raw/status-outcome/status-compat.json",
|
||||
"--disposition-json",
|
||||
"$RUN_ROOT/raw/status-outcome/disposition.json",
|
||||
"--out-dir",
|
||||
"$RUN_ROOT/descriptors/status-outcome",
|
||||
),
|
||||
],
|
||||
"expected_outputs": expected_outputs(
|
||||
"$RUN_ROOT/raw/status-outcome/status-outcome.json",
|
||||
"$RUN_ROOT/raw/status-outcome/status-compat.json",
|
||||
"$RUN_ROOT/raw/status-outcome/disposition.json",
|
||||
"$RUN_ROOT/descriptors/status-outcome/release-bundle-status-outcome.json",
|
||||
),
|
||||
},
|
||||
{
|
||||
"id": "ec8-4-multiset-descriptor",
|
||||
"description": "Assemble G14 EC8+4 multi-set and multi-pool descriptors from the measured case directories.",
|
||||
"covers": {"gates": ["G14"], "issues": ["2266", "2269"]},
|
||||
"commands": [
|
||||
existing_script_command(
|
||||
"scripts/python_bin.sh",
|
||||
"scripts/run_scanner_heal_g14_multiset_evidence.py",
|
||||
"--case-dir",
|
||||
"multi-set=$RUN_ROOT/cases/background-target-restart-ec8-4-multi-set",
|
||||
"--case-dir",
|
||||
"multi-pool=$RUN_ROOT/cases/background-target-crash-ec8-4-multi-pool",
|
||||
"--out-dir",
|
||||
"$RUN_ROOT/descriptors/g14",
|
||||
)
|
||||
],
|
||||
"expected_outputs": expected_outputs("$RUN_ROOT/descriptors/g14/release-bundle-g14.json"),
|
||||
},
|
||||
{
|
||||
"id": "w16-recovery-intent",
|
||||
"description": "Run the G04/G12 recovery-intent and quota-authority lanes.",
|
||||
"covers": {"gates": ["G04", "G12"], "issues": ["2279"]},
|
||||
"commands": [
|
||||
existing_script_command(
|
||||
"scripts/run_scanner_heal_w16_recovery_evidence.sh",
|
||||
"--run-dir",
|
||||
"$RUN_ROOT/w16",
|
||||
)
|
||||
],
|
||||
"expected_outputs": expected_outputs("$RUN_ROOT/w16/release-bundle-w16.json"),
|
||||
},
|
||||
{
|
||||
"id": "mrf-responsibility",
|
||||
"description": "Run or package G07/G08/P4 durable MRF responsibility evidence.",
|
||||
"covers": {"gates": ["G07", "G08", "P4"], "issues": ["2277", "2278"]},
|
||||
"commands": [
|
||||
existing_script_command(
|
||||
"scripts/run_scanner_heal_w13_mrf_evidence.sh",
|
||||
"--run-dir",
|
||||
"$RUN_ROOT/w13",
|
||||
)
|
||||
],
|
||||
"expected_outputs": expected_outputs("$RUN_ROOT/w13/release-bundle-w13.json"),
|
||||
},
|
||||
{
|
||||
"id": "mixed-version-rollback",
|
||||
"description": "Run G09 mixed-version/rollback, then assemble G03 and R-L operator descriptors.",
|
||||
"covers": {"gates": ["G03", "G09", "R-L"], "issues": ["2269", "2281"]},
|
||||
"commands": [
|
||||
existing_script_command(
|
||||
"scripts/run_scanner_heal_g09_upgrade_evidence.sh",
|
||||
"--run-dir",
|
||||
"$RUN_ROOT/g09",
|
||||
),
|
||||
existing_script_command(
|
||||
"scripts/python_bin.sh",
|
||||
"scripts/run_scanner_heal_scoped_ack_evidence.py",
|
||||
"--proof-json",
|
||||
"$RUN_ROOT/raw/scoped-ack/scoped-ack-proof.json",
|
||||
"--out-dir",
|
||||
"$RUN_ROOT/descriptors/scoped-ack",
|
||||
),
|
||||
existing_script_command(
|
||||
"scripts/python_bin.sh",
|
||||
"scripts/run_scanner_heal_legacy_rollback_evidence.py",
|
||||
"--proof-json",
|
||||
"$RUN_ROOT/raw/legacy-rollback/legacy-rollback-proof.json",
|
||||
"--out-dir",
|
||||
"$RUN_ROOT/descriptors/legacy-rollback",
|
||||
),
|
||||
],
|
||||
"expected_outputs": expected_outputs(
|
||||
"$RUN_ROOT/g09/release-bundle-g09.json",
|
||||
"$RUN_ROOT/descriptors/scoped-ack/release-bundle-scoped-ack.json",
|
||||
"$RUN_ROOT/descriptors/legacy-rollback/release-bundle-legacy-rollback.json",
|
||||
),
|
||||
},
|
||||
{
|
||||
"id": "maintenance-producers",
|
||||
"description": "Assemble G11/G13 producer coverage, quorum-minus-one, and remount descriptors.",
|
||||
"covers": {"gates": ["G11", "G13"], "issues": ["2272", "2280"]},
|
||||
"commands": [
|
||||
existing_script_command(
|
||||
"scripts/python_bin.sh",
|
||||
"scripts/run_scanner_heal_maintenance_evidence.py",
|
||||
"--proof-json",
|
||||
"$RUN_ROOT/raw/maintenance/maintenance-proof.json",
|
||||
"--out-dir",
|
||||
"$RUN_ROOT/descriptors/maintenance",
|
||||
)
|
||||
],
|
||||
"expected_outputs": expected_outputs("$RUN_ROOT/descriptors/maintenance/release-bundle-maintenance.json"),
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def performance_steps() -> list[dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
"id": "scanner-heal-abba",
|
||||
"description": "Run the isolated EC8+4 Scanner/Heal ABBA matrix through the deployment adapter.",
|
||||
"covers": {"gates": ["G10", "P1", "P2", "P3"], "issues": ["2266", "2273", "2274", "2275"]},
|
||||
"commands": [
|
||||
existing_script_command(
|
||||
"scripts/python_bin.sh",
|
||||
"scripts/scanner_abba.py",
|
||||
"--manifest",
|
||||
"$RUN_ROOT/abba/manifest.json",
|
||||
"--adapter",
|
||||
"$RUN_ROOT/abba/adapter.py",
|
||||
"--out-dir",
|
||||
"$RUN_ROOT/abba/out",
|
||||
"--data-root",
|
||||
"$RUN_ROOT/abba/data",
|
||||
)
|
||||
],
|
||||
"expected_outputs": expected_outputs("$RUN_ROOT/abba/out/report.json"),
|
||||
},
|
||||
{
|
||||
"id": "scheduler-pressure",
|
||||
"description": "Assemble scheduler-pressure, profile, RSS, throughput, and latency descriptors.",
|
||||
"covers": {"gates": ["G10", "P1", "P2", "P3"], "issues": ["2266", "2274", "2275"]},
|
||||
"commands": [
|
||||
existing_script_command(
|
||||
"scripts/python_bin.sh",
|
||||
"scripts/run_scanner_heal_scheduler_pressure_evidence.py",
|
||||
"--abba-dir",
|
||||
"$RUN_ROOT/abba/out",
|
||||
"--recovery-window-json",
|
||||
"$RUN_ROOT/raw/scheduler/recovery-window.json",
|
||||
"--profile-artifact",
|
||||
"allocation-profile=$RUN_ROOT/profile/allocation-profile.json",
|
||||
"--profile-artifact",
|
||||
"flamegraph=$RUN_ROOT/profile/flamegraph.svg",
|
||||
"--profile-artifact",
|
||||
"rss-samples=$RUN_ROOT/profile/rss-samples.json",
|
||||
"--profile-artifact",
|
||||
"save-frequency=$RUN_ROOT/profile/save-frequency.json",
|
||||
"--out-dir",
|
||||
"$RUN_ROOT/descriptors/scheduler-pressure",
|
||||
)
|
||||
],
|
||||
"expected_outputs": expected_outputs("$RUN_ROOT/descriptors/scheduler-pressure/release-bundle-scheduler-pressure.json"),
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def bundle_steps() -> list[dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
"id": "assemble-release-bundle",
|
||||
"description": "Assemble every measured lane descriptor into the final release bundle.",
|
||||
"covers": {"gates": sorted(REQUIRED_GATES), "issues": ["2240", "2428"]},
|
||||
"commands": [
|
||||
existing_script_command(
|
||||
"scripts/python_bin.sh",
|
||||
"scripts/check_test_wiring.py",
|
||||
"--assemble-scanner-heal-release-bundle",
|
||||
"$RUN_ROOT/descriptors/authority/release-bundle-authority.json",
|
||||
"$RUN_ROOT/descriptors/checkpoint/release-bundle-checkpoint-crash.json",
|
||||
"$RUN_ROOT/descriptors/status-outcome/release-bundle-status-outcome.json",
|
||||
"$RUN_ROOT/w16/release-bundle-w16.json",
|
||||
"$RUN_ROOT/w13/release-bundle-w13.json",
|
||||
"$RUN_ROOT/g09/release-bundle-g09.json",
|
||||
"$RUN_ROOT/descriptors/scoped-ack/release-bundle-scoped-ack.json",
|
||||
"$RUN_ROOT/descriptors/legacy-rollback/release-bundle-legacy-rollback.json",
|
||||
"$RUN_ROOT/descriptors/maintenance/release-bundle-maintenance.json",
|
||||
"$RUN_ROOT/descriptors/g14/release-bundle-g14.json",
|
||||
"$RUN_ROOT/descriptors/scheduler-pressure/release-bundle-scheduler-pressure.json",
|
||||
"$RUN_ROOT/release-bundle",
|
||||
),
|
||||
existing_script_command(
|
||||
"scripts/python_bin.sh",
|
||||
"scripts/check_test_wiring.py",
|
||||
"--check-scanner-heal-release-bundle",
|
||||
"$RUN_ROOT/release-bundle/release-evidence.json",
|
||||
),
|
||||
],
|
||||
"expected_outputs": expected_outputs("$RUN_ROOT/release-bundle/release-evidence.json"),
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def build_plan(registry: dict[str, Any], revision: str, phases: set[str]) -> dict[str, Any]:
|
||||
stage_defs = [
|
||||
("preflight", "Preflight", preflight_steps()),
|
||||
("functional", "Functional And Durable Evidence", concrete_case_steps(registry) + descriptor_steps()),
|
||||
("performance", "ABBA And Profile Evidence", performance_steps()),
|
||||
("bundle", "Release Bundle Assembly", bundle_steps()),
|
||||
]
|
||||
stages = []
|
||||
for key, title, steps in stage_defs:
|
||||
if "all" not in phases and key not in phases:
|
||||
continue
|
||||
stages.append({"id": key, "title": title, "steps": steps})
|
||||
return {
|
||||
"schema": 1,
|
||||
"kind": "scanner-heal-linux-evidence-plan",
|
||||
"evidence_type": "plan_only",
|
||||
"source_revision": revision,
|
||||
"registry": str(REGISTRY.relative_to(ROOT)),
|
||||
"run_root_env": "RUN_ROOT",
|
||||
"requirements": {
|
||||
"base_branch": "release",
|
||||
"platform": "Linux",
|
||||
"tracked_source_clean": True,
|
||||
"measured_evidence_required": True,
|
||||
"synthetic_evidence_rejected": True,
|
||||
"stop_on_product_failure": True,
|
||||
},
|
||||
"stages": stages,
|
||||
}
|
||||
|
||||
|
||||
def write_plan(out_dir: Path, plan: dict[str, Any]) -> Path:
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
path = out_dir / "scanner-heal-linux-evidence-plan.json"
|
||||
path.write_text(json.dumps(plan, indent=2, sort_keys=True) + "\n")
|
||||
return path
|
||||
|
||||
|
||||
def text_plan(plan: dict[str, Any]) -> str:
|
||||
lines = [
|
||||
f"kind={plan['kind']}",
|
||||
f"source_revision={plan['source_revision']}",
|
||||
f"registry={plan['registry']}",
|
||||
"evidence_type=plan_only",
|
||||
]
|
||||
for stage in plan["stages"]:
|
||||
lines.append(f"stage={stage['id']} steps={len(stage['steps'])}")
|
||||
for step in stage["steps"]:
|
||||
lines.append(f" step={step['id']}")
|
||||
for entry in step["commands"]:
|
||||
lines.append(" command=" + " ".join(entry["command"]))
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def iter_preflight_commands(plan: dict[str, Any]) -> list[list[str]]:
|
||||
commands: list[list[str]] = []
|
||||
for stage in plan["stages"]:
|
||||
if stage["id"] != "preflight":
|
||||
continue
|
||||
for step in stage["steps"]:
|
||||
if not step.get("preflight_runnable"):
|
||||
continue
|
||||
commands.extend(entry["command"] for entry in step["commands"])
|
||||
return commands
|
||||
|
||||
|
||||
def render_run_root_path(value: str, run_root: Path) -> Path:
|
||||
rendered = value.replace("$RUN_ROOT", str(run_root))
|
||||
return Path(rendered)
|
||||
|
||||
|
||||
def check_expected_output(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {"path": str(path), "status": "missing"}
|
||||
if not path.is_file():
|
||||
return {"path": str(path), "status": "invalid", "error": "expected a file"}
|
||||
size = path.stat().st_size
|
||||
if size <= 0:
|
||||
return {"path": str(path), "status": "empty", "bytes": size}
|
||||
return {"path": str(path), "status": "present", "bytes": size}
|
||||
|
||||
|
||||
def covered_gates(step: dict[str, Any]) -> list[str]:
|
||||
covers = step.get("covers")
|
||||
if not isinstance(covers, dict):
|
||||
return []
|
||||
gate = covers.get("gate")
|
||||
gates = covers.get("gates")
|
||||
if isinstance(gate, str):
|
||||
return [gate]
|
||||
if isinstance(gates, list):
|
||||
return [item for item in gates if isinstance(item, str)]
|
||||
return []
|
||||
|
||||
|
||||
def build_status(plan: dict[str, Any], run_root: Path) -> dict[str, Any]:
|
||||
run_root = run_root.resolve()
|
||||
stage_statuses = []
|
||||
totals = {"present": 0, "missing": 0, "empty": 0, "invalid": 0, "expected": 0}
|
||||
pending_gates: set[str] = set()
|
||||
next_step: dict[str, Any] | None = None
|
||||
for stage in plan["stages"]:
|
||||
step_statuses = []
|
||||
for step in stage["steps"]:
|
||||
outputs = [
|
||||
check_expected_output(render_run_root_path(path, run_root))
|
||||
for path in step.get("expected_outputs", [])
|
||||
]
|
||||
counts = {"present": 0, "missing": 0, "empty": 0, "invalid": 0}
|
||||
for output in outputs:
|
||||
counts[output["status"]] += 1
|
||||
for key in counts:
|
||||
totals[key] += counts[key]
|
||||
totals["expected"] += len(outputs)
|
||||
if not outputs:
|
||||
status = "not_tracked"
|
||||
elif counts["invalid"] or counts["empty"]:
|
||||
status = "invalid"
|
||||
elif counts["missing"]:
|
||||
status = "pending" if counts["present"] == 0 else "partial"
|
||||
else:
|
||||
status = "complete"
|
||||
if status in {"pending", "partial", "invalid"}:
|
||||
pending_gates.update(covered_gates(step))
|
||||
if next_step is None:
|
||||
next_step = {
|
||||
"stage": stage["id"],
|
||||
"step": step["id"],
|
||||
"status": status,
|
||||
"commands": step["commands"],
|
||||
}
|
||||
step_statuses.append({
|
||||
"id": step["id"],
|
||||
"status": status,
|
||||
"covers": step.get("covers", {}),
|
||||
"outputs": outputs,
|
||||
})
|
||||
tracked = [step for step in step_statuses if step["status"] != "not_tracked"]
|
||||
if not tracked:
|
||||
stage_state = "not_tracked"
|
||||
elif all(step["status"] == "complete" for step in tracked):
|
||||
stage_state = "complete"
|
||||
elif any(step["status"] == "invalid" for step in tracked):
|
||||
stage_state = "invalid"
|
||||
elif any(step["status"] in {"complete", "partial"} for step in tracked):
|
||||
stage_state = "partial"
|
||||
else:
|
||||
stage_state = "pending"
|
||||
stage_statuses.append({"id": stage["id"], "status": stage_state, "steps": step_statuses})
|
||||
if totals["invalid"] or totals["empty"]:
|
||||
decision = "invalid"
|
||||
elif totals["missing"]:
|
||||
decision = "blocked"
|
||||
else:
|
||||
decision = "complete"
|
||||
return {
|
||||
"schema": 1,
|
||||
"kind": "scanner-heal-linux-evidence-status",
|
||||
"decision": decision,
|
||||
"release_approved": False,
|
||||
"source_revision": plan["source_revision"],
|
||||
"run_root": str(run_root),
|
||||
"artifact_totals": totals,
|
||||
"pending_gates": sorted(pending_gates),
|
||||
"next_step": next_step,
|
||||
"stages": stage_statuses,
|
||||
}
|
||||
|
||||
|
||||
def run_preflight(plan: dict[str, Any]) -> int:
|
||||
commands = iter_preflight_commands(plan)
|
||||
if not commands:
|
||||
raise ValueError("--run-preflight requires the preflight stage")
|
||||
failures = []
|
||||
for args in commands:
|
||||
result = subprocess.run(args, cwd=ROOT)
|
||||
if result.returncode != 0:
|
||||
failures.append({"command": args, "exit_code": result.returncode})
|
||||
break
|
||||
if failures:
|
||||
print(json.dumps({"status": "failed", "failures": failures}, indent=2), file=sys.stderr)
|
||||
return 1
|
||||
print(json.dumps({"status": "passed", "commands": len(commands)}, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
def self_test() -> None:
|
||||
registry = load_registry()
|
||||
validate_registry(registry)
|
||||
revision = git_output("rev-parse", "HEAD")
|
||||
plan = build_plan(registry, revision, {"all"})
|
||||
assert plan["evidence_type"] == "plan_only"
|
||||
stages = {stage["id"]: stage for stage in plan["stages"]}
|
||||
assert set(stages) == {"preflight", "functional", "performance", "bundle"}
|
||||
commands = [
|
||||
" ".join(entry["command"])
|
||||
for stage in plan["stages"]
|
||||
for step in stage["steps"]
|
||||
for entry in step["commands"]
|
||||
]
|
||||
assert not any("--case release" in item for item in commands)
|
||||
assert any("run_scanner_heal_g09_upgrade_evidence.sh" in item for item in commands)
|
||||
assert any("scanner_abba.py" in item for item in commands)
|
||||
assert any("--check-scanner-heal-release-bundle" in item for item in commands)
|
||||
preflight = iter_preflight_commands(plan)
|
||||
assert preflight
|
||||
assert all(args[0].startswith("scripts/") for args in preflight)
|
||||
out_dir = Path(os.environ.get("TMPDIR", "/tmp")) / "rustfs-scanner-heal-linux-plan-self-test"
|
||||
path = write_plan(out_dir, plan)
|
||||
loaded = json.loads(path.read_text())
|
||||
assert loaded["source_revision"] == revision
|
||||
status = build_status(plan, out_dir / "empty-run")
|
||||
assert status["decision"] == "blocked"
|
||||
assert status["release_approved"] is False
|
||||
assert status["pending_gates"]
|
||||
complete_root = out_dir / "complete-run"
|
||||
for stage in plan["stages"]:
|
||||
for step in stage["steps"]:
|
||||
for output in step.get("expected_outputs", []):
|
||||
path = render_run_root_path(output, complete_root)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text("{}\n")
|
||||
complete = build_status(plan, complete_root)
|
||||
assert complete["decision"] == "complete"
|
||||
assert complete["release_approved"] is False
|
||||
print("PASS: scanner/heal Linux evidence plan self-test")
|
||||
|
||||
|
||||
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--phase", action="append", choices=("all", "preflight", "functional", "performance", "bundle"), default=[])
|
||||
parser.add_argument("--source-revision")
|
||||
parser.add_argument("--out-dir", type=Path, default=DEFAULT_OUTPUT_ROOT)
|
||||
parser.add_argument("--write-plan", action="store_true")
|
||||
parser.add_argument("--format", choices=("text", "json"), default="text")
|
||||
parser.add_argument("--run-preflight", action="store_true")
|
||||
parser.add_argument("--status-root", type=Path)
|
||||
parser.add_argument("--self-test", action="store_true")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = parse_args(argv)
|
||||
if args.self_test:
|
||||
self_test()
|
||||
return 0
|
||||
try:
|
||||
registry = load_registry()
|
||||
validate_registry(registry)
|
||||
phases = set(args.phase or ["all"])
|
||||
if "all" in phases and len(phases) > 1:
|
||||
raise ValueError("--phase all cannot be combined with another phase")
|
||||
if args.status_root is not None and (args.write_plan or args.run_preflight):
|
||||
raise ValueError("--status-root cannot be combined with --write-plan or --run-preflight")
|
||||
plan = build_plan(registry, source_revision(args.source_revision), phases)
|
||||
if args.status_root is not None:
|
||||
status = build_status(plan, args.status_root)
|
||||
print(json.dumps(status, indent=2, sort_keys=True))
|
||||
return 0 if status["decision"] == "complete" else 3
|
||||
if args.write_plan:
|
||||
path = write_plan(args.out_dir.resolve(), plan)
|
||||
print(path)
|
||||
elif args.run_preflight:
|
||||
pass
|
||||
elif args.format == "json":
|
||||
print(json.dumps(plan, indent=2, sort_keys=True))
|
||||
else:
|
||||
print(text_plan(plan))
|
||||
if args.run_preflight:
|
||||
return run_preflight(plan)
|
||||
return 0
|
||||
except (AssertionError, ValueError, OSError, subprocess.CalledProcessError) as error:
|
||||
print(f"ERROR: {error}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -146,13 +146,16 @@ def build_descriptor(args: argparse.Namespace) -> Path:
|
||||
"gates": gates,
|
||||
})
|
||||
for gate in ("G11", "G13"):
|
||||
subprocess.check_call([
|
||||
check = subprocess.run([
|
||||
sys.executable,
|
||||
str(ROOT / "scripts/check_test_wiring.py"),
|
||||
"--check-scanner-heal-release-bundle-gate",
|
||||
str(descriptor),
|
||||
gate,
|
||||
], cwd=ROOT)
|
||||
], cwd=ROOT, capture_output=True, text=True)
|
||||
if check.returncode != 0:
|
||||
details = "\n".join(part for part in (check.stdout.strip(), check.stderr.strip()) if part)
|
||||
raise ValueError(details or f"{gate} release bundle gate check failed")
|
||||
return descriptor
|
||||
|
||||
|
||||
@@ -188,8 +191,8 @@ def write_self_test_proof(path: Path, source_revision: str) -> None:
|
||||
"unknown_producer_excluded": True,
|
||||
},
|
||||
"segment_activation_preflight": {
|
||||
"production_activation": False,
|
||||
"scanner_segment_reuse_activated": False,
|
||||
"production_activation": True,
|
||||
"scanner_segment_reuse_activated": True,
|
||||
"proof_inputs": list(wiring.SCANNER_HEAL_SEGMENT_ACTIVATION_PROOF_INPUTS),
|
||||
"fail_closed_checks": list(wiring.SCANNER_HEAL_SEGMENT_ACTIVATION_FAIL_CLOSED_CHECKS),
|
||||
},
|
||||
@@ -243,6 +246,21 @@ def run_self_test() -> None:
|
||||
else:
|
||||
raise ValueError("self-test accepted synthetic proof")
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
source_revision = git_head()
|
||||
proof = root / "maintenance-proof.json"
|
||||
write_self_test_proof(proof, source_revision)
|
||||
payload = wiring.read_json(proof)
|
||||
payload["segment_activation_preflight"]["scanner_segment_reuse_activated"] = False
|
||||
wiring.write_json(proof, payload)
|
||||
try:
|
||||
build_descriptor(parse_args(["--proof-json", str(proof), "--out-dir", str(root / "out")]))
|
||||
except ValueError as err:
|
||||
wiring.require("runtime activation gate is enabled" in str(err), "wrong self-test failure for inactive segment reuse")
|
||||
else:
|
||||
raise ValueError("self-test accepted inactive segment reuse proof")
|
||||
|
||||
|
||||
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
|
||||
+470
@@ -0,0 +1,470 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Collect Scanner/Heal status-and-outcome raw evidence from live observations.
|
||||
|
||||
This helper normalizes operator-collected live observation JSON into the three
|
||||
measured raw artifacts consumed by run_scanner_heal_status_outcome_evidence.py.
|
||||
It does not approve a release bundle by itself.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import check_test_wiring as wiring
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def git_head() -> str:
|
||||
return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip()
|
||||
|
||||
|
||||
def timestamp(value: Any, name: str) -> str:
|
||||
wiring.require(isinstance(value, str) and value.endswith("Z"), f"invalid {name}")
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
wiring.require(parsed.tzinfo is not None, f"{name} must include timezone")
|
||||
return parsed.isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def identity_string(value: Any, name: str) -> str:
|
||||
wiring.require(
|
||||
isinstance(value, str) and re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._:-]{7,127}", value),
|
||||
f"invalid {name}",
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
def measured_observation(path: Path, source_revision: str) -> dict[str, Any]:
|
||||
payload = wiring.read_json(path.resolve())
|
||||
wiring.require(isinstance(payload, dict), "observation must be a JSON object")
|
||||
for marker in ("fixture", "fixture_only", "dry_run", "synthetic"):
|
||||
wiring.require(payload.get(marker) is not True, f"observation is {marker}")
|
||||
wiring.require(payload.get("schema") == 1, "observation schema must be 1")
|
||||
wiring.require(payload.get("evidence_type") == "measured", "observation must be measured")
|
||||
wiring.require(payload.get("source_revision") == source_revision, "observation source revision mismatch")
|
||||
run_id = identity_string(payload.get("run_id"), "run_id")
|
||||
window_id = identity_string(payload.get("measurement_window_id"), "measurement_window_id")
|
||||
wiring.require(run_id != window_id, "run/window identities must differ")
|
||||
started = timestamp(payload.get("started_at"), "started_at")
|
||||
finished = timestamp(payload.get("finished_at"), "finished_at")
|
||||
wiring.require(
|
||||
datetime.fromisoformat(finished.replace("Z", "+00:00"))
|
||||
>= datetime.fromisoformat(started.replace("Z", "+00:00")),
|
||||
"finished_at precedes started_at",
|
||||
)
|
||||
wiring.require(isinstance(payload.get("command"), list) and payload["command"], "missing command provenance")
|
||||
return payload
|
||||
|
||||
|
||||
def object_items(payload: dict[str, Any], key: str) -> list[dict[str, Any]]:
|
||||
items = payload.get(key)
|
||||
wiring.require(isinstance(items, list) and items, f"missing {key}")
|
||||
for index, item in enumerate(items):
|
||||
wiring.require(isinstance(item, dict), f"{key}[{index}] must be an object")
|
||||
return items
|
||||
|
||||
|
||||
def case_map(items: list[dict[str, Any]], key: str, expected: tuple[str, ...]) -> dict[str, dict[str, Any]]:
|
||||
observed: dict[str, dict[str, Any]] = {}
|
||||
for item in items:
|
||||
case = item.get("case")
|
||||
wiring.require(isinstance(case, str) and case, f"{key} item missing case")
|
||||
wiring.require(case not in observed, f"{key} duplicate case: {case}")
|
||||
observed[case] = item
|
||||
missing = [case for case in expected if case not in observed]
|
||||
unknown = [case for case in observed if case not in expected]
|
||||
wiring.require(not missing, f"{key} missing cases: {', '.join(missing)}")
|
||||
wiring.require(not unknown, f"{key} unknown cases: {', '.join(unknown)}")
|
||||
return observed
|
||||
|
||||
|
||||
def bool_true(value: Any, name: str) -> None:
|
||||
wiring.release_bundle_bool_true(value, name)
|
||||
|
||||
|
||||
def status_outcome(payload: dict[str, Any], common: dict[str, Any]) -> dict[str, Any]:
|
||||
outcome_items = case_map(
|
||||
object_items(payload, "status_outcomes"),
|
||||
"status_outcomes",
|
||||
wiring.SCANNER_HEAL_RELEASE_G05_PER_OBJECT_OUTCOME_CASES,
|
||||
)
|
||||
counts = {"repaired": 0, "healthy": 0, "skipped": 0, "failed": 0}
|
||||
for case, item in outcome_items.items():
|
||||
outcome = item.get("outcome")
|
||||
wiring.require(outcome in counts, f"status_outcomes.{case} has invalid outcome")
|
||||
bool_true(item.get("status_matches_object_oracle"), f"status_outcomes.{case}.status_matches_object_oracle")
|
||||
counts[outcome] += 1
|
||||
for outcome, count in counts.items():
|
||||
wiring.require(count > 0, f"missing measured {outcome} outcome")
|
||||
|
||||
retention = case_map(
|
||||
object_items(payload, "terminal_retention_samples"),
|
||||
"terminal_retention_samples",
|
||||
wiring.SCANNER_HEAL_RELEASE_G05_TERMINAL_RETENTION_CASES,
|
||||
)
|
||||
window = wiring.evidence_integer(
|
||||
payload.get("terminal_retention_window_seconds"),
|
||||
"terminal_retention_window_seconds",
|
||||
1,
|
||||
86400,
|
||||
)
|
||||
max_age = 0
|
||||
pruned = 0
|
||||
for case, item in retention.items():
|
||||
bool_true(item.get("retained_until_window"), f"terminal_retention_samples.{case}.retained_until_window")
|
||||
max_age = max(
|
||||
max_age,
|
||||
wiring.evidence_integer(
|
||||
item.get("max_age_seconds"),
|
||||
f"terminal_retention_samples.{case}.max_age_seconds",
|
||||
0,
|
||||
86400,
|
||||
),
|
||||
)
|
||||
if item.get("pruned_after_window") is True:
|
||||
pruned += 1
|
||||
wiring.require(max_age <= window, "terminal retention max age exceeds window")
|
||||
wiring.require(pruned > 0, "no terminal records were pruned after the retention window")
|
||||
|
||||
return {
|
||||
**common,
|
||||
"per_object_outcome_cases": list(wiring.SCANNER_HEAL_RELEASE_G05_PER_OBJECT_OUTCOME_CASES),
|
||||
"outcome_counts": counts,
|
||||
"status_matches_object_oracle": True,
|
||||
"terminal_retention_cases": list(wiring.SCANNER_HEAL_RELEASE_G05_TERMINAL_RETENTION_CASES),
|
||||
"terminal_retention_window_seconds": window,
|
||||
"max_terminal_record_age_seconds": max_age,
|
||||
"terminal_records_pruned_after_window": pruned,
|
||||
}
|
||||
|
||||
|
||||
def status_compat(payload: dict[str, Any], common: dict[str, Any]) -> dict[str, Any]:
|
||||
status = case_map(
|
||||
object_items(payload, "status_samples"),
|
||||
"status_samples",
|
||||
wiring.SCANNER_HEAL_RELEASE_G06_CONCURRENT_STATUS_CASES,
|
||||
)
|
||||
status_samples = 0
|
||||
degraded = False
|
||||
for case, item in status.items():
|
||||
bool_true(item.get("http_success"), f"status_samples.{case}.http_success")
|
||||
status_samples += wiring.evidence_integer(item.get("samples"), f"status_samples.{case}.samples", 1, 2**31 - 1)
|
||||
degraded = degraded or item.get("degraded") is True
|
||||
|
||||
legacy = case_map(
|
||||
object_items(payload, "legacy_client_samples"),
|
||||
"legacy_client_samples",
|
||||
wiring.SCANNER_HEAL_RELEASE_G06_LEGACY_CLIENT_CASES,
|
||||
)
|
||||
for case, item in legacy.items():
|
||||
bool_true(item.get("accepted"), f"legacy_client_samples.{case}.accepted")
|
||||
rustfs_and_minio = (
|
||||
legacy["rustfs-admin-v3-background-heal-status"].get("path_compatible") is True
|
||||
and legacy["minio-admin-v3-background-heal-status"].get("path_compatible") is True
|
||||
)
|
||||
empty_body = legacy["heal-client-token-empty-body"].get("empty_body_accepted") is True
|
||||
bool_true(rustfs_and_minio, "rustfs and minio status path compatibility")
|
||||
bool_true(empty_body, "empty body heal status request")
|
||||
|
||||
truncation = case_map(
|
||||
object_items(payload, "truncation_samples"),
|
||||
"truncation_samples",
|
||||
wiring.SCANNER_HEAL_RELEASE_G06_TRUNCATION_CASES,
|
||||
)
|
||||
max_payload = 1
|
||||
for case, item in truncation.items():
|
||||
bool_true(item.get("rejected"), f"truncation_samples.{case}.rejected")
|
||||
max_payload = max(
|
||||
max_payload,
|
||||
wiring.evidence_integer(item.get("payload_bytes"), f"truncation_samples.{case}.payload_bytes", 1, 2**20),
|
||||
)
|
||||
|
||||
return {
|
||||
**common,
|
||||
"concurrent_status_cases": list(wiring.SCANNER_HEAL_RELEASE_G06_CONCURRENT_STATUS_CASES),
|
||||
"status_samples": status_samples,
|
||||
"all_status_responses_http_success": True,
|
||||
"partial_status_reports_degraded": degraded,
|
||||
"legacy_client_cases": list(wiring.SCANNER_HEAL_RELEASE_G06_LEGACY_CLIENT_CASES),
|
||||
"rustfs_and_minio_paths_compatible": rustfs_and_minio,
|
||||
"empty_body_status_requests_accepted": empty_body,
|
||||
"truncation_cases": list(wiring.SCANNER_HEAL_RELEASE_G06_TRUNCATION_CASES),
|
||||
"truncated_payloads_rejected": True,
|
||||
"max_status_payload_bytes": max_payload,
|
||||
}
|
||||
|
||||
|
||||
def disposition(payload: dict[str, Any], common: dict[str, Any]) -> dict[str, Any]:
|
||||
managers = case_map(
|
||||
object_items(payload, "manager_dispositions"),
|
||||
"manager_dispositions",
|
||||
wiring.SCANNER_HEAL_RELEASE_RD_MANAGER_CASES,
|
||||
)
|
||||
for case, item in managers.items():
|
||||
bool_true(item.get("terminal"), f"manager_dispositions.{case}.terminal")
|
||||
|
||||
events = case_map(
|
||||
object_items(payload, "event_dispositions"),
|
||||
"event_dispositions",
|
||||
wiring.SCANNER_HEAL_RELEASE_RD_EVENT_CASES,
|
||||
)
|
||||
for case, item in events.items():
|
||||
bool_true(item.get("correlates_to_manager"), f"event_dispositions.{case}.correlates_to_manager")
|
||||
|
||||
ledgers = case_map(
|
||||
object_items(payload, "ledger_dispositions"),
|
||||
"ledger_dispositions",
|
||||
wiring.SCANNER_HEAL_RELEASE_RD_LEDGER_CASES,
|
||||
)
|
||||
for case, item in ledgers.items():
|
||||
bool_true(item.get("correlates_to_events"), f"ledger_dispositions.{case}.correlates_to_events")
|
||||
bool_true(item.get("replay_preserves_terminal"), f"ledger_dispositions.{case}.replay_preserves_terminal")
|
||||
|
||||
grace = case_map(
|
||||
object_items(payload, "grace_samples"),
|
||||
"grace_samples",
|
||||
wiring.SCANNER_HEAL_RELEASE_RD_GRACE_CASES,
|
||||
)
|
||||
grace_window = wiring.evidence_integer(payload.get("grace_window_seconds"), "grace_window_seconds", 1, 86400)
|
||||
retained = False
|
||||
pruned = False
|
||||
for item in grace.values():
|
||||
retained = retained or item.get("retention_observed") is True
|
||||
pruned = pruned or item.get("expiry_pruned_terminal_records") is True
|
||||
bool_true(retained, "grace retention observed")
|
||||
bool_true(pruned, "grace expiry pruned terminal records")
|
||||
|
||||
return {
|
||||
**common,
|
||||
"manager_disposition_cases": list(wiring.SCANNER_HEAL_RELEASE_RD_MANAGER_CASES),
|
||||
"manager_dispositions_are_terminal": True,
|
||||
"event_disposition_cases": list(wiring.SCANNER_HEAL_RELEASE_RD_EVENT_CASES),
|
||||
"events_correlate_to_manager_dispositions": True,
|
||||
"ledger_disposition_cases": list(wiring.SCANNER_HEAL_RELEASE_RD_LEDGER_CASES),
|
||||
"ledger_correlates_to_events": True,
|
||||
"ledger_replay_preserves_terminal_disposition": True,
|
||||
"grace_cases": list(wiring.SCANNER_HEAL_RELEASE_RD_GRACE_CASES),
|
||||
"grace_window_seconds": grace_window,
|
||||
"grace_retention_observed": retained,
|
||||
"grace_expiry_pruned_terminal_records": pruned,
|
||||
}
|
||||
|
||||
|
||||
def common_raw(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"schema": 1,
|
||||
"evidence_type": "measured",
|
||||
"source_revision": payload["source_revision"],
|
||||
"run_id": payload["run_id"],
|
||||
"measurement_window_id": payload["measurement_window_id"],
|
||||
"started_at": payload["started_at"],
|
||||
"finished_at": payload["finished_at"],
|
||||
"command": payload["command"],
|
||||
}
|
||||
|
||||
|
||||
def collect(args: argparse.Namespace) -> tuple[Path, Path, Path]:
|
||||
out_dir = args.out_dir.resolve()
|
||||
wiring.require(not out_dir.exists(), "output directory must be new")
|
||||
source_revision = args.source_revision or git_head()
|
||||
observed = measured_observation(args.observations_json, source_revision)
|
||||
common = common_raw(observed)
|
||||
out_dir.mkdir(parents=True)
|
||||
status_outcome_path = out_dir / "status-outcome.json"
|
||||
status_compat_path = out_dir / "status-compat.json"
|
||||
disposition_path = out_dir / "disposition.json"
|
||||
wiring.write_json(status_outcome_path, status_outcome(observed, common))
|
||||
wiring.write_json(status_compat_path, status_compat(observed, common))
|
||||
wiring.write_json(disposition_path, disposition(observed, common))
|
||||
return status_outcome_path, status_compat_path, disposition_path
|
||||
|
||||
|
||||
def write_self_test_observation(path: Path, source_revision: str) -> None:
|
||||
now = datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
||||
payload = {
|
||||
"schema": 1,
|
||||
"evidence_type": "measured",
|
||||
"source_revision": source_revision,
|
||||
"run_id": f"status-probe-{source_revision[:12]}",
|
||||
"measurement_window_id": f"status-probe-window-{source_revision[:12]}",
|
||||
"started_at": now,
|
||||
"finished_at": now,
|
||||
"command": ["scripts/run_scanner_heal_status_outcome_probe.py", "--observations-json", "<live-observations>"],
|
||||
"terminal_retention_window_seconds": 3600,
|
||||
"grace_window_seconds": 300,
|
||||
"status_outcomes": [
|
||||
{"case": "object-repaired", "outcome": "repaired", "status_matches_object_oracle": True},
|
||||
{"case": "object-already-healthy", "outcome": "healthy", "status_matches_object_oracle": True},
|
||||
{"case": "object-skipped-by-policy", "outcome": "skipped", "status_matches_object_oracle": True},
|
||||
{"case": "object-failed-and-retained", "outcome": "failed", "status_matches_object_oracle": True},
|
||||
],
|
||||
"terminal_retention_samples": [
|
||||
{"case": "finished-retained-until-window", "retained_until_window": True, "max_age_seconds": 1200},
|
||||
{"case": "failed-retained-until-window", "retained_until_window": True, "max_age_seconds": 1300},
|
||||
{"case": "canceled-retained-until-window", "retained_until_window": True, "max_age_seconds": 1400},
|
||||
{
|
||||
"case": "expired-terminal-pruned-after-window",
|
||||
"retained_until_window": True,
|
||||
"max_age_seconds": 3599,
|
||||
"pruned_after_window": True,
|
||||
},
|
||||
],
|
||||
"status_samples": [
|
||||
{"case": "status-during-admin-heal", "samples": 2, "http_success": True},
|
||||
{"case": "status-during-background-heal", "samples": 2, "http_success": True},
|
||||
{"case": "status-while-peer-down", "samples": 2, "http_success": True, "degraded": True},
|
||||
{"case": "status-after-peer-rejoin", "samples": 2, "http_success": True},
|
||||
],
|
||||
"legacy_client_samples": [
|
||||
{"case": "rustfs-admin-v3-background-heal-status", "accepted": True, "path_compatible": True},
|
||||
{"case": "minio-admin-v3-background-heal-status", "accepted": True, "path_compatible": True},
|
||||
{"case": "heal-client-token-empty-body", "accepted": True, "empty_body_accepted": True},
|
||||
{"case": "node-heal-status-v1-wire", "accepted": True},
|
||||
],
|
||||
"truncation_samples": [
|
||||
{"case": "oversize-node-status-reject", "rejected": True, "payload_bytes": 1048576},
|
||||
{"case": "truncated-node-status-reject", "rejected": True, "payload_bytes": 4096},
|
||||
{"case": "trailing-data-node-status-reject", "rejected": True, "payload_bytes": 4096},
|
||||
],
|
||||
"manager_dispositions": [
|
||||
{"case": "accepted", "terminal": True},
|
||||
{"case": "coalesced-duplicate", "terminal": True},
|
||||
{"case": "rejected-policy", "terminal": True},
|
||||
{"case": "terminal-retained", "terminal": True},
|
||||
],
|
||||
"event_dispositions": [
|
||||
{"case": "event-repaired", "correlates_to_manager": True},
|
||||
{"case": "event-failed", "correlates_to_manager": True},
|
||||
{"case": "event-skipped", "correlates_to_manager": True},
|
||||
{"case": "event-grace-retained", "correlates_to_manager": True},
|
||||
],
|
||||
"ledger_dispositions": [
|
||||
{"case": "ledger-recorded", "correlates_to_events": True, "replay_preserves_terminal": True},
|
||||
{"case": "ledger-replayed", "correlates_to_events": True, "replay_preserves_terminal": True},
|
||||
{"case": "ledger-discharged", "correlates_to_events": True, "replay_preserves_terminal": True},
|
||||
{"case": "ledger-pruned-after-grace", "correlates_to_events": True, "replay_preserves_terminal": True},
|
||||
],
|
||||
"grace_samples": [
|
||||
{"case": "grace-open-retains-disposition", "retention_observed": True},
|
||||
{"case": "grace-expired-prunes-terminal", "expiry_pruned_terminal_records": True},
|
||||
{"case": "restart-preserves-grace-clock", "retention_observed": True},
|
||||
],
|
||||
}
|
||||
wiring.write_json(path, payload)
|
||||
|
||||
|
||||
def run_self_test() -> None:
|
||||
import tempfile
|
||||
|
||||
source_revision = git_head()
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
observation = root / "observation.json"
|
||||
write_self_test_observation(observation, source_revision)
|
||||
raw_paths = collect(parse_args([
|
||||
"--observations-json",
|
||||
str(observation),
|
||||
"--out-dir",
|
||||
str(root / "raw"),
|
||||
]))
|
||||
descriptor = root / "descriptor"
|
||||
subprocess.check_call([
|
||||
sys.executable,
|
||||
str(ROOT / "scripts/run_scanner_heal_status_outcome_evidence.py"),
|
||||
"--status-outcome-json",
|
||||
str(raw_paths[0]),
|
||||
"--status-compat-json",
|
||||
str(raw_paths[1]),
|
||||
"--disposition-json",
|
||||
str(raw_paths[2]),
|
||||
"--out-dir",
|
||||
str(descriptor),
|
||||
], cwd=ROOT)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
observation = root / "observation.json"
|
||||
write_self_test_observation(observation, source_revision)
|
||||
payload = wiring.read_json(observation)
|
||||
payload["status_outcomes"].pop()
|
||||
wiring.write_json(observation, payload)
|
||||
try:
|
||||
collect(parse_args(["--observations-json", str(observation), "--out-dir", str(root / "raw")]))
|
||||
except ValueError as err:
|
||||
wiring.require("status_outcomes missing cases" in str(err), "wrong self-test failure for missing outcome")
|
||||
else:
|
||||
raise ValueError("self-test accepted incomplete status outcome observations")
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
observation = root / "observation.json"
|
||||
write_self_test_observation(observation, source_revision)
|
||||
payload = wiring.read_json(observation)
|
||||
payload["synthetic"] = True
|
||||
wiring.write_json(observation, payload)
|
||||
try:
|
||||
collect(parse_args(["--observations-json", str(observation), "--out-dir", str(root / "raw")]))
|
||||
except ValueError as err:
|
||||
wiring.require("observation is synthetic" in str(err), "wrong self-test failure for synthetic observation")
|
||||
else:
|
||||
raise ValueError("self-test accepted synthetic status/outcome observations")
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
observation = root / "observation.json"
|
||||
write_self_test_observation(observation, source_revision)
|
||||
payload = wiring.read_json(observation)
|
||||
payload["measurement_window_id"] = payload["run_id"]
|
||||
wiring.write_json(observation, payload)
|
||||
try:
|
||||
collect(parse_args(["--observations-json", str(observation), "--out-dir", str(root / "raw")]))
|
||||
except ValueError as err:
|
||||
wiring.require("run/window identities must differ" in str(err), "wrong self-test failure for identity reuse")
|
||||
else:
|
||||
raise ValueError("self-test accepted reused run/window identities")
|
||||
|
||||
|
||||
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--observations-json", type=Path)
|
||||
parser.add_argument("--out-dir", type=Path)
|
||||
parser.add_argument("--source-revision")
|
||||
parser.add_argument("--self-test", action="store_true")
|
||||
args = parser.parse_args(argv)
|
||||
if not args.self_test:
|
||||
if args.observations_json is None:
|
||||
parser.error("--observations-json is required unless --self-test is used")
|
||||
if args.out_dir is None:
|
||||
parser.error("--out-dir is required unless --self-test is used")
|
||||
return args
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
if args.self_test:
|
||||
run_self_test()
|
||||
return
|
||||
paths = collect(args)
|
||||
json.dump(
|
||||
{
|
||||
"status_outcome_json": str(paths[0]),
|
||||
"status_compat_json": str(paths[1]),
|
||||
"disposition_json": str(paths[2]),
|
||||
},
|
||||
sys.stdout,
|
||||
indent=2,
|
||||
allow_nan=False,
|
||||
)
|
||||
sys.stdout.write("\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -4,6 +4,8 @@ import unittest
|
||||
|
||||
from diagnose_scanner_enumeration_restart import (
|
||||
converged,
|
||||
final_complete_recheck_after_full_retention,
|
||||
fully_retained,
|
||||
replays_raw_window,
|
||||
validate_recoverable_quantum,
|
||||
validate_report,
|
||||
@@ -29,6 +31,7 @@ class ReportTests(unittest.TestCase):
|
||||
report = self.report()
|
||||
self.validate(report)
|
||||
self.assertTrue(converged(report, 4))
|
||||
self.assertTrue(fully_retained(report, 4))
|
||||
|
||||
def test_incomplete_or_inexact_coverage_cannot_pass(self):
|
||||
for key, value in (("snapshot_complete", False), ("objects_retained", 3),
|
||||
@@ -132,6 +135,9 @@ class ReportTests(unittest.TestCase):
|
||||
previous["versions_retained"] = 0
|
||||
previous["bytes_retained"] = 0
|
||||
previous["objects_processed"] = 0
|
||||
previous["raw_page_index_committed_entries"] = 1
|
||||
previous["raw_page_index_indexed_entries"] = 1
|
||||
previous["raw_page_index_complete"] = False
|
||||
previous["snapshot_complete"] = False
|
||||
previous["outcome"] = "cancelled_without_cache"
|
||||
current = dict(previous, round=1, pid=124, objects_before=0)
|
||||
@@ -140,6 +146,10 @@ class ReportTests(unittest.TestCase):
|
||||
advanced = dict(current, objects_retained=1)
|
||||
self.assertFalse(replays_raw_window(previous, advanced))
|
||||
|
||||
indexed = dict(current, raw_page_index_committed_entries=2,
|
||||
raw_page_index_indexed_entries=2)
|
||||
self.assertFalse(replays_raw_window(previous, indexed))
|
||||
|
||||
def test_recoverable_quantum_rejects_replayed_raw_window(self):
|
||||
previous = self.report()
|
||||
previous.update(objects_retained=0, versions_retained=0, bytes_retained=0,
|
||||
@@ -149,6 +159,35 @@ class ReportTests(unittest.TestCase):
|
||||
with self.assertRaisesRegex(ValueError, "raw enumeration window replayed"):
|
||||
validate_recoverable_quantum([previous, current], objects=4, budget=16, require_converged=False)
|
||||
|
||||
def test_recoverable_quantum_allows_final_complete_round_after_full_retention(self):
|
||||
previous = self.report()
|
||||
previous.update(raw_entries=8, raw_page_index_committed_entries=4,
|
||||
raw_page_index_indexed_entries=4, objects_before=2,
|
||||
objects_processed=2, objects_retained=4,
|
||||
versions_retained=4, bytes_retained=4,
|
||||
snapshot_complete=False, outcome="partial")
|
||||
current = dict(previous, round=1, pid=124, objects_before=4,
|
||||
snapshot_complete=True, outcome="complete")
|
||||
|
||||
self.assertTrue(replays_raw_window(previous, current))
|
||||
self.assertTrue(final_complete_recheck_after_full_retention(previous, current, 4))
|
||||
validate_recoverable_quantum([previous, current], objects=4, budget=16, require_converged=True)
|
||||
|
||||
def test_recoverable_quantum_rejects_partial_replay_after_full_retention(self):
|
||||
previous = self.report()
|
||||
previous.update(raw_entries=8, raw_page_index_committed_entries=4,
|
||||
raw_page_index_indexed_entries=4, objects_before=2,
|
||||
objects_processed=2, objects_retained=4,
|
||||
versions_retained=4, bytes_retained=4,
|
||||
snapshot_complete=False, outcome="partial")
|
||||
current = dict(previous, round=1, pid=124, objects_before=4,
|
||||
snapshot_complete=False, outcome="partial")
|
||||
|
||||
self.assertTrue(replays_raw_window(previous, current))
|
||||
self.assertFalse(final_complete_recheck_after_full_retention(previous, current, 4))
|
||||
with self.assertRaisesRegex(ValueError, "raw enumeration window replayed"):
|
||||
validate_recoverable_quantum([previous, current], objects=4, budget=16, require_converged=False)
|
||||
|
||||
def test_recoverable_quantum_requires_three_stage_progress_and_convergence(self):
|
||||
first = self.report()
|
||||
first.update(round=0, pid=123, raw_entries=2, raw_page_index_committed_entries=2,
|
||||
|
||||
+123
@@ -0,0 +1,123 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
RUNNER="$SCRIPT_DIR/run_scanner_heal_linux_evidence_plan.py"
|
||||
TMP_DIR="$(mktemp -d)"
|
||||
|
||||
cleanup() {
|
||||
rm -rf "$TMP_DIR"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
"${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" --self-test
|
||||
|
||||
"${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" --phase preflight >"$TMP_DIR/preflight.out"
|
||||
rg -q "stage=preflight" "$TMP_DIR/preflight.out"
|
||||
rg -q "scripts/check_test_wiring.py --self-test" "$TMP_DIR/preflight.out"
|
||||
|
||||
"${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" \
|
||||
--phase functional \
|
||||
--source-revision aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
|
||||
--format json >"$TMP_DIR/functional.json"
|
||||
|
||||
"${RUSTFS_PYTHON_BIN:-python3}" - "$TMP_DIR/functional.json" <<'PY'
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
plan = json.loads(pathlib.Path(sys.argv[1]).read_text())
|
||||
assert plan["schema"] == 1
|
||||
assert plan["source_revision"] == "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
assert [stage["id"] for stage in plan["stages"]] == ["functional"]
|
||||
commands = [
|
||||
" ".join(entry["command"])
|
||||
for stage in plan["stages"]
|
||||
for step in stage["steps"]
|
||||
for entry in step["commands"]
|
||||
]
|
||||
assert any("background-target-crash-ec8-4-multi-pool" in command for command in commands)
|
||||
assert any("run_scanner_heal_status_outcome_probe.py" in command for command in commands)
|
||||
assert all("--case release" not in command for command in commands)
|
||||
PY
|
||||
|
||||
"${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" --write-plan --out-dir "$TMP_DIR/plan" >"$TMP_DIR/path.out"
|
||||
PLAN_PATH="$(tr -d '\n' <"$TMP_DIR/path.out")"
|
||||
test -s "$PLAN_PATH"
|
||||
rg -q '"evidence_type": "plan_only"' "$PLAN_PATH"
|
||||
rg -q '"stop_on_product_failure": true' "$PLAN_PATH"
|
||||
|
||||
if "${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" \
|
||||
--phase functional \
|
||||
--source-revision aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
|
||||
--status-root "$TMP_DIR/missing-run" >"$TMP_DIR/status-missing.json"; then
|
||||
echo "missing evidence status should fail closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
rg -q '"decision": "blocked"' "$TMP_DIR/status-missing.json"
|
||||
rg -q '"release_approved": false' "$TMP_DIR/status-missing.json"
|
||||
rg -q '"next_step"' "$TMP_DIR/status-missing.json"
|
||||
|
||||
if "${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" \
|
||||
--phase functional \
|
||||
--status-root "$TMP_DIR/missing-run" \
|
||||
--run-preflight >/dev/null 2>"$TMP_DIR/status-mode.err"; then
|
||||
echo "status mode should reject preflight execution" >&2
|
||||
exit 1
|
||||
fi
|
||||
rg -q "status-root cannot be combined" "$TMP_DIR/status-mode.err"
|
||||
|
||||
"${RUSTFS_PYTHON_BIN:-python3}" - "$RUNNER" "$TMP_DIR/complete-run" <<'PY'
|
||||
import json
|
||||
import pathlib
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
runner = pathlib.Path(sys.argv[1])
|
||||
run_root = pathlib.Path(sys.argv[2])
|
||||
plan = json.loads(subprocess.check_output([
|
||||
sys.executable,
|
||||
str(runner),
|
||||
"--phase",
|
||||
"functional",
|
||||
"--source-revision",
|
||||
"a" * 40,
|
||||
"--format",
|
||||
"json",
|
||||
], text=True))
|
||||
for stage in plan["stages"]:
|
||||
for step in stage["steps"]:
|
||||
for output in step.get("expected_outputs", []):
|
||||
path = pathlib.Path(output.replace("$RUN_ROOT", str(run_root)))
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text("{}\n")
|
||||
status = json.loads(subprocess.check_output([
|
||||
sys.executable,
|
||||
str(runner),
|
||||
"--phase",
|
||||
"functional",
|
||||
"--source-revision",
|
||||
"a" * 40,
|
||||
"--status-root",
|
||||
str(run_root),
|
||||
], text=True))
|
||||
assert status["decision"] == "complete"
|
||||
assert status["release_approved"] is False
|
||||
assert status["artifact_totals"]["missing"] == 0
|
||||
PY
|
||||
|
||||
if "${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" \
|
||||
--phase performance \
|
||||
--run-preflight >/dev/null 2>"$TMP_DIR/no-preflight.err"; then
|
||||
echo "preflight execution without the preflight stage should fail" >&2
|
||||
exit 1
|
||||
fi
|
||||
rg -q "requires the preflight stage" "$TMP_DIR/no-preflight.err"
|
||||
|
||||
if "${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" --source-revision bad >/dev/null 2>"$TMP_DIR/bad.err"; then
|
||||
echo "invalid source revision should fail" >&2
|
||||
exit 1
|
||||
fi
|
||||
rg -q "source-revision" "$TMP_DIR/bad.err"
|
||||
@@ -3,5 +3,7 @@ set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
RUNNER="$SCRIPT_DIR/run_scanner_heal_status_outcome_evidence.py"
|
||||
PROBE="$SCRIPT_DIR/run_scanner_heal_status_outcome_probe.py"
|
||||
|
||||
"${RUSTFS_PYTHON_BIN:-python3}" "$PROBE" --self-test
|
||||
"${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" --self-test
|
||||
|
||||
Reference in New Issue
Block a user