mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-11 13:29:12 +00:00
Compare commits
14 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| c8a9c4087c | |||
| 853ae63b6a | |||
| f02bc947cd | |||
| 0cbc3ffe61 | |||
| b1cc286cac | |||
| 50b31bc75b | |||
| 5cd58319ed | |||
| c677590d50 | |||
| 94d875ea07 | |||
| 2f5db4e8f2 | |||
| 5c17012fe2 | |||
| aeaba86d73 | |||
| 8f150d1d8e | |||
| da762e0b02 |
@@ -1,2 +1,2 @@
|
||||
sha256-linux=563bff8f1171d6dbe166ff8440310dbe98430e466aa3ecd8dc39e3c872b320f7
|
||||
sha256-darwin=563bff8f1171d6dbe166ff8440310dbe98430e466aa3ecd8dc39e3c872b320f7
|
||||
sha256-linux=775825dcb2b4997c4fa24bd9ba9c0546316c503f4d5e369c39abff0678c95e8e
|
||||
sha256-darwin=775825dcb2b4997c4fa24bd9ba9c0546316c503f4d5e369c39abff0678c95e8e
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=15cb0cf9909bfbfc5a835fb08bd3675516c641db1b92ecc1e170a1cfa0fa2fd5
|
||||
sha256-linux=3163fdd29df5def86cf511ca7db05880d5c0caaa608a7031a71394327f7c217a
|
||||
sha256-darwin=874c881d7b45f12378a5817c7f42c95c4981960a2ec9ce12dcf4af239ae1f9d5
|
||||
sha256-linux=9351e25b45bf7dfce18b951a5e3740225f457cacc53b8bf9f500f6947763ec0e
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=5fbb230b89212b7c3d7229d6cef3e7e2d16f0ecfec62237ebc770785706f67d9
|
||||
sha256=6d18f9cce820c51d5589de944e8cc185f73eeca0ea9a9916651943e3759169d0
|
||||
|
||||
@@ -539,13 +539,6 @@ path = "junit.xml"
|
||||
filter = 'package(e2e_test)'
|
||||
test-group = 'e2e-cluster-nightly'
|
||||
|
||||
# The EC8+4 multi-set heal proof deliberately uploads a larger workload so the
|
||||
# background root-heal pass can be interrupted after targeting the replacement
|
||||
# drive's erasure slots. Keep the extended budget scoped to this proof case.
|
||||
[[profile.e2e-nightly.overrides]]
|
||||
filter = 'package(e2e_test) & test(=heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_across_multi_set_after_background_target_restart)'
|
||||
slow-timeout = { period = "120s", terminate-after = 12, grace-period = "10s" }
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# e2e-distributed profile — 4-node 4-disk Actions suite
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -259,9 +259,6 @@ jobs:
|
||||
} > artifacts/test-and-lint/doctest-diagnostics.txt
|
||||
exit "${status}"
|
||||
|
||||
- name: Check offline enrollment E2E root boundary
|
||||
run: ./scripts/check_offline_enrollment_e2e.sh
|
||||
|
||||
- name: Upload test reports and diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
@@ -296,6 +293,36 @@ jobs:
|
||||
- name: Run rebalance/decommission migration proofs
|
||||
run: ./scripts/check_migration_gate_count.sh
|
||||
|
||||
# The root boundary requires fresh CLI and integration-test builds. Give it
|
||||
# its own time budget instead of sharing the workspace lint/test budget.
|
||||
offline-enrollment-root-boundary:
|
||||
name: Offline Enrollment Root Boundary
|
||||
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-dev
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Protect Connect test home
|
||||
run: chmod go-w "$(realpath "$HOME")"
|
||||
|
||||
- name: Check offline enrollment E2E root boundary
|
||||
run: ./scripts/check_offline_enrollment_e2e.sh
|
||||
|
||||
# Dedicated serial lane for the ILM / lifecycle integration tests. These tests
|
||||
# drive the object layer through process-global singletons (the GLOBAL_ENV
|
||||
# ECStore, the global tier-config manager, background-expiry workers) and bind
|
||||
@@ -335,7 +362,11 @@ jobs:
|
||||
# re-enabled by backlog#1304 (restore accepts serialize on a short CAS
|
||||
# guard; the copy-back no longer holds the #4877 whole-copy-back lock,
|
||||
# so the mid-restore ongoing read and fast 409 rejection it asserts are
|
||||
# the implemented contract).
|
||||
# the implemented contract). The remaining exclusions each hit a
|
||||
# DIFFERENT, independent issue (all tracked under rustfs/backlog#1148;
|
||||
# they keep #[ignore] with a backlog reference):
|
||||
# - test_noncurrent_{expiry,transition}_still_works_after_immediate_compensation_transition:
|
||||
# noncurrent transition/expiry after an immediate compensation transition.
|
||||
- name: Run ignored ILM integration tests serially
|
||||
env:
|
||||
# Match the measured Test and Lint link budget. The default exposed
|
||||
@@ -348,7 +379,7 @@ jobs:
|
||||
NEXTEST_HIDE_PROGRESS_BAR=1 timeout --verbose --signal=TERM --kill-after=30s 80m \
|
||||
cargo nextest run -j1 --run-ignored ignored-only \
|
||||
-p rustfs-scanner -p rustfs \
|
||||
-E 'binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))' \
|
||||
-E '(binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))) and not (test(test_noncurrent_expiry_still_works_after_immediate_compensation_transition) or test(test_noncurrent_transition_still_works_after_immediate_compensation_transition))' \
|
||||
--status-level all --final-status-level all \
|
||||
2>&1 | tee artifacts/ilm-integration/nextest.log
|
||||
status=${PIPESTATUS[0]}
|
||||
@@ -966,6 +997,7 @@ jobs:
|
||||
# debug binary; each test spawns its own rustfs server on a random port.
|
||||
- name: Run e2e full suite
|
||||
env:
|
||||
CARGO_BIN_EXE_rustfs: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs
|
||||
RUSTFS_E2E_STARTUP_CAS_BINARY: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs
|
||||
RUSTFS_E2E_STARTUP_CAS_BUILD_MANIFEST: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs.e2e-startup-cas-build.json
|
||||
RUSTFS_E2E_STARTUP_CAS_ARTIFACT_DIR: ${{ runner.temp }}/rustfs-startup-cas-evidence
|
||||
@@ -1185,6 +1217,7 @@ jobs:
|
||||
- typos
|
||||
- quick-checks
|
||||
- test-and-lint
|
||||
- offline-enrollment-root-boundary
|
||||
- test-ilm-integration-serial
|
||||
- test-and-lint-rio-v2
|
||||
- connect-short-credential-boundary
|
||||
@@ -1218,6 +1251,7 @@ jobs:
|
||||
- typos
|
||||
- quick-checks
|
||||
- test-and-lint
|
||||
- offline-enrollment-root-boundary
|
||||
- test-ilm-integration-serial
|
||||
- test-and-lint-rio-v2
|
||||
- test-and-lint-protocols
|
||||
|
||||
@@ -156,6 +156,7 @@ jobs:
|
||||
|
||||
- name: Run cluster fault e2e nightly suite
|
||||
env:
|
||||
CARGO_BIN_EXE_rustfs: ${{ github.workspace }}/target/debug/rustfs
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-nightly-logs
|
||||
run: cargo nextest run --profile e2e-nightly -p e2e_test
|
||||
|
||||
|
||||
Generated
+79
-79
@@ -271,7 +271,7 @@ version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -282,7 +282,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"once_cell_polyfill",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -688,9 +688,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "async-compat"
|
||||
version = "0.2.6"
|
||||
version = "0.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4c97d7ff3c25d6c10d64170c12acaf5d4245e76dece3779c1d92b153a64f11df"
|
||||
checksum = "a1ba85bc55464dcbf728b56d97e119d673f4cf9062be330a9a26f3acf504a590"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-io",
|
||||
@@ -1906,7 +1906,7 @@ dependencies = [
|
||||
"maybe-owned",
|
||||
"rustix",
|
||||
"rustix-linux-procfs",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
"winx",
|
||||
]
|
||||
|
||||
@@ -3942,7 +3942,7 @@ dependencies = [
|
||||
"libc",
|
||||
"option-ext",
|
||||
"redox_users",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3986,7 +3986,7 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
|
||||
|
||||
[[package]]
|
||||
name = "e2e_test"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"astral-tokio-tar",
|
||||
@@ -4287,7 +4287,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5155,9 +5155,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hickory-net"
|
||||
version = "0.26.3"
|
||||
version = "0.26.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c480823ed7c2c5d0f09c41020cb6b7c28029ce60ec42dc942158dcf22f8e0a4d"
|
||||
checksum = "084e7bd6a377435d568f652153e571b50970d7ccc1d1eeec0519f834632287e1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"cfg-if",
|
||||
@@ -5179,9 +5179,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hickory-proto"
|
||||
version = "0.26.3"
|
||||
version = "0.26.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "12b92608f679a6fa515dd1d15c1ff89443026e391200a2c840c7afcba482893d"
|
||||
checksum = "7e2da0694c15b44c6f68a6b05e0233617008c54080e31d6eb848d858a9c5b38d"
|
||||
dependencies = [
|
||||
"data-encoding",
|
||||
"idna",
|
||||
@@ -5199,9 +5199,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hickory-resolver"
|
||||
version = "0.26.3"
|
||||
version = "0.26.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f3da5255c95d5a716857d54b5b8f4e8d67c3484d3beaaaae2ce25063b3ba981"
|
||||
checksum = "0e4f9f4603319422d482738f3f6fe5aac03157fdbfed1cd85a3ff45adb09072f"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"futures-util",
|
||||
@@ -6963,7 +6963,7 @@ version = "0.50.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -8712,7 +8712,7 @@ dependencies = [
|
||||
"once_cell",
|
||||
"socket2",
|
||||
"tracing",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -9476,7 +9476,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"anyhow",
|
||||
@@ -9624,7 +9624,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-audit"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"const-str",
|
||||
"futures",
|
||||
@@ -9646,7 +9646,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-checksums"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -9662,7 +9662,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-common"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"metrics",
|
||||
@@ -9675,7 +9675,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-concurrency"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"insta",
|
||||
@@ -9688,7 +9688,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-config"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"const-str",
|
||||
"hotpath",
|
||||
@@ -9698,7 +9698,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-credentials"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"hmac 0.13.0",
|
||||
@@ -9712,7 +9712,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-crypto"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"argon2",
|
||||
@@ -9733,7 +9733,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-data-usage"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"rmp-serde",
|
||||
@@ -9743,7 +9743,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-ecstore"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-channel",
|
||||
@@ -9879,7 +9879,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-extension-schema"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"serde",
|
||||
@@ -9889,7 +9889,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-filemeta"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"byteorder",
|
||||
@@ -9917,7 +9917,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-heal"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64-simd",
|
||||
@@ -9954,7 +9954,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-heal-contracts"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -9964,7 +9964,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-iam"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
@@ -10013,7 +10013,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-io-core"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"hotpath",
|
||||
@@ -10025,7 +10025,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-io-metrics"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"criterion",
|
||||
"hotpath",
|
||||
@@ -10089,7 +10089,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-keystone"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures",
|
||||
@@ -10116,7 +10116,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-kms"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"anyhow",
|
||||
@@ -10166,14 +10166,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-license"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"thiserror 2.0.20",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-lifecycle"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"hotpath",
|
||||
@@ -10195,7 +10195,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-lock"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"compact_str",
|
||||
@@ -10218,7 +10218,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-log-analyzer"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"flate2",
|
||||
@@ -10237,7 +10237,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-madmin"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"http 1.5.0",
|
||||
@@ -10275,7 +10275,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-notify"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
@@ -10310,7 +10310,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-object-capacity"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"criterion",
|
||||
"futures",
|
||||
@@ -10329,7 +10329,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-object-data-cache"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"criterion",
|
||||
@@ -10346,7 +10346,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-obs"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"crossbeam-channel",
|
||||
@@ -10404,7 +10404,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-policy"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64-simd",
|
||||
@@ -10435,7 +10435,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-protocols"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"astral-tokio-tar",
|
||||
"async-compression",
|
||||
@@ -10497,7 +10497,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-protos"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"flatbuffers",
|
||||
"hotpath",
|
||||
@@ -10522,7 +10522,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-replication"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"bytes",
|
||||
@@ -10540,7 +10540,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-rio"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"arc-swap",
|
||||
@@ -10584,7 +10584,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-rio-v2"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"bytes",
|
||||
@@ -10607,7 +10607,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3-client"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -10651,7 +10651,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3-ops"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"rustfs-s3-types",
|
||||
@@ -10659,7 +10659,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3-types"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"serde",
|
||||
@@ -10668,7 +10668,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3select-api"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-compression",
|
||||
@@ -10703,7 +10703,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3select-query"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"async-recursion",
|
||||
"async-trait",
|
||||
@@ -10724,7 +10724,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-scanner"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
@@ -10770,7 +10770,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-scanner-metrics"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"jiff",
|
||||
@@ -10785,7 +10785,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-security-governance"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"thiserror 2.0.20",
|
||||
@@ -10793,7 +10793,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-signer"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -10811,7 +10811,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-storage-api"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"hotpath",
|
||||
@@ -10826,7 +10826,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-targets"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-nats",
|
||||
@@ -10880,7 +10880,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-test-utils"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"rustfs-data-usage",
|
||||
@@ -10896,7 +10896,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-tls-runtime"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"hotpath",
|
||||
@@ -10917,7 +10917,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-trusted-proxies"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
@@ -10954,7 +10954,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-utils"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"blake2",
|
||||
@@ -10996,7 +10996,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-zip"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
dependencies = [
|
||||
"astral-tokio-tar",
|
||||
"async-compression",
|
||||
@@ -11065,7 +11065,7 @@ dependencies = [
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11148,7 +11148,7 @@ dependencies = [
|
||||
"security-framework",
|
||||
"security-framework-sys",
|
||||
"webpki-root-certs",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11205,7 +11205,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
|
||||
[[package]]
|
||||
name = "s3s"
|
||||
version = "0.15.0"
|
||||
source = "git+https://github.com/rustfs/s3s.git?rev=bdcb6259339c41369f9f1c60e3a42b5ab8da607b#bdcb6259339c41369f9f1c60e3a42b5ab8da607b"
|
||||
source = "git+https://github.com/s3s-project/s3s.git?rev=f3e17541f366696bf0cbaf380fcbd8b44c17eba4#f3e17541f366696bf0cbaf380fcbd8b44c17eba4"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrayvec",
|
||||
@@ -11263,7 +11263,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "s3s-rfc2047"
|
||||
version = "0.16.0-alpha.1"
|
||||
source = "git+https://github.com/rustfs/s3s.git?rev=bdcb6259339c41369f9f1c60e3a42b5ab8da607b#bdcb6259339c41369f9f1c60e3a42b5ab8da607b"
|
||||
source = "git+https://github.com/s3s-project/s3s.git?rev=f3e17541f366696bf0cbaf380fcbd8b44c17eba4#f3e17541f366696bf0cbaf380fcbd8b44c17eba4"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"thiserror 2.0.20",
|
||||
@@ -11272,7 +11272,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "s3s-sigv2"
|
||||
version = "0.16.0-alpha.1"
|
||||
source = "git+https://github.com/rustfs/s3s.git?rev=bdcb6259339c41369f9f1c60e3a42b5ab8da607b#bdcb6259339c41369f9f1c60e3a42b5ab8da607b"
|
||||
source = "git+https://github.com/s3s-project/s3s.git?rev=f3e17541f366696bf0cbaf380fcbd8b44c17eba4#f3e17541f366696bf0cbaf380fcbd8b44c17eba4"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"hmac 0.13.0",
|
||||
@@ -11285,7 +11285,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "s3s-sigv4"
|
||||
version = "0.16.0-alpha.1"
|
||||
source = "git+https://github.com/rustfs/s3s.git?rev=bdcb6259339c41369f9f1c60e3a42b5ab8da607b#bdcb6259339c41369f9f1c60e3a42b5ab8da607b"
|
||||
source = "git+https://github.com/s3s-project/s3s.git?rev=f3e17541f366696bf0cbaf380fcbd8b44c17eba4#f3e17541f366696bf0cbaf380fcbd8b44c17eba4"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"base64-simd",
|
||||
@@ -11949,7 +11949,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -12100,7 +12100,7 @@ dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"psm",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -12405,10 +12405,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.4.3",
|
||||
"getrandom 0.3.4",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -12761,9 +12761,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "toml_edit"
|
||||
version = "0.25.14+spec-1.1.0"
|
||||
version = "0.25.13+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d2195eec204e2764644a4ea619704f9fbe5e0673038eded55ad9956f24fca0cc"
|
||||
checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b"
|
||||
dependencies = [
|
||||
"indexmap 2.14.2",
|
||||
"toml_datetime",
|
||||
@@ -13510,7 +13510,7 @@ version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
+52
-52
@@ -73,7 +73,7 @@ edition = "2024"
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/rustfs/rustfs"
|
||||
rust-version = "1.98.0"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.6"
|
||||
homepage = "https://rustfs.com"
|
||||
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
||||
keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
|
||||
@@ -90,56 +90,56 @@ redundant_clone = "warn"
|
||||
|
||||
[workspace.dependencies]
|
||||
# RustFS Internal Crates
|
||||
rustfs = { path = "./rustfs", version = "1.0.0-rc.5" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.5" }
|
||||
rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.0-rc.5" }
|
||||
rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.0-rc.5" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.5" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.5" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0-rc.5" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.5" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0-rc.5" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.5" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.5" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.5" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.5" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.5" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.5" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.5" }
|
||||
rustfs-license = { path = "crates/license", version = "1.0.0-rc.5" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.5" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.5" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.5" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.5" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.5" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.5" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.5" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.5" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.5", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.5" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.5" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.5" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.5" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.5" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.5" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.5" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.5" }
|
||||
rustfs-s3-client = { path = "crates/s3-client", version = "1.0.0-rc.5" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.5" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.5" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.5" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.5" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.5" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.5" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.5" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.5" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.5" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.5" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.5" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.5" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.5" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.5" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.5" }
|
||||
rustfs = { path = "./rustfs", version = "1.0.0-rc.6" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.6" }
|
||||
rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.0-rc.6" }
|
||||
rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.0-rc.6" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.6" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.6" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0-rc.6" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.6" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0-rc.6" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.6" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.6" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.6" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.6" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.6" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.6" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.6" }
|
||||
rustfs-license = { path = "crates/license", version = "1.0.0-rc.6" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.6" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.6" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.6" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.6" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.6" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.6" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.6" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.6" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.6", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.6" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.6" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.6" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.6" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.6" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.6" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.6" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.6" }
|
||||
rustfs-s3-client = { path = "crates/s3-client", version = "1.0.0-rc.6" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.6" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.6" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.6" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.6" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.6" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.6" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.6" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.6" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.6" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.6" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.6" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.6" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.6" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.6" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.6" }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
@@ -312,7 +312,7 @@ rustify = { version = "0.7", default-features = false }
|
||||
rustix = { version = "1.1.4" }
|
||||
rust-embed = { version = "8.12.0" }
|
||||
rustc-hash = { version = "2.1.3" }
|
||||
s3s = { git = "https://github.com/rustfs/s3s.git", rev = "bdcb6259339c41369f9f1c60e3a42b5ab8da607b", version = "0.15.0", features = ["minio"] }
|
||||
s3s = { git = "https://github.com/s3s-project/s3s.git", rev = "f3e17541f366696bf0cbaf380fcbd8b44c17eba4", version = "0.15.0", features = ["minio"] }
|
||||
serial_test = "4.0.1"
|
||||
shadow-rs = { default-features = false, version = "2.0.0" }
|
||||
siphasher = "1.0.3"
|
||||
|
||||
@@ -141,7 +141,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# Using specific version
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.5
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.6
|
||||
```
|
||||
|
||||
If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
|
||||
|
||||
+1
-1
@@ -138,7 +138,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# 使用指定版本运行
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.5
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.6
|
||||
```
|
||||
|
||||
如果您通过绑定挂载启用 TLS 证书目录,也请用同样方式准备该目录:
|
||||
|
||||
@@ -62,6 +62,37 @@ Current guidance:
|
||||
- `RUSTFS_CORS_ALLOWED_ORIGINS` defaults to empty, so the S3 endpoint emits no generic CORS headers unless configured. Set `*` for wildcard origins without credentials, or a comma-separated allow-list for credentialed explicit origins.
|
||||
- `RUSTFS_CONSOLE_CORS_ALLOWED_ORIGINS` defaults to `*` for the console service.
|
||||
|
||||
## Console URL prefix
|
||||
|
||||
`RUSTFS_CONSOLE_PREFIX` changes the embedded console URL prefix. The default is
|
||||
`/rustfs/console`. For example, `RUSTFS_CONSOLE_PREFIX=/console` serves the UI at
|
||||
`http://localhost:9001/console/`. Nested prefixes such as `/management/console`
|
||||
are supported; one trailing slash is removed. Restart the server after changing it.
|
||||
|
||||
The prefix must be a non-root absolute path of at most 256 bytes, with nonempty
|
||||
segments containing only ASCII letters, digits, `-`, `_`, `.`, or `~`. Dot
|
||||
segments, encoded characters, and overlaps with reserved admin, RPC, health,
|
||||
profiling, browser entry, and icon routes are rejected at startup. `/` is not supported.
|
||||
Choose a prefix that does not collide with S3 bucket paths.
|
||||
|
||||
The console routes, embedded frontend asset URLs, browser redirects, and OIDC
|
||||
console redirects use this prefix. Admin API paths and the identity provider's
|
||||
`/rustfs/admin/v3/oidc/callback/...` URL remain unchanged. `RUSTFS_CONSOLE_ADDRESS`
|
||||
continues to control only the listening address and port. The server adapts bundled
|
||||
console asset references from their build-time base path to the runtime prefix.
|
||||
|
||||
OEM builds can set `RUSTFS_CONSOLE_BASE_PATH` when compiling RustFS to embed a
|
||||
different default, such as `/nuofans/console`. Build the bundled console with the
|
||||
same `NEXT_PUBLIC_BASE_PATH`. An unset or empty build variable retains
|
||||
`/rustfs/console`. The build path must satisfy the validation rules above and must
|
||||
not have a trailing slash.
|
||||
|
||||
At startup, `RUSTFS_CONSOLE_PREFIX` takes precedence over the compiled default.
|
||||
Changing `RUSTFS_CONSOLE_BASE_PATH` when starting an existing binary has no effect;
|
||||
rebuild both components to change the embedded default. If a runtime prefix is
|
||||
configured, asset adaptation uses the compiled base path as its source, including
|
||||
when restoring `/rustfs/console` for a custom OEM build.
|
||||
|
||||
## Browser redirect environment variables
|
||||
|
||||
- `RUSTFS_BROWSER_REDIRECT_URL` sets the externally reachable browser origin used for OIDC callback, console success redirect, and logout fallback URLs. Configure it to the public scheme and authority without a path, for example `https://console.example.com`. In load-balancer deployments, keep OIDC authorize and callback requests on the same backend node because the in-flight OIDC `state` is local to the RustFS node.
|
||||
|
||||
@@ -213,6 +213,11 @@ pub const ENV_RUSTFS_CONSOLE_ENABLE: &str = "RUSTFS_CONSOLE_ENABLE";
|
||||
/// Environment variable for console server address.
|
||||
pub const ENV_RUSTFS_CONSOLE_ADDRESS: &str = "RUSTFS_CONSOLE_ADDRESS";
|
||||
|
||||
/// URL path prefix for the embedded console, read once at server startup.
|
||||
pub const ENV_RUSTFS_CONSOLE_PREFIX: &str = "RUSTFS_CONSOLE_PREFIX";
|
||||
/// Default embedded console URL path prefix.
|
||||
pub const DEFAULT_CONSOLE_PREFIX: &str = "/rustfs/console";
|
||||
|
||||
/// Public browser entrypoint used to build OIDC callback and console redirects.
|
||||
///
|
||||
/// This should be the externally reachable scheme and authority, without a path.
|
||||
|
||||
@@ -64,6 +64,14 @@ pub const MAX_HEAL_REQUEST_SIZE: usize = 1024 * 1024; // 1 MB
|
||||
/// memory exhaustion from malicious or misconfigured remote services.
|
||||
pub const MAX_S3_CLIENT_RESPONSE_SIZE: usize = 10 * 1024 * 1024; // 10 MB
|
||||
|
||||
/// Maximum body size accepted by a single `PutObject` or `UploadPart` request (5 GiB).
|
||||
/// Used for: the s3s streaming-body limit and the request-header admission check.
|
||||
/// Rationale: matches the AWS S3 single-PUT / single-part ceiling. Larger objects
|
||||
/// must use multipart upload. The header check rejects an oversize
|
||||
/// `Content-Length` before any body byte is read so the client gets
|
||||
/// `EntityTooLarge` immediately instead of streaming 5 GiB into a mid-stream failure.
|
||||
pub const MAX_SINGLE_PUT_OBJECT_SIZE: u64 = 5 * 1024 * 1024 * 1024; // 5 GiB
|
||||
|
||||
/// Maximum size for OIDC provider response bodies (1 MB)
|
||||
/// Used for: discovery documents, JWKS documents and token endpoint responses
|
||||
/// Rationale: a hostile or compromised identity provider must not be able to exhaust
|
||||
|
||||
@@ -38,6 +38,13 @@ All commands assume repo root. `cargo test` triggers an on-demand build of the
|
||||
`rustfs` binary from [`src/common.rs`](src/common.rs) (`rustfs_binary_path`) on
|
||||
first use — the first invocation is slow, later ones reuse the binary.
|
||||
|
||||
Root-heal interruption scenarios use a test-only commit barrier. Prebuild with `e2e-test-hooks` and pin that binary so concurrent cases do not replace it through on-demand builds:
|
||||
|
||||
```bash
|
||||
cargo build -p rustfs --bin rustfs --features e2e-test-hooks
|
||||
CARGO_BIN_EXE_rustfs="$PWD/target/debug/rustfs" cargo nextest run -p e2e_test -E 'test(heal_erasure_disk_rebuild_test)'
|
||||
```
|
||||
|
||||
```bash
|
||||
# Whole crate (default = ignored tests skipped)
|
||||
cargo nextest run -p e2e_test
|
||||
|
||||
@@ -61,7 +61,6 @@ pub(crate) struct VersionShardCensus {
|
||||
pub has_xl_meta: bool,
|
||||
pub data_dir: Option<String>,
|
||||
pub erasure_index: Option<usize>,
|
||||
pub erasure_distribution: Option<Vec<usize>>,
|
||||
pub data_blocks: Option<usize>,
|
||||
pub parity_blocks: Option<usize>,
|
||||
pub expected_part_numbers: BTreeSet<usize>,
|
||||
@@ -91,7 +90,6 @@ impl VersionShardCensus {
|
||||
&& manifest.is_complete()
|
||||
&& self.data_dir == manifest.data_dir
|
||||
&& self.erasure_index == manifest.erasure_index
|
||||
&& self.erasure_distribution == manifest.erasure_distribution
|
||||
&& self.data_blocks == manifest.data_blocks
|
||||
&& self.parity_blocks == manifest.parity_blocks
|
||||
&& self.expected_part_numbers == manifest.expected_part_numbers
|
||||
@@ -319,7 +317,6 @@ pub(crate) fn census_object_version_on_disk(
|
||||
has_xl_meta: false,
|
||||
data_dir: None,
|
||||
erasure_index: None,
|
||||
erasure_distribution: None,
|
||||
data_blocks: None,
|
||||
parity_blocks: None,
|
||||
expected_part_numbers: BTreeSet::new(),
|
||||
@@ -337,7 +334,6 @@ pub(crate) fn census_object_version_on_disk(
|
||||
};
|
||||
let data_dir = file_info.data_dir.map(|id| id.to_string());
|
||||
let erasure_index = Some(file_info.erasure.index);
|
||||
let erasure_distribution = Some(file_info.erasure.distribution.clone());
|
||||
let inline_data_fingerprint = file_info.data.as_deref().map(shard_fingerprint).transpose()?;
|
||||
let part_dir = data_dir.as_ref().map_or_else(|| object_dir.clone(), |id| object_dir.join(id));
|
||||
let present_part_fingerprints = match std::fs::read_dir(&part_dir) {
|
||||
@@ -370,7 +366,6 @@ pub(crate) fn census_object_version_on_disk(
|
||||
has_xl_meta: true,
|
||||
data_dir,
|
||||
erasure_index,
|
||||
erasure_distribution,
|
||||
data_blocks: Some(file_info.erasure.data_blocks),
|
||||
parity_blocks: Some(file_info.erasure.parity_blocks),
|
||||
expected_part_numbers,
|
||||
@@ -426,7 +421,6 @@ mod tests {
|
||||
has_xl_meta: true,
|
||||
data_dir: Some("data-dir".to_string()),
|
||||
erasure_index: Some(3),
|
||||
erasure_distribution: Some(vec![1, 2, 3, 4]),
|
||||
data_blocks: Some(2),
|
||||
parity_blocks: Some(2),
|
||||
expected_part_numbers: BTreeSet::from([1]),
|
||||
|
||||
@@ -32,7 +32,6 @@ const EC84_NODE_COUNT: usize = 3;
|
||||
const EC84_DRIVES_PER_NODE: usize = 4;
|
||||
const EC84_DATA_BLOCKS: usize = 8;
|
||||
const EC84_PARITY_BLOCKS: usize = 4;
|
||||
const EC84_ERASURE_SET_DRIVE_COUNT: usize = EC84_DATA_BLOCKS + EC84_PARITY_BLOCKS;
|
||||
const EC84_TARGET_DRIVE_RESTART_CASE: &str = "ec84-target-drive-restart";
|
||||
const EC84_TARGET_DRIVE_RESTART_ORACLE: &str = "ec84-target-drive-restart.json";
|
||||
const EC84_HEAL_CONTROL_READY_TIMEOUT: Duration = Duration::from_secs(45);
|
||||
@@ -186,7 +185,6 @@ async fn write_scanner_heal_evidence(context: ScannerHealEvidenceContext, payloa
|
||||
"binary_sha256": string_field(&context.run, "binary.sha256")?,
|
||||
"test_binary_sha256": string_field(&context.run, "test_binary.sha256")?,
|
||||
"topology": {"nodes": EC84_NODE_COUNT, "drives_per_node": EC84_DRIVES_PER_NODE},
|
||||
"erasure_set_drive_count": EC84_ERASURE_SET_DRIVE_COUNT,
|
||||
"pid_before": payload.pid_before,
|
||||
"pid_after": payload.pid_after,
|
||||
"unclean_shutdown_marker": false,
|
||||
@@ -253,7 +251,6 @@ async fn put_large_inventory(client: &Client, bucket: &str) -> TestResult<Vec<Ex
|
||||
has_xl_meta: false,
|
||||
data_dir: None,
|
||||
erasure_index: None,
|
||||
erasure_distribution: None,
|
||||
data_blocks: None,
|
||||
parity_blocks: None,
|
||||
expected_part_numbers: Default::default(),
|
||||
@@ -413,16 +410,4 @@ mod tests {
|
||||
"admin POST failed: 503 Service Unavailable cluster heal coordination unavailable".into();
|
||||
assert!(!is_cluster_heal_coordination_unavailable(wrong_status.as_ref()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ec84_drive_restart_evidence_shape_matches_registry() {
|
||||
assert_eq!(EC84_ERASURE_SET_DRIVE_COUNT, EC84_NODE_COUNT * EC84_DRIVES_PER_NODE);
|
||||
|
||||
let registry: Value = serde_json::from_str(include_str!("../../../../.config/scanner-heal-required-tests.json"))
|
||||
.expect("scanner/heal registry is valid JSON");
|
||||
let case = ®istry["cases"][EC84_TARGET_DRIVE_RESTART_CASE];
|
||||
assert_eq!(case["erasure_set_drive_count"], EC84_ERASURE_SET_DRIVE_COUNT);
|
||||
assert_eq!(case["topology"]["nodes"], EC84_NODE_COUNT);
|
||||
assert_eq!(case["topology"]["drives_per_node"], EC84_DRIVES_PER_NODE);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ mod harness;
|
||||
mod heal_test;
|
||||
mod object_lock_test;
|
||||
mod observability_test;
|
||||
mod replication_delete_marker_test;
|
||||
mod replication_quota_test;
|
||||
mod s3_basic_test;
|
||||
mod s3_during_data_movement_test;
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Functional REP-105 (rustfs/backlog#2195 item 4): a delete marker created
|
||||
//! on a multi-node source cluster must replicate to the bucket-replication
|
||||
//! target. Objects converged in seconds while delete markers did not arrive
|
||||
//! within 180 s on the shared 3-node functional environment; the single-node
|
||||
//! e2e never saw it.
|
||||
|
||||
use super::harness::{
|
||||
DistCluster, DistLayout, TestResult, enable_versioning, put_bucket_replication, put_object, set_remote_target, unique_bucket,
|
||||
wait_for_replicated_bytes, wait_until,
|
||||
};
|
||||
use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, RustFSTestEnvironment, init_logging, replication_fast_env, signed_request};
|
||||
use crate::replication_extension_test::LOOPBACK_REPLICATION_TARGET_ENV;
|
||||
use aws_sdk_s3::Client;
|
||||
use http::{Method, StatusCode};
|
||||
use std::time::Duration;
|
||||
|
||||
async fn target_has_delete_marker(client: &Client, bucket: &str, key: &str) -> TestResult<bool> {
|
||||
let versions = client.list_object_versions().bucket(bucket).prefix(key).send().await?;
|
||||
Ok(versions.delete_markers().iter().any(|marker| marker.key() == Some(key)))
|
||||
}
|
||||
|
||||
async fn delete_marker_replicates(
|
||||
source: &DistCluster,
|
||||
source_bucket: &str,
|
||||
target_client: &Client,
|
||||
target_bucket: &str,
|
||||
) -> TestResult {
|
||||
let key = "delete-marker/object.bin";
|
||||
let body = b"delete marker replication payload".to_vec();
|
||||
// Write through one node, delete through another: behind a load
|
||||
// balancer consecutive requests land on different nodes.
|
||||
put_object(&source.client(1)?, source_bucket, key, body.clone()).await?;
|
||||
wait_for_replicated_bytes(target_client, target_bucket, key, &body, Duration::from_secs(60)).await?;
|
||||
|
||||
let delete = source
|
||||
.client(2)?
|
||||
.delete_object()
|
||||
.bucket(source_bucket)
|
||||
.key(key)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
delete.delete_marker(),
|
||||
Some(true),
|
||||
"a versioned DELETE without versionId must create a marker"
|
||||
);
|
||||
|
||||
wait_until(
|
||||
Duration::from_secs(90),
|
||||
|| async { target_has_delete_marker(target_client, target_bucket, key).await },
|
||||
"delete marker replicated to the target bucket",
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_bucket_replication_replicates_delete_marker_to_peer_cluster() -> TestResult {
|
||||
init_logging();
|
||||
let (source, target) = DistCluster::start_replication_pair().await?;
|
||||
let source_bucket = unique_bucket("dm-src");
|
||||
let target_bucket = unique_bucket("dm-dst");
|
||||
source.create_bucket(&source_bucket).await?;
|
||||
target.create_bucket(&target_bucket).await?;
|
||||
enable_versioning(&source.client(0)?, &source_bucket).await?;
|
||||
enable_versioning(&target.client(0)?, &target_bucket).await?;
|
||||
|
||||
let arn = set_remote_target(&source.cluster, &source_bucket, &target.cluster, &target_bucket).await?;
|
||||
put_bucket_replication(&source.cluster, &source_bucket, &arn).await?;
|
||||
|
||||
delete_marker_replicates(&source, &source_bucket, &target.client(0)?, &target_bucket).await
|
||||
}
|
||||
|
||||
/// The functional environment replicates from a 3-node site to a single-node
|
||||
/// target; keep that shape as its own case.
|
||||
#[tokio::test]
|
||||
async fn four_node_bucket_replication_replicates_delete_marker_to_single_node_target() -> TestResult {
|
||||
init_logging();
|
||||
let mut extra: Vec<(&str, &str)> = replication_fast_env();
|
||||
extra.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
extra.extend_from_slice(FAST_DATA_USAGE_SCANNER_ENV);
|
||||
let source = DistCluster::start_with_env(DistLayout::FourNodeFourDisk, &extra).await?;
|
||||
let mut target = RustFSTestEnvironment::new().await?;
|
||||
target.start_rustfs_server_without_cleanup(vec![]).await?;
|
||||
|
||||
let source_bucket = unique_bucket("dm-src");
|
||||
let target_bucket = unique_bucket("dm-dst");
|
||||
source.create_bucket(&source_bucket).await?;
|
||||
let target_client = target.create_s3_client();
|
||||
target_client.create_bucket().bucket(&target_bucket).send().await?;
|
||||
enable_versioning(&source.client(0)?, &source_bucket).await?;
|
||||
enable_versioning(&target_client, &target_bucket).await?;
|
||||
|
||||
let body = serde_json::json!({
|
||||
"endpoint": target.address,
|
||||
"credentials": { "accessKey": target.access_key, "secretKey": target.secret_key },
|
||||
"targetbucket": target_bucket,
|
||||
"secure": false,
|
||||
"type": "replication"
|
||||
});
|
||||
let url = format!(
|
||||
"{}/rustfs/admin/v3/set-remote-target?bucket={}",
|
||||
source.cluster.nodes[0].url,
|
||||
urlencoding::encode(&source_bucket)
|
||||
);
|
||||
let response = signed_request(
|
||||
Method::PUT,
|
||||
&url,
|
||||
&source.cluster.access_key,
|
||||
&source.cluster.secret_key,
|
||||
Some(body.to_string().into_bytes()),
|
||||
Some("application/json"),
|
||||
)
|
||||
.await?;
|
||||
if response.status() != StatusCode::OK {
|
||||
let status = response.status();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
return Err(format!("set remote target failed: {status} {body}").into());
|
||||
}
|
||||
let arn: String = serde_json::from_slice(&response.bytes().await?)?;
|
||||
put_bucket_replication(&source.cluster, &source_bucket, &arn).await?;
|
||||
|
||||
delete_marker_replicates(&source, &source_bucket, &target_client, &target_bucket).await
|
||||
}
|
||||
@@ -126,3 +126,165 @@ async fn four_node_site_replication_replicates_object_to_peer_site() -> TestResu
|
||||
wait_for_replicated_bytes(&site_a.client(3)?, &bucket, reverse_key, &reverse_body, Duration::from_secs(60)).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn node_admin(
|
||||
cluster: &crate::common::RustFSTestClusterEnvironment,
|
||||
node_idx: usize,
|
||||
method: Method,
|
||||
path_and_query: &str,
|
||||
body: Option<String>,
|
||||
) -> TestResult<(StatusCode, String)> {
|
||||
crate::common::admin_request(
|
||||
&cluster.nodes[node_idx].url,
|
||||
method,
|
||||
path_and_query,
|
||||
body,
|
||||
&cluster.access_key,
|
||||
&cluster.secret_key,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Pair two clusters through site A's first node and wait until both report
|
||||
/// the two-site topology as enabled.
|
||||
async fn pair_sites(site_a: &DistCluster, site_b: &DistCluster) -> TestResult {
|
||||
let sites = vec![
|
||||
PeerSite {
|
||||
name: "site-a".to_string(),
|
||||
endpoint: site_a.cluster.nodes[0].url.clone(),
|
||||
access_key: site_a.cluster.access_key.clone(),
|
||||
secret_key: site_a.cluster.secret_key.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
PeerSite {
|
||||
name: "site-b".to_string(),
|
||||
endpoint: site_b.cluster.nodes[0].url.clone(),
|
||||
access_key: site_b.cluster.access_key.clone(),
|
||||
secret_key: site_b.cluster.secret_key.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
];
|
||||
let add_status = site_replication_add(&site_a.cluster, &sites).await?;
|
||||
assert!(
|
||||
add_status.success && add_status.err_detail.is_empty() && add_status.initial_sync_error_message.is_empty(),
|
||||
"site replication add reported failure: {add_status:?}"
|
||||
);
|
||||
wait_for_site_replication_enabled(&site_a.cluster).await?;
|
||||
wait_for_site_replication_enabled(&site_b.cluster).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn list_users_contains(
|
||||
cluster: &crate::common::RustFSTestClusterEnvironment,
|
||||
node_idx: usize,
|
||||
access_key: &str,
|
||||
) -> TestResult<bool> {
|
||||
let (status, body) = node_admin(cluster, node_idx, Method::GET, "/rustfs/admin/v3/list-users", None).await?;
|
||||
if !status.is_success() {
|
||||
return Err(format!("list-users on node {node_idx} failed: {status} {body}").into());
|
||||
}
|
||||
let users: serde_json::Value = serde_json::from_str(&body)?;
|
||||
Ok(users.get(access_key).is_some())
|
||||
}
|
||||
|
||||
/// backlog#2367 A-7 / functional SITE-102: an IAM change handled by a node
|
||||
/// other than the one that ran `site-replication/add` must still reach the
|
||||
/// peer site. Behind a load balancer every admin call may land on a
|
||||
/// different node, so the coordinator node is not special.
|
||||
#[tokio::test]
|
||||
async fn four_node_site_replication_converges_iam_user_created_on_a_non_coordinator_node() -> TestResult {
|
||||
init_logging();
|
||||
let (site_a, site_b) = DistCluster::start_replication_pair().await?;
|
||||
pair_sites(&site_a, &site_b).await?;
|
||||
|
||||
let user = format!("siteuser-{}", &uuid::Uuid::new_v4().simple().to_string()[..8]);
|
||||
let body = serde_json::json!({ "secretKey": "siteuser-secret-key-1234", "status": "enabled" }).to_string();
|
||||
let (status, response) = node_admin(
|
||||
&site_a.cluster,
|
||||
1,
|
||||
Method::PUT,
|
||||
&format!("/rustfs/admin/v3/add-user?accessKey={user}"),
|
||||
Some(body),
|
||||
)
|
||||
.await?;
|
||||
assert!(status.is_success(), "add-user on site A node 1 failed: {status} {response}");
|
||||
|
||||
let site_b_cluster = &site_b.cluster;
|
||||
let user_ref = user.as_str();
|
||||
wait_until(
|
||||
Duration::from_secs(90),
|
||||
|| async move { list_users_contains(site_b_cluster, 0, user_ref).await },
|
||||
"user created on site A node 1 visible on site B",
|
||||
)
|
||||
.await?;
|
||||
assert!(
|
||||
list_users_contains(&site_a.cluster, 2, &user).await?,
|
||||
"the user must be visible on every site A node"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// backlog#2367 A-5 / functional SITE-105: a resync started right after
|
||||
/// pairing must not report buckets as failed. The bucket carrying an
|
||||
/// operator-configured bucket-replication target to the peer (the shape the
|
||||
/// functional suite leaves behind) and a plain versioned bucket are both
|
||||
/// wired by the pairing itself.
|
||||
#[tokio::test]
|
||||
async fn four_node_site_replication_resync_start_right_after_pairing_reports_no_failed_bucket() -> TestResult {
|
||||
init_logging();
|
||||
let (site_a, site_b) = DistCluster::start_replication_pair().await?;
|
||||
|
||||
let pre_src = unique_bucket("pre-src");
|
||||
let pre_dst = unique_bucket("pre-dst");
|
||||
let plain = unique_bucket("plain");
|
||||
site_a.create_bucket(&pre_src).await?;
|
||||
site_b.create_bucket(&pre_dst).await?;
|
||||
site_a.create_bucket(&plain).await?;
|
||||
enable_versioning(&site_a.client(0)?, &pre_src).await?;
|
||||
enable_versioning(&site_b.client(0)?, &pre_dst).await?;
|
||||
enable_versioning(&site_a.client(0)?, &plain).await?;
|
||||
let arn = super::harness::set_remote_target(&site_a.cluster, &pre_src, &site_b.cluster, &pre_dst).await?;
|
||||
super::harness::put_bucket_replication(&site_a.cluster, &pre_src, &arn).await?;
|
||||
|
||||
pair_sites(&site_a, &site_b).await?;
|
||||
|
||||
let (status, info) = node_admin(&site_a.cluster, 1, Method::GET, "/rustfs/admin/v3/site-replication/info", None).await?;
|
||||
assert!(status.is_success(), "site-replication/info failed: {status} {info}");
|
||||
let info: serde_json::Value = serde_json::from_str(&info)?;
|
||||
let peer = info["sites"]
|
||||
.as_array()
|
||||
.and_then(|sites| sites.iter().find(|site| site["name"] == "site-b"))
|
||||
.cloned()
|
||||
.ok_or_else(|| format!("site-b peer missing from info: {info}"))?;
|
||||
|
||||
// Through a non-coordinator node, like a load-balanced admin call.
|
||||
let (status, response) = node_admin(
|
||||
&site_a.cluster,
|
||||
1,
|
||||
Method::PUT,
|
||||
"/rustfs/admin/v3/site-replication/resync/op?operation=start",
|
||||
Some(peer.to_string()),
|
||||
)
|
||||
.await?;
|
||||
assert!(status.is_success(), "resync start failed: {status} {response}");
|
||||
let resync: rustfs_madmin::SRResyncOpStatus = serde_json::from_str(&response)?;
|
||||
let failed: Vec<String> = resync
|
||||
.buckets
|
||||
.iter()
|
||||
.filter(|bucket| bucket.status == "failed")
|
||||
.map(|bucket| format!("{}: {}", bucket.bucket, bucket.err_detail))
|
||||
.collect();
|
||||
assert!(
|
||||
failed.is_empty(),
|
||||
"resync right after pairing reported failed buckets: {failed:?} (status={}, detail={})",
|
||||
resync.status,
|
||||
resync.err_detail
|
||||
);
|
||||
assert!(
|
||||
resync.buckets.iter().any(|bucket| bucket.bucket == pre_src)
|
||||
&& resync.buckets.iter().any(|bucket| bucket.bucket == plain),
|
||||
"both buckets must be part of the resync: {:?}",
|
||||
resync.buckets
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -24,7 +24,7 @@ mod tests {
|
||||
use crate::storage_api::RUSTFS_META_BUCKET;
|
||||
use aws_sdk_s3::{
|
||||
error::{ProvideErrorMetadata, SdkError},
|
||||
operation::{delete_object::DeleteObjectError, put_object::PutObjectError},
|
||||
operation::put_object::PutObjectError,
|
||||
primitives::ByteStream,
|
||||
};
|
||||
use http::Method;
|
||||
@@ -45,7 +45,6 @@ mod tests {
|
||||
|
||||
struct ReplacementDriveSelection {
|
||||
replaced_disk: PathBuf,
|
||||
drive_index: usize,
|
||||
replacement_format_path: PathBuf,
|
||||
replacement_format: Vec<u8>,
|
||||
expected_pool_metadata: Option<VersionShardCensus>,
|
||||
@@ -462,12 +461,6 @@ mod tests {
|
||||
shard_census: VersionShardCensus,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct OutagePeerManifest {
|
||||
erasure_indices: HashSet<usize>,
|
||||
erasure_distribution: Option<Vec<usize>>,
|
||||
}
|
||||
|
||||
fn deterministic_object_body(len: usize, seed: u8) -> Vec<u8> {
|
||||
let mut value = seed;
|
||||
std::iter::repeat_with(|| {
|
||||
@@ -493,81 +486,6 @@ mod tests {
|
||||
Ok(matching)
|
||||
}
|
||||
|
||||
fn collect_outage_peer_manifest(
|
||||
cluster: &RustFSTestClusterEnvironment,
|
||||
offline_node_index: usize,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
erasure_set_drive_count: usize,
|
||||
) -> Result<OutagePeerManifest, Box<dyn Error + Send + Sync>> {
|
||||
let mut manifest = OutagePeerManifest::default();
|
||||
for (node_index, node) in cluster.nodes.iter().enumerate() {
|
||||
if node_index == offline_node_index {
|
||||
continue;
|
||||
}
|
||||
for (drive_index, drive) in node.data_dirs.iter().enumerate() {
|
||||
let census = census_object_version_on_disk(Path::new(drive), bucket, key, None)?;
|
||||
if !census.has_xl_meta {
|
||||
continue;
|
||||
}
|
||||
assert!(
|
||||
census.is_complete(),
|
||||
"online node {node_index} drive {drive_index} must hold a complete outage-object shard: {census:?}"
|
||||
);
|
||||
let erasure_index = census.erasure_index.ok_or_else(|| {
|
||||
format!("online node {node_index} drive {drive_index} outage-object shard has no erasure index: {census:?}")
|
||||
})?;
|
||||
assert!(
|
||||
(1..=erasure_set_drive_count).contains(&erasure_index),
|
||||
"online node {node_index} drive {drive_index} outage-object erasure index is out of range: {census:?}"
|
||||
);
|
||||
let distribution = census.erasure_distribution.as_ref().ok_or_else(|| {
|
||||
format!(
|
||||
"online node {node_index} drive {drive_index} outage-object shard has no erasure distribution: {census:?}"
|
||||
)
|
||||
})?;
|
||||
assert_eq!(
|
||||
distribution.len(),
|
||||
erasure_set_drive_count,
|
||||
"online node {node_index} drive {drive_index} outage-object distribution must match the erasure set: {census:?}"
|
||||
);
|
||||
match &manifest.erasure_distribution {
|
||||
Some(existing) => {
|
||||
assert_eq!(existing, distribution, "outage-object shards must agree on one erasure distribution")
|
||||
}
|
||||
None => manifest.erasure_distribution = Some(distribution.clone()),
|
||||
}
|
||||
assert!(
|
||||
manifest.erasure_indices.insert(erasure_index),
|
||||
"outage-object erasure index {erasure_index} is duplicated across online drives"
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(manifest)
|
||||
}
|
||||
|
||||
fn outage_candidate_replacement_erasure_index(
|
||||
peer_manifest: &OutagePeerManifest,
|
||||
erasure_set_drive_count: usize,
|
||||
replacement_set_slot: usize,
|
||||
) -> Option<usize> {
|
||||
let distribution = peer_manifest.erasure_distribution.as_ref()?;
|
||||
distribution.get(replacement_set_slot).copied().filter(|replacement_index| {
|
||||
(1..=erasure_set_drive_count).contains(replacement_index)
|
||||
&& !peer_manifest.erasure_indices.contains(replacement_index)
|
||||
})
|
||||
}
|
||||
|
||||
fn outage_candidate_targets_replacement(
|
||||
peer_manifest: &OutagePeerManifest,
|
||||
erasure_set_drive_count: usize,
|
||||
replacement_set_slot: usize,
|
||||
) -> bool {
|
||||
let min_online_data_shards = erasure_set_drive_count.saturating_sub(4);
|
||||
peer_manifest.erasure_indices.len() >= min_online_data_shards
|
||||
&& outage_candidate_replacement_erasure_index(peer_manifest, erasure_set_drive_count, replacement_set_slot).is_some()
|
||||
}
|
||||
|
||||
fn metadata_count(disk: &Path, bucket: &str, expected_manifests: &[PhysicalObjectManifest]) -> usize {
|
||||
expected_manifests
|
||||
.iter()
|
||||
@@ -683,10 +601,6 @@ mod tests {
|
||||
error.as_service_error().and_then(ProvideErrorMetadata::code) == Some("ServiceUnavailable")
|
||||
}
|
||||
|
||||
fn is_service_unavailable_delete(error: &SdkError<DeleteObjectError>) -> bool {
|
||||
error.as_service_error().and_then(ProvideErrorMetadata::code) == Some("ServiceUnavailable")
|
||||
}
|
||||
|
||||
fn select_replacement_drive(
|
||||
cluster: &RustFSTestClusterEnvironment,
|
||||
node_index: usize,
|
||||
@@ -698,7 +612,7 @@ mod tests {
|
||||
.ok_or_else(|| format!("replacement node {node_index} is absent"))?;
|
||||
let mut incomplete_pool_metadata = Vec::new();
|
||||
|
||||
for (drive_index, drive) in node.data_dirs.iter().enumerate() {
|
||||
for drive in &node.data_dirs {
|
||||
let replaced_disk = PathBuf::from(drive);
|
||||
let replacement_format_path = replaced_disk.join(".rustfs.sys").join("format.json");
|
||||
let replacement_format = std::fs::read(&replacement_format_path).map_err(|err| {
|
||||
@@ -707,7 +621,6 @@ mod tests {
|
||||
if !require_pool_metadata {
|
||||
return Ok(ReplacementDriveSelection {
|
||||
replaced_disk,
|
||||
drive_index,
|
||||
replacement_format_path,
|
||||
replacement_format,
|
||||
expected_pool_metadata: None,
|
||||
@@ -718,7 +631,6 @@ mod tests {
|
||||
if census.is_complete() {
|
||||
return Ok(ReplacementDriveSelection {
|
||||
replaced_disk,
|
||||
drive_index,
|
||||
replacement_format_path,
|
||||
replacement_format,
|
||||
expected_pool_metadata: Some(census),
|
||||
@@ -1268,7 +1180,7 @@ mod tests {
|
||||
async fn test_cluster_root_heal_recovers_ec84_shards_across_multi_set_after_background_target_restart()
|
||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
timeout(
|
||||
Duration::from_secs(900),
|
||||
Duration::from_secs(600),
|
||||
run_cluster_root_heal_interruption(InterruptionScenario::BackgroundTargetRestartEc84MultiSet),
|
||||
)
|
||||
.await?
|
||||
@@ -1420,13 +1332,10 @@ mod tests {
|
||||
|
||||
let ReplacementDriveSelection {
|
||||
replaced_disk,
|
||||
drive_index: replacement_drive_index,
|
||||
replacement_format_path,
|
||||
replacement_format,
|
||||
expected_pool_metadata,
|
||||
} = select_replacement_drive(&cluster, 1, background_enabled)?;
|
||||
let replacement_global_drive_index = topology.drives_per_node + replacement_drive_index;
|
||||
let replacement_set_slot = replacement_global_drive_index % erasure_set_drive_count;
|
||||
let default_online_object_count = if !outage_target_manifest_required { 64 } else { 24 };
|
||||
let online_object_count = std::env::var("RUSTFS_HEAL_CHAOS_OBJECT_COUNT")
|
||||
.ok()
|
||||
@@ -1482,23 +1391,6 @@ mod tests {
|
||||
expected_manifests.push(PhysicalObjectManifest { key, shard_census });
|
||||
attempt_count += 1;
|
||||
}
|
||||
for manifest in &expected_manifests {
|
||||
let distribution = manifest
|
||||
.shard_census
|
||||
.erasure_distribution
|
||||
.as_ref()
|
||||
.ok_or_else(|| format!("replacement baseline manifest has no erasure distribution: {manifest:?}"))?;
|
||||
assert_eq!(
|
||||
distribution.len(),
|
||||
erasure_set_drive_count,
|
||||
"replacement baseline distribution must match the erasure set: {manifest:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
manifest.shard_census.erasure_index,
|
||||
distribution.get(replacement_set_slot).copied(),
|
||||
"replacement baseline shard must match the selected drive's erasure-set slot"
|
||||
);
|
||||
}
|
||||
|
||||
if background_enabled {
|
||||
wait_for_scanner_cycle_after(&cluster, 0).await?;
|
||||
@@ -1523,9 +1415,6 @@ mod tests {
|
||||
let mut outage_write_deferred_until_rejoin = false;
|
||||
let mut service_unavailable_outage_writes = 0usize;
|
||||
let mut last_service_unavailable = None;
|
||||
let mut outage_peer_manifest = OutagePeerManifest::default();
|
||||
let mut replacement_outage_erasure_index = None;
|
||||
let mut rejected_outage_keys = Vec::new();
|
||||
for attempt in 0..max_outage_write_attempts {
|
||||
let candidate_key = format!("cluster/written-while-node-down-{attempt:04}.bin");
|
||||
let put_result = timeout(
|
||||
@@ -1540,24 +1429,6 @@ mod tests {
|
||||
.await;
|
||||
match put_result {
|
||||
Ok(Ok(_)) => {
|
||||
if outage_target_manifest_required {
|
||||
let candidate_peer_manifest =
|
||||
collect_outage_peer_manifest(&cluster, 1, bucket, &candidate_key, erasure_set_drive_count)?;
|
||||
if !outage_candidate_targets_replacement(
|
||||
&candidate_peer_manifest,
|
||||
erasure_set_drive_count,
|
||||
replacement_set_slot,
|
||||
) {
|
||||
rejected_outage_keys.push(candidate_key);
|
||||
continue;
|
||||
}
|
||||
replacement_outage_erasure_index = outage_candidate_replacement_erasure_index(
|
||||
&candidate_peer_manifest,
|
||||
erasure_set_drive_count,
|
||||
replacement_set_slot,
|
||||
);
|
||||
outage_peer_manifest = candidate_peer_manifest;
|
||||
}
|
||||
outage_key = Some(candidate_key);
|
||||
break;
|
||||
}
|
||||
@@ -1585,37 +1456,57 @@ mod tests {
|
||||
}
|
||||
};
|
||||
|
||||
if !outage_write_deferred_until_rejoin && outage_peer_manifest.erasure_indices.is_empty() {
|
||||
outage_peer_manifest = collect_outage_peer_manifest(&cluster, 1, bucket, &outage_key, erasure_set_drive_count)?;
|
||||
replacement_outage_erasure_index =
|
||||
outage_candidate_replacement_erasure_index(&outage_peer_manifest, erasure_set_drive_count, replacement_set_slot);
|
||||
let mut outage_peer_erasure_indices = HashSet::new();
|
||||
if !outage_write_deferred_until_rejoin {
|
||||
for (node_index, node) in cluster.nodes.iter().enumerate() {
|
||||
if node_index == 1 {
|
||||
continue;
|
||||
}
|
||||
for (drive_index, drive) in node.data_dirs.iter().enumerate() {
|
||||
let census = census_object_version_on_disk(Path::new(drive), bucket, &outage_key, None)?;
|
||||
if !census.has_xl_meta {
|
||||
continue;
|
||||
}
|
||||
assert!(
|
||||
census.is_complete(),
|
||||
"online node {node_index} drive {drive_index} must hold a complete outage-object shard: {census:?}"
|
||||
);
|
||||
let erasure_index = census.erasure_index.ok_or_else(|| {
|
||||
format!(
|
||||
"online node {node_index} drive {drive_index} outage-object shard has no erasure index: {census:?}"
|
||||
)
|
||||
})?;
|
||||
assert!(
|
||||
(1..=erasure_set_drive_count).contains(&erasure_index),
|
||||
"online node {node_index} drive {drive_index} outage-object erasure index is out of range: {census:?}"
|
||||
);
|
||||
assert!(
|
||||
outage_peer_erasure_indices.insert(erasure_index),
|
||||
"outage-object erasure index {erasure_index} is duplicated across online drives"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
outage_write_deferred_until_rejoin
|
||||
|| (!outage_peer_manifest.erasure_indices.is_empty()
|
||||
&& outage_peer_manifest.erasure_indices.len() <= erasure_set_drive_count),
|
||||
|| (!outage_peer_erasure_indices.is_empty() && outage_peer_erasure_indices.len() <= erasure_set_drive_count),
|
||||
"outage-object must occupy one non-empty erasure set"
|
||||
);
|
||||
if outage_target_manifest_required {
|
||||
let min_online_data_shards = erasure_set_drive_count.saturating_sub(4);
|
||||
assert!(
|
||||
outage_peer_manifest.erasure_indices.len() >= min_online_data_shards,
|
||||
outage_peer_erasure_indices.len() >= min_online_data_shards,
|
||||
"online drives in the selected erasure set must retain at least the EC data quorum"
|
||||
);
|
||||
}
|
||||
let missing_outage_erasure_indices = (1..=erasure_set_drive_count)
|
||||
.filter(|index| !outage_peer_manifest.erasure_indices.contains(index))
|
||||
.filter(|index| !outage_peer_erasure_indices.contains(index))
|
||||
.collect::<HashSet<_>>();
|
||||
if outage_target_manifest_required {
|
||||
assert!(
|
||||
!missing_outage_erasure_indices.is_empty(),
|
||||
"the stopped target must account for at least one missing outage-object erasure index"
|
||||
);
|
||||
assert_eq!(
|
||||
replacement_outage_erasure_index.filter(|index| missing_outage_erasure_indices.contains(index)),
|
||||
replacement_outage_erasure_index,
|
||||
"the outage object must target the selected replacement drive's erasure-set slot"
|
||||
);
|
||||
}
|
||||
|
||||
let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
|
||||
@@ -1639,25 +1530,17 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
// Keep the partial-repair checkpoint stable across readiness and admin
|
||||
// requests. Endpoint-blackhole tests must prove their own network stall.
|
||||
let commit_barrier = if scenario != InterruptionScenario::TargetEndpointBlackhole {
|
||||
let barrier = replaced_disk.join(".rustfs.sys/e2e-heal-commit-barrier");
|
||||
std::fs::create_dir_all(barrier.parent().ok_or("commit barrier has no parent")?)?;
|
||||
std::fs::write(&barrier, format!("{bucket}/cluster/online/"))?;
|
||||
Some(barrier)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
cluster.start_node_from_binary(1, &server_binary).await?;
|
||||
for rejected_key in rejected_outage_keys {
|
||||
let delete_deadline = Instant::now() + Duration::from_secs(60);
|
||||
loop {
|
||||
let delete_result = timeout(
|
||||
Duration::from_secs(30),
|
||||
clients[0].delete_object().bucket(bucket).key(&rejected_key).send(),
|
||||
)
|
||||
.await;
|
||||
match delete_result {
|
||||
Ok(Ok(_)) => break,
|
||||
Ok(Err(error)) if is_service_unavailable_delete(&error) && Instant::now() < delete_deadline => {
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
Ok(Err(error)) => return Err(error.into()),
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let status_url = format!("{}/rustfs/admin/v3/background-heal/status", cluster.nodes[0].url);
|
||||
let recovery_deadline = Instant::now() + Duration::from_secs(60);
|
||||
@@ -1790,6 +1673,12 @@ mod tests {
|
||||
sleep(Duration::from_millis(10)).await;
|
||||
};
|
||||
|
||||
if let Some(barrier) = &commit_barrier {
|
||||
assert!(
|
||||
barrier.with_extension("admitted").is_file(),
|
||||
"interruption tests require a server built with e2e-test-hooks"
|
||||
);
|
||||
}
|
||||
let pre_interrupt_status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
|
||||
let pre_interrupt_status: serde_json::Value = serde_json::from_str(&pre_interrupt_status_body)
|
||||
.map_err(|err| format!("pre-interrupt background heal status is not JSON ({err}): {pre_interrupt_status_body}"))?;
|
||||
@@ -1995,6 +1884,9 @@ mod tests {
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(barrier) = &commit_barrier {
|
||||
std::fs::remove_file(barrier)?;
|
||||
}
|
||||
cluster.start_node_from_binary(interruption_node, &server_binary).await?;
|
||||
if interruption_node == 0 {
|
||||
let target = cluster.nodes[1]
|
||||
@@ -2122,9 +2014,9 @@ mod tests {
|
||||
assert_eq!(
|
||||
outage_census
|
||||
.erasure_index
|
||||
.filter(|index| replacement_outage_erasure_index == Some(*index)),
|
||||
.filter(|index| missing_outage_erasure_indices.contains(index)),
|
||||
outage_census.erasure_index,
|
||||
"the outage object must be rebuilt into the selected replacement drive's erasure-set slot"
|
||||
"the outage object must be rebuilt into one of the stopped node's missing erasure slots"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -2404,28 +2296,4 @@ mod tests {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn outage_candidate_must_target_replacement_erasure_index() {
|
||||
let distribution = vec![4, 7, 10, 1, 5, 8, 11, 2, 6, 9, 12, 3];
|
||||
let replacement_set_slot = 8;
|
||||
let replacement_index = distribution[replacement_set_slot];
|
||||
let peers_missing_replacement = OutagePeerManifest {
|
||||
erasure_indices: (1..=12).filter(|index| *index != replacement_index).collect(),
|
||||
erasure_distribution: Some(distribution.clone()),
|
||||
};
|
||||
assert!(outage_candidate_targets_replacement(&peers_missing_replacement, 12, replacement_set_slot));
|
||||
|
||||
let peers_missing_other_slot = OutagePeerManifest {
|
||||
erasure_indices: (1..=12).filter(|index| *index != 9).collect(),
|
||||
erasure_distribution: Some(distribution),
|
||||
};
|
||||
assert!(!outage_candidate_targets_replacement(&peers_missing_other_slot, 12, replacement_set_slot));
|
||||
|
||||
let insufficient_peer_shards = OutagePeerManifest {
|
||||
erasure_indices: [1, 3, 4, 5, 6, 7, 8].into_iter().collect(),
|
||||
erasure_distribution: Some(vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]),
|
||||
};
|
||||
assert!(!outage_candidate_targets_replacement(&insufficient_peer_shards, 12, replacement_set_slot));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,8 +52,8 @@ use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
BucketLifecycleConfiguration, BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, ExpirationStatus,
|
||||
LifecycleRule, LifecycleRuleFilter, MetadataDirective, NoncurrentVersionTransition, RestoreRequest, Transition,
|
||||
TransitionStorageClass, VersioningConfiguration,
|
||||
LifecycleRule, LifecycleRuleFilter, NoncurrentVersionTransition, RestoreRequest, Transition, TransitionStorageClass,
|
||||
VersioningConfiguration,
|
||||
};
|
||||
use http::Method;
|
||||
use serde::Deserialize;
|
||||
@@ -963,81 +963,6 @@ async fn test_hermetic_transition_main_path() -> TestResult {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// PUT and materialized self-copy must retain cleanup ownership of a replaced
|
||||
/// transitioned null version while publishing the new bytes and metadata.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn test_hermetic_transition_overwrite_and_self_copy() -> TestResult {
|
||||
let mut cold = RustFSTestEnvironment::new().await?;
|
||||
cold.access_key = "coldtieradmin".to_string();
|
||||
cold.secret_key = "coldtiersecret".to_string();
|
||||
cold.start_rustfs_server_without_cleanup(vec![]).await?;
|
||||
let cold_client = cold.create_s3_client();
|
||||
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
let mut hot = RustFSTestEnvironment::new().await?;
|
||||
start_tier_source(&mut hot, crate::common::FAST_DATA_USAGE_SCANNER_ENV).await?;
|
||||
let hot_client = hot.create_s3_client();
|
||||
add_rustfs_tier(&hot, &cold).await?;
|
||||
hot_client.create_bucket().bucket(SOURCE_BUCKET).send().await?;
|
||||
|
||||
let data = payload();
|
||||
for self_copy in [false, true] {
|
||||
hot_client
|
||||
.put_bucket_lifecycle_configuration()
|
||||
.bucket(SOURCE_BUCKET)
|
||||
.lifecycle_configuration(BucketLifecycleConfiguration::builder().rules(transition_rule()?).build()?)
|
||||
.send()
|
||||
.await?;
|
||||
put_multipart_object(&hot_client, SOURCE_BUCKET, OBJECT_KEY, &data).await?;
|
||||
wait_for_transition(&hot_client, SOURCE_BUCKET, OBJECT_KEY, StdDuration::from_secs(90)).await?;
|
||||
assert_eq!(cold_tier_object_count(&cold_client).await?, 1);
|
||||
// Keep the replacement local so disappearance of the old remote
|
||||
// object cannot be confused with another automatic transition.
|
||||
hot_client.delete_bucket_lifecycle().bucket(SOURCE_BUCKET).send().await?;
|
||||
|
||||
let expected = if self_copy { data.clone() } else { vec![0x73; 513] };
|
||||
if self_copy {
|
||||
hot_client
|
||||
.copy_object()
|
||||
.bucket(SOURCE_BUCKET)
|
||||
.key(OBJECT_KEY)
|
||||
.copy_source(format!("{SOURCE_BUCKET}/{}", urlencoding::encode(OBJECT_KEY)))
|
||||
.metadata_directive(MetadataDirective::Replace)
|
||||
.content_type("text/plain")
|
||||
.metadata("replacement", "kept")
|
||||
.send()
|
||||
.await?;
|
||||
} else {
|
||||
hot_client
|
||||
.put_object()
|
||||
.bucket(SOURCE_BUCKET)
|
||||
.key(OBJECT_KEY)
|
||||
.body(ByteStream::from(expected.clone()))
|
||||
.content_type("text/plain")
|
||||
.metadata("replacement", "kept")
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
wait_for_cold_tier_empty(&cold_client, StdDuration::from_secs(90)).await?;
|
||||
let current = hot_client.get_object().bucket(SOURCE_BUCKET).key(OBJECT_KEY).send().await?;
|
||||
assert_eq!(current.content_type(), Some("text/plain"));
|
||||
assert_eq!(current.metadata().and_then(|m| m.get("replacement")).map(String::as_str), Some("kept"));
|
||||
assert!(
|
||||
current
|
||||
.metadata()
|
||||
.is_none_or(|metadata| !metadata.contains_key(USER_META_KEY))
|
||||
);
|
||||
assert_eq!(current.body.collect().await?.into_bytes().as_ref(), expected.as_slice());
|
||||
hot_client
|
||||
.delete_object()
|
||||
.bucket(SOURCE_BUCKET)
|
||||
.key(OBJECT_KEY)
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Restore a transitioned object through a real RustFS remote tier.
|
||||
///
|
||||
/// The test covers the externally visible copy-back contract that a mock tier
|
||||
|
||||
@@ -6383,6 +6383,18 @@ async fn test_site_replication_edit_and_status_peer_state_real_three_node() -> R
|
||||
let relayed_key = "after-edit-from-relay.txt";
|
||||
let relayed_payload = b"site replication after endpoint edit from relay".to_vec();
|
||||
|
||||
// The first joining receiver owns data before the third site has the
|
||||
// shared account. Initial probes and backfill must wait for every join.
|
||||
target_client.create_bucket().bucket(bucket).send().await?;
|
||||
enable_bucket_versioning(&target_env, bucket).await?;
|
||||
target_client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(baseline_key)
|
||||
.body(ByteStream::from(baseline_payload.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let add_status = site_replication_add(
|
||||
&source_env,
|
||||
&[
|
||||
@@ -6410,7 +6422,10 @@ async fn test_site_replication_edit_and_status_peer_state_real_three_node() -> R
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
assert!(add_status.success, "unexpected site add result: {:?}", add_status);
|
||||
assert!(
|
||||
add_status.success && add_status.err_detail.is_empty() && add_status.initial_sync_error_message.is_empty(),
|
||||
"unexpected site add result: {add_status:?}"
|
||||
);
|
||||
|
||||
let source_info = wait_for_site_replication_enabled(&source_env, 3).await?;
|
||||
let _target_info = wait_for_site_replication_enabled(&target_env, 3).await?;
|
||||
@@ -6421,19 +6436,11 @@ async fn test_site_replication_edit_and_status_peer_state_real_three_node() -> R
|
||||
.find(|peer| peer.endpoint == target_env.url)
|
||||
.ok_or("target peer missing from source site replication info")?;
|
||||
|
||||
source_client.create_bucket().bucket(bucket).send().await?;
|
||||
enable_bucket_versioning(&source_env, bucket).await?;
|
||||
wait_for_bucket_on_target(&target_client, bucket).await?;
|
||||
wait_for_bucket_on_target(&relay_client, bucket).await?;
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(baseline_key)
|
||||
.body(ByteStream::from(baseline_payload.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
let replicated_baseline = wait_for_object_on_target(&target_client, bucket, baseline_key).await?;
|
||||
assert_eq!(replicated_baseline, baseline_payload);
|
||||
for client in [&source_client, &relay_client] {
|
||||
wait_for_bucket_on_target(client, bucket).await?;
|
||||
let backfilled = wait_for_object_on_target(client, bucket, baseline_key).await?;
|
||||
assert_eq!(backfilled, baseline_payload);
|
||||
}
|
||||
|
||||
let old_target_address = target_env.address.clone();
|
||||
let new_target_port = RustFSTestEnvironment::find_available_port().await?;
|
||||
|
||||
@@ -27,9 +27,7 @@ mod tests {
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, VersioningConfiguration,
|
||||
};
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
|
||||
use tracing::info;
|
||||
|
||||
fn create_s3_client(env: &RustFSTestEnvironment) -> Client {
|
||||
@@ -85,156 +83,6 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_version_tags(client: &Client, bucket: &str, key: &str, version: Option<&str>, value: Option<&str>) {
|
||||
let tags = client
|
||||
.get_object_tagging()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.set_version_id(version.map(str::to_owned))
|
||||
.send()
|
||||
.await
|
||||
.expect("GetObjectTagging must accept the exact version selector");
|
||||
let expected = value
|
||||
.map(|value| vec![Tag::builder().key("generation").value(value).build().expect("valid tag")])
|
||||
.unwrap_or_default();
|
||||
assert_eq!(tags.tag_set(), expected, "version selector: {version:?}");
|
||||
}
|
||||
|
||||
async fn assert_null_tagging_across_versioning_changes(client: &Client, bucket: &str, key: &str) {
|
||||
assert_version_tags(client, bucket, key, None, None).await;
|
||||
assert_version_tags(client, bucket, key, Some("null"), None).await;
|
||||
client
|
||||
.put_object_tagging()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.version_id("null")
|
||||
.tagging(
|
||||
Tagging::builder()
|
||||
.tag_set(
|
||||
Tag::builder()
|
||||
.key("generation")
|
||||
.value("original-null")
|
||||
.build()
|
||||
.expect("valid tag"),
|
||||
)
|
||||
.build()
|
||||
.expect("valid tagging"),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.expect("tag the original null version");
|
||||
|
||||
enable_versioning(client, bucket)
|
||||
.await
|
||||
.expect("enable versioning over a null version");
|
||||
let versioned = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.tagging("generation=versioned")
|
||||
.body(ByteStream::from_static(b"new version"))
|
||||
.send()
|
||||
.await
|
||||
.expect("write a newer UUID version");
|
||||
let version = versioned.version_id().expect("versioned PUT must return a UUID");
|
||||
assert_ne!(version, "null");
|
||||
assert_version_tags(client, bucket, key, None, Some("versioned")).await;
|
||||
assert_version_tags(client, bucket, key, Some(version), Some("versioned")).await;
|
||||
assert_version_tags(client, bucket, key, Some("null"), Some("original-null")).await;
|
||||
|
||||
client
|
||||
.put_object_tagging()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.version_id("null")
|
||||
.tagging(
|
||||
Tagging::builder()
|
||||
.tag_set(
|
||||
Tag::builder()
|
||||
.key("generation")
|
||||
.value("updated-null")
|
||||
.build()
|
||||
.expect("valid tag"),
|
||||
)
|
||||
.build()
|
||||
.expect("valid tagging"),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.expect("update tags on the noncurrent null version");
|
||||
assert_version_tags(client, bucket, key, Some("null"), Some("updated-null")).await;
|
||||
assert_version_tags(client, bucket, key, None, Some("versioned")).await;
|
||||
client
|
||||
.delete_object_tagging()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.version_id("null")
|
||||
.send()
|
||||
.await
|
||||
.expect("delete only the noncurrent null version tags");
|
||||
assert_version_tags(client, bucket, key, Some("null"), None).await;
|
||||
assert_version_tags(client, bucket, key, Some(version), Some("versioned")).await;
|
||||
|
||||
suspend_versioning(client, bucket).await.expect("suspend versioning");
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.tagging("generation=suspended-null")
|
||||
.body(ByteStream::from_static(b"replacement null version"))
|
||||
.send()
|
||||
.await
|
||||
.expect("replace the null version while suspended");
|
||||
assert_version_tags(client, bucket, key, None, Some("suspended-null")).await;
|
||||
assert_version_tags(client, bucket, key, Some("null"), Some("suspended-null")).await;
|
||||
assert_version_tags(client, bucket, key, Some(version), Some("versioned")).await;
|
||||
|
||||
enable_versioning(client, bucket).await.expect("re-enable versioning");
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.tagging("generation=latest")
|
||||
.body(ByteStream::from_static(b"latest version"))
|
||||
.send()
|
||||
.await
|
||||
.expect("write a new latest version");
|
||||
assert_version_tags(client, bucket, key, Some("null"), Some("suspended-null")).await;
|
||||
assert_version_tags(client, bucket, key, None, Some("latest")).await;
|
||||
client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.version_id("null")
|
||||
.send()
|
||||
.await
|
||||
.expect("remove only the null version");
|
||||
assert_version_tags(client, bucket, key, None, Some("latest")).await;
|
||||
let absent_version = uuid::Uuid::new_v4().to_string();
|
||||
for (missing_key, selector, expected_code) in [
|
||||
(key, Some("null"), "NoSuchVersion"),
|
||||
(key, Some(absent_version.as_str()), "NoSuchVersion"),
|
||||
("never-created", None, "NoSuchKey"),
|
||||
("never-created", Some("null"), "NoSuchVersion"),
|
||||
("never-created", Some(absent_version.as_str()), "NoSuchVersion"),
|
||||
] {
|
||||
let missing = client
|
||||
.get_object_tagging()
|
||||
.bucket(bucket)
|
||||
.key(missing_key)
|
||||
.set_version_id(selector.map(str::to_owned))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a missing version must not fall back to latest");
|
||||
assert_eq!(
|
||||
missing.as_service_error().and_then(ProvideErrorMetadata::code),
|
||||
Some(expected_code),
|
||||
"key: {missing_key}, version selector: {selector:?}"
|
||||
);
|
||||
}
|
||||
assert_version_tags(client, bucket, key, None, Some("latest")).await;
|
||||
}
|
||||
|
||||
/// Test 1: PutObject should return version_id when versioning is enabled
|
||||
/// This directly addresses the Veeam issue from #1066
|
||||
#[tokio::test]
|
||||
@@ -414,9 +262,7 @@ mod tests {
|
||||
info!("🧪 TEST: PutObject behavior without versioning (no regression)");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
|
||||
env.start_rustfs_server_without_cleanup(vec![])
|
||||
.await
|
||||
.expect("Failed to start isolated RustFS");
|
||||
env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
|
||||
|
||||
let client = create_s3_client(&env);
|
||||
let bucket = "test-no-versioning";
|
||||
@@ -444,9 +290,6 @@ mod tests {
|
||||
output.version_id().is_none() || output.version_id() == Some("null"),
|
||||
"non-versioned PUT must omit version ID or return the S3 null version"
|
||||
);
|
||||
// Reuse this unversioned fixture to prove explicit null never becomes
|
||||
// an implicit latest-version read after enable/suspend transitions.
|
||||
assert_null_tagging_across_versioning_changes(&client, bucket, key).await;
|
||||
info!("✅ PASSED: PutObject works correctly without versioning");
|
||||
}
|
||||
|
||||
|
||||
@@ -202,7 +202,9 @@ pub mod bucket {
|
||||
}
|
||||
|
||||
pub mod migration {
|
||||
pub use crate::bucket::migration::{LegacyBlobDecryptFn, try_migrate_bucket_metadata, try_migrate_iam_config};
|
||||
pub use crate::bucket::migration::{
|
||||
LegacyBlobDecryptFn, migration_startup_error, try_migrate_bucket_metadata, try_migrate_iam_config,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod object_lock {
|
||||
|
||||
@@ -822,7 +822,7 @@ async fn scan_exact_free_version_targets(
|
||||
let mut targets = Vec::new();
|
||||
for pool in &api.pools {
|
||||
for set in &pool.disk_set {
|
||||
let versions = match set.load_file_info_versions_for_tier_cleanup(&oi.bucket, &oi.name).await {
|
||||
let versions = match set.load_file_info_versions_exact(&oi.bucket, &oi.name).await {
|
||||
Ok(Some(versions)) => versions,
|
||||
Ok(None) => continue,
|
||||
Err(err) if is_err_strict_volume_not_found(&err) => continue,
|
||||
@@ -8100,75 +8100,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn tier_overwrite_cleanup_retains_a_minority_live_remote_reference() {
|
||||
let (disk_paths, ecstore) = setup_test_env().await;
|
||||
let bucket = format!("overwrite-minority-{}", Uuid::new_v4());
|
||||
let object = "still-referenced";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
let (backend, identity) = register_recovery_mock_tier(&ecstore).await;
|
||||
seed_recoverable_free_version(&disk_paths, &bucket, object, None, Some(identity.clone())).await;
|
||||
let page = list_tier_free_versions(Arc::clone(&ecstore), 100, None, None, CancellationToken::new())
|
||||
.await
|
||||
.expect("list persisted cleanup owner");
|
||||
let owner = page.items.into_iter().find(|oi| oi.bucket == bucket).expect("seeded owner");
|
||||
backend
|
||||
.set_put_remote_version(Some(owner.transitioned_object.version_id.clone()))
|
||||
.await;
|
||||
let lease = TierConfigMgr::acquire_operation_lease(&ecstore.tier_config_mgr(), "WARM")
|
||||
.await
|
||||
.expect("remote fixture lease");
|
||||
lease
|
||||
.put(
|
||||
&owner.transitioned_object.name,
|
||||
rustfs_s3_client::transition_api::ReaderImpl::Body(bytes::Bytes::from_static(b"old")),
|
||||
3,
|
||||
)
|
||||
.await
|
||||
.expect("seed referenced remote bytes");
|
||||
drop(lease);
|
||||
let path = disk_paths[0].join(&bucket).join(object).join(STORAGE_FORMAT_FILE);
|
||||
let cleanup_metadata = fs::read(&path).await.expect("save completed replica");
|
||||
let mut live = FileInfo::new(object, 2, 2);
|
||||
live.volume = bucket.clone();
|
||||
live.erasure.index = 1;
|
||||
live.data_dir = Some(Uuid::new_v4());
|
||||
live.mod_time = Some(OffsetDateTime::now_utc());
|
||||
live.size = 3;
|
||||
live.add_object_part(1, "149603e6c03516362a8da23f624db945".to_string(), 3, live.mod_time, 3, None, None);
|
||||
live.transition_status = TRANSITION_COMPLETE.to_string();
|
||||
live.transition_tier = "WARM".to_string();
|
||||
live.transitioned_objname = owner.transitioned_object.name.clone();
|
||||
live.transition_version = Some(owner.transitioned_object.version_id.clone());
|
||||
live.transition_version_state = rustfs_filemeta::TransitionVersionState::Exact;
|
||||
rustfs_utils::http::insert_str(&mut live.metadata, rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID, identity);
|
||||
let mut old_metadata = FileMeta::new();
|
||||
old_metadata.add_version(live).expect("prepare minority live source");
|
||||
fs::write(&path, old_metadata.marshal_msg().expect("encode live source"))
|
||||
.await
|
||||
.expect("model one replica retained by an interrupted overwrite");
|
||||
|
||||
let err = super::cleanup_free_version_exact(Arc::clone(&ecstore), &owner, &CancellationToken::new())
|
||||
.await
|
||||
.expect_err("quorum free versions cannot erase a minority live reference");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::WouldBlock);
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
assert!(backend.contains(&owner.transitioned_object.name).await);
|
||||
|
||||
fs::write(&path, cleanup_metadata)
|
||||
.await
|
||||
.expect("complete replica convergence");
|
||||
assert!(
|
||||
super::cleanup_free_version_exact(Arc::clone(&ecstore), &owner, &CancellationToken::new())
|
||||
.await
|
||||
.expect("converged cleanup can delete the exact remote owner")
|
||||
);
|
||||
assert_eq!(backend.remove_count().await, 1);
|
||||
assert!(!backend.contains(&owner.transitioned_object.name).await);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
use crate::bucket::metadata::BUCKET_METADATA_FILE;
|
||||
use crate::bucket::replication::ReplicationMigrationBridge;
|
||||
use crate::disk::{BUCKET_META_PREFIX, MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
|
||||
use crate::error::Error;
|
||||
use crate::error::{Error, Result, is_err_strict_not_found, is_err_strict_volume_not_found};
|
||||
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
|
||||
use crate::storage_api_contracts::{
|
||||
bucket::{BucketOperations, BucketOptions},
|
||||
@@ -33,7 +33,7 @@ use rustfs_utils::path::SLASH_SEPARATOR;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::sync::Arc;
|
||||
use time::OffsetDateTime;
|
||||
use tracing::{debug, info, warn};
|
||||
use tracing::{debug, info};
|
||||
|
||||
/// IAM config prefix under meta bucket (e.g. config/iam/).
|
||||
const IAM_CONFIG_PREFIX: &str = "config/iam";
|
||||
@@ -53,6 +53,39 @@ type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
|
||||
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
|
||||
|
||||
#[derive(Clone, Debug, thiserror::Error)]
|
||||
enum MigrationMetadataError {
|
||||
#[error("empty legacy metadata: {0}")]
|
||||
Empty(String),
|
||||
#[error("incompatible legacy metadata: {0}")]
|
||||
Incompatible(String),
|
||||
}
|
||||
|
||||
impl From<MigrationMetadataError> for Error {
|
||||
fn from(error: MigrationMetadataError) -> Self {
|
||||
let message = match &error {
|
||||
MigrationMetadataError::Empty(_) => "empty legacy metadata",
|
||||
MigrationMetadataError::Incompatible(_) => "incompatible legacy metadata",
|
||||
};
|
||||
// Keep the record path in the typed source, not in the quorum grouping key.
|
||||
Self::other_with_context(message, error)
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts a migration failure at the startup boundary, rendering the safe
|
||||
/// record path while leaving storage-layer error grouping stable.
|
||||
pub fn migration_startup_error(error: Error) -> std::io::Error {
|
||||
if let Error::Io(io_error) = &error
|
||||
&& let Some(metadata_error) = io_error
|
||||
.get_ref()
|
||||
.and_then(|context| context.source())
|
||||
.and_then(|source| source.downcast_ref::<MigrationMetadataError>())
|
||||
{
|
||||
return std::io::Error::other(metadata_error.clone());
|
||||
}
|
||||
std::io::Error::other(error)
|
||||
}
|
||||
|
||||
/// Callback used to decrypt an at-rest config blob during MinIO -> RustFS migration.
|
||||
///
|
||||
/// MinIO encrypts IAM identity/service-account files and the server config at rest
|
||||
@@ -211,7 +244,7 @@ fn normalize_bucket_meta_blob(path: &str, data: &[u8]) -> std::result::Result<Op
|
||||
/// Uses list_bucket (from disk volumes) to get bucket names, since list_objects_v2 on the legacy
|
||||
/// meta bucket may not work (legacy format differs from object layer expectations).
|
||||
/// Skips buckets that already exist in RustFS (idempotent).
|
||||
pub async fn try_migrate_bucket_metadata<S>(store: Arc<S>)
|
||||
pub async fn try_migrate_bucket_metadata<S>(store: Arc<S>) -> Result<()>
|
||||
where
|
||||
S: BucketOperations<Error = crate::error::Error>
|
||||
+ ObjectIO<
|
||||
@@ -231,25 +264,18 @@ where
|
||||
DeletedObject = DeletedObject,
|
||||
>,
|
||||
{
|
||||
let buckets_list = match store
|
||||
let buckets_list = store
|
||||
.list_bucket(&BucketOptions {
|
||||
no_metadata: true,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(b) => b,
|
||||
Err(e) => {
|
||||
warn!("list buckets failed (skip migration): {e}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
.await?;
|
||||
|
||||
let buckets: Vec<String> = buckets_list.into_iter().map(|b| b.name).collect();
|
||||
|
||||
if buckets.is_empty() {
|
||||
debug!("No migrating bucket metadata found");
|
||||
return;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
debug!("Found {} migrating bucket metadata, migrating...", buckets.len());
|
||||
@@ -263,26 +289,40 @@ where
|
||||
|
||||
for bucket in buckets {
|
||||
let meta_path = format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}{bucket}{SLASH_SEPARATOR}{BUCKET_METADATA_FILE}");
|
||||
migrate_one_if_missing(store.clone(), &opts, &h, &meta_path, &format!("bucket metadata: {bucket}")).await;
|
||||
migrate_one_if_missing(store.clone(), &opts, &h, &meta_path, &format!("bucket metadata: {bucket}")).await?;
|
||||
|
||||
let resync_path = format!(
|
||||
"{BUCKET_META_PREFIX}{SLASH_SEPARATOR}{bucket}{SLASH_SEPARATOR}{REPLICATION_META_DIR}{SLASH_SEPARATOR}{RESYNC_META_FILE}"
|
||||
);
|
||||
migrate_one_if_missing(store.clone(), &opts, &h, &resync_path, &format!("bucket replication resync: {bucket}")).await;
|
||||
migrate_one_if_missing(store.clone(), &opts, &h, &resync_path, &format!("bucket replication resync: {bucket}")).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn migration_target_exists<S: EcstoreObjectOperations>(store: &S, path: &str) -> Result<bool> {
|
||||
match store
|
||||
.get_object_info(RUSTFS_META_BUCKET, path, &ObjectOptions::default())
|
||||
.await
|
||||
{
|
||||
Ok(_) => Ok(true),
|
||||
Err(err) if is_err_strict_not_found(&err) || is_err_strict_volume_not_found(&err) => Ok(false),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
async fn migrate_one_if_missing<S>(store: Arc<S>, opts: &ObjectOptions, headers: &HeaderMap, path: &str, label: &str)
|
||||
async fn migrate_one_if_missing<S>(
|
||||
store: Arc<S>,
|
||||
opts: &ObjectOptions,
|
||||
headers: &HeaderMap,
|
||||
path: &str,
|
||||
label: &str,
|
||||
) -> Result<()>
|
||||
where
|
||||
S: EcstoreObjectIO + EcstoreObjectOperations,
|
||||
{
|
||||
if store
|
||||
.get_object_info(RUSTFS_META_BUCKET, path, &ObjectOptions::default())
|
||||
.await
|
||||
.is_ok()
|
||||
{
|
||||
if migration_target_exists(store.as_ref(), path).await? {
|
||||
debug!("{label} already exists in RustFS, skip");
|
||||
return;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut rd = match store
|
||||
@@ -290,43 +330,31 @@ where
|
||||
.await
|
||||
{
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
debug!("read migrating {label}: {e}");
|
||||
return;
|
||||
}
|
||||
// Ordinary RustFS deployments have no legacy bucket, and optional
|
||||
// legacy settings (such as replication resync) may not exist.
|
||||
Err(err) if is_err_strict_not_found(&err) || is_err_strict_volume_not_found(&err) => return Ok(()),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
|
||||
let data = match rd.read_all().await {
|
||||
Ok(d) if !d.is_empty() => d,
|
||||
Ok(_) => return,
|
||||
Err(e) => {
|
||||
debug!("read migrating {label} body: {e}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let data = match normalize_bucket_meta_blob(path, &data) {
|
||||
Ok(Some(normalized)) => normalized,
|
||||
Ok(None) => data,
|
||||
Err(e) => {
|
||||
warn!("skip {label} migration due to incompatible format: {e}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let data = rd.read_all().await?;
|
||||
if data.is_empty() {
|
||||
return Err(MigrationMetadataError::Empty(path.to_owned()).into());
|
||||
}
|
||||
let data = normalize_bucket_meta_blob(path, &data)
|
||||
.map_err(|_| MigrationMetadataError::Incompatible(path.to_owned()))?
|
||||
.unwrap_or(data);
|
||||
|
||||
let mut put_data = PutObjReader::from_vec(data);
|
||||
if let Err(e) = store.put_object(RUSTFS_META_BUCKET, path, &mut put_data, opts).await {
|
||||
warn!("write {label}: {e}");
|
||||
} else {
|
||||
info!("Migrated {label}");
|
||||
}
|
||||
store.put_object(RUSTFS_META_BUCKET, path, &mut put_data, opts).await?;
|
||||
info!("Migrated {label}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Migrates IAM config from legacy meta bucket `config/iam/` to RustFS meta bucket.
|
||||
/// Lists all objects under the IAM prefix in the source, copies each to the target if not present.
|
||||
/// Skips objects that already exist in RustFS (idempotent).
|
||||
/// If list_objects_v2 on the legacy bucket fails (e.g. format differs), migration is skipped.
|
||||
pub async fn try_migrate_iam_config<S>(store: Arc<S>, decrypt_fn: Option<LegacyBlobDecryptFn>)
|
||||
/// An absent legacy bucket is a no-op; migration errors prevent startup readiness.
|
||||
pub async fn try_migrate_iam_config<S>(store: Arc<S>, decrypt_fn: Option<LegacyBlobDecryptFn>) -> Result<()>
|
||||
where
|
||||
S: ListOperations<
|
||||
Error = crate::error::Error,
|
||||
@@ -366,47 +394,36 @@ where
|
||||
loop {
|
||||
let list_result = match store
|
||||
.clone()
|
||||
.list_objects_v2(MIGRATING_META_BUCKET, &prefix, continuation, None, 500, false, None, false)
|
||||
.list_objects_v2(MIGRATING_META_BUCKET, &prefix, continuation.clone(), None, 500, false, None, false)
|
||||
.await
|
||||
{
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
debug!("list IAM config from legacy bucket failed (skip migration): {e}");
|
||||
return;
|
||||
}
|
||||
Err(err) if is_err_strict_volume_not_found(&err) => return Ok(()),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
|
||||
for obj in list_result.objects {
|
||||
let path = &obj.name;
|
||||
if path.is_empty() || path.ends_with('/') {
|
||||
// Unsupported records must not trigger target lookups, reads, or decryption.
|
||||
if path != IAM_FORMAT_FILE_PATH
|
||||
&& !is_identity_path(path)
|
||||
&& !is_group_path(path)
|
||||
&& !is_policy_doc_path(path)
|
||||
&& !is_policy_mapping_path(path)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if store
|
||||
.get_object_info(RUSTFS_META_BUCKET, path, &ObjectOptions::default())
|
||||
.await
|
||||
.is_ok()
|
||||
{
|
||||
if migration_target_exists(store.as_ref(), path).await? {
|
||||
debug!("IAM config already exists in RustFS, skip: {path}");
|
||||
continue;
|
||||
}
|
||||
let mut rd = match store
|
||||
let mut rd = store
|
||||
.get_object_reader(MIGRATING_META_BUCKET, path, None, h.clone(), &opts)
|
||||
.await
|
||||
{
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
debug!("read migrating IAM config {path}: {e}");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let data = match rd.read_all().await {
|
||||
Ok(d) if !d.is_empty() => d,
|
||||
Ok(_) => continue,
|
||||
Err(e) => {
|
||||
debug!("read migrating IAM config {path} body: {e}");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
.await?;
|
||||
let data = rd.read_all().await?;
|
||||
if data.is_empty() {
|
||||
return Err(MigrationMetadataError::Empty(path.to_owned()).into());
|
||||
}
|
||||
// MinIO encrypts IAM identity/service-account files at rest. Decrypt
|
||||
// before normalizing; fall back to the raw bytes when no key applies
|
||||
// (plaintext blobs, or nothing to decrypt) so existing behavior holds.
|
||||
@@ -420,22 +437,17 @@ where
|
||||
debug!("skip unsupported IAM config path during migration: {path}");
|
||||
continue;
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("skip IAM config migration due to incompatible format, path: {path}, err: {e}");
|
||||
continue;
|
||||
}
|
||||
// Parser errors may contain credential data. Report only the path.
|
||||
Err(_) => return Err(MigrationMetadataError::Incompatible(path.to_owned()).into()),
|
||||
};
|
||||
let mut put_data = PutObjReader::from_vec(data);
|
||||
if let Err(e) = store.put_object(RUSTFS_META_BUCKET, path, &mut put_data, &opts).await {
|
||||
warn!("write IAM config {path}: {e}");
|
||||
} else {
|
||||
info!("Migrated IAM config: {path}");
|
||||
total_migrated += 1;
|
||||
}
|
||||
store.put_object(RUSTFS_META_BUCKET, path, &mut put_data, &opts).await?;
|
||||
info!("Migrated IAM config: {path}");
|
||||
total_migrated += 1;
|
||||
}
|
||||
|
||||
continuation = list_result.next_continuation_token.or(list_result.continuation_token);
|
||||
if !list_result.is_truncated || continuation.is_none() {
|
||||
continuation = next_iam_migration_page(list_result.is_truncated, continuation, list_result.next_continuation_token)?;
|
||||
if continuation.is_none() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -443,10 +455,74 @@ where
|
||||
if total_migrated > 0 {
|
||||
info!("IAM migration complete: {} object(s) migrated", total_migrated);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn next_iam_migration_page(truncated: bool, previous: Option<String>, next: Option<String>) -> Result<Option<String>> {
|
||||
if !truncated {
|
||||
return Ok(None);
|
||||
}
|
||||
let next = next.filter(|token| !token.is_empty());
|
||||
if next.is_none() || next == previous {
|
||||
return Err(Error::other("legacy IAM migration listing did not advance"));
|
||||
}
|
||||
Ok(next)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn migration_errors_group_by_cause_and_retain_typed_record_context() {
|
||||
use super::{Error, MigrationMetadataError};
|
||||
|
||||
for (make_error, message) in [
|
||||
(
|
||||
MigrationMetadataError::Empty as fn(String) -> MigrationMetadataError,
|
||||
"empty legacy metadata",
|
||||
),
|
||||
(MigrationMetadataError::Incompatible, "incompatible legacy metadata"),
|
||||
] {
|
||||
let first: Error = make_error("buckets/first/.metadata.bin".into()).into();
|
||||
let second: Error = make_error("buckets/second/.metadata.bin".into()).into();
|
||||
assert_eq!(first, second, "record paths must not fragment error grouping");
|
||||
assert_eq!(first.clone(), second, "cloning must preserve error grouping");
|
||||
|
||||
let io_error = std::io::Error::from(first);
|
||||
let detail = io_error
|
||||
.get_ref()
|
||||
.and_then(|context| context.source())
|
||||
.expect("record context must remain in the error source");
|
||||
assert!(detail.downcast_ref::<MigrationMetadataError>().is_some());
|
||||
assert!(detail.to_string().contains("buckets/first/.metadata.bin"));
|
||||
|
||||
let startup_error = super::migration_startup_error(make_error("buckets/startup/.metadata.bin".into()).into());
|
||||
assert!(
|
||||
startup_error
|
||||
.get_ref()
|
||||
.is_some_and(|source| source.is::<MigrationMetadataError>())
|
||||
);
|
||||
assert_eq!(startup_error.to_string(), format!("{message}: buckets/startup/.metadata.bin"));
|
||||
}
|
||||
assert_ne!(
|
||||
Error::from(MigrationMetadataError::Empty("record".into())),
|
||||
Error::from(MigrationMetadataError::Incompatible("record".into())),
|
||||
"different migration failures must remain distinguishable"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncated_iam_listing_cannot_report_completed_migration() {
|
||||
use super::next_iam_migration_page;
|
||||
assert_eq!(next_iam_migration_page(false, Some("old".into()), None).expect("final page"), None);
|
||||
assert_eq!(
|
||||
next_iam_migration_page(true, Some("old".into()), Some("next".into())).expect("advancing page"),
|
||||
Some("next".into())
|
||||
);
|
||||
for next in [None, Some(String::new()), Some("old".into())] {
|
||||
assert!(next_iam_migration_page(true, Some("old".into()), next).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
use super::{normalize_bucket_meta_blob, normalize_iam_config_blob};
|
||||
use crate::bucket::replication::{
|
||||
BucketReplicationResyncStatus, ReplicationMigrationBridge, ResyncStatusType, TargetReplicationResyncStatus,
|
||||
@@ -659,6 +735,13 @@ mod tests {
|
||||
.collect();
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), existing).await;
|
||||
|
||||
super::try_migrate_bucket_metadata(ecstore.clone())
|
||||
.await
|
||||
.expect("fresh stores do not require a legacy metadata bucket");
|
||||
super::try_migrate_iam_config(ecstore.clone(), None)
|
||||
.await
|
||||
.expect("fresh stores do not require a legacy IAM bucket");
|
||||
|
||||
let meta_path = format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}interop{SLASH_SEPARATOR}{BUCKET_METADATA_FILE}");
|
||||
let put_opts = ObjectOptions::default();
|
||||
|
||||
@@ -680,8 +763,31 @@ mod tests {
|
||||
.await
|
||||
.expect("seed .minio.sys bucket metadata");
|
||||
|
||||
// --- Run the real startup migration. ---
|
||||
super::try_migrate_bucket_metadata(ecstore.clone()).await;
|
||||
// A partial import must report failure, even if the main bucket
|
||||
// metadata copied successfully before an incompatible resync record.
|
||||
let resync_path = format!("{BUCKET_META_PREFIX}/interop/.replication/resync.bin");
|
||||
ecstore
|
||||
.put_object(
|
||||
MIGRATING_META_BUCKET,
|
||||
&resync_path,
|
||||
&mut PutObjReader::from_vec(b"invalid resync metadata".to_vec()),
|
||||
&put_opts,
|
||||
)
|
||||
.await
|
||||
.expect("seed malformed legacy resync metadata");
|
||||
assert!(
|
||||
super::try_migrate_bucket_metadata(ecstore.clone()).await.is_err(),
|
||||
"incompatible native metadata must not be reported as a completed migration"
|
||||
);
|
||||
ecstore
|
||||
.delete_object(MIGRATING_META_BUCKET, &resync_path, ObjectOptions::default())
|
||||
.await
|
||||
.expect("remove invalid optional legacy resync record");
|
||||
|
||||
// Retry the real startup migration after repairing the source.
|
||||
super::try_migrate_bucket_metadata(ecstore.clone())
|
||||
.await
|
||||
.expect("native bucket metadata migration completes");
|
||||
|
||||
// --- The migrated `.rustfs.sys` blob must carry every MinIO config, ---
|
||||
// byte-identical to the source (typed XML/JSON parsing of these fields is
|
||||
|
||||
@@ -6409,6 +6409,10 @@ impl LocalDisk {
|
||||
// A missing or still-populated directory is benign here; see
|
||||
// is_benign_object_rmdir_error (handles the illumos/Solaris EEXIST
|
||||
// convention, rustfs/rustfs#4978).
|
||||
if is_dir_not_empty_error(&err) {
|
||||
// A populated directory keeps its ancestors populated; no further pruning is needed.
|
||||
return Ok(());
|
||||
}
|
||||
if !is_benign_object_rmdir_error(&err) {
|
||||
warn!(
|
||||
event = EVENT_DISK_LOCAL_DELETE_FAILED,
|
||||
@@ -11222,6 +11226,176 @@ mod test {
|
||||
(disk, dir)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_pruning_stops_at_live_metadata_below_a_guarded_ancestor() {
|
||||
// Tuple fields drop in order, releasing the disk's root handle before the temporary directory.
|
||||
let fixture = new_disk().await;
|
||||
let (disk, _dir) = &fixture;
|
||||
let base = disk.get_bucket_path(RUSTFS_META_BUCKET).expect("resolve metadata volume");
|
||||
let shared = base.join("buckets");
|
||||
let guard = Arc::new(
|
||||
os::mkdir_all_below_existing_base_std(&shared, &base, &disk.publication_root)
|
||||
.expect("retain the shared publication directory"),
|
||||
);
|
||||
|
||||
for owned in [false, true] {
|
||||
for missing_backup in [false, true] {
|
||||
let transaction = Uuid::new_v4();
|
||||
let object = shared.join(".bloomcycle.bin");
|
||||
let rollback = object.join(transaction.to_string());
|
||||
let metadata = object.join(STORAGE_FORMAT_FILE);
|
||||
let backup = rollback.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
fs::create_dir_all(&rollback).await.expect("create rollback directory");
|
||||
fs::write(&metadata, b"committed metadata")
|
||||
.await
|
||||
.expect("write live metadata");
|
||||
if !missing_backup {
|
||||
fs::write(&backup, b"old metadata").await.expect("write rollback backup");
|
||||
}
|
||||
let owner: Option<Arc<dyn Send + Sync>> = if owned { Some(guard.clone()) } else { None };
|
||||
let result = disk
|
||||
.delete_with_namespace_owner(
|
||||
RUSTFS_META_BUCKET,
|
||||
&format!("buckets/.bloomcycle.bin/{transaction}/{STORAGE_FORMAT_FILE_BACKUP}"),
|
||||
DeleteOptions::default(),
|
||||
owner,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(!backup.exists(), "backup must be absent, owned={owned}, missing={missing_backup}");
|
||||
assert!(!rollback.exists(), "empty rollback directory must be pruned");
|
||||
assert_eq!(fs::read(&metadata).await.expect("read committed metadata"), b"committed metadata");
|
||||
result.expect("a nonempty object must stop pruning before the guarded ancestor");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_pruning_removes_empty_and_missing_ancestors_but_keeps_the_volume() {
|
||||
let fixture = new_disk().await;
|
||||
let (disk, _dir) = &fixture;
|
||||
ensure_test_volume(disk, "pruning").await;
|
||||
let base = disk.get_bucket_path("pruning").expect("resolve test volume");
|
||||
|
||||
for missing in [false, true] {
|
||||
let parent = base.join("parent");
|
||||
let rollback = parent.join("object/transaction");
|
||||
fs::create_dir_all(&rollback).await.expect("create empty ancestor chain");
|
||||
let path = if missing {
|
||||
"parent/object/transaction/missing/xl.meta.bkp"
|
||||
} else {
|
||||
fs::write(rollback.join(STORAGE_FORMAT_FILE_BACKUP), b"backup")
|
||||
.await
|
||||
.expect("create backup");
|
||||
"parent/object/transaction/xl.meta.bkp"
|
||||
};
|
||||
|
||||
disk.delete("pruning", path, DeleteOptions::default())
|
||||
.await
|
||||
.expect("empty and missing ancestors should be pruned");
|
||||
assert!(!parent.exists(), "the whole empty chain should be removed");
|
||||
assert!(base.is_dir(), "pruning must stop at the volume boundary");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_pruning_does_not_remove_the_base_or_an_outside_path() {
|
||||
let fixture = new_disk().await;
|
||||
let (disk, dir) = &fixture;
|
||||
let base = dir.path().join("base");
|
||||
let outside = dir.path().join("outside");
|
||||
fs::create_dir(&base).await.expect("create base");
|
||||
fs::write(&outside, b"outside data").await.expect("create outside file");
|
||||
|
||||
disk.delete_file(&base, &base, false, false)
|
||||
.await
|
||||
.expect("base path is protected");
|
||||
disk.delete_file(&base, &outside, false, false)
|
||||
.await
|
||||
.expect("outside path is protected");
|
||||
assert!(base.is_dir(), "the base must not be removed even when empty");
|
||||
assert_eq!(fs::read(&outside).await.expect("read outside file"), b"outside data");
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[tokio::test]
|
||||
async fn delete_pruning_propagates_a_locked_backup_error() {
|
||||
use std::os::windows::fs::OpenOptionsExt;
|
||||
use windows_sys::Win32::{Foundation::ERROR_SHARING_VIOLATION, Storage::FileSystem::FILE_SHARE_READ};
|
||||
|
||||
let fixture = new_disk().await;
|
||||
let (disk, _dir) = &fixture;
|
||||
ensure_test_volume(disk, "pruning").await;
|
||||
let base = disk.get_bucket_path("pruning").expect("resolve test volume");
|
||||
let backup = base.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
fs::write(&backup, b"backup").await.expect("write backup");
|
||||
let guard = std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.share_mode(FILE_SHARE_READ)
|
||||
.open(&backup)
|
||||
.expect("hold the backup without delete sharing");
|
||||
|
||||
let err = disk
|
||||
.delete("pruning", STORAGE_FORMAT_FILE_BACKUP, DeleteOptions::default())
|
||||
.await
|
||||
.expect_err("a genuine target-file deletion failure must propagate");
|
||||
let DiskError::Io(err) = err else {
|
||||
panic!("expected contextual I/O error, got {err:?}");
|
||||
};
|
||||
let context = err
|
||||
.get_ref()
|
||||
.and_then(|err| err.downcast_ref::<FileAccessDeniedWithContext>())
|
||||
.expect("preserve the failing path and original OS error");
|
||||
assert_eq!(context.path, backup);
|
||||
assert_eq!(
|
||||
context.source.raw_os_error(),
|
||||
Some(i32::try_from(ERROR_SHARING_VIOLATION).expect("OS code fits"))
|
||||
);
|
||||
assert_eq!(fs::read(&backup).await.expect("backup remains readable"), b"backup");
|
||||
drop(guard);
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[tokio::test]
|
||||
async fn delete_pruning_propagates_a_locked_empty_parent_error() {
|
||||
use windows_sys::Win32::Foundation::ERROR_SHARING_VIOLATION;
|
||||
|
||||
let fixture = new_disk().await;
|
||||
let (disk, _dir) = &fixture;
|
||||
ensure_test_volume(disk, "pruning").await;
|
||||
let base = disk.get_bucket_path("pruning").expect("resolve test volume");
|
||||
let parent = base.join("parent");
|
||||
let guard = os::mkdir_all_below_existing_base_std(&parent, &base, &disk.publication_root)
|
||||
.expect("retain an empty parent without delete sharing");
|
||||
let backup = parent.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
fs::write(&backup, b"backup").await.expect("write backup");
|
||||
|
||||
let err = disk
|
||||
.delete("pruning", "parent/xl.meta.bkp", DeleteOptions::default())
|
||||
.await
|
||||
.expect_err("a real parent failure without a nonempty boundary must still propagate");
|
||||
let DiskError::Io(err) = err else {
|
||||
panic!("expected contextual I/O error, got {err:?}");
|
||||
};
|
||||
let context = err
|
||||
.get_ref()
|
||||
.and_then(|err| err.downcast_ref::<FileAccessDeniedWithContext>())
|
||||
.expect("preserve parent failure context");
|
||||
assert_eq!(context.path, parent);
|
||||
assert_eq!(
|
||||
context.source.raw_os_error(),
|
||||
Some(i32::try_from(ERROR_SHARING_VIOLATION).expect("OS code fits"))
|
||||
);
|
||||
assert!(!backup.exists(), "the target was removed before the parent error");
|
||||
assert!(parent.is_dir(), "the guarded parent remains");
|
||||
drop(guard);
|
||||
disk.delete("pruning", "parent/xl.meta.bkp", DeleteOptions::default())
|
||||
.await
|
||||
.expect("pruning should succeed once the actual guard is released");
|
||||
assert!(!parent.exists());
|
||||
assert!(base.is_dir());
|
||||
}
|
||||
|
||||
// #948: a genuinely missing source is benign and must still return Ok.
|
||||
#[tokio::test]
|
||||
async fn windows_and_unix_move_to_trash_missing_source_is_ok() {
|
||||
@@ -11705,14 +11879,59 @@ mod test {
|
||||
/// stale deterministically, instead of sleeping and hoping the filesystem
|
||||
/// timestamp granularity (or a backward wall-clock step) cooperates.
|
||||
fn backdate_mtime(path: &Path, age: Duration) {
|
||||
use std::fs::{File, FileTimes};
|
||||
use std::fs::{FileTimes, OpenOptions};
|
||||
let mtime = std::time::SystemTime::now() - age;
|
||||
File::open(path)
|
||||
let mut options = OpenOptions::new();
|
||||
options.read(true);
|
||||
#[cfg(windows)]
|
||||
{
|
||||
use std::os::windows::fs::OpenOptionsExt;
|
||||
use windows_sys::Win32::Storage::FileSystem::{FILE_FLAG_BACKUP_SEMANTICS, FILE_WRITE_ATTRIBUTES};
|
||||
|
||||
// Directories need backup semantics, and changing mtime needs attribute-write access.
|
||||
options
|
||||
.access_mode(FILE_WRITE_ATTRIBUTES)
|
||||
.custom_flags(FILE_FLAG_BACKUP_SEMANTICS);
|
||||
}
|
||||
options
|
||||
.open(path)
|
||||
.expect("path should open to backdate its mtime")
|
||||
.set_times(FileTimes::new().set_modified(mtime))
|
||||
.expect("mtime should rewind into the past");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cleanup_tmp_on_startup_backdate_mtime_preserves_files_and_directory_contents() {
|
||||
use std::time::SystemTime;
|
||||
|
||||
let root = tempfile::tempdir().expect("create timestamp fixture root");
|
||||
let directory = root.path().join("directory");
|
||||
let file = directory.join("payload");
|
||||
std::fs::create_dir(&directory).expect("create timestamp fixture directory");
|
||||
std::fs::write(&file, b"unchanged payload").expect("write timestamp fixture payload");
|
||||
let age = Duration::from_secs(60);
|
||||
// Filesystems may round stored timestamps; do not require subsecond precision or sleep.
|
||||
let rounding = Duration::from_secs(2);
|
||||
|
||||
for path in [&file, &directory] {
|
||||
let earliest = SystemTime::now() - age - rounding;
|
||||
backdate_mtime(path, age);
|
||||
let latest = SystemTime::now() - age + rounding;
|
||||
let modified = std::fs::metadata(path)
|
||||
.expect("read backdated path metadata")
|
||||
.modified()
|
||||
.expect("read backdated modification time");
|
||||
assert!(modified >= earliest && modified <= latest, "mtime must be backdated for {path:?}");
|
||||
}
|
||||
|
||||
let moved = root.path().join("moved");
|
||||
std::fs::rename(&directory, &moved).expect("mtime helper must release its handles before cleanup");
|
||||
assert_eq!(
|
||||
std::fs::read(moved.join("payload")).expect("read preserved payload"),
|
||||
b"unchanged payload"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn startup_cleanup_barrier_and_tmp_trash_cleanup_cover_noop_and_delete_paths() {
|
||||
use tempfile::tempdir;
|
||||
|
||||
@@ -47,6 +47,43 @@ use tokio::fs;
|
||||
use tracing::{info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Hold later repair publications after admitting one baseline object. The
|
||||
/// fixture arms this on one replacement disk before rejoining the cluster.
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
async fn wait_for_heal_commit_test_barrier(root: &Path, bucket: &str, object: &str) -> Result<()> {
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
let barrier = root.join(".rustfs.sys/e2e-heal-commit-barrier");
|
||||
let prefix = match fs::read_to_string(&barrier).await {
|
||||
Ok(prefix) => prefix,
|
||||
Err(error) if error.kind() == ErrorKind::NotFound => return Ok(()),
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
let key = format!("{bucket}/{object}");
|
||||
if prefix.is_empty() || !key.starts_with(&prefix) {
|
||||
return Ok(());
|
||||
}
|
||||
let admitted = barrier.with_extension("admitted");
|
||||
match fs::OpenOptions::new().write(true).create_new(true).open(&admitted).await {
|
||||
Ok(mut file) => {
|
||||
file.write_all(key.as_bytes()).await?;
|
||||
return Ok(());
|
||||
}
|
||||
Err(error) if error.kind() == ErrorKind::AlreadyExists => {}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(120);
|
||||
loop {
|
||||
if !fs::try_exists(&barrier).await? || fs::read_to_string(&admitted).await? == key {
|
||||
return Ok(());
|
||||
}
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Err(std::io::Error::new(ErrorKind::TimedOut, "heal commit test barrier was not released").into());
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
}
|
||||
}
|
||||
|
||||
fn rollback_committed_rename_std(
|
||||
dst_file_path: &Path,
|
||||
new_data_path: Option<&Path>,
|
||||
@@ -253,6 +290,10 @@ impl LocalDisk {
|
||||
state: &mut RenameDataState,
|
||||
) -> Result<RenameDataResp> {
|
||||
crate::hp_guard!("LocalDisk::rename_data");
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
if fi.is_healing() {
|
||||
wait_for_heal_commit_test_barrier(&self.root, dst_volume, dst_path).await?;
|
||||
}
|
||||
let mut fi = fi;
|
||||
// A non-force DeleteBucket must not remove a directory while a local
|
||||
// object commit is publishing into it. The peer's empty scan remains
|
||||
|
||||
@@ -1626,6 +1626,72 @@ mod tests {
|
||||
aborting_full_queue_settles_pending_send().await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn delayed_reader_error_keeps_source_and_drops_every_encode_path() {
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("injected request body inactivity")]
|
||||
struct BodyInactivity;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct StalledReader {
|
||||
data: Cursor<Vec<u8>>,
|
||||
timer: Option<Pin<Box<tokio::time::Sleep>>>,
|
||||
dropped: Arc<std::sync::atomic::AtomicBool>,
|
||||
}
|
||||
|
||||
impl AsyncRead for StalledReader {
|
||||
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
|
||||
if self.data.position() < self.data.get_ref().len() as u64 {
|
||||
return Pin::new(&mut self.data).poll_read(cx, buf);
|
||||
}
|
||||
let timer = self
|
||||
.timer
|
||||
.get_or_insert_with(|| Box::pin(tokio::time::sleep(Duration::from_secs(300))));
|
||||
std::task::ready!(timer.as_mut().poll(cx));
|
||||
Poll::Ready(Err(std::io::Error::other(BodyInactivity)))
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for StalledReader {
|
||||
fn drop(&mut self) {
|
||||
self.dropped.store(true, std::sync::atomic::Ordering::Release);
|
||||
}
|
||||
}
|
||||
|
||||
// Explicit entry points select the paths; environment caches and input
|
||||
// size heuristics cannot silently turn this into repeated Vec coverage.
|
||||
for path in ["direct", "vec", "bytesmut", "batched"] {
|
||||
let dropped = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let reader = StalledReader {
|
||||
data: Cursor::new(vec![7; 64]),
|
||||
timer: None,
|
||||
dropped: Arc::clone(&dropped),
|
||||
};
|
||||
let committed = Arc::new(Mutex::new(Vec::new()));
|
||||
let mut writers = (0..4)
|
||||
.map(|_| Some(bitrot_writer(DeferredCommitWriter::new(Arc::clone(&committed)), 32)))
|
||||
.collect::<Vec<_>>();
|
||||
let erasure = Arc::new(Erasure::new(2, 2, 64));
|
||||
let result = match path {
|
||||
"direct" => erasure.encode_single_block_non_inline(reader, &mut writers, 2).await,
|
||||
"vec" => erasure.encode_with_ingest_mode(reader, &mut writers, 2, false).await,
|
||||
"bytesmut" => erasure.encode_with_ingest_mode(reader, &mut writers, 2, true).await,
|
||||
"batched" => erasure.encode_batched(reader, &mut writers, 2).await,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
let error = result.expect_err("stalled input must fail before shard commit");
|
||||
assert!(error.get_ref().is_some_and(|source| source.is::<BodyInactivity>()), "{path}: {error:?}");
|
||||
assert!(
|
||||
dropped.load(std::sync::atomic::Ordering::Acquire),
|
||||
"{path} must release its reader/producer before returning"
|
||||
);
|
||||
assert!(
|
||||
committed.lock().expect("committed bytes").is_empty(),
|
||||
"{path} must not commit partial shards"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn helper_writers_cover_flush_and_shutdown_paths() {
|
||||
let mut failing_write = FailingWriteWriter;
|
||||
|
||||
@@ -3076,23 +3076,6 @@ impl SetDisks {
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
) -> Result<Option<rustfs_filemeta::FileInfoVersions>> {
|
||||
self.load_file_info_versions_for_cleanup(bucket, object, false).await
|
||||
}
|
||||
|
||||
pub(crate) async fn load_file_info_versions_for_tier_cleanup(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
) -> Result<Option<rustfs_filemeta::FileInfoVersions>> {
|
||||
self.load_file_info_versions_for_cleanup(bucket, object, true).await
|
||||
}
|
||||
|
||||
async fn load_file_info_versions_for_cleanup(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
retain_unconfirmed_tier_references: bool,
|
||||
) -> Result<Option<rustfs_filemeta::FileInfoVersions>> {
|
||||
let disk_object = rustfs_utils::path::encode_dir_object(object);
|
||||
let disks = self.get_disks_internal().await;
|
||||
@@ -3169,24 +3152,12 @@ impl SetDisks {
|
||||
)));
|
||||
}
|
||||
|
||||
let mut file_info_versions = FileMeta {
|
||||
let file_info_versions = FileMeta {
|
||||
versions,
|
||||
..Default::default()
|
||||
}
|
||||
.get_all_file_info_versions(bucket, object, true)
|
||||
.map_err(decode_error)?;
|
||||
if retain_unconfirmed_tier_references {
|
||||
// A failed overwrite may leave its live source on a minority
|
||||
// of disks. Preserve that reference even if quorum merging
|
||||
// selects only the replacement and its cleanup owner.
|
||||
file_info_versions.versions.extend(
|
||||
transition_copies
|
||||
.into_values()
|
||||
.flatten()
|
||||
.map(|(version, _)| version)
|
||||
.filter(|version| !version.tier_free_version()),
|
||||
);
|
||||
}
|
||||
|
||||
for file_info in file_info_versions
|
||||
.versions
|
||||
@@ -12243,64 +12214,6 @@ mod tests {
|
||||
assert!(result.is_err(), "missing disks must prevent metadata write quorum");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tier_overwrite_cleanup_rejects_unreadable_disk_despite_metadata_quorum() {
|
||||
let bucket = "tier-unreadable-disk";
|
||||
let object = "object";
|
||||
let mut dirs = Vec::new();
|
||||
let mut disks = Vec::new();
|
||||
let mut fi = metadata_test_fileinfo(object);
|
||||
fi.mod_time = Some(OffsetDateTime::now_utc());
|
||||
for index in 1..=3 {
|
||||
let (dir, disk) = read_multiple_test_disk(bucket, &[]).await;
|
||||
fi.erasure.index = index;
|
||||
disk.write_metadata(bucket, bucket, object, fi.clone())
|
||||
.await
|
||||
.expect("seed metadata quorum");
|
||||
dirs.push(dir);
|
||||
disks.push(Some(disk));
|
||||
}
|
||||
disks.push(None);
|
||||
let set = io_primitives_test_set(disks, 2).await;
|
||||
assert!(
|
||||
set.load_file_info_versions_exact(bucket, object).await.is_err(),
|
||||
"exact reads must preserve release's unreadable-replica fence"
|
||||
);
|
||||
assert!(
|
||||
set.load_file_info_versions_for_tier_cleanup(bucket, object).await.is_err(),
|
||||
"unreadable replica may still reference the old remote object"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tier_overwrite_cleanup_rejects_minority_metadata_in_an_absent_set() {
|
||||
let bucket = "tier-minority-metadata";
|
||||
let object = "object";
|
||||
let mut dirs = Vec::new();
|
||||
let mut disks = Vec::new();
|
||||
for index in 1..=4 {
|
||||
let (dir, disk) = read_multiple_test_disk(bucket, &[]).await;
|
||||
if index == 1 {
|
||||
let mut fi = metadata_test_fileinfo(object);
|
||||
fi.mod_time = Some(OffsetDateTime::now_utc());
|
||||
disk.write_metadata(bucket, bucket, object, fi)
|
||||
.await
|
||||
.expect("seed minority metadata");
|
||||
}
|
||||
dirs.push(dir);
|
||||
disks.push(Some(disk));
|
||||
}
|
||||
let set = io_primitives_test_set(disks, 2).await;
|
||||
assert!(
|
||||
set.load_file_info_versions_exact(bucket, object).await.is_err(),
|
||||
"exact reads must preserve release's minority-ownership fence"
|
||||
);
|
||||
assert!(
|
||||
set.load_file_info_versions_for_tier_cleanup(bucket, object).await.is_err(),
|
||||
"absence on a majority cannot prove this physical set has no remote reference"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn load_file_info_versions_exact_returns_versions_from_read_quorum() {
|
||||
let bucket = "exact-versions-bucket";
|
||||
|
||||
@@ -3280,65 +3280,6 @@ mod heal_result_report_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn deep_heal_rebuilds_missing_part_when_metadata_remains_current() {
|
||||
let (temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "deep-heal-missing-part-current-meta";
|
||||
let object = "object.bin";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
}
|
||||
|
||||
let payload = vec![0x7b; 1024 * 1024];
|
||||
let mut reader = PutObjReader::from_vec(payload);
|
||||
set.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("source object should be written before shard loss");
|
||||
let source = disks[2]
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("source metadata should be readable");
|
||||
let data_dir = source.data_dir.expect("non-inline source should have a data directory");
|
||||
let missing_part = temp_dirs[1]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(data_dir.to_string())
|
||||
.join("part.1");
|
||||
tokio::fs::remove_file(&missing_part)
|
||||
.await
|
||||
.expect("target shard should be removed while xl.meta remains current");
|
||||
|
||||
let (result, error) = set
|
||||
.heal_object(
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
no_lock: true,
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("deep heal should finish after a single shard is removed");
|
||||
|
||||
assert!(error.is_none(), "deep heal should recover the missing shard: {error:?}");
|
||||
assert_eq!(result.after.drives[1].state, DriveState::Ok.to_string());
|
||||
assert!(
|
||||
missing_part.exists(),
|
||||
"deep heal must reconstruct the missing shard on the original disk slot"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replacement_target_readback_checks_the_requested_historical_version() {
|
||||
let (temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
|
||||
@@ -5767,6 +5767,61 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn capped_staging_queue_does_not_poll_the_part_reader() {
|
||||
use futures::StreamExt;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-staging-body-demand";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disks, bucket).await;
|
||||
let mut options = ObjectOptions::default();
|
||||
insert_str(&mut options.user_defined, "max-total-object-size", "1024".to_owned());
|
||||
let upload = set_disks
|
||||
.new_multipart_upload(bucket, object, &options)
|
||||
.await
|
||||
.expect("capped upload");
|
||||
let upload_path = SetDisks::get_upload_id_dir(bucket, object, &upload.upload_id);
|
||||
let semaphore = capped_multipart_staging_semaphore(&upload_path);
|
||||
let held = Arc::clone(&semaphore).acquire_owned().await.expect("hold staging permit");
|
||||
let owners = Arc::strong_count(&semaphore);
|
||||
let polls = Arc::new(AtomicUsize::new(0));
|
||||
let body_polls = Arc::clone(&polls);
|
||||
let stream = futures::stream::iter([Ok::<Bytes, std::io::Error>(Bytes::from(vec![7; 512]))]).inspect(move |_| {
|
||||
body_polls.fetch_add(1, Ordering::Relaxed);
|
||||
});
|
||||
let input = tokio_util::io::StreamReader::new(stream);
|
||||
let mut reader = PutObjReader::new(HashReader::from_stream(input, 512, 512, None, None, false).expect("part reader"));
|
||||
let task = tokio::spawn(async move {
|
||||
set_disks
|
||||
.put_object_part(bucket, object, &upload.upload_id, 1, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
while Arc::strong_count(&semaphore) == owners {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("part must reach the actual staging semaphore");
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(600)).await;
|
||||
tokio::time::resume();
|
||||
assert_eq!(polls.load(Ordering::Relaxed), 0, "staging admission must not create read demand");
|
||||
assert!(!task.is_finished());
|
||||
drop(held);
|
||||
let part = tokio::time::timeout(Duration::from_secs(10), task)
|
||||
.await
|
||||
.expect("staging permit released")
|
||||
.expect("part task")
|
||||
.expect("queued part");
|
||||
assert_eq!(part.size, 512);
|
||||
assert_eq!(polls.load(Ordering::Relaxed), 1);
|
||||
drop(semaphore);
|
||||
remove_capped_multipart_staging_semaphore(&upload_path);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn put_object_part_recovers_transaction_with_one_faulty_disk_at_write_quorum() {
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
|
||||
@@ -3821,13 +3821,6 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
fi.metadata = user_defined;
|
||||
if fi.version_id.is_none_or(|id| id.is_nil()) && !opts.data_movement && expected_restore_operation_id.is_none() {
|
||||
// Every disk must publish the same cleanup owner alongside a
|
||||
// replaced null version. This transient key is not persisted
|
||||
// on the new object; recovery discovers the free-version in
|
||||
// the committed xl.meta even if this request is cancelled.
|
||||
fi.set_tier_free_version_id(&Uuid::new_v4().to_string());
|
||||
}
|
||||
fi.mod_time = mod_time;
|
||||
fi.size = w_size as i64;
|
||||
fi.versioned = opts.versioned || opts.version_suspended;
|
||||
@@ -18181,102 +18174,6 @@ mod put_object_tmp_cleanup_tests {
|
||||
drop(temp_dirs);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn tier_overwrite_failed_quorum_and_cancellation_preserve_live_source() {
|
||||
for cancel_before_rename in [false, true] {
|
||||
let (dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "tier-overwrite-failure";
|
||||
let object = "still-live";
|
||||
make_completion_test_bucket(&disks, bucket).await;
|
||||
let old_body = vec![0x31; TEST_OBJECT_SIZE];
|
||||
let mut metadata = HashMap::from([(
|
||||
"x-amz-restore".to_string(),
|
||||
"ongoing-request=\"false\", expiry-date=\"2099-01-01T00:00:00Z\"".to_string(),
|
||||
)]);
|
||||
for (suffix, value) in [
|
||||
(rustfs_utils::http::SUFFIX_TRANSITION_STATUS, "complete".to_string()),
|
||||
(rustfs_utils::http::SUFFIX_TRANSITION_TIER, "WARM".to_string()),
|
||||
(rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME, "remote/still-live".to_string()),
|
||||
(rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID, "exact-live-version".to_string()),
|
||||
(rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE, "exact".to_string()),
|
||||
(rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID, "ab".repeat(32)),
|
||||
] {
|
||||
rustfs_utils::http::insert_str(&mut metadata, suffix, value);
|
||||
}
|
||||
set.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(old_body.clone()),
|
||||
&ObjectOptions {
|
||||
user_defined: metadata,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed live transitioned source");
|
||||
wait_for_tmp_workspace_to_drain(&dirs, "seed write must drain").await;
|
||||
let before = set
|
||||
.load_file_info_versions_exact(bucket, object)
|
||||
.await
|
||||
.expect("read original metadata")
|
||||
.expect("original exists");
|
||||
|
||||
if cancel_before_rename {
|
||||
let barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::AfterQuotaReservation);
|
||||
let writer = Arc::clone(&set);
|
||||
let put = tokio::spawn(async move {
|
||||
writer
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![0x32; TEST_OBJECT_SIZE]),
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
put.abort();
|
||||
assert!(put.await.expect_err("cancel paused replacement").is_cancelled());
|
||||
wait_for_tmp_workspace_to_drain(&dirs, "cancelled replacement must roll back").await;
|
||||
drop(barrier);
|
||||
} else {
|
||||
let _fault = rename_fault_injection::fail_rename_on(object, &[2, 3]);
|
||||
let err = set
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![0x32; TEST_OBJECT_SIZE]),
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("two disk commits cannot satisfy write quorum three");
|
||||
assert!(matches!(err, Error::ErasureWriteQuorum | Error::InsufficientWriteQuorum(_, _)), "{err}");
|
||||
}
|
||||
let after = set
|
||||
.load_file_info_versions_exact(bucket, object)
|
||||
.await
|
||||
.expect("read rolled-back metadata")
|
||||
.expect("live source must survive");
|
||||
assert_eq!(after.versions, before.versions, "failed replacement must preserve the live version");
|
||||
assert_eq!(
|
||||
after.free_versions, before.free_versions,
|
||||
"failed replacement must not publish a cleanup owner"
|
||||
);
|
||||
let mut reader = set
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("live source remains readable");
|
||||
let mut actual = Vec::new();
|
||||
reader.stream.read_to_end(&mut actual).await.expect("read original bytes");
|
||||
assert_eq!(actual, old_body);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cooperative_cancellation_while_waiting_for_namespace_lock_cleans_tmp_workspace() {
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
|
||||
@@ -2832,224 +2832,6 @@ mod tests {
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn tier_overwrite_put_and_self_copy_recover_persisted_cleanup_owners() {
|
||||
use crate::bucket::lifecycle::bucket_lifecycle_ops::ExpiryState;
|
||||
use crate::bucket::lifecycle::tier_free_version_recovery::recover_tier_free_versions;
|
||||
use rustfs_filemeta::TransitionVersionState::{Exact, KnownDisabled, SuspendedNull};
|
||||
use rustfs_s3_client::transition_api::ReaderImpl;
|
||||
use rustfs_utils::http::{
|
||||
SUFFIX_TRANSITION_STATUS, SUFFIX_TRANSITION_TIER, SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
SUFFIX_TRANSITIONED_OBJECTNAME, SUFFIX_TRANSITIONED_VERSION_ID, SUFFIX_TRANSITIONED_VERSION_STATE, insert_str,
|
||||
};
|
||||
|
||||
let temp_dir = tempfile::tempdir().expect("create tier overwrite store");
|
||||
let (mut ctx, mut store, mut shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-overwrite", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let tier = "OVERWRITE-TIER";
|
||||
let backend = register_mock_tier(&ctx.tier_config_mgr(), tier).await;
|
||||
let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier)
|
||||
.await
|
||||
.expect("tier identity");
|
||||
let identity = rustfs_utils::crypto::hex(lease.backend_identity());
|
||||
drop(lease);
|
||||
|
||||
for state in [Exact, KnownDisabled, SuspendedNull] {
|
||||
for suspended in [false, true] {
|
||||
for self_copy in [false, true] {
|
||||
let bucket = format!("tier-overwrite-{}", Uuid::new_v4());
|
||||
let object = "object";
|
||||
let remote = format!("remote/{bucket}");
|
||||
let version = match state {
|
||||
Exact => "opaque-overwrite-version",
|
||||
SuspendedNull => "null",
|
||||
_ => "",
|
||||
};
|
||||
let payload = vec![0x5b; if suspended { 512 * 1024 } else { 257 }];
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
backend.set_put_remote_version(Some(version.to_string())).await;
|
||||
let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier)
|
||||
.await
|
||||
.expect("seed tier lease");
|
||||
lease
|
||||
.put(
|
||||
&remote,
|
||||
ReaderImpl::Body(bytes::Bytes::from(payload.clone())),
|
||||
payload.len().try_into().expect("payload size"),
|
||||
)
|
||||
.await
|
||||
.expect("seed remote bytes");
|
||||
drop(lease);
|
||||
let mut metadata = HashMap::from([
|
||||
("content-type".to_string(), "application/octet-stream".to_string()),
|
||||
(
|
||||
"x-amz-restore".to_string(),
|
||||
"ongoing-request=\"false\", expiry-date=\"2099-01-01T00:00:00Z\"".to_string(),
|
||||
),
|
||||
]);
|
||||
for (suffix, value) in [
|
||||
(SUFFIX_TRANSITION_STATUS, "complete"),
|
||||
(SUFFIX_TRANSITION_TIER, tier),
|
||||
(SUFFIX_TRANSITION_TIER_DESTINATION_ID, identity.as_str()),
|
||||
(SUFFIX_TRANSITIONED_OBJECTNAME, remote.as_str()),
|
||||
(SUFFIX_TRANSITIONED_VERSION_STATE, state.as_str()),
|
||||
] {
|
||||
insert_str(&mut metadata, suffix, value.to_string());
|
||||
}
|
||||
if !version.is_empty() {
|
||||
insert_str(&mut metadata, SUFFIX_TRANSITIONED_VERSION_ID, version.to_string());
|
||||
}
|
||||
let options = ObjectOptions {
|
||||
version_suspended: suspended,
|
||||
..Default::default()
|
||||
};
|
||||
store
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(payload.clone()),
|
||||
&ObjectOptions {
|
||||
user_defined: metadata,
|
||||
..options.clone()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed transitioned source with locally restored bytes");
|
||||
let expected = if self_copy {
|
||||
payload.clone()
|
||||
} else {
|
||||
vec![0x73; payload.len()]
|
||||
};
|
||||
let new_metadata = HashMap::from([
|
||||
("content-type".to_string(), "text/plain".to_string()),
|
||||
("x-amz-meta-replacement".to_string(), "kept".to_string()),
|
||||
]);
|
||||
if self_copy {
|
||||
let mut source = store
|
||||
.get_object_info(&bucket, object, &options)
|
||||
.await
|
||||
.expect("self-copy source");
|
||||
source.metadata_only = false;
|
||||
source.user_defined = Arc::new(new_metadata);
|
||||
source.put_object_reader = Some(PutObjReader::from_vec(expected.clone()));
|
||||
store
|
||||
.copy_object(&bucket, object, &bucket, object, &mut source, &options, &options)
|
||||
.await
|
||||
.expect("materialized self-copy");
|
||||
} else {
|
||||
store
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(expected.clone()),
|
||||
&ObjectOptions {
|
||||
user_defined: new_metadata,
|
||||
..options.clone()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("overwrite transitioned null version");
|
||||
}
|
||||
|
||||
let set = store.pools[0].get_disks_by_key(object);
|
||||
let versions = set
|
||||
.load_file_info_versions_exact(&bucket, object)
|
||||
.await
|
||||
.expect("read committed disk metadata")
|
||||
.expect("replacement metadata exists");
|
||||
let free: Vec<_> = versions
|
||||
.versions
|
||||
.iter()
|
||||
.chain(versions.free_versions.iter())
|
||||
.filter(|fi| fi.tier_free_version())
|
||||
.collect();
|
||||
assert_eq!(free.len(), 1, "{state:?}, suspended={suspended}, copy={self_copy}");
|
||||
assert_eq!(free[0].transitioned_objname, remote);
|
||||
assert_eq!(free[0].transition_version_state, state);
|
||||
assert!(backend.contains(&remote).await, "commit must not delete remote bytes before cleanup");
|
||||
let removed_before = backend.remove_count().await;
|
||||
|
||||
// Restart before queue delivery. The new runtime must
|
||||
// reconstruct ownership solely from the committed xl.meta.
|
||||
let tier_config = ctx
|
||||
.tier_config_mgr()
|
||||
.read()
|
||||
.await
|
||||
.tiers
|
||||
.get(tier)
|
||||
.expect("tier configuration survives restart")
|
||||
.clone_with_credentials();
|
||||
drop(set);
|
||||
shutdown.cancel();
|
||||
drop(store);
|
||||
drop(ctx);
|
||||
(ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-overwrite-restart", &[4]))
|
||||
.await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
{
|
||||
let manager = ctx.tier_config_mgr();
|
||||
let mut manager = manager.write().await;
|
||||
manager.tiers.insert(tier.to_string(), tier_config);
|
||||
manager
|
||||
.install_test_driver(tier, Box::new(backend.clone()))
|
||||
.expect("rebind the same remote destination after restart");
|
||||
}
|
||||
let set = store.pools[0].get_disks_by_key(object);
|
||||
ExpiryState::resize_workers(1, Arc::clone(&store)).await;
|
||||
let recovered = recover_tier_free_versions(Arc::clone(&store), 100, None, None)
|
||||
.await
|
||||
.expect("recover persisted cleanup owner");
|
||||
assert!(recovered.enqueued >= 1);
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
let versions = set
|
||||
.load_file_info_versions_exact(&bucket, object)
|
||||
.await
|
||||
.expect("read cleanup progress")
|
||||
.expect("new object must survive cleanup");
|
||||
if versions
|
||||
.versions
|
||||
.iter()
|
||||
.chain(versions.free_versions.iter())
|
||||
.all(|fi| !fi.tier_free_version())
|
||||
{
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("cleanup must converge");
|
||||
assert!(!backend.contains(&remote).await);
|
||||
assert_eq!(backend.remove_count().await, removed_before + 1, "one remote DELETE per owner");
|
||||
assert_eq!(backend.remove_versions().await.last(), Some(&(remote.clone(), version.to_string())));
|
||||
let mut reader = store
|
||||
.get_object_reader(&bucket, object, None, HeaderMap::new(), &options)
|
||||
.await
|
||||
.expect("replacement remains readable");
|
||||
let mut actual = Vec::new();
|
||||
reader.stream.read_to_end(&mut actual).await.expect("read replacement bytes");
|
||||
assert_eq!(actual, expected);
|
||||
let current = store
|
||||
.get_object_info(&bucket, object, &options)
|
||||
.await
|
||||
.expect("replacement metadata");
|
||||
assert_eq!(current.user_defined.get("content-type").map(String::as_str), Some("text/plain"));
|
||||
assert_eq!(current.user_defined.get("x-amz-meta-replacement").map(String::as_str), Some("kept"));
|
||||
assert!(current.transitioned_object.status.is_empty());
|
||||
}
|
||||
}
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
@@ -15721,20 +15503,12 @@ mod tests {
|
||||
);
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
let metadata_absent = {
|
||||
// Synchronize with cleanup so the snapshot cannot span per-disk marker removal.
|
||||
let mut read_opts = ObjectOptions::default();
|
||||
let _guards = store
|
||||
.acquire_all_physical_object_read_locks("batch_transitioned_delete_test", bucket, causal, &mut read_opts)
|
||||
.await
|
||||
.expect("causal batch cleanup observation should acquire object read locks");
|
||||
store.pools[0]
|
||||
.get_disks_by_key(causal)
|
||||
.load_file_info_versions_exact(bucket, causal)
|
||||
.await
|
||||
.expect("causal batch cleanup metadata should remain readable")
|
||||
.is_none()
|
||||
};
|
||||
let metadata_absent = store.pools[0]
|
||||
.get_disks_by_key(causal)
|
||||
.load_file_info_versions_exact(bucket, causal)
|
||||
.await
|
||||
.expect("causal batch cleanup metadata should remain readable")
|
||||
.is_none();
|
||||
if metadata_absent && backend.remove_versions().await.len() >= 2 {
|
||||
return;
|
||||
}
|
||||
@@ -15813,24 +15587,12 @@ mod tests {
|
||||
);
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
let metadata_absent = {
|
||||
let mut read_opts = ObjectOptions::default();
|
||||
let _guards = store
|
||||
.acquire_all_physical_object_read_locks(
|
||||
"batch_transitioned_delete_test",
|
||||
bucket,
|
||||
versioned_causal,
|
||||
&mut read_opts,
|
||||
)
|
||||
.await
|
||||
.expect("versioned causal batch cleanup observation should acquire object read locks");
|
||||
store.pools[0]
|
||||
.get_disks_by_key(versioned_causal)
|
||||
.load_file_info_versions_exact(bucket, versioned_causal)
|
||||
.await
|
||||
.expect("versioned causal batch cleanup metadata should remain readable")
|
||||
.is_none()
|
||||
};
|
||||
let metadata_absent = store.pools[0]
|
||||
.get_disks_by_key(versioned_causal)
|
||||
.load_file_info_versions_exact(bucket, versioned_causal)
|
||||
.await
|
||||
.expect("versioned causal batch cleanup metadata should remain readable")
|
||||
.is_none();
|
||||
if metadata_absent && backend.remove_versions().await.len() == 3 {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -480,110 +480,7 @@ impl FileMeta {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn add_version(&mut self, fi: FileInfo) -> Result<()> {
|
||||
if let Some(free_version) = self.overwritten_tier_free_version(&fi)? {
|
||||
// The replacement and its cleanup owner must share one xl.meta
|
||||
// commit. Keep the original intact if either insertion fails.
|
||||
let mut next = self.clone();
|
||||
next.add_version_inner(fi)?;
|
||||
next.add_version_filemata(free_version)?;
|
||||
*self = next;
|
||||
return Ok(());
|
||||
}
|
||||
self.add_version_inner(fi)
|
||||
}
|
||||
|
||||
fn overwritten_tier_free_version(&self, fi: &FileInfo) -> Result<Option<FileMetaVersion>> {
|
||||
use rustfs_utils::http::{
|
||||
SUFFIX_TIER_FV_ID, SUFFIX_TRANSITION_STATUS, SUFFIX_TRANSITION_TIER, SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
SUFFIX_TRANSITIONED_OBJECTNAME, SUFFIX_TRANSITIONED_VERSION_ID, SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
get_consistent_bytes, get_consistent_str, has_internal_suffix, strip_internal_prefix_preserving_case,
|
||||
};
|
||||
|
||||
if fi.version_id.is_some_and(|id| !id.is_nil()) || !contains_key_str(&fi.metadata, SUFFIX_TIER_FV_ID) {
|
||||
return Ok(None);
|
||||
}
|
||||
let Some(existing) = self
|
||||
.versions
|
||||
.iter()
|
||||
.find(|v| v.header.version_id.is_none_or(|id| id.is_nil()))
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let old = existing.parse_version_meta()?;
|
||||
let Some(mut object) = old.object else {
|
||||
return Ok(None);
|
||||
};
|
||||
let status = get_consistent_bytes(&object.meta_sys, SUFFIX_TRANSITION_STATUS);
|
||||
if status.is_none()
|
||||
&& object
|
||||
.meta_sys
|
||||
.keys()
|
||||
.any(|key| has_internal_suffix(key, SUFFIX_TRANSITION_STATUS))
|
||||
{
|
||||
// Empty status is a valid local object. The reader distinguishes
|
||||
// it from conflicting aliases before the ordinary overwrite.
|
||||
object.into_fileinfo(&fi.volume, &fi.name, false)?;
|
||||
return Ok(None);
|
||||
}
|
||||
if status != Some(TRANSITION_COMPLETE.as_bytes()) {
|
||||
return Ok(None);
|
||||
}
|
||||
// Reuse the reader's alias/state validation. A legacy empty remote
|
||||
// version is valid and must not be mistaken for conflicting aliases.
|
||||
object.into_fileinfo(&fi.volume, &fi.name, false)?;
|
||||
if object
|
||||
.meta_sys
|
||||
.keys()
|
||||
.any(|key| has_internal_suffix(key, SUFFIX_TRANSITION_TIER_DESTINATION_ID))
|
||||
&& get_consistent_bytes(&object.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID).is_none()
|
||||
{
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
let transition_suffixes = [
|
||||
SUFFIX_TRANSITION_STATUS,
|
||||
SUFFIX_TRANSITION_TIER,
|
||||
SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
SUFFIX_TRANSITIONED_OBJECTNAME,
|
||||
SUFFIX_TRANSITIONED_VERSION_ID,
|
||||
SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
];
|
||||
let replacement = MetaObject::from(fi.clone());
|
||||
if transition_suffixes
|
||||
.iter()
|
||||
.all(|suffix| get_consistent_bytes(&object.meta_sys, suffix) == get_consistent_bytes(&replacement.meta_sys, suffix))
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
let id = get_consistent_str(&fi.metadata, SUFFIX_TIER_FV_ID).ok_or(Error::FileCorrupt)?;
|
||||
let id = Uuid::parse_str(id)?;
|
||||
if id.is_nil() || self.versions.iter().any(|version| version.header.version_id == Some(id)) {
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
// The reader also accepts legacy key casing. Canonicalize only this
|
||||
// cleanup source so init_free_version preserves every accepted field,
|
||||
// including an explicitly empty unversioned remote version.
|
||||
for suffix in transition_suffixes {
|
||||
let value = object
|
||||
.meta_sys
|
||||
.iter()
|
||||
.find(|(key, _)| {
|
||||
strip_internal_prefix_preserving_case(key).is_some_and(|found| found.eq_ignore_ascii_case(suffix))
|
||||
})
|
||||
.map(|(_, value)| value.clone());
|
||||
if let Some(value) = value {
|
||||
rustfs_utils::http::insert_bytes(&mut object.meta_sys, suffix, value);
|
||||
}
|
||||
}
|
||||
let (free_version, created) = object.init_free_version(fi)?;
|
||||
if !created {
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
Ok(Some(free_version))
|
||||
}
|
||||
|
||||
fn add_version_inner(&mut self, mut fi: FileInfo) -> Result<()> {
|
||||
rustfs_utils::http::remove_str(&mut fi.metadata, rustfs_utils::http::SUFFIX_TIER_FV_ID);
|
||||
pub fn add_version(&mut self, mut fi: FileInfo) -> Result<()> {
|
||||
// empty version_id means "null" (versioning disabled/suspended)
|
||||
if fi.version_id.is_none() {
|
||||
fi.version_id = Some(Uuid::nil());
|
||||
@@ -1566,235 +1463,6 @@ mod test {
|
||||
});
|
||||
}
|
||||
|
||||
fn tier_overwrite_fixture(state: crate::TransitionVersionState) -> (FileMeta, FileInfo) {
|
||||
let mut source = FileInfo::new("object", 2, 2);
|
||||
source.erasure.index = 1;
|
||||
source.mod_time = Some(OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixture timestamp"));
|
||||
source.data_dir = Some(Uuid::new_v4());
|
||||
source.transition_status = TRANSITION_COMPLETE.to_string();
|
||||
source.transition_tier = "WARM".to_string();
|
||||
source.transitioned_objname = "remote/old-object".to_string();
|
||||
source.transition_version_state = state;
|
||||
source.transition_version = match state {
|
||||
crate::TransitionVersionState::Exact => Some("opaque-provider-version".to_string()),
|
||||
crate::TransitionVersionState::SuspendedNull => Some("null".to_string()),
|
||||
_ => None,
|
||||
};
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut source.metadata,
|
||||
rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
"ab".repeat(32),
|
||||
);
|
||||
if state == crate::TransitionVersionState::KnownDisabled {
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut source.metadata,
|
||||
rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID,
|
||||
String::new(),
|
||||
);
|
||||
}
|
||||
let mut meta = FileMeta::new();
|
||||
meta.add_version(source.clone()).expect("seed transitioned null version");
|
||||
(meta, source)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_overwrite_preserves_exact_cleanup_owner_across_reload() {
|
||||
use crate::TransitionVersionState::{Exact, KnownDisabled, SuspendedNull, Unknown};
|
||||
use rustfs_utils::http::{MINIO_INTERNAL_PREFIX, RUSTFS_INTERNAL_PREFIX, SUFFIX_TIER_FV_ID};
|
||||
|
||||
for state in [Exact, KnownDisabled, SuspendedNull, Unknown] {
|
||||
for inline in [false, true] {
|
||||
let (mut meta, _) = tier_overwrite_fixture(state);
|
||||
let old = meta.versions[0].parse_version_meta().expect("old metadata");
|
||||
let old = old.object.expect("old object");
|
||||
let id = Uuid::new_v4();
|
||||
let mut replacement = FileInfo::new("object", 2, 2);
|
||||
replacement.version_id = inline.then_some(Uuid::nil());
|
||||
replacement.mod_time = Some(OffsetDateTime::from_unix_timestamp(1_700_000_001).expect("fixture timestamp"));
|
||||
replacement.data_dir = Some(Uuid::new_v4());
|
||||
replacement.size = 3;
|
||||
if inline {
|
||||
replacement.data = Some(Bytes::from_static(b"new"));
|
||||
}
|
||||
replacement.set_tier_free_version_id(&id.to_string());
|
||||
|
||||
meta.add_version(replacement.clone())
|
||||
.expect("replace transitioned null version");
|
||||
let bytes = meta.marshal_msg().expect("persist replacement and cleanup owner");
|
||||
let mut reopened = FileMeta::load(&bytes).expect("reopen committed metadata");
|
||||
assert_eq!(reopened.versions.len(), 2);
|
||||
let (_, free) = reopened.find_version(Some(id)).expect("durable cleanup owner");
|
||||
assert!(free.free_version());
|
||||
let marker = free.delete_marker.expect("cleanup marker");
|
||||
for suffix in [
|
||||
rustfs_utils::http::SUFFIX_TRANSITION_TIER,
|
||||
rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME,
|
||||
rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID,
|
||||
rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
] {
|
||||
for prefix in [RUSTFS_INTERNAL_PREFIX, MINIO_INTERNAL_PREFIX] {
|
||||
let key = format!("{prefix}{suffix}");
|
||||
assert_eq!(marker.meta_sys.get(&key), old.meta_sys.get(&key), "{state:?}: {key}");
|
||||
}
|
||||
}
|
||||
let (_, current) = reopened.find_version(None).expect("replacement survives restart");
|
||||
let current = current.object.expect("replacement object");
|
||||
assert_eq!(current.size, 3);
|
||||
assert!(!rustfs_utils::http::contains_key_bytes(¤t.meta_sys, SUFFIX_TIER_FV_ID));
|
||||
reopened
|
||||
.add_version(replacement)
|
||||
.expect("replaying replacement is idempotent");
|
||||
assert_eq!(reopened.versions.len(), 2);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_overwrite_rejects_cleanup_failure_without_mutating_source() {
|
||||
for id in ["not-a-uuid".to_string(), Uuid::nil().to_string()] {
|
||||
let (mut meta, _) = tier_overwrite_fixture(crate::TransitionVersionState::Exact);
|
||||
let before = meta.clone();
|
||||
let mut replacement = FileInfo::new("object", 2, 2);
|
||||
replacement.mod_time = Some(OffsetDateTime::now_utc());
|
||||
replacement.set_tier_free_version_id(&id);
|
||||
assert!(meta.add_version(replacement).is_err());
|
||||
assert_eq!(meta, before, "invalid cleanup identity must preserve source");
|
||||
}
|
||||
with_object_max_versions_for_test(1, || {
|
||||
let (mut meta, _) = tier_overwrite_fixture(crate::TransitionVersionState::KnownDisabled);
|
||||
let before = meta.clone();
|
||||
let mut replacement = FileInfo::new("object", 2, 2);
|
||||
replacement.mod_time = Some(OffsetDateTime::now_utc());
|
||||
replacement.data = Some(Bytes::from_static(b"new"));
|
||||
replacement.set_tier_free_version_id(&Uuid::new_v4().to_string());
|
||||
assert_eq!(
|
||||
meta.add_version(replacement)
|
||||
.expect_err("cleanup owner exceeds version limit"),
|
||||
Error::MaxVersionsExceeded
|
||||
);
|
||||
assert_eq!(meta, before, "failed cleanup insertion must also preserve inline bytes");
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_overwrite_allows_empty_transition_status_on_local_source() {
|
||||
let mut source = FileInfo::new("object", 2, 2);
|
||||
source.mod_time = Some(OffsetDateTime::now_utc());
|
||||
source.data = Some(Bytes::from_static(b"old"));
|
||||
rustfs_utils::http::insert_str(&mut source.metadata, rustfs_utils::http::SUFFIX_TRANSITION_STATUS, String::new());
|
||||
let mut meta = FileMeta::new();
|
||||
meta.add_version(source)
|
||||
.expect("seed readable local metadata with empty status");
|
||||
let mut replacement = FileInfo::new("object", 2, 2);
|
||||
replacement.mod_time = Some(OffsetDateTime::now_utc());
|
||||
replacement.size = 3;
|
||||
replacement.data = Some(Bytes::from_static(b"new"));
|
||||
replacement.set_tier_free_version_id(&Uuid::new_v4().to_string());
|
||||
meta.add_version(replacement)
|
||||
.expect("an ordinary overwrite must still succeed");
|
||||
assert_eq!(meta.versions.len(), 1);
|
||||
let (_, current) = meta.find_version(None).expect("replacement remains visible");
|
||||
assert_eq!(current.object.expect("ordinary object").size, 3);
|
||||
assert!(!meta.versions[0].header.free_version());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_overwrite_preserves_legacy_metadata_casing() {
|
||||
use rustfs_utils::http::{MINIO_INTERNAL_PREFIX, RUSTFS_INTERNAL_PREFIX};
|
||||
|
||||
for state in [
|
||||
crate::TransitionVersionState::Exact,
|
||||
crate::TransitionVersionState::KnownDisabled,
|
||||
] {
|
||||
let (mut meta, _) = tier_overwrite_fixture(state);
|
||||
let mut source = meta.versions[0].parse_version_meta().expect("seeded source");
|
||||
let object = source.object.as_mut().expect("transitioned source");
|
||||
let expected = object.meta_sys.clone();
|
||||
object.meta_sys = object
|
||||
.meta_sys
|
||||
.drain()
|
||||
.map(|(key, value)| (key.to_ascii_uppercase(), value))
|
||||
.collect();
|
||||
meta.versions[0] = FileMetaShallowVersion::try_from(source).expect("legacy key casing");
|
||||
let id = Uuid::new_v4();
|
||||
let mut replacement = FileInfo::new("object", 2, 2);
|
||||
replacement.mod_time = Some(OffsetDateTime::now_utc());
|
||||
replacement.set_tier_free_version_id(&id.to_string());
|
||||
meta.add_version(replacement).expect("overwrite readable legacy source");
|
||||
let reopened = FileMeta::load(&meta.marshal_msg().expect("persist overwrite")).expect("reopen overwrite");
|
||||
let (_, owner) = reopened
|
||||
.find_version(Some(id))
|
||||
.expect("legacy source must retain cleanup ownership");
|
||||
let marker = owner.delete_marker.expect("cleanup marker");
|
||||
for suffix in [
|
||||
rustfs_utils::http::SUFFIX_TRANSITION_TIER,
|
||||
rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME,
|
||||
rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID,
|
||||
rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
] {
|
||||
for prefix in [RUSTFS_INTERNAL_PREFIX, MINIO_INTERNAL_PREFIX] {
|
||||
let key = format!("{prefix}{suffix}");
|
||||
assert_eq!(marker.meta_sys.get(&key), expected.get(&key), "legacy {state:?}: {key}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_overwrite_rejects_conflicting_remote_metadata_aliases() {
|
||||
use rustfs_utils::http::{
|
||||
MINIO_INTERNAL_PREFIX, SUFFIX_TRANSITION_STATUS, SUFFIX_TRANSITION_TIER, SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
SUFFIX_TRANSITIONED_OBJECTNAME, SUFFIX_TRANSITIONED_VERSION_ID, SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
};
|
||||
for suffix in [
|
||||
SUFFIX_TRANSITION_STATUS,
|
||||
SUFFIX_TRANSITION_TIER,
|
||||
SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
SUFFIX_TRANSITIONED_OBJECTNAME,
|
||||
SUFFIX_TRANSITIONED_VERSION_ID,
|
||||
SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
] {
|
||||
let (mut meta, _) = tier_overwrite_fixture(crate::TransitionVersionState::Exact);
|
||||
let mut old = meta.versions[0].parse_version_meta().expect("seeded source metadata");
|
||||
old.object
|
||||
.as_mut()
|
||||
.expect("transitioned source")
|
||||
.meta_sys
|
||||
.insert(format!("{MINIO_INTERNAL_PREFIX}{suffix}"), b"conflicting-value".to_vec());
|
||||
meta.versions[0] = FileMetaShallowVersion::try_from(old).expect("encode conflicting aliases");
|
||||
let before = meta.clone();
|
||||
let mut replacement = FileInfo::new("object", 2, 2);
|
||||
replacement.mod_time = Some(OffsetDateTime::now_utc());
|
||||
replacement.set_tier_free_version_id(&Uuid::new_v4().to_string());
|
||||
assert_eq!(
|
||||
meta.add_version(replacement)
|
||||
.expect_err("ambiguous ownership must fail closed"),
|
||||
Error::FileCorrupt
|
||||
);
|
||||
assert_eq!(meta, before, "conflicting {suffix} must not erase the old remote tuple");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_overwrite_keeps_retained_remote_and_versioned_copy_ownership() {
|
||||
let (mut meta, mut source) = tier_overwrite_fixture(crate::TransitionVersionState::Exact);
|
||||
source.set_tier_free_version_id(&Uuid::new_v4().to_string());
|
||||
meta.add_version(source.clone())
|
||||
.expect("restore retains the same remote owner");
|
||||
assert_eq!(meta.versions.len(), 1);
|
||||
source.version_id = Some(Uuid::new_v4());
|
||||
source.transition_status.clear();
|
||||
source.transition_tier.clear();
|
||||
source.transitioned_objname.clear();
|
||||
source.transition_version = None;
|
||||
source.transition_version_state = crate::TransitionVersionState::Unknown;
|
||||
meta.add_version(source).expect("versioned write retains historical source");
|
||||
assert_eq!(meta.versions.len(), 2);
|
||||
assert!(meta.versions.iter().all(|version| !version.header.free_version()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn add_version_filemata_uses_canonical_equal_time_order() {
|
||||
let mod_time = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid test timestamp");
|
||||
|
||||
@@ -548,7 +548,10 @@ fn retry_budget_for_result(task: &HealTask, result: &Result<()>, retryable_batch
|
||||
}
|
||||
|
||||
let error = err.to_string();
|
||||
if !err.is_recoverable_heal() {
|
||||
// Batch aggregation preserves the typed classification in its counters,
|
||||
// while the returned task error retains only the first error's display text.
|
||||
let retryable_batch_result = retryable_batch_failure && matches!(err, Error::TaskExecutionFailed { .. });
|
||||
if !retryable_batch_result && !err.is_recoverable_heal() {
|
||||
return None;
|
||||
}
|
||||
|
||||
|
||||
@@ -2614,27 +2614,62 @@ fn test_retry_request_for_recoverable_error_stops_at_limit() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_retry_request_rescans_batch_when_all_exhausted_objects_are_retryable() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let task = HealTask::from_request(HealRequest::bucket("bucket".to_string()), storage);
|
||||
let result = Err(task
|
||||
.record_batch_failure(BatchHealFailure {
|
||||
scope: "bucket:bucket".to_string(),
|
||||
failed: 1,
|
||||
retryable: 1,
|
||||
permanent: 0,
|
||||
first_object: "object".to_string(),
|
||||
first_error: "Lock acquisition timeout".to_string(),
|
||||
})
|
||||
.await);
|
||||
for source_error in [
|
||||
Error::Disk(DiskError::FaultyDisk),
|
||||
Error::Disk(DiskError::FaultyRemoteDisk),
|
||||
Error::Storage(EcstoreError::SlowDown),
|
||||
Error::TaskExecutionFailed {
|
||||
message: "Lock acquisition timeout".to_string(),
|
||||
},
|
||||
] {
|
||||
assert!(source_error.is_recoverable_heal());
|
||||
let first_error = source_error.to_string();
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let task = HealTask::from_request(HealRequest::bucket("bucket".to_string()), storage);
|
||||
let result = Err(task
|
||||
.record_batch_failure(BatchHealFailure {
|
||||
scope: "bucket:bucket".to_string(),
|
||||
failed: 1,
|
||||
retryable: 1,
|
||||
permanent: 0,
|
||||
first_object: "object".to_string(),
|
||||
first_error: first_error.clone(),
|
||||
})
|
||||
.await);
|
||||
|
||||
let (retry_request, retry_delay, error) = retry_request_for_result_with_budget(&task, &result)
|
||||
.await
|
||||
.expect("all-retryable batch failure should rescan within the manager retry budget");
|
||||
let (retry_request, retry_delay, error) = retry_request_for_result_with_budget(&task, &result)
|
||||
.await
|
||||
.expect("all-retryable batch failure should rescan within the manager retry budget");
|
||||
|
||||
assert_eq!(retry_request.id, task.id);
|
||||
assert_eq!(retry_request.retry_attempts, 1);
|
||||
assert!(retry_delay > Duration::ZERO);
|
||||
assert!(error.contains("Lock acquisition timeout"));
|
||||
assert_eq!(retry_request.id, task.id);
|
||||
assert_eq!(retry_request.retry_attempts, 1);
|
||||
assert!(retry_delay > Duration::ZERO);
|
||||
assert!(error.contains(&first_error));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_retry_request_does_not_rescan_cancelled_or_timed_out_retryable_batch() {
|
||||
for terminal_error in [Error::TaskCancelled, Error::TaskTimeout] {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let task = HealTask::from_request(HealRequest::bucket("bucket".to_string()), storage);
|
||||
let _ = task
|
||||
.record_batch_failure(BatchHealFailure {
|
||||
scope: "bucket:bucket".to_string(),
|
||||
failed: 1,
|
||||
retryable: 1,
|
||||
permanent: 0,
|
||||
first_object: "object".to_string(),
|
||||
first_error: Error::Disk(DiskError::FaultyDisk).to_string(),
|
||||
})
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
retry_request_for_result_with_budget(&task, &Err(terminal_error))
|
||||
.await
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -100,6 +100,34 @@ async fn minio_permanent_identities_survive_migration_and_repeated_iam_loads() {
|
||||
.await;
|
||||
env.make_bucket(LEGACY_META_BUCKET, false).await;
|
||||
|
||||
for (path, body) in [
|
||||
("config/iam/empty.json", Vec::new()),
|
||||
("config/iam/users/ignored/extra.json", b"not JSON".to_vec()),
|
||||
] {
|
||||
env.put_object_bytes(LEGACY_META_BUCKET, path, body).await;
|
||||
}
|
||||
try_migrate_iam_config(
|
||||
env.ecstore.clone(),
|
||||
Some(std::sync::Arc::new(|_| panic!("unsupported IAM records must not be decrypted"))),
|
||||
)
|
||||
.await
|
||||
.expect("unsupported IAM records, including empty objects, must be skipped");
|
||||
|
||||
let format_path = "config/iam/format.json";
|
||||
for body in [Vec::new(), b"invalid IAM format".to_vec()] {
|
||||
env.put_object_bytes(LEGACY_META_BUCKET, format_path, body).await;
|
||||
let error = try_migrate_iam_config(env.ecstore.clone(), None)
|
||||
.await
|
||||
.expect_err("empty or incompatible supported IAM metadata must prevent startup readiness");
|
||||
let io_error = std::io::Error::from(error);
|
||||
let detail = io_error
|
||||
.get_ref()
|
||||
.and_then(|context| context.source())
|
||||
.expect("failure must retain the supported record in its source");
|
||||
assert!(detail.to_string().contains(format_path), "failure must identify the supported record");
|
||||
}
|
||||
seed_legacy_iam_object(&env, format_path, &json!({"version": 1})).await;
|
||||
|
||||
let regular_source = json!({
|
||||
"version": 1,
|
||||
"credentials": {
|
||||
@@ -155,7 +183,12 @@ async fn minio_permanent_identities_survive_migration_and_repeated_iam_loads() {
|
||||
)
|
||||
.await;
|
||||
|
||||
try_migrate_iam_config(env.ecstore.clone(), None).await;
|
||||
try_migrate_iam_config(env.ecstore.clone(), None)
|
||||
.await
|
||||
.expect("legacy IAM migration completes after source repair");
|
||||
try_migrate_iam_config(env.ecstore.clone(), None)
|
||||
.await
|
||||
.expect("completed legacy IAM migration is idempotent");
|
||||
|
||||
let store = ObjectStore::new(env.ecstore);
|
||||
assert_identity_survives(
|
||||
|
||||
@@ -730,12 +730,16 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
|
||||
}
|
||||
}
|
||||
|
||||
// Highest priority first, like MinIO's `FilterActionableRules`. The
|
||||
// tie-breakers make this a total order: a comparator that only
|
||||
// orders same-destination pairs is not transitive, and the standard
|
||||
// library sort panics on such inputs past its insertion-sort
|
||||
// threshold (backlog#2367 C-1).
|
||||
rules.sort_by(|a, b| {
|
||||
if a.destination == b.destination {
|
||||
b.priority.cmp(&a.priority)
|
||||
} else {
|
||||
std::cmp::Ordering::Equal
|
||||
}
|
||||
b.priority
|
||||
.cmp(&a.priority)
|
||||
.then_with(|| a.destination.bucket.cmp(&b.destination.bucket))
|
||||
.then_with(|| a.id.cmp(&b.id))
|
||||
});
|
||||
|
||||
rules
|
||||
@@ -813,24 +817,19 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
|
||||
return vec![role.to_string()];
|
||||
}
|
||||
|
||||
let mut arns = Vec::new();
|
||||
let mut targets_map: HashSet<String> = HashSet::new();
|
||||
let rules = self.filter_actionable_rules(obj);
|
||||
|
||||
for rule in rules {
|
||||
// Rule order (priority descending) is the ARN order: callers that
|
||||
// iterate targets see the highest-priority destination first.
|
||||
let mut arns: Vec<String> = Vec::new();
|
||||
for rule in self.filter_actionable_rules(obj) {
|
||||
if rule.status == ReplicationRuleStatus::from_static(ReplicationRuleStatus::DISABLED) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let arn = rule.destination.bucket.trim();
|
||||
if !arn.is_empty() && !targets_map.contains(arn) {
|
||||
targets_map.insert(arn.to_string());
|
||||
if !arn.is_empty() && !arns.iter().any(|seen| seen == arn) {
|
||||
arns.push(arn.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
for arn in targets_map {
|
||||
arns.push(arn);
|
||||
}
|
||||
arns
|
||||
}
|
||||
|
||||
@@ -1908,6 +1907,84 @@ mod tests {
|
||||
assert_eq!(decisions, vec![(target_a.to_string(), false), (target_b.to_string(), true)]);
|
||||
}
|
||||
|
||||
// backlog#2367 C-1: the actionable-rule sort must be a total order. A
|
||||
// comparator that answers `Equal` for different destinations but orders
|
||||
// same-destination rules by priority is not transitive, and the standard
|
||||
// library sort panics on such inputs once the slice is past the
|
||||
// insertion-sort threshold (> 20 rules).
|
||||
#[test]
|
||||
fn actionable_rule_sort_is_a_total_order_across_destinations() {
|
||||
let targets = ["arn:target:a", "arn:target:b", "arn:target:c"];
|
||||
let mut seed: u64 = 0x2367;
|
||||
for _ in 0..200 {
|
||||
let rule_count = 21 + (seed % 200) as usize;
|
||||
let rules = (0..rule_count)
|
||||
.map(|index| {
|
||||
seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
|
||||
let target = targets[(seed >> 33) as usize % targets.len()];
|
||||
delete_marker_rule(&format!("r{index}"), target, "", index as i32, true)
|
||||
})
|
||||
.collect();
|
||||
let config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules,
|
||||
};
|
||||
let ordered = config.filter_actionable_rules(&ObjectOpts {
|
||||
name: "logs/app.log".to_string(),
|
||||
op_type: ReplicationType::Object,
|
||||
..Default::default()
|
||||
});
|
||||
assert_eq!(ordered.len(), rule_count);
|
||||
assert!(
|
||||
ordered.windows(2).all(|pair| pair[0].priority >= pair[1].priority),
|
||||
"actionable rules must be ordered by descending priority"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// backlog#2367 C-2: a V1 rule carries its prefix at the top level (no
|
||||
// <Filter>). Ignoring it made `<Prefix>logs/</Prefix>` match every object.
|
||||
#[test]
|
||||
fn top_level_rule_prefix_scopes_matching_without_a_filter() {
|
||||
let arn = "arn:target:a";
|
||||
let config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![delete_marker_rule("v1-prefix", arn, "logs/", 1, true)],
|
||||
};
|
||||
assert_eq!(config.rules[0].prefix(), "logs/");
|
||||
|
||||
let matching = config.filter_actionable_rules(&ObjectOpts {
|
||||
name: "logs/app.log".to_string(),
|
||||
op_type: ReplicationType::Object,
|
||||
..Default::default()
|
||||
});
|
||||
assert_eq!(matching.len(), 1);
|
||||
|
||||
let outside = config.filter_actionable_rules(&ObjectOpts {
|
||||
name: "data/app.log".to_string(),
|
||||
op_type: ReplicationType::Object,
|
||||
..Default::default()
|
||||
});
|
||||
assert!(outside.is_empty(), "an object outside the V1 prefix must not match: {outside:?}");
|
||||
assert!(
|
||||
config
|
||||
.filter_target_arns(&ObjectOpts {
|
||||
name: "data/app.log".to_string(),
|
||||
op_type: ReplicationType::Object,
|
||||
..Default::default()
|
||||
})
|
||||
.is_empty()
|
||||
);
|
||||
|
||||
// A <Filter> still wins over the deprecated top-level element.
|
||||
let mut filtered = delete_marker_rule("filtered", arn, "logs/", 1, true);
|
||||
filtered.filter = Some(s3s::dto::ReplicationRuleFilter {
|
||||
prefix: Some("photos/".to_string()),
|
||||
..Default::default()
|
||||
});
|
||||
assert_eq!(filtered.prefix(), "photos/");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn force_delete_targets_use_overlapping_rules_and_highest_priority_switch() {
|
||||
let target_a = "arn:target:a";
|
||||
|
||||
@@ -22,6 +22,10 @@ pub trait ReplicationRuleExt {
|
||||
}
|
||||
|
||||
impl ReplicationRuleExt for ReplicationRule {
|
||||
/// The rule's key prefix: `Filter.Prefix`, else `Filter.And.Prefix`, else
|
||||
/// the deprecated top-level `Prefix` of a V1 rule written without a
|
||||
/// `<Filter>` (backlog#2367 C-2). A rule that carries both keeps AWS's
|
||||
/// precedence: the `<Filter>` is authoritative.
|
||||
fn prefix(&self) -> &str {
|
||||
if let Some(filter) = &self.filter {
|
||||
if let Some(prefix) = &filter.prefix {
|
||||
@@ -32,7 +36,7 @@ impl ReplicationRuleExt for ReplicationRule {
|
||||
""
|
||||
}
|
||||
} else {
|
||||
""
|
||||
self.prefix.as_deref().unwrap_or("")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1691,7 +1691,7 @@ mod serial_tests {
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
#[ignore = "FAILING on main: excluded from the serial ILM lane pending a fix, see rustfs/backlog#1148 (ilm-1 partial)"]
|
||||
async fn test_noncurrent_expiry_still_works_after_immediate_compensation_transition() {
|
||||
let (disk_paths, ecstore) = setup_isolated_test_env(true).await;
|
||||
|
||||
@@ -1775,7 +1775,7 @@ mod serial_tests {
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
#[ignore = "FAILING on main: excluded from the serial ILM lane pending a fix, see rustfs/backlog#1148 (ilm-1 partial)"]
|
||||
async fn test_noncurrent_transition_still_works_after_immediate_compensation_transition() {
|
||||
let (disk_paths, ecstore) = setup_isolated_test_env(true).await;
|
||||
|
||||
|
||||
@@ -30,23 +30,11 @@ These are approved-target invariants. A protocol's explicitly labeled current ex
|
||||
| Remote PUT is in flight or its response is unknown | Transition transaction | Only cleanup of its own canonical candidate, subject to the transaction recovery predicate | Durable transaction identity plus a known remote-version state; the approved target also requires expiry and durable takeover of the creator fence |
|
||||
| Local transition commit is complete | Exact transitioned version in `xl.meta` | No | Recovery finds the transaction's logical bucket/object/version and requires the complete recorded source identity (version ID, data directory, modification time, size, and ETag), `TRANSITION_COMPLETE`, and the same remote object, tier, and remote version before removing only the transaction record |
|
||||
| An ordinary delete removes that transitioned version | Hidden `xl.meta` free-version | Yes | Metadata quorum atomically removes the visible version and preserves its exact tier tuple in the free-version |
|
||||
| PUT or materialized self-copy replaces a transitioned null version | Hidden `xl.meta` free-version | Yes, after replacement commit and complete physical reference checks | The coordinator supplies one cleanup UUID to every disk; replacement metadata and the old remote tuple are written in the same `xl.meta` commit |
|
||||
| A recursive prefix/delete-all operation cannot preserve per-object markers | v6 journal bound to an immutable single dispatch manifest or a chunk-parent-bound child manifest | Yes, but only after child/manifest completion and all-pool absence proof | `DispatchAuthorized`, exact local destructive mutation, every journal `Committed`, then child/manifest `Completed`; a chunk parent advances only after that child completion |
|
||||
| Tier configuration mutation, manual job, or decommission receipt | Intent/admission/copy proof only | No | These records gate configuration, scheduling, or migration; they never become remote-object cleanup owners |
|
||||
|
||||
An old journal and a free-version can coexist during compatibility recovery. That coexistence is evidence of multiple possible owners, not permission to choose one: the journal path must retain its record until the version-specific recovery rule proves which owner is authoritative.
|
||||
|
||||
Null-version replacement uses the existing free-version format and recovery
|
||||
worker. Failed metadata preparation preserves both the old version and its
|
||||
inline bytes; rename rollback restores the complete previous metadata. Recovery
|
||||
must retain cleanup while any physical replica still references the remote tuple,
|
||||
including a minority version omitted by quorum merging, or any disk cannot be
|
||||
checked. A successful replacement needs no in-memory queue receipt to survive
|
||||
restart: the normal free-version sweep discovers its committed owner. Restores
|
||||
that retain the same remote tuple and ordinary versioned writes retain their
|
||||
existing ownership. Older binaries can read this format, but all writers and
|
||||
cleanup workers need the overwrite fix before these guarantees cover the fleet.
|
||||
|
||||
## Persisted record inventory
|
||||
|
||||
All keys below are objects in the internal metadata bucket. The table gives the canonical target form. Transition-transaction runtime recovery extracts the final 32 hexadecimal characters and UUID while ignoring shard directories and accepting uppercase hex. Manual-job runtime recovery requires exactly two shards matching the filename prefix, but accepts an uppercase UUID when the shards use the same uppercase text; it then loads the lowercase canonical job by UUID. The decommission validator recomputes and rejects a noncanonical manual-job path, but currently inherits the weaker transition parser. Exact runtime canonical-path validation for both protocols is an approved target.
|
||||
|
||||
@@ -102,10 +102,16 @@ Cloudflare's proxy may buffer the entire request body before forwarding and can
|
||||
1. Bypass the proxy. Send the failing request to `http://<host>:9000` directly. Success confirms the fault is in the proxy/CDN path.
|
||||
2. Bypass the CDN, keep the proxy. Point the proxy straight at the origin (Cloudflare grey cloud / direct DNS). If it now works, the CDN was buffering or re-chunking the body.
|
||||
3. Check idle reuse. Intermittent failures that correlate with upload size are almost always the keep-alive mismatch. Lower the proxy keepalive (or disable it) and retry.
|
||||
4. Check for a truncated body. If the upload hangs indefinitely rather than resetting, the proxy is forwarding a partial body and then going silent without closing the connection. RustFS bounds this wait with `RUSTFS_HTTP_REQUEST_BODY_READ_TIMEOUT` (`DEFAULT_HTTP_REQUEST_BODY_READ_TIMEOUT`, 300; `0` disables) and on timeout logs `put_object_body_read_stalled` with the received/expected byte counts.
|
||||
4. Check for a stalled body. A client or intermediary can stop forwarding data without closing the connection. `RUSTFS_HTTP_REQUEST_BODY_READ_TIMEOUT` defaults to 300 seconds. `PutObject` logs `put_object_body_read_stalled` when its body-read guard expires. `UploadPart` logs `upload_part_body_read_stalled` and returns `RequestTimeout` (HTTP 400); its log records `raw_bytes_received`, `expected_decoded_bytes`, `timeout_secs`, bucket, key, and request ID. The event identifies missing input progress, without attributing the cause to a particular proxy.
|
||||
5. Compare bytes. Confirm the proxy forwards exactly `Content-Length` body bytes with no compression or transformation.
|
||||
6. Confirm signed headers survive. `Host` and `x-amz-*` must reach RustFS unchanged; a `SignatureDoesNotMatch` (rather than a hang) points here.
|
||||
|
||||
For HTTP `UploadPart`, the inactivity budget counts time waiting for raw request-body bytes while storage is requesting input. Positive raw bytes reset the budget, including fragments of a signed AWS chunk that has not yet finished decoding. Foreground admission, capped-session staging, and storage backpressure do not consume the budget. Finishing the declared payload does not bypass the signed terminator, required trailers, or final body validation. This is an inactivity limit, so an upload making progress can take longer than 300 seconds overall.
|
||||
|
||||
Setting the timeout to `0` disables it for ordinary uploads. Multipart sessions with an explicit total-object-size cap retain a minimum 300-second timeout, including when the configured value is `0`. After a body-stall timeout, HTTP/1 uses the existing raw-body drain and closes the connection; HTTP/2 releases the affected stream and keeps the connection usable.
|
||||
|
||||
`UploadPart` requires a known logical byte length. RustFS uses the length normalized by S3S after authentication and decoding, with an exact logical stream length as a fallback. A bare `x-amz-decoded-content-length` or `Content-Encoding: aws-chunked` declaration cannot supply this length by itself. Requests reaching an ordinary upload session without a known length return `MissingContentLength` (HTTP 411) before body ingestion; capped sessions retain their `UnexpectedContent` rejection. Preserve the client's framing and signed headers through the proxy. The 5 GiB limit applies to each part request, not the combined size of an ordinary multipart upload.
|
||||
|
||||
## Known failure signatures
|
||||
|
||||
| Symptom | Forwarding fault | Issue |
|
||||
|
||||
@@ -77,7 +77,7 @@
|
||||
|
||||
rustfs = rustPlatform.buildRustPackage {
|
||||
pname = "rustfs";
|
||||
version = "1.0.0-rc.5";
|
||||
version = "1.0.0-rc.6";
|
||||
|
||||
src = ./.;
|
||||
|
||||
|
||||
@@ -2,8 +2,8 @@ apiVersion: v2
|
||||
name: rustfs
|
||||
description: RustFS helm chart to deploy RustFS on kubernetes cluster.
|
||||
type: application
|
||||
version: "1.0.0-rc.5"
|
||||
appVersion: "1.0.0-rc.5"
|
||||
version: "1.0.0-rc.6"
|
||||
appVersion: "1.0.0-rc.6"
|
||||
home: https://rustfs.com
|
||||
icon: https://media.sys.truenas.net/apps/rustfs/icons/icon.svg
|
||||
maintainers:
|
||||
|
||||
+5
-2
@@ -1,9 +1,9 @@
|
||||
%global _enable_debug_packages 0
|
||||
%global _empty_manifest_terminate_build 0
|
||||
%global prerelease rc.5
|
||||
%global prerelease rc.6
|
||||
Name: rustfs
|
||||
Version: 1.0.0
|
||||
Release: rc.5
|
||||
Release: rc.6
|
||||
Summary: High-performance distributed object storage for MinIO alternative
|
||||
|
||||
License: Apache-2.0
|
||||
@@ -58,6 +58,9 @@ install %_builddir/%{name}-%{version}-%{prerelease}/target/%_arch/%_arch-unknown
|
||||
%_bindir/rustfs
|
||||
|
||||
%changelog
|
||||
* Thu Sep 10 2026 overtrue <anzhengchao@gmail.com>
|
||||
- Update RPM package to RustFS 1.0.0-rc.6
|
||||
|
||||
* Mon Aug 31 2026 overtrue <anzhengchao@gmail.com>
|
||||
- Update RPM package to RustFS 1.0.0-rc.5
|
||||
|
||||
|
||||
+168
-21
@@ -22,9 +22,9 @@ use crate::server::rate_limit::{
|
||||
apply_throttle_headers, client_ip,
|
||||
};
|
||||
use crate::server::{
|
||||
APPLE_TOUCH_ICON_PATH, APPLE_TOUCH_ICON_PRECOMPOSED_PATH, CONSOLE_PREFIX, FAVICON_PATH, HEALTH_PREFIX, HEALTH_READY_PATH,
|
||||
HeaderMapCarrier, HealthProbe, LICENSE, RUSTFS_ADMIN_PREFIX, RequestContextLayer, VERSION, build_health_response_parts,
|
||||
collect_probe_readiness,
|
||||
APPLE_TOUCH_ICON_PATH, APPLE_TOUCH_ICON_PRECOMPOSED_PATH, FAVICON_PATH, HEALTH_PREFIX, HEALTH_READY_PATH, HeaderMapCarrier,
|
||||
HealthProbe, LICENSE, RUSTFS_ADMIN_PREFIX, RequestContextLayer, VERSION, build_health_response_parts,
|
||||
collect_probe_readiness, console_prefix,
|
||||
};
|
||||
use crate::version::{self, build};
|
||||
use axum::{
|
||||
@@ -83,7 +83,7 @@ async fn static_handler(uri: Uri) -> impl IntoResponse {
|
||||
return Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header("Content-Type", mime_type.to_string())
|
||||
.body(Body::from(file.data))
|
||||
.body(Body::from(rewrite_console_asset(path, file.data, crate::server::console_prefix())))
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
@@ -95,7 +95,7 @@ async fn static_handler(uri: Uri) -> impl IntoResponse {
|
||||
return Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header("Content-Type", mime_type.to_string())
|
||||
.body(Body::from(file.data))
|
||||
.body(Body::from(rewrite_console_asset(&index_path, file.data, crate::server::console_prefix())))
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
@@ -106,7 +106,11 @@ async fn static_handler(uri: Uri) -> impl IntoResponse {
|
||||
Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header("Content-Type", mime_type.to_string())
|
||||
.body(Body::from(file.data))
|
||||
.body(Body::from(rewrite_console_asset(
|
||||
"index.html",
|
||||
file.data,
|
||||
crate::server::console_prefix(),
|
||||
)))
|
||||
.unwrap()
|
||||
} else {
|
||||
Response::builder()
|
||||
@@ -116,6 +120,57 @@ async fn static_handler(uri: Uri) -> impl IntoResponse {
|
||||
}
|
||||
}
|
||||
|
||||
// Next exports bake the base path into HTML, chunk loaders, and RSC text payloads.
|
||||
// Rewrite only path references, preserving external URLs such as the source repository.
|
||||
fn rewrite_console_asset<'a>(path: &str, data: std::borrow::Cow<'a, [u8]>, prefix: &str) -> std::borrow::Cow<'a, [u8]> {
|
||||
use std::borrow::Cow;
|
||||
|
||||
if prefix == crate::server::CONSOLE_PREFIX
|
||||
|| !matches!(
|
||||
path.rsplit('.').next(),
|
||||
Some("html" | "js" | "css" | "json" | "txt" | "webmanifest" | "svg")
|
||||
)
|
||||
{
|
||||
return data;
|
||||
}
|
||||
let Ok(text) = std::str::from_utf8(&data) else {
|
||||
return data;
|
||||
};
|
||||
let mut rewritten = Cow::Borrowed(text);
|
||||
let escaped_default = crate::server::CONSOLE_PREFIX.replace('/', "\\/");
|
||||
let escaped_prefix = prefix.replace('/', "\\/");
|
||||
for (source, target) in [
|
||||
(crate::server::CONSOLE_PREFIX, prefix),
|
||||
(escaped_default.as_str(), escaped_prefix.as_str()),
|
||||
] {
|
||||
let mut output = String::new();
|
||||
let mut copied = 0;
|
||||
for (offset, _) in rewritten.match_indices(source) {
|
||||
let end = offset + source.len();
|
||||
let before = rewritten.as_bytes().get(offset.wrapping_sub(1)).copied();
|
||||
let after = rewritten.as_bytes().get(end).copied();
|
||||
let starts_path = before
|
||||
.is_none_or(|byte| byte.is_ascii_whitespace() || matches!(byte, b'"' | b'\'' | b'`' | b'(' | b'=' | b'}' | b'>'));
|
||||
let ends_prefix = after.is_none_or(|byte| {
|
||||
byte.is_ascii_whitespace() || matches!(byte, b'/' | b'\\' | b'"' | b'\'' | b'`' | b'?' | b'#' | b')' | b'<')
|
||||
});
|
||||
if starts_path && ends_prefix {
|
||||
output.push_str(&rewritten[copied..offset]);
|
||||
output.push_str(target);
|
||||
copied = end;
|
||||
}
|
||||
}
|
||||
if copied != 0 {
|
||||
output.push_str(&rewritten[copied..]);
|
||||
rewritten = Cow::Owned(output);
|
||||
}
|
||||
}
|
||||
match rewritten {
|
||||
Cow::Borrowed(_) => data,
|
||||
Cow::Owned(text) => Cow::Owned(text.into_bytes()),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Clone)]
|
||||
pub(crate) struct Config {
|
||||
#[serde(skip)]
|
||||
@@ -468,7 +523,7 @@ fn get_console_config_from_env() -> (bool, u32, u64, String) {
|
||||
/// - `true` if the path is for console access, `false` otherwise.
|
||||
pub fn is_console_path(path: &str) -> bool {
|
||||
matches!(path, FAVICON_PATH | APPLE_TOUCH_ICON_PATH | APPLE_TOUCH_ICON_PRECOMPOSED_PATH)
|
||||
|| has_path_prefix(path, CONSOLE_PREFIX)
|
||||
|| has_path_prefix(path, console_prefix())
|
||||
}
|
||||
|
||||
/// Setup comprehensive middleware stack with tower-http features
|
||||
@@ -487,34 +542,35 @@ fn setup_console_middleware_stack(
|
||||
rate_limit_rpm: u32,
|
||||
auth_timeout: u64,
|
||||
) -> Router {
|
||||
let console_prefix = console_prefix();
|
||||
let mut app = Router::new()
|
||||
.route(FAVICON_PATH, get(static_handler))
|
||||
.route(&format!("{CONSOLE_PREFIX}{LICENSE}"), get(license_handler))
|
||||
.route(&format!("{CONSOLE_PREFIX}{VERSION}"), get(version_handler))
|
||||
.nest(CONSOLE_PREFIX, Router::new().fallback_service(get(static_handler)))
|
||||
.route(&format!("{console_prefix}{LICENSE}"), get(license_handler))
|
||||
.route(&format!("{console_prefix}{VERSION}"), get(version_handler))
|
||||
.nest(console_prefix, Router::new().fallback_service(get(static_handler)))
|
||||
.fallback_service(get(static_handler));
|
||||
|
||||
if rustfs_utils::get_env_bool(rustfs_config::ENV_HEALTH_ENDPOINT_ENABLE, rustfs_config::DEFAULT_HEALTH_ENDPOINT_ENABLE) {
|
||||
app = app
|
||||
.route(&format!("{CONSOLE_PREFIX}{HEALTH_PREFIX}"), get(health_check).head(health_check))
|
||||
.route(&format!("{console_prefix}{HEALTH_PREFIX}"), get(health_check).head(health_check))
|
||||
.route(
|
||||
&format!("{CONSOLE_PREFIX}{}", crate::server::HEALTH_COMPAT_LIVE_PATH),
|
||||
&format!("{console_prefix}{}", crate::server::HEALTH_COMPAT_LIVE_PATH),
|
||||
get(health_check).head(health_check),
|
||||
)
|
||||
.route(&format!("{CONSOLE_PREFIX}{HEALTH_READY_PATH}"), get(health_check).head(health_check));
|
||||
.route(&format!("{console_prefix}{HEALTH_READY_PATH}"), get(health_check).head(health_check));
|
||||
} else {
|
||||
// Keep disabled health probes from falling through to the SPA fallback.
|
||||
app = app
|
||||
.route(
|
||||
&format!("{CONSOLE_PREFIX}{HEALTH_PREFIX}"),
|
||||
&format!("{console_prefix}{HEALTH_PREFIX}"),
|
||||
get(health_route_disabled).head(health_route_disabled),
|
||||
)
|
||||
.route(
|
||||
&format!("{CONSOLE_PREFIX}{}", crate::server::HEALTH_COMPAT_LIVE_PATH),
|
||||
&format!("{console_prefix}{}", crate::server::HEALTH_COMPAT_LIVE_PATH),
|
||||
get(health_route_disabled).head(health_route_disabled),
|
||||
)
|
||||
.route(
|
||||
&format!("{CONSOLE_PREFIX}{HEALTH_READY_PATH}"),
|
||||
&format!("{console_prefix}{HEALTH_READY_PATH}"),
|
||||
get(health_route_disabled).head(health_route_disabled),
|
||||
);
|
||||
}
|
||||
@@ -624,7 +680,7 @@ async fn health_check(
|
||||
uri: Uri,
|
||||
server_ctx: Option<Extension<Arc<crate::runtime_sources::ServerContextSlot>>>,
|
||||
) -> Response {
|
||||
let probe = if uri.path().strip_prefix(CONSOLE_PREFIX) == Some(HEALTH_READY_PATH) {
|
||||
let probe = if uri.path().strip_prefix(console_prefix()) == Some(HEALTH_READY_PATH) {
|
||||
HealthProbe::Readiness
|
||||
} else {
|
||||
HealthProbe::Liveness
|
||||
@@ -786,6 +842,7 @@ pub(crate) fn make_console_server() -> Router {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::server::CONSOLE_PREFIX;
|
||||
use axum::body::Body;
|
||||
use axum::routing::get;
|
||||
use http::{Request, StatusCode};
|
||||
@@ -867,9 +924,14 @@ mod tests {
|
||||
async fn console_config_handler_serializes_admin_discovery_paths() {
|
||||
init_console_cfg(IpAddr::V4(Ipv4Addr::LOCALHOST), 9001);
|
||||
|
||||
let response = config_handler(Uri::from_static("http://127.0.0.1:9001/rustfs/console/api/v1/config"), HeaderMap::new())
|
||||
.await
|
||||
.into_response();
|
||||
let response = config_handler(
|
||||
format!("http://127.0.0.1:9001{CONSOLE_PREFIX}/api/v1/config")
|
||||
.parse()
|
||||
.expect("console URI"),
|
||||
HeaderMap::new(),
|
||||
)
|
||||
.await
|
||||
.into_response();
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = response.into_body();
|
||||
@@ -889,7 +951,8 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn external_admin_paths_are_not_console_paths() {
|
||||
assert!(is_console_path("/rustfs/console/"));
|
||||
assert!(is_console_path(&format!("{CONSOLE_PREFIX}/")));
|
||||
assert!(!is_console_path(&format!("{CONSOLE_PREFIX}-other/index.html")));
|
||||
assert!(is_console_path("/apple-touch-icon.png"));
|
||||
assert!(is_console_path("/apple-touch-icon-precomposed.png"));
|
||||
assert!(!is_console_path("/minio/admin/v3/info"));
|
||||
@@ -1273,3 +1336,87 @@ mod tests {
|
||||
assert!(value.get("expired").is_none());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod console_asset_prefix_tests {
|
||||
use super::rewrite_console_asset;
|
||||
use crate::server::CONSOLE_PREFIX;
|
||||
use std::borrow::Cow;
|
||||
|
||||
#[test]
|
||||
fn rewrites_exported_assets_and_client_routes() {
|
||||
let fixtures = [
|
||||
(
|
||||
"index.html",
|
||||
r#"<script src="/rustfs/console/_next/app.js"></script>"#,
|
||||
r#"<script src="/console/_next/app.js"></script>"#,
|
||||
),
|
||||
(
|
||||
"app.js",
|
||||
r#"let base="/rustfs/console";fetch(`${host}/rustfs/console/version`)"#,
|
||||
r#"let base="/console";fetch(`${host}/console/version`)"#,
|
||||
),
|
||||
("app.css", "url(/rustfs/console/logo.svg)", "url(/console/logo.svg)"),
|
||||
(
|
||||
"route.txt",
|
||||
r#"2:I[1,["/rustfs/console/_next/app.js"],"default"]"#,
|
||||
r#"2:I[1,["/console/_next/app.js"],"default"]"#,
|
||||
),
|
||||
("config.json", r#"{"url":"\/rustfs\/console\/login"}"#, r#"{"url":"\/console\/login"}"#),
|
||||
("site.webmanifest", r#"{"start_url":"/rustfs/console/"}"#, r#"{"start_url":"/console/"}"#),
|
||||
(
|
||||
"logo.svg",
|
||||
r#"<image href="/rustfs/console/logo.png"/>"#,
|
||||
r#"<image href="/console/logo.png"/>"#,
|
||||
),
|
||||
];
|
||||
for (path, input, expected) in fixtures {
|
||||
let input = input
|
||||
.replace("/rustfs/console", CONSOLE_PREFIX)
|
||||
.replace("\\/rustfs\\/console", &CONSOLE_PREFIX.replace('/', "\\/"));
|
||||
assert_eq!(
|
||||
rewrite_console_asset(path, Cow::Borrowed(input.as_bytes()), "/console").as_ref(),
|
||||
expected.as_bytes(),
|
||||
"{path}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restores_the_standard_path_from_an_oem_build() {
|
||||
let input = format!(r#"<script src="{CONSOLE_PREFIX}/_next/app.js"></script>"#);
|
||||
let output = rewrite_console_asset("index.html", Cow::Borrowed(input.as_bytes()), rustfs_config::DEFAULT_CONSOLE_PREFIX);
|
||||
assert_eq!(output.as_ref(), br#"<script src="/rustfs/console/_next/app.js"></script>"#);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preserves_unrelated_urls_paths_and_default_bytes() {
|
||||
let input = format!(
|
||||
r#"["https://github.com/rustfs/console","/other{CONSOLE_PREFIX}","{CONSOLE_PREFIX}-extra","{CONSOLE_PREFIX}/index.html"]"#
|
||||
);
|
||||
let expected = format!(
|
||||
r#"["https://github.com/rustfs/console","/other{CONSOLE_PREFIX}","{CONSOLE_PREFIX}-extra","/console/index.html"]"#
|
||||
);
|
||||
assert_eq!(
|
||||
rewrite_console_asset("app.js", Cow::Borrowed(input.as_bytes()), "/console").as_ref(),
|
||||
expected.as_bytes()
|
||||
);
|
||||
let unchanged = rewrite_console_asset("app.js", Cow::Borrowed(input.as_bytes()), CONSOLE_PREFIX);
|
||||
assert!(matches!(unchanged, Cow::Borrowed(_)));
|
||||
assert_eq!(unchanged.as_ref(), input.as_bytes());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preserves_binary_invalid_utf8_and_text_without_paths() {
|
||||
let invalid_utf8 = [b"\xff".as_slice(), CONSOLE_PREFIX.as_bytes()].concat();
|
||||
for (path, bytes) in [
|
||||
("image.png", CONSOLE_PREFIX.as_bytes()),
|
||||
("app.js", invalid_utf8.as_slice()),
|
||||
("app.js", b"https://github.com/rustfs/console".as_slice()),
|
||||
] {
|
||||
let output = rewrite_console_asset(path, Cow::Borrowed(bytes), "/console");
|
||||
assert!(matches!(output, Cow::Borrowed(_)), "{path}");
|
||||
assert_eq!(output.as_ref(), bytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -38,7 +38,7 @@ use super::supervise_admin_mutation;
|
||||
use crate::admin::auth::validate_admin_request;
|
||||
use crate::admin::router::{AdminOperation, Operation, S3Router};
|
||||
use crate::admin::runtime_sources::{current_action_credentials, current_ready_iam_handle, object_store_from_req};
|
||||
use crate::admin::service::caller_identity::CallerIdentity;
|
||||
use crate::admin::service::caller_identity::{CallerIdentity, oidc_profile_fields};
|
||||
use crate::admin::storage_api::s3::{self, Body, S3ErrorCode, S3Request, S3Response, S3Result};
|
||||
use crate::admin::utils::read_compatible_admin_body;
|
||||
use crate::auth::constant_time_eq;
|
||||
@@ -73,6 +73,16 @@ pub fn register_account_route(r: &mut S3Router<AdminOperation>) -> std::io::Resu
|
||||
/// `GET /rustfs/admin/v3/account/info`
|
||||
pub struct SelfAccountInfoHandler {}
|
||||
|
||||
#[derive(Debug, serde::Serialize)]
|
||||
struct SelfAccountInfoResponse {
|
||||
#[serde(flatten)]
|
||||
account: SelfAccountInfo,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
username: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
email: Option<String>,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for SelfAccountInfoHandler {
|
||||
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
@@ -124,17 +134,22 @@ impl Operation for SelfAccountInfoHandler {
|
||||
None => return Err(s3::error(S3ErrorCode::ServiceUnavailable, "the object store is not ready")),
|
||||
};
|
||||
|
||||
let info = SelfAccountInfo {
|
||||
access_key: caller.access_key.clone(),
|
||||
identity_type: caller.identity_type,
|
||||
session_access_key: caller.session_access_key.clone(),
|
||||
is_admin: caller.is_owner,
|
||||
status,
|
||||
member_of,
|
||||
policies,
|
||||
credentials_source: caller.credentials_source,
|
||||
mutable: caller.mutability(),
|
||||
mfa,
|
||||
let (username, email) = oidc_profile_fields(&caller.credentials);
|
||||
let info = SelfAccountInfoResponse {
|
||||
account: SelfAccountInfo {
|
||||
access_key: caller.access_key.clone(),
|
||||
identity_type: caller.identity_type,
|
||||
session_access_key: caller.session_access_key.clone(),
|
||||
is_admin: caller.is_owner,
|
||||
status,
|
||||
member_of,
|
||||
policies,
|
||||
credentials_source: caller.credentials_source,
|
||||
mutable: caller.mutability(),
|
||||
mfa,
|
||||
},
|
||||
username,
|
||||
email,
|
||||
};
|
||||
|
||||
admin_json_response(req.uri.path(), &caller.credentials.secret_key, StatusCode::OK, &info)
|
||||
@@ -548,6 +563,38 @@ fn validate_new_secret_key(request: &ChangePasswordRequest) -> S3Result<()> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::server::ADMIN_PREFIX;
|
||||
use rustfs_madmin::account::{AccountMutability, CredentialsSource};
|
||||
|
||||
#[test]
|
||||
fn self_account_info_response_adds_oidc_display_fields_without_changing_base_type() {
|
||||
let mut response = SelfAccountInfoResponse {
|
||||
account: SelfAccountInfo {
|
||||
access_key: "virtual-parent".to_string(),
|
||||
identity_type: IdentityType::Sts,
|
||||
session_access_key: Some("temporary-key".to_string()),
|
||||
is_admin: false,
|
||||
status: "enabled".to_string(),
|
||||
member_of: Vec::new(),
|
||||
policies: Vec::new(),
|
||||
credentials_source: CredentialsSource::Iam,
|
||||
mutable: AccountMutability::default(),
|
||||
mfa: AccountMfaSummary::default(),
|
||||
},
|
||||
username: Some("oidc-user".to_string()),
|
||||
email: Some("oidc-user@example.test".to_string()),
|
||||
};
|
||||
|
||||
let value = serde_json::to_value(&response).expect("serialize account response");
|
||||
assert_eq!(value["access_key"], "virtual-parent");
|
||||
assert_eq!(value["username"], "oidc-user");
|
||||
assert_eq!(value["email"], "oidc-user@example.test");
|
||||
|
||||
response.username = None;
|
||||
response.email = None;
|
||||
let legacy_shape = serde_json::to_value(&response).expect("serialize account response without OIDC fields");
|
||||
assert!(!legacy_shape.as_object().unwrap().contains_key("username"));
|
||||
assert!(!legacy_shape.as_object().unwrap().contains_key("email"));
|
||||
}
|
||||
|
||||
fn change_request(current: &str, new: &str) -> ChangePasswordRequest {
|
||||
ChangePasswordRequest {
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
use crate::admin::auth::authenticate_request;
|
||||
use crate::admin::router::{AdminOperation, Operation, S3Router};
|
||||
use crate::admin::runtime_sources::{current_action_credentials, object_store_from_req};
|
||||
use crate::admin::service::caller_identity::oidc_profile_fields;
|
||||
use crate::admin::storage_api::bucket::versioning_sys::BucketVersioningSys;
|
||||
use crate::admin::storage_api::contract::admin::StorageAdminApi;
|
||||
use crate::admin::storage_api::contract::bucket::{BucketOperations, BucketOptions};
|
||||
@@ -52,6 +53,16 @@ pub struct AccountInfo {
|
||||
|
||||
pub struct AccountInfoHandler {}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct AccountInfoResponse {
|
||||
#[serde(flatten)]
|
||||
account: rustfs_madmin::AccountInfo,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
username: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
email: Option<String>,
|
||||
}
|
||||
|
||||
pub fn register_account_info_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
|
||||
r.insert(
|
||||
Method::GET,
|
||||
@@ -242,6 +253,7 @@ impl Operation for AccountInfoHandler {
|
||||
let policy_str = serde_json::to_string(&effective_policy)
|
||||
.map_err(|_e| S3Error::with_message(S3ErrorCode::InternalError, "parse policy failed"))?;
|
||||
|
||||
let (username, email) = oidc_profile_fields(&cred);
|
||||
let mut account_info = rustfs_madmin::AccountInfo {
|
||||
account_name,
|
||||
server: StorageAdminApi::backend_info(store.as_ref()).await,
|
||||
@@ -288,8 +300,12 @@ impl Operation for AccountInfoHandler {
|
||||
}
|
||||
}
|
||||
|
||||
let data = serde_json::to_vec(&account_info)
|
||||
.map_err(|_e| S3Error::with_message(S3ErrorCode::InternalError, "parse accountInfo failed"))?;
|
||||
let data = serde_json::to_vec(&AccountInfoResponse {
|
||||
account: account_info,
|
||||
username,
|
||||
email,
|
||||
})
|
||||
.map_err(|_e| S3Error::with_message(S3ErrorCode::InternalError, "parse accountInfo failed"))?;
|
||||
|
||||
let mut header = HeaderMap::new();
|
||||
header.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||
@@ -305,6 +321,25 @@ mod tests {
|
||||
use rustfs_policy::policy::BucketPolicy;
|
||||
use s3s::dto::{Destination, ReplicationRule};
|
||||
|
||||
#[test]
|
||||
fn accountinfo_response_adds_optional_oidc_display_fields() {
|
||||
let mut response = AccountInfoResponse {
|
||||
account: rustfs_madmin::AccountInfo::default(),
|
||||
username: Some("oidc-user".to_string()),
|
||||
email: Some("oidc-user@example.test".to_string()),
|
||||
};
|
||||
|
||||
let value = serde_json::to_value(&response).expect("serialize accountinfo response");
|
||||
assert_eq!(value["username"], "oidc-user");
|
||||
assert_eq!(value["email"], "oidc-user@example.test");
|
||||
|
||||
response.username = None;
|
||||
response.email = None;
|
||||
let legacy_shape = serde_json::to_value(&response).expect("serialize accountinfo response without OIDC fields");
|
||||
assert!(!legacy_shape.as_object().unwrap().contains_key("username"));
|
||||
assert!(!legacy_shape.as_object().unwrap().contains_key("email"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_account_info_structure() {
|
||||
// Test AccountInfo struct creation and serialization
|
||||
|
||||
@@ -17,7 +17,6 @@ use crate::admin::router::{AdminOperation, Operation, S3Router};
|
||||
use crate::admin::runtime_sources::app_context_from_req;
|
||||
use crate::admin::storage_api::bucket::is_reserved_or_invalid_bucket;
|
||||
use crate::admin::storage_api::bucket::utils::is_valid_object_prefix;
|
||||
use crate::error::ApiError;
|
||||
use crate::server::ADMIN_PREFIX;
|
||||
use crate::server::RemoteAddr;
|
||||
use crate::storage::rpc::node_service::heal::{
|
||||
@@ -29,7 +28,6 @@ use futures_util::future::join_all;
|
||||
use http::{HeaderMap, HeaderValue, Uri};
|
||||
use hyper::{Method, StatusCode};
|
||||
use matchit::Params;
|
||||
use percent_encoding::percent_decode_str;
|
||||
use rustfs_config::MAX_HEAL_REQUEST_SIZE;
|
||||
use rustfs_heal::heal::utils::format_set_disk_id;
|
||||
use rustfs_heal_contracts::heal_channel::{
|
||||
@@ -37,11 +35,13 @@ use rustfs_heal_contracts::heal_channel::{
|
||||
};
|
||||
use rustfs_policy::policy::action::{Action, AdminAction};
|
||||
use rustfs_scanner::scanner::{BackgroundHealInfo, read_background_heal_info};
|
||||
use rustfs_utils::path::path_join;
|
||||
use s3s::header::{CONTENT_LENGTH, CONTENT_TYPE};
|
||||
use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::{BTreeMap, BTreeSet, HashSet};
|
||||
use std::future::Future;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
use tokio::time::{Duration, timeout};
|
||||
@@ -71,17 +71,9 @@ struct HealInitParams {
|
||||
}
|
||||
|
||||
fn extract_heal_init_params(body: &Bytes, uri: &Uri, params: Params<'_, '_>) -> S3Result<HealInitParams> {
|
||||
// matchit captures the original URI bytes. Decode once before validation
|
||||
// so literal %2F keys remain distinct from actual path separators.
|
||||
let mut hip = HealInitParams {
|
||||
bucket: percent_decode_str(params.get("bucket").unwrap_or_default())
|
||||
.decode_utf8()
|
||||
.map_err(|_| ApiError::invalid_request("invalid bucket name encoding"))?
|
||||
.into_owned(),
|
||||
obj_prefix: percent_decode_str(params.get("prefix").unwrap_or_default())
|
||||
.decode_utf8()
|
||||
.map_err(|_| ApiError::invalid_request("invalid object name encoding"))?
|
||||
.into_owned(),
|
||||
bucket: params.get("bucket").map(|s| s.to_string()).unwrap_or_default(),
|
||||
obj_prefix: params.get("prefix").map(|s| s.to_string()).unwrap_or_default(),
|
||||
..Default::default()
|
||||
};
|
||||
validate_heal_target(&hip.bucket, &hip.obj_prefix)?;
|
||||
@@ -172,13 +164,13 @@ fn validate_heal_target(bucket: &str, obj_prefix: &str) -> S3Result<()> {
|
||||
}
|
||||
|
||||
fn encode_heal_control_path(bucket: &str, obj_prefix: &str) -> String {
|
||||
if obj_prefix.is_empty() {
|
||||
return bucket.to_owned();
|
||||
if bucket.is_empty() && obj_prefix.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
|
||||
// This identifies an S3 target, not a filesystem path. In particular,
|
||||
// a leading slash in the object must not alias the sibling without it.
|
||||
format!("{bucket}/{obj_prefix}")
|
||||
path_join(&[PathBuf::from(bucket), PathBuf::from(obj_prefix)])
|
||||
.to_string_lossy()
|
||||
.into_owned()
|
||||
}
|
||||
|
||||
fn heal_control_response_id(heal_path: &str, client_token: &str) -> String {
|
||||
@@ -208,7 +200,7 @@ pub fn register_heal_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<
|
||||
|
||||
r.insert(
|
||||
Method::POST,
|
||||
format!("{}{}", ADMIN_PREFIX, "/v3/heal/{bucket}/{*prefix}").as_str(),
|
||||
format!("{}{}", ADMIN_PREFIX, "/v3/heal/{bucket}/{prefix}").as_str(),
|
||||
AdminOperation(&HealHandler {}),
|
||||
)?;
|
||||
|
||||
@@ -1624,135 +1616,6 @@ mod tests {
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::time::Duration;
|
||||
|
||||
fn parse_registered_heal_request(uri: &Uri) -> s3s::S3Result<HealInitParams> {
|
||||
let mut registered = super::S3Router::new(false);
|
||||
super::register_heal_route(&mut registered).expect("register production Heal routes");
|
||||
let mut router = Router::new();
|
||||
for route in registered.registered_routes() {
|
||||
router.insert(route.clone(), ()).expect("replay production route");
|
||||
}
|
||||
let path = format!("POST|{}", uri.path());
|
||||
let matched = router.at(&path).expect("request must match a production Heal route");
|
||||
let body = Bytes::from_static(
|
||||
br#"{"recursive":false,"dryRun":true,"remove":false,"recreate":false,"scanMode":2,"updateParity":false,"nolock":false,"readRepair":false,"pool":0,"set":0}"#,
|
||||
);
|
||||
extract_heal_init_params(&body, uri, matched.params)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_routes_accept_nested_and_encoded_object_paths() {
|
||||
let mut router = super::S3Router::new(false);
|
||||
super::register_heal_route(&mut router).expect("register production Heal routes");
|
||||
for prefix in ["/rustfs/admin", "/minio/admin"] {
|
||||
for target in [
|
||||
"",
|
||||
"test-bucket",
|
||||
"test-bucket/object.bin",
|
||||
"test-bucket/dir/sub/object.bin",
|
||||
"test-bucket/dir%2Fobject.bin",
|
||||
] {
|
||||
let path = format!("{prefix}/v3/heal/{target}");
|
||||
assert!(router.contains_compatible_route(http::Method::POST, &path), "{path}");
|
||||
assert!(!router.contains_compatible_route(http::Method::GET, &path), "{path}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_target_decodes_once_and_keeps_start_status_stop_identity() {
|
||||
for (wire, object) in [
|
||||
("object.bin", "object.bin"),
|
||||
("dir/sub/object.bin", "dir/sub/object.bin"),
|
||||
("dir%2Fsub%2Fobject.bin", "dir/sub/object.bin"),
|
||||
("dir%2fsub/object.bin", "dir/sub/object.bin"),
|
||||
("%2Fobject.bin", "/object.bin"),
|
||||
("dir/", "dir/"),
|
||||
("dir%2F", "dir/"),
|
||||
("literal%252Fslash", "literal%2Fslash"),
|
||||
("space%20key%2Bplus", "space key+plus"),
|
||||
("literal+plus", "literal+plus"),
|
||||
("%E4%B8%AD%E6%96%87%2F%E6%96%87%E4%BB%B6", "中文/文件"),
|
||||
("query%3Fhash%23percent%25", "query?hash#percent%"),
|
||||
] {
|
||||
for query in ["", "?clientToken=task", "?clientToken=task&forceStop=true"] {
|
||||
let uri = format!("/rustfs/admin/v3/heal/test%2Dbucket/{wire}{query}")
|
||||
.parse()
|
||||
.expect("valid encoded URI");
|
||||
let parsed = parse_registered_heal_request(&uri).expect("valid Heal target");
|
||||
assert_eq!(parsed.bucket, "test-bucket");
|
||||
assert_eq!(parsed.obj_prefix, object, "wire target: {wire}");
|
||||
assert_eq!(
|
||||
encode_heal_control_path(&parsed.bucket, &parsed.obj_prefix),
|
||||
format!("test-bucket/{object}")
|
||||
);
|
||||
assert_eq!(parsed.client_token, if query.is_empty() { "" } else { "task" });
|
||||
assert_eq!(parsed.force_stop, query.ends_with("forceStop=true"));
|
||||
if query.is_empty() {
|
||||
let request = build_heal_channel_request(&parsed);
|
||||
assert_eq!(request.bucket, "test-bucket");
|
||||
assert_eq!(request.object_prefix.as_deref(), Some(object));
|
||||
assert_eq!(request.pool_index, Some(0));
|
||||
assert_eq!(request.set_index, Some(0));
|
||||
assert_eq!(request.dry_run, Some(true));
|
||||
assert_eq!(request.scan_mode, Some(HealScanMode::Deep));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_target_validates_decoded_paths_before_admission() {
|
||||
for target in [
|
||||
"test%2Fbucket/object",
|
||||
"test%00bucket/object",
|
||||
"test%FFbucket/object",
|
||||
"test-bucket/dir%2F..%2Fobject",
|
||||
"test-bucket/dir/%2e/object",
|
||||
"test-bucket/dir%5C..%5Cobject",
|
||||
"test-bucket/dir%2F%2Fobject",
|
||||
"test-bucket/object%00",
|
||||
"test-bucket/object%FF",
|
||||
] {
|
||||
let uri = format!("/rustfs/admin/v3/heal/{target}").parse().expect("encoded URI");
|
||||
let err = parse_registered_heal_request(&uri).expect_err("decoded invalid target must fail closed");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest, "target: {target}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_nested_heal_routes_still_require_authentication() {
|
||||
use s3s::route::S3Route;
|
||||
|
||||
let mut router = super::S3Router::new(false);
|
||||
super::register_heal_route(&mut router).expect("register production Heal routes");
|
||||
for prefix in ["/rustfs/admin", "/minio/admin"] {
|
||||
for object in ["dir/object.bin", "dir%2Fobject.bin", "literal%252Fslash"] {
|
||||
let mut req = s3s::S3Request {
|
||||
input: s3s::Body::empty(),
|
||||
method: http::Method::POST,
|
||||
uri: format!("{prefix}/v3/heal/test-bucket/{object}").parse().expect("Heal URI"),
|
||||
headers: http::HeaderMap::new(),
|
||||
extensions: http::Extensions::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
};
|
||||
let err = router
|
||||
.check_access(&mut req)
|
||||
.await
|
||||
.expect_err("router must require a signature");
|
||||
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
|
||||
let err = router
|
||||
.call(req)
|
||||
.await
|
||||
.expect_err("handler must independently require authentication");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
assert!(err.to_string().contains("authentication required"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn replacement_record(task_id: &str) -> rustfs_heal::ReplacementRecoveryRecord {
|
||||
rustfs_heal::ReplacementRecoveryRecord {
|
||||
task_id: task_id.to_string(),
|
||||
|
||||
@@ -24,7 +24,7 @@ use crate::admin::storage_api::config::{
|
||||
read_admin_config_without_migrate, read_admin_server_config_snapshot, save_admin_server_config_snapshot,
|
||||
};
|
||||
use crate::admin::utils::json_response;
|
||||
use crate::server::{ADMIN_PREFIX, CONSOLE_PREFIX, MINIO_ADMIN_PREFIX};
|
||||
use crate::server::{ADMIN_PREFIX, MINIO_ADMIN_PREFIX, console_prefix};
|
||||
use http::StatusCode;
|
||||
use hyper::Method;
|
||||
use matchit::Params;
|
||||
@@ -824,7 +824,8 @@ fn build_console_redirect(
|
||||
let fragment =
|
||||
build_console_callback_fragment(access_key, secret_key, session_token, expiration, redirect_after, logout_token);
|
||||
|
||||
let callback_path = format!("{CONSOLE_PREFIX}{CONSOLE_OIDC_CALLBACK_SUFFIX}");
|
||||
let console_prefix = console_prefix();
|
||||
let callback_path = format!("{console_prefix}{CONSOLE_OIDC_CALLBACK_SUFFIX}");
|
||||
if let Some(base_url) = browser_redirect_url(&callback_path)? {
|
||||
return Ok(format!("{base_url}#{fragment}"));
|
||||
}
|
||||
@@ -836,7 +837,8 @@ fn build_console_redirect(
|
||||
}
|
||||
|
||||
fn build_console_login_redirect(req: &S3Request<Body>) -> S3Result<String> {
|
||||
let login_path = format!("{CONSOLE_PREFIX}{CONSOLE_LOGIN_SUFFIX}");
|
||||
let console_prefix = console_prefix();
|
||||
let login_path = format!("{console_prefix}{CONSOLE_LOGIN_SUFFIX}");
|
||||
if let Some(url) = browser_redirect_url(&login_path)? {
|
||||
return Ok(url);
|
||||
}
|
||||
@@ -1672,6 +1674,26 @@ mod tests {
|
||||
assert_eq!(callback, "https://internal:9000/rustfs/admin/v3/oidc/callback/default");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn console_prefix_process_case_oidc() {
|
||||
if std::env::var_os("RUSTFS_TEST_CONSOLE_PREFIX_PROCESS").is_none() {
|
||||
return;
|
||||
}
|
||||
crate::server::init_console_prefix().expect("initialize console prefix");
|
||||
let prefix = console_prefix();
|
||||
let req = build_oidc_request("http://internal/rustfs/admin/v3/oidc/callback/default", Some("internal:9000"), None);
|
||||
assert_eq!(
|
||||
build_console_login_redirect(&req).expect("login URL"),
|
||||
format!("https://console.example.com{prefix}/auth/login")
|
||||
);
|
||||
let redirect = build_console_redirect(&req, "access", "secret", "token", None, None, None).expect("console callback URL");
|
||||
assert!(redirect.starts_with(&format!("https://console.example.com{prefix}/auth/oidc-callback/#")));
|
||||
assert_eq!(
|
||||
derive_callback_uri(&req, "default").expect("admin callback URL"),
|
||||
"https://console.example.com/rustfs/admin/v3/oidc/callback/default"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_console_redirect_uses_browser_redirect_url() {
|
||||
let req = build_oidc_request("http://internal/rustfs/admin/v3/oidc/callback/default", Some("internal:9000"), None);
|
||||
@@ -1681,7 +1703,7 @@ mod tests {
|
||||
.expect("console redirect should use browser redirect URL")
|
||||
});
|
||||
|
||||
assert!(redirect.starts_with("https://console.example.com/rustfs/console/auth/oidc-callback/#"));
|
||||
assert!(redirect.starts_with(&format!("https://console.example.com{}/auth/oidc-callback/#", console_prefix())));
|
||||
assert!(redirect.contains("redirect=%2Fbuckets"));
|
||||
assert!(redirect.contains("logoutToken=logout-token"));
|
||||
}
|
||||
@@ -1694,7 +1716,7 @@ mod tests {
|
||||
build_console_login_redirect(&req).expect("login redirect should use browser redirect URL")
|
||||
});
|
||||
|
||||
assert_eq!(redirect, "https://console.example.com/rustfs/console/auth/login");
|
||||
assert_eq!(redirect, format!("https://console.example.com{}/auth/login", console_prefix()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -44,7 +44,8 @@ use crate::admin::utils::{empty_response, json_response, read_compatible_admin_b
|
||||
use crate::error::ApiError;
|
||||
use crate::server::ADMIN_PREFIX;
|
||||
use crate::site_replication::identity::{
|
||||
canonical_endpoint, is_https_endpoint, mark_unknown_peer_sync_enabled, same_identity_endpoint, site_identity_key,
|
||||
canonical_endpoint, deployment_id_for_endpoint, is_https_endpoint, mark_unknown_peer_sync_enabled, same_identity_endpoint,
|
||||
site_identity_key,
|
||||
};
|
||||
use crate::storage::storage_api::{lock_bucket_targets_metadata, with_config_object_write_lock};
|
||||
use base64_simd::URL_SAFE_NO_PAD;
|
||||
@@ -312,6 +313,8 @@ struct SRPeerJoinResponse {
|
||||
peer: PeerInfo,
|
||||
#[serde(rename = "initialSyncErrorMessage", default, skip_serializing_if = "String::is_empty")]
|
||||
initial_sync_error_message: String,
|
||||
#[serde(rename = "initialSyncDeferred", default, skip_serializing_if = "std::ops::Not::not")]
|
||||
initial_sync_deferred: bool,
|
||||
/// Whether the receiving site actually applied this join.
|
||||
///
|
||||
/// Three-valued on purpose. `None` means the peer did not report — MinIO
|
||||
@@ -330,6 +333,8 @@ struct SRPeerJoinEnvelope {
|
||||
request: SRPeerJoinReq,
|
||||
#[serde(rename = "deferSyncStateEnable", default, skip_serializing_if = "std::ops::Not::not")]
|
||||
defer_sync_state_enable: bool,
|
||||
#[serde(rename = "deferInitialSync", default, skip_serializing_if = "std::ops::Not::not")]
|
||||
defer_initial_sync: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
@@ -2792,6 +2797,19 @@ fn prune_in_sync_status_details(status: &mut SRStatusInfo, opts: &SRStatusOption
|
||||
}
|
||||
}
|
||||
|
||||
fn peer_states_from_infos(
|
||||
site_infos: BTreeMap<String, SRInfo>,
|
||||
reachable_peers: &HashSet<String>,
|
||||
) -> BTreeMap<String, SRStateInfo> {
|
||||
// Failed metainfo fetches leave default entries in site_infos for comparison;
|
||||
// they must not become fabricated peer state. PeerErrors describes the failure.
|
||||
site_infos
|
||||
.into_iter()
|
||||
.filter(|(deployment_id, _)| reachable_peers.contains(deployment_id))
|
||||
.map(|(deployment_id, info)| (deployment_id, info.state))
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn build_status_info(state: &SiteReplicationState, local_peer: &PeerInfo, uri: &Uri) -> S3Result<SRStatusInfo> {
|
||||
let opts = sr_status_options(uri);
|
||||
let mut local_info = Some(filter_sr_info(build_sr_info(state, local_peer).await?, &opts));
|
||||
@@ -2920,17 +2938,7 @@ async fn build_status_info(state: &SiteReplicationState, local_peer: &PeerInfo,
|
||||
}
|
||||
|
||||
if opts.peer_state {
|
||||
for (deployment_id, peer) in &state.peers {
|
||||
status.peer_states.insert(
|
||||
deployment_id.clone(),
|
||||
SRStateInfo {
|
||||
name: peer.name.clone(),
|
||||
peers: state.peers.clone(),
|
||||
updated_at: state.updated_at,
|
||||
api_version: Some(SITE_REPL_API_VERSION.to_string()),
|
||||
},
|
||||
);
|
||||
}
|
||||
status.peer_states = peer_states_from_infos(site_infos, &reachable_peers);
|
||||
}
|
||||
|
||||
Ok(status)
|
||||
@@ -2940,6 +2948,7 @@ fn merge_add_sites(
|
||||
mut state: SiteReplicationState,
|
||||
local_peer: PeerInfo,
|
||||
sites: Vec<PeerSite>,
|
||||
preflight_infos: &[SiteReplicationAddPreflightInfo],
|
||||
service_account_access_key: String,
|
||||
service_account_parent: String,
|
||||
replicate_ilm_expiry: bool,
|
||||
@@ -2949,11 +2958,36 @@ fn merge_add_sites(
|
||||
state.service_account_parent = service_account_parent;
|
||||
state.updated_at = Some(OffsetDateTime::now_utc());
|
||||
state.peers = build_join_peers(&state, &local_peer, sites, replicate_ilm_expiry);
|
||||
// Every join must carry the verified identities, including peers that
|
||||
// have not joined yet. Fixing only the coordinator after each reply
|
||||
// leaves the other sites holding endpoint-derived placeholders.
|
||||
for info in preflight_infos {
|
||||
if let Some(mut peer) = existing_peer_for_endpoint(&state, &info.endpoint) {
|
||||
peer.deployment_id = info.deployment_id.clone();
|
||||
state = reconcile_peer_with_actual_identity(state, peer);
|
||||
}
|
||||
}
|
||||
state
|
||||
}
|
||||
|
||||
fn update_peer(mut state: SiteReplicationState, incoming: PeerInfo, ilm_expiry_override: Option<bool>) -> SiteReplicationState {
|
||||
let mut peer = normalize_peer_info(incoming);
|
||||
// An older sender may still hold a placeholder after this site has
|
||||
// learned the real ID. Do not let that delivery downgrade the identity.
|
||||
if peer.deployment_id == deployment_id_for_endpoint(&peer.endpoint)
|
||||
&& let Some(existing) = state.peers.values().find(|existing| {
|
||||
same_identity_endpoint(&existing.endpoint, &peer.endpoint)
|
||||
&& existing.deployment_id != deployment_id_for_endpoint(&existing.endpoint)
|
||||
})
|
||||
{
|
||||
peer.deployment_id = existing.deployment_id.clone();
|
||||
}
|
||||
// Remove the placeholder before persistence normalizes duplicate
|
||||
// endpoints; otherwise map ordering can discard the real identity.
|
||||
state.peers.retain(|_, existing| {
|
||||
!same_identity_endpoint(&existing.endpoint, &peer.endpoint)
|
||||
|| existing.deployment_id != deployment_id_for_endpoint(&existing.endpoint)
|
||||
});
|
||||
if let Some(enabled) = ilm_expiry_override {
|
||||
peer.replicate_ilm_expiry = enabled;
|
||||
}
|
||||
@@ -3539,6 +3573,13 @@ fn align_peer_edit_deployment_id(state: &SiteReplicationState, incoming: &mut Pe
|
||||
return;
|
||||
};
|
||||
if matches.next().is_none() {
|
||||
if same_identity_endpoint(&peer.endpoint, &incoming.endpoint)
|
||||
&& peer.deployment_id == deployment_id_for_endpoint(&peer.endpoint)
|
||||
&& !incoming.deployment_id.is_empty()
|
||||
&& incoming.deployment_id != deployment_id_for_endpoint(&incoming.endpoint)
|
||||
{
|
||||
return;
|
||||
}
|
||||
incoming.deployment_id = peer.deployment_id.clone();
|
||||
}
|
||||
}
|
||||
@@ -3853,6 +3894,30 @@ fn pending_remote_peer_ids(peers: &BTreeMap<String, PeerInfo>, local_peer: &Peer
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The peers a pending remove / rotation still has to notify: every remote
|
||||
/// peer that has not acked, with the local site excluded by the same
|
||||
/// deployment-id-or-endpoint identity [`pending_remote_peer_ids`] finalizes
|
||||
/// on. The tick-driven `local_peer` carries the node's own listen address
|
||||
/// rather than the registered site endpoint (and a handler's carries the
|
||||
/// request `Host`, which behind a load balancer differs too), so an
|
||||
/// endpoint-only check dialed the site itself, timed out against the
|
||||
/// lifecycle lock this very request holds, and reported the operation as
|
||||
/// `Partial` (backlog#2367 A-4).
|
||||
fn pending_peers_awaiting_notification<'a>(
|
||||
peers: &'a BTreeMap<String, PeerInfo>,
|
||||
local_peer: &PeerInfo,
|
||||
acked_deployment_ids: &BTreeSet<String>,
|
||||
) -> Vec<&'a PeerInfo> {
|
||||
peers
|
||||
.values()
|
||||
.filter(|peer| {
|
||||
peer.deployment_id != local_peer.deployment_id
|
||||
&& !same_identity_endpoint(&peer.endpoint, &local_peer.endpoint)
|
||||
&& !acked_deployment_ids.contains(&peer.deployment_id)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn pending_all_remote_peers_acked(
|
||||
peers: &BTreeMap<String, PeerInfo>,
|
||||
local_peer: &PeerInfo,
|
||||
@@ -4068,12 +4133,7 @@ async fn drive_pending_rotation(pending: &PendingRotation, local_peer: &PeerInfo
|
||||
};
|
||||
|
||||
let mut peer_errors = Vec::new();
|
||||
for peer in pending.peers.values() {
|
||||
if same_identity_endpoint(&peer.endpoint, &local_peer.endpoint)
|
||||
|| pending.acked_deployment_ids.contains(&peer.deployment_id)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
for peer in pending_peers_awaiting_notification(&pending.peers, local_peer, &pending.acked_deployment_ids) {
|
||||
// A superseded join returns BEFORE `apply_iam`, so a no-op answer
|
||||
// means the peer never installed the new secret. Acking it would
|
||||
// finalize a rotation half the mesh cannot authenticate against
|
||||
@@ -4376,12 +4436,9 @@ async fn drive_pending_remove(pending_remove: &PendingRemove, local_peer: &PeerI
|
||||
if secret_candidates.is_empty() {
|
||||
peer_errors.push("site replication service account secret unavailable".to_string());
|
||||
} else {
|
||||
for peer in pending_remove.original_peers.values() {
|
||||
if same_identity_endpoint(&peer.endpoint, &local_peer.endpoint)
|
||||
|| pending_remove.acked_deployment_ids.contains(&peer.deployment_id)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
for peer in
|
||||
pending_peers_awaiting_notification(&pending_remove.original_peers, local_peer, &pending_remove.acked_deployment_ids)
|
||||
{
|
||||
if let Err(err) = PeerAdminRequest::put(
|
||||
&runtime_peer_connection(peer)?,
|
||||
SITE_REPLICATION_PEER_REMOVE_PATH,
|
||||
@@ -4931,7 +4988,7 @@ async fn ensure_site_replication_bucket_targets(bucket: &str) -> S3Result<()> {
|
||||
return Ok(());
|
||||
};
|
||||
let config = bucket_replication_config_for_target_refresh(bucket).await?;
|
||||
ensure_site_replication_bucket_targets_with_runtime(
|
||||
let written = ensure_site_replication_bucket_targets_with_runtime(
|
||||
bucket,
|
||||
&runtime.state,
|
||||
&runtime.local_peer,
|
||||
@@ -4939,7 +4996,11 @@ async fn ensure_site_replication_bucket_targets(bucket: &str) -> S3Result<()> {
|
||||
&runtime.service_account_secret_key,
|
||||
expected_incarnation_id,
|
||||
)
|
||||
.await
|
||||
.await?;
|
||||
if written {
|
||||
reload_bucket_metadata_on_peers(bucket, "site_replication_bucket_targets", false).await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn ensure_site_replication_bucket_setup(bucket: &str) -> S3Result<bool> {
|
||||
@@ -5009,6 +5070,9 @@ async fn cleanup_removed_site_replication_bucket(bucket: &str, removed_deploymen
|
||||
Err(err) => return Err(ApiError::from(err).into()),
|
||||
}
|
||||
|
||||
if removed > 0 {
|
||||
reload_bucket_metadata_on_peers(bucket, "site_replication_bucket_cleanup", true).await;
|
||||
}
|
||||
Ok(removed)
|
||||
}
|
||||
|
||||
@@ -5325,7 +5389,7 @@ async fn refresh_bucket_targets_after_endpoint_edit(pending_id: &str, service_ac
|
||||
let local_peer = current_local_runtime_peer(&target_state);
|
||||
let _targets_guard = lock_bucket_targets_metadata(&bucket.name).await;
|
||||
let replication_config = bucket_replication_config_for_target_refresh(&bucket.name).await?;
|
||||
ensure_site_replication_bucket_targets_with_runtime(
|
||||
let written = ensure_site_replication_bucket_targets_with_runtime(
|
||||
&bucket.name,
|
||||
&target_state,
|
||||
&local_peer,
|
||||
@@ -5334,6 +5398,9 @@ async fn refresh_bucket_targets_after_endpoint_edit(pending_id: &str, service_ac
|
||||
expected_incarnation_id,
|
||||
)
|
||||
.await?;
|
||||
if written {
|
||||
reload_bucket_metadata_on_peers(&bucket.name, "site_replication_endpoint_refresh", false).await;
|
||||
}
|
||||
|
||||
rewritten.push(bucket.name.clone());
|
||||
|
||||
@@ -5377,16 +5444,12 @@ async fn site_bucket_resync_manifest_entry(bucket: &str, peer: &PeerInfo, now: O
|
||||
..Default::default()
|
||||
};
|
||||
let _targets_guard = lock_bucket_targets_metadata(bucket).await;
|
||||
let (config, _) = match metadata_sys::get_replication_config(bucket).await {
|
||||
Ok(config) => config,
|
||||
Err(err) => {
|
||||
entry.status = "failed".to_string();
|
||||
entry.err_detail = summarize_peer_error_detail(&err.to_string());
|
||||
return entry;
|
||||
}
|
||||
};
|
||||
let targets = match metadata_sys::list_bucket_targets(bucket).await {
|
||||
Ok(targets) => targets,
|
||||
// Read what is persisted, not this node's cache: the wiring may have
|
||||
// been written by another node moments ago (`start_site_bucket_resync`
|
||||
// already reads its targets from disk), and an operator resync must see
|
||||
// the same records the drive will use.
|
||||
let (config, targets) = match site_bucket_resync_persisted_wiring(bucket).await {
|
||||
Ok(wiring) => wiring,
|
||||
Err(err) => {
|
||||
entry.status = "failed".to_string();
|
||||
entry.err_detail = summarize_peer_error_detail(&err.to_string());
|
||||
@@ -5417,6 +5480,15 @@ async fn site_bucket_resync_manifest_entry(bucket: &str, peer: &PeerInfo, now: O
|
||||
entry
|
||||
}
|
||||
|
||||
/// The persisted replication configuration and bucket targets, bypassing the
|
||||
/// node-local metadata cache. `ConfigNotFound` surfaces for a bucket without
|
||||
/// a replication configuration, matching the cached read's error.
|
||||
async fn site_bucket_resync_persisted_wiring(bucket: &str) -> Result<(ReplicationConfiguration, BucketTargets), StorageError> {
|
||||
let metadata = metadata_sys::get_config_from_disk(bucket).await?;
|
||||
let config = metadata.replication_config.ok_or(StorageError::ConfigNotFound)?;
|
||||
Ok((config, metadata.bucket_target_config.unwrap_or_default()))
|
||||
}
|
||||
|
||||
async fn start_site_bucket_resync(bucket: &str, target_arn: &str, resync_id: &str) -> ResyncBucketStatus {
|
||||
let mut bucket_status = ResyncBucketStatus {
|
||||
bucket: bucket.to_string(),
|
||||
@@ -5439,17 +5511,8 @@ async fn start_site_bucket_resync(bucket: &str, target_arn: &str, resync_id: &st
|
||||
}
|
||||
};
|
||||
|
||||
let (config, _) = match metadata_sys::get_replication_config(bucket).await {
|
||||
Ok(config) => config,
|
||||
Err(err) => {
|
||||
bucket_status.status = "failed".to_string();
|
||||
bucket_status.err_detail = err.to_string();
|
||||
return bucket_status;
|
||||
}
|
||||
};
|
||||
|
||||
let targets = match metadata_sys::list_bucket_targets_from_disk(bucket).await {
|
||||
Ok(targets) => targets,
|
||||
let (config, targets) = match site_bucket_resync_persisted_wiring(bucket).await {
|
||||
Ok(wiring) => wiring,
|
||||
Err(err) => {
|
||||
bucket_status.status = "failed".to_string();
|
||||
bucket_status.err_detail = err.to_string();
|
||||
@@ -6046,6 +6109,10 @@ async fn apply_bucket_meta_item(item: SRBucketMeta) -> S3Result<()> {
|
||||
drop(lifecycle_guard);
|
||||
drop(targets_guard);
|
||||
|
||||
if !skip_config_write {
|
||||
reload_bucket_metadata_on_peers(&item.bucket, "site_replication_bucket_meta", item.r#type == "lc-config").await;
|
||||
}
|
||||
|
||||
if item.r#type == "replication-config" {
|
||||
// Rebuild the local outbound rules too: a site that joined an already-replicated
|
||||
// bucket receives this item before it has any `site-repl-*` rule of its own.
|
||||
@@ -6684,6 +6751,7 @@ fn parse_peer_join_response(body: &[u8], fallback_peer: PeerInfo) -> Result<SRPe
|
||||
return Ok(SRPeerJoinResponse {
|
||||
peer: fallback_peer,
|
||||
initial_sync_error_message: String::new(),
|
||||
initial_sync_deferred: false,
|
||||
applied: None,
|
||||
});
|
||||
}
|
||||
@@ -6775,6 +6843,7 @@ impl Operation for SiteReplicationAddHandler {
|
||||
current_state,
|
||||
local_peer.clone(),
|
||||
sites.clone(),
|
||||
&preflight_infos,
|
||||
service_account_access_key.clone(),
|
||||
admin_access_key,
|
||||
replicate_ilm_expiry,
|
||||
@@ -6789,66 +6858,86 @@ impl Operation for SiteReplicationAddHandler {
|
||||
updated_at: state.updated_at,
|
||||
},
|
||||
defer_sync_state_enable: true,
|
||||
defer_initial_sync: true,
|
||||
};
|
||||
let peer_join_path = with_site_replication_bootstrap_token(
|
||||
SITE_REPLICATION_PEER_JOIN_PATH,
|
||||
&add_in_progress_guard.token.to_string(),
|
||||
);
|
||||
|
||||
let mut joined_endpoints = HashSet::new();
|
||||
// Install every site's service account before any receiver probes or
|
||||
// backfills to a peer that may not have joined yet. Reuse the join
|
||||
// snapshot for the second pass without repeating IAM/topology writes.
|
||||
// Only peers acknowledging deferral get a second request; older
|
||||
// receivers retain their one-pass behavior and reported errors.
|
||||
let initial_sync_path = format!("{peer_join_path}&initial-sync=true");
|
||||
let mut initial_sync_errors = SiteReplicationErrorSummary::default();
|
||||
for (site, preflight) in sites.iter().zip(preflight_infos.iter()) {
|
||||
if same_identity_endpoint(&site.endpoint, &local_peer.endpoint)
|
||||
|| !joined_endpoints.insert(site_identity_key(&site.endpoint))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let mut deferred_endpoints = HashSet::new();
|
||||
for (path, defer_initial_sync) in [(&peer_join_path, true), (&initial_sync_path, false)] {
|
||||
let mut joined_endpoints = HashSet::new();
|
||||
for (site, preflight) in sites.iter().zip(preflight_infos.iter()) {
|
||||
if same_identity_endpoint(&site.endpoint, &local_peer.endpoint)
|
||||
|| (!defer_initial_sync && !deferred_endpoints.contains(&site_identity_key(&site.endpoint)))
|
||||
|| !joined_endpoints.insert(site_identity_key(&site.endpoint))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut peer_join_req = join_req.clone();
|
||||
peer_join_req.request.svc_acct_parent = site.access_key.clone();
|
||||
let connection = PeerConnection::try_from(site)?;
|
||||
let body = PeerAdminRequest::put(&connection, &peer_join_path, &site.access_key)
|
||||
.send(&site.secret_key, &peer_join_req)
|
||||
.await?;
|
||||
let mut peer_join_req = join_req.clone();
|
||||
peer_join_req.defer_initial_sync = defer_initial_sync;
|
||||
peer_join_req.request.svc_acct_parent = site.access_key.clone();
|
||||
let connection = PeerConnection::try_from(site)?;
|
||||
let body = PeerAdminRequest::put(&connection, path, &site.access_key)
|
||||
.send(&site.secret_key, &peer_join_req)
|
||||
.await?;
|
||||
|
||||
let mut fallback_peer = existing_peer_for_endpoint(&state, &site.endpoint)
|
||||
.unwrap_or_else(|| normalize_peer_site(site.clone(), replicate_ilm_expiry));
|
||||
fallback_peer.deployment_id = preflight.deployment_id.clone();
|
||||
let join_response = parse_peer_join_response(&body, fallback_peer).map_err(|e| {
|
||||
S3Error::with_message(
|
||||
S3ErrorCode::InternalError,
|
||||
format!("parse peer join response from {} failed: {e}", site.endpoint),
|
||||
)
|
||||
})?;
|
||||
if !join_response.initial_sync_error_message.is_empty() {
|
||||
initial_sync_errors.push(format!("{}: {}", site.endpoint, join_response.initial_sync_error_message));
|
||||
let mut fallback_peer = existing_peer_for_endpoint(&state, &site.endpoint)
|
||||
.unwrap_or_else(|| normalize_peer_site(site.clone(), replicate_ilm_expiry));
|
||||
fallback_peer.deployment_id = preflight.deployment_id.clone();
|
||||
let join_response = parse_peer_join_response(&body, fallback_peer).map_err(|e| {
|
||||
S3Error::with_message(
|
||||
S3ErrorCode::InternalError,
|
||||
format!("parse peer join response from {} failed: {e}", site.endpoint),
|
||||
)
|
||||
})?;
|
||||
if join_response.initial_sync_deferred {
|
||||
if defer_initial_sync {
|
||||
deferred_endpoints.insert(site_identity_key(&site.endpoint));
|
||||
} else {
|
||||
initial_sync_errors.push(format!("{}: peer did not complete initial sync", site.endpoint));
|
||||
}
|
||||
}
|
||||
if !join_response.initial_sync_error_message.is_empty() {
|
||||
initial_sync_errors.push(format!("{}: {}", site.endpoint, join_response.initial_sync_error_message));
|
||||
}
|
||||
// An explicit no-op join. The peer answered 200 but wrote nothing —
|
||||
// its persisted state is already newer than the snapshot it was
|
||||
// sent — so the add is only PARTIALLY configured and saying
|
||||
// "configured successfully" would be a lie (rustfs/rustfs#5963).
|
||||
// `None` (a MinIO peer, or one older than the field) is not a
|
||||
// no-op signal and is deliberately not reported.
|
||||
if join_response.applied == Some(false) {
|
||||
let phase = if defer_initial_sync { "join" } else { "initial sync" };
|
||||
initial_sync_errors.push(format!(
|
||||
"{}: peer did not apply the {phase} (its site replication state is newer than the snapshot it was sent); \
|
||||
the site is not configured against this peer",
|
||||
site.endpoint
|
||||
));
|
||||
}
|
||||
state = reconcile_peer_with_actual_identity(state, join_response.peer);
|
||||
let reconciled_peer = existing_peer_for_endpoint(&state, &site.endpoint).ok_or_else(|| {
|
||||
S3Error::with_message(
|
||||
S3ErrorCode::InternalError,
|
||||
format!("peer join response from {} did not identify the requested site", site.endpoint),
|
||||
)
|
||||
})?;
|
||||
validate_proposed_peer(&reconciled_peer).map_err(|err| {
|
||||
S3Error::with_message(
|
||||
S3ErrorCode::InvalidRequest,
|
||||
format!("invalid peer join response from {}: {err}", site.endpoint),
|
||||
)
|
||||
})?;
|
||||
}
|
||||
// An explicit no-op join. The peer answered 200 but wrote nothing —
|
||||
// its persisted state is already newer than the snapshot it was
|
||||
// sent — so the add is only PARTIALLY configured and saying
|
||||
// "configured successfully" would be a lie (rustfs/rustfs#5963).
|
||||
// `None` (a MinIO peer, or one older than the field) is not a
|
||||
// no-op signal and is deliberately not reported.
|
||||
if join_response.applied == Some(false) {
|
||||
initial_sync_errors.push(format!(
|
||||
"{}: peer did not apply the join (its site replication state is newer than the snapshot it was sent); \
|
||||
the site is not configured against this peer",
|
||||
site.endpoint
|
||||
));
|
||||
}
|
||||
state = reconcile_peer_with_actual_identity(state, join_response.peer);
|
||||
let reconciled_peer = existing_peer_for_endpoint(&state, &site.endpoint).ok_or_else(|| {
|
||||
S3Error::with_message(
|
||||
S3ErrorCode::InternalError,
|
||||
format!("peer join response from {} did not identify the requested site", site.endpoint),
|
||||
)
|
||||
})?;
|
||||
validate_proposed_peer(&reconciled_peer).map_err(|err| {
|
||||
S3Error::with_message(
|
||||
S3ErrorCode::InvalidRequest,
|
||||
format!("invalid peer join response from {}: {err}", site.endpoint),
|
||||
)
|
||||
})?;
|
||||
}
|
||||
|
||||
mark_unknown_peer_sync_enabled(&mut state.peers);
|
||||
@@ -7122,6 +7211,24 @@ impl Operation for SiteReplicationNetPerfHandler {
|
||||
|
||||
pub struct SRPeerJoinHandler {}
|
||||
|
||||
fn ensure_initial_sync_join_current(state: &SiteReplicationState, join_req: &SRPeerJoinReq) -> S3Result<()> {
|
||||
if !state.enabled()
|
||||
|| join_req.updated_at.is_none()
|
||||
|| state.updated_at != join_req.updated_at
|
||||
|| state.service_account_access_key.is_empty()
|
||||
|| state.service_account_access_key != join_req.svc_acct_access_key
|
||||
|| state.pending_remove.is_some()
|
||||
|| state.pending_rotation.is_some()
|
||||
|| pending_endpoint_refresh(state).is_some()
|
||||
{
|
||||
return Err(s3_error!(
|
||||
InvalidRequest,
|
||||
"site replication changed before initial sync; re-run replicate add"
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// What the join admission decided about an incoming peer join. The verdict —
|
||||
/// and the committed state the back-fill afterwards needs — travel out of
|
||||
/// [`admit_peer_join`] instead of being answered where they are decided.
|
||||
@@ -7284,6 +7391,7 @@ fn superseded_join_response(peer: PeerInfo) -> SRPeerJoinResponse {
|
||||
SRPeerJoinResponse {
|
||||
peer,
|
||||
initial_sync_error_message: String::new(),
|
||||
initial_sync_deferred: false,
|
||||
applied: Some(false),
|
||||
}
|
||||
}
|
||||
@@ -7293,6 +7401,7 @@ fn applied_join_response(peer: PeerInfo, initial_sync_error_message: String) ->
|
||||
SRPeerJoinResponse {
|
||||
peer,
|
||||
initial_sync_error_message,
|
||||
initial_sync_deferred: false,
|
||||
applied: Some(true),
|
||||
}
|
||||
}
|
||||
@@ -7302,17 +7411,30 @@ impl Operation for SRPeerJoinHandler {
|
||||
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
let cred = validate_site_replication_admin_request(&req, AdminAction::SiteReplicationAddAction).await?;
|
||||
let bootstrap_token = site_replication_bootstrap_token(&req.uri);
|
||||
let initial_sync_only = query_pairs(&req.uri).get("initial-sync").is_some_and(|value| value == "true");
|
||||
let local_endpoint = site_replication_local_endpoint(&req.uri, &req.headers);
|
||||
// The body is fully read before the admission takes the lifecycle
|
||||
// guard: a sender that stalls mid-body must not block this node's
|
||||
// add/remove/rotate/reconciler.
|
||||
let join_envelope: SRPeerJoinEnvelope = read_site_replication_json(req, &cred.secret_key, true).await?;
|
||||
let defer_sync_state_enable = join_envelope.defer_sync_state_enable;
|
||||
let defer_initial_sync = join_envelope.defer_initial_sync;
|
||||
let join_req = join_envelope.request;
|
||||
validate_join_peer_snapshot(&join_req.peers)?;
|
||||
|
||||
let committed =
|
||||
admit_peer_join(local_endpoint, join_req, defer_sync_state_enable, apply_peer_join_service_account).await?;
|
||||
let _initial_sync_guard = if initial_sync_only {
|
||||
Some(SiteReplicationLifecycleGuard::acquire().await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let committed = if initial_sync_only {
|
||||
let state = load_site_replication_state().await?;
|
||||
ensure_initial_sync_join_current(&state, &join_req)?;
|
||||
let local_peer = local_peer_at_endpoint(local_endpoint, &state);
|
||||
PeerJoinOutcome::Applied(Box::new(state), local_peer)
|
||||
} else {
|
||||
admit_peer_join(local_endpoint, join_req, defer_sync_state_enable, apply_peer_join_service_account).await?
|
||||
};
|
||||
// Committed; the reverse-reachability probe and the bucket back-fill
|
||||
// run outside the transaction — their transport helpers' retry-event
|
||||
// bookkeeping re-enters it (P1-15).
|
||||
@@ -7329,6 +7451,12 @@ impl Operation for SRPeerJoinHandler {
|
||||
return json_response(StatusCode::OK, &superseded_join_response(peer));
|
||||
}
|
||||
};
|
||||
if defer_initial_sync && !initial_sync_only {
|
||||
let mut response =
|
||||
applied_join_response(state.peers.get(&local_peer.deployment_id).cloned().unwrap_or(local_peer), String::new());
|
||||
response.initial_sync_deferred = true;
|
||||
return json_response(StatusCode::OK, &response);
|
||||
}
|
||||
// Fix 1 (receiving side): ensure the joining peer also sets up replication for any
|
||||
// buckets it already owns so the reverse direction works from the start. Per-bucket
|
||||
// failures are logged (BUG2) so a reverse-direction back-fill gap is observable.
|
||||
@@ -7432,6 +7560,7 @@ impl Operation for SRPeerBucketOpsHandler {
|
||||
)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
reload_bucket_metadata_on_peers(&bucket, "site_replication_make_bucket", false).await;
|
||||
}
|
||||
"configure-replication" => {
|
||||
store
|
||||
@@ -8455,9 +8584,92 @@ impl Operation for SRRotateServiceAccountHandler {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::site_replication::identity::deployment_id_for_endpoint;
|
||||
use rustfs_madmin::SRSessionPolicy;
|
||||
|
||||
#[test]
|
||||
fn peer_states_preserve_each_sites_actual_membership_and_metadata() {
|
||||
let local = SRStateInfo {
|
||||
name: "local".to_string(),
|
||||
peers: BTreeMap::from([(
|
||||
"actual-remote".to_string(),
|
||||
PeerInfo {
|
||||
deployment_id: "actual-remote".to_string(),
|
||||
endpoint: "http://remote:9000".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
)]),
|
||||
updated_at: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
api_version: Some("1".to_string()),
|
||||
};
|
||||
let remote = SRStateInfo {
|
||||
name: "remote-reported-name".to_string(),
|
||||
peers: BTreeMap::from([(
|
||||
"legacy-placeholder".to_string(),
|
||||
PeerInfo {
|
||||
deployment_id: "legacy-placeholder".to_string(),
|
||||
endpoint: "http://local:9000".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
)]),
|
||||
updated_at: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(10)),
|
||||
api_version: None,
|
||||
};
|
||||
let infos = BTreeMap::from([
|
||||
(
|
||||
"local".to_string(),
|
||||
SRInfo {
|
||||
state: local.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
(
|
||||
"remote".to_string(),
|
||||
SRInfo {
|
||||
state: remote.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
]);
|
||||
let states = peer_states_from_infos(infos, &HashSet::from(["local".to_string(), "remote".to_string()]));
|
||||
assert_eq!(states.len(), 2);
|
||||
assert_eq!(serde_json::to_value(&states["local"]).unwrap(), serde_json::to_value(local).unwrap());
|
||||
assert_eq!(serde_json::to_value(&states["remote"]).unwrap(), serde_json::to_value(remote).unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_states_omit_unreachable_peers_instead_of_defaulting_them() {
|
||||
let infos = BTreeMap::from([
|
||||
("local".to_string(), SRInfo::default()),
|
||||
("offline".to_string(), SRInfo::default()),
|
||||
]);
|
||||
let states = peer_states_from_infos(infos, &HashSet::from(["local".to_string()]));
|
||||
assert_eq!(states.len(), 1);
|
||||
assert!(states.contains_key("local"));
|
||||
assert!(!states.contains_key("offline"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_states_preserve_a_reachable_peers_empty_membership() {
|
||||
let states = peer_states_from_infos(
|
||||
BTreeMap::from([(
|
||||
"remote".to_string(),
|
||||
SRInfo {
|
||||
enabled: false,
|
||||
state: SRStateInfo {
|
||||
name: "remote".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
},
|
||||
)]),
|
||||
&HashSet::from(["remote".to_string()]),
|
||||
);
|
||||
assert_eq!(states["remote"].name, "remote");
|
||||
assert!(states["remote"].peers.is_empty());
|
||||
assert!(states["remote"].updated_at.is_none());
|
||||
assert!(states["remote"].api_version.is_none());
|
||||
}
|
||||
|
||||
/// A peer the status probe could not reach must render as offline.
|
||||
///
|
||||
/// Regression: `build_metrics_summary` used to hardcode `online: true` and
|
||||
@@ -10889,6 +11101,7 @@ mod tests {
|
||||
secret_key: "remote-sk".to_string(),
|
||||
..PeerSite::default()
|
||||
}],
|
||||
&[],
|
||||
"svc-ak".to_string(),
|
||||
"root".to_string(),
|
||||
true,
|
||||
@@ -10922,6 +11135,7 @@ mod tests {
|
||||
..PeerSite::default()
|
||||
},
|
||||
],
|
||||
&[],
|
||||
"svc-ak".to_string(),
|
||||
"root".to_string(),
|
||||
true,
|
||||
@@ -12128,6 +12342,173 @@ mod tests {
|
||||
assert!(normalized.contains_key("hash-remote"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_peer_identity_join_snapshot_uses_verified_ids() {
|
||||
let actual = ["site-a", "site-b", "site-c"].map(|name| PeerInfo {
|
||||
deployment_id: format!("{name}-deployment"),
|
||||
..peer(name, &format!("https://{name}.example.com:9000"))
|
||||
});
|
||||
let preflight = actual
|
||||
.iter()
|
||||
.map(|peer| preflight_site("reported-name", &peer.endpoint, &peer.deployment_id, 0))
|
||||
.collect::<Vec<_>>();
|
||||
let sites = actual
|
||||
.iter()
|
||||
.map(|peer| PeerSite {
|
||||
name: peer.name.clone(),
|
||||
endpoint: peer.endpoint.clone(),
|
||||
skip_tls_verify: true,
|
||||
ca_cert_pem: "requested-ca".to_string(),
|
||||
..Default::default()
|
||||
})
|
||||
.collect();
|
||||
let state = merge_add_sites(
|
||||
SiteReplicationState::default(),
|
||||
actual[0].clone(),
|
||||
sites,
|
||||
&preflight,
|
||||
"svc-ak".to_string(),
|
||||
"root".to_string(),
|
||||
false,
|
||||
);
|
||||
assert_eq!(state.peers.len(), actual.len());
|
||||
for expected in &actual {
|
||||
let stored = state
|
||||
.peers
|
||||
.get(&expected.deployment_id)
|
||||
.expect("verified ID in initial join map");
|
||||
assert_eq!(stored.name, expected.name);
|
||||
assert_eq!(stored.endpoint, expected.endpoint);
|
||||
assert!(stored.skip_tls_verify);
|
||||
assert_eq!(stored.ca_cert_pem, "requested-ca");
|
||||
}
|
||||
for local in &actual[1..] {
|
||||
let mut joined = SiteReplicationState::default();
|
||||
apply_peer_join(
|
||||
&mut joined,
|
||||
local,
|
||||
SRPeerJoinReq {
|
||||
peers: state.peers.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
true,
|
||||
);
|
||||
assert_eq!(joined.peers.keys().collect::<Vec<_>>(), state.peers.keys().collect::<Vec<_>>());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_peer_identity_legacy_edit_does_not_restore_placeholder() {
|
||||
let actual = PeerInfo {
|
||||
deployment_id: "actual-remote".to_string(),
|
||||
..peer("remote", "http://remote.example.com:9000")
|
||||
};
|
||||
for name in ["remote", ""] {
|
||||
let state = SiteReplicationState {
|
||||
peers: BTreeMap::from([(actual.deployment_id.clone(), actual.clone())]),
|
||||
..Default::default()
|
||||
};
|
||||
let mut incoming = PeerInfo {
|
||||
deployment_id: deployment_id_for_endpoint("https://REMOTE.example.com:9000/"),
|
||||
sync_state: SyncStatus::Enable,
|
||||
..peer(name, "https://REMOTE.example.com:9000/")
|
||||
};
|
||||
align_peer_edit_deployment_id(&state, &mut incoming);
|
||||
let state = update_peer(state, incoming, None);
|
||||
assert_eq!(state.peers.len(), 1);
|
||||
assert_eq!(state.peers[&actual.deployment_id].sync_state, SyncStatus::Enable);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_peer_identity_finalization_repairs_legacy_three_site_join() {
|
||||
let actual = ["site-a", "site-b", "site-c"].map(|name| PeerInfo {
|
||||
deployment_id: format!("{name}-deployment"),
|
||||
..peer(name, &format!("http://{name}.example.com:9000"))
|
||||
});
|
||||
let sites = actual
|
||||
.iter()
|
||||
.map(|peer| PeerSite {
|
||||
name: peer.name.clone(),
|
||||
endpoint: peer.endpoint.clone(),
|
||||
..Default::default()
|
||||
})
|
||||
.collect();
|
||||
let mut coordinator = merge_add_sites(
|
||||
SiteReplicationState::default(),
|
||||
actual[0].clone(),
|
||||
sites,
|
||||
&[],
|
||||
"svc-ak".to_string(),
|
||||
"root".to_string(),
|
||||
true,
|
||||
);
|
||||
let join = SRPeerJoinReq {
|
||||
peers: coordinator.peers.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
for remote in &actual[1..] {
|
||||
coordinator = reconcile_peer_with_actual_identity(coordinator, remote.clone());
|
||||
}
|
||||
mark_unknown_peer_sync_enabled(&mut coordinator.peers);
|
||||
|
||||
for local in &actual[1..] {
|
||||
let mut state = SiteReplicationState::default();
|
||||
apply_peer_join(&mut state, local, join.clone(), true);
|
||||
for mut incoming in coordinator.peers.values().cloned() {
|
||||
align_peer_edit_deployment_id(&state, &mut incoming);
|
||||
state = apply_internal_peer_edit(state, local, incoming, None).expect("finalize peer identity");
|
||||
}
|
||||
assert_eq!(state.peers.len(), actual.len(), "finalization must not retain placeholder peers");
|
||||
for expected in &actual {
|
||||
let stored = existing_peer_for_endpoint(&state, &expected.endpoint).expect("peer remains configured");
|
||||
assert_eq!(stored.deployment_id, expected.deployment_id, "observer: {}", local.name);
|
||||
assert_eq!(stored.sync_state, SyncStatus::Enable);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_peer_identity_edit_replaces_placeholder_for_canonical_endpoint() {
|
||||
let local = PeerInfo {
|
||||
deployment_id: "local-deployment".to_string(),
|
||||
..peer("local", "https://local.example.com:9000")
|
||||
};
|
||||
let endpoint = "http://remote.example.com:9000";
|
||||
let placeholder = PeerInfo {
|
||||
deployment_id: deployment_id_for_endpoint(endpoint),
|
||||
..peer("remote", endpoint)
|
||||
};
|
||||
for deployment_id in ["00000000-0000-4000-8000-000000000001", "ffffffff-ffff-4fff-bfff-ffffffffffff"] {
|
||||
for name in ["remote", ""] {
|
||||
for already_present in [false, true] {
|
||||
let mut incoming = PeerInfo {
|
||||
deployment_id: deployment_id.to_string(),
|
||||
sync_state: SyncStatus::Enable,
|
||||
..peer(name, "https://REMOTE.example.com:9000/")
|
||||
};
|
||||
let mut state = SiteReplicationState {
|
||||
peers: BTreeMap::from([
|
||||
(local.deployment_id.clone(), local.clone()),
|
||||
(placeholder.deployment_id.clone(), placeholder.clone()),
|
||||
]),
|
||||
..Default::default()
|
||||
};
|
||||
if already_present {
|
||||
state.peers.insert(incoming.deployment_id.clone(), incoming.clone());
|
||||
}
|
||||
align_peer_edit_deployment_id(&state, &mut incoming);
|
||||
let state = apply_internal_peer_edit(state, &local, incoming, None).expect("repair peer identity");
|
||||
assert_eq!(state.peers.len(), 2, "repair must replace, not duplicate, the placeholder");
|
||||
assert!(!state.peers.contains_key(&placeholder.deployment_id));
|
||||
assert!(state.peers.contains_key(deployment_id));
|
||||
let normalized = normalize_peer_map_by_identity(state.peers);
|
||||
assert!(normalized.contains_key(deployment_id), "normalization must retain the actual ID");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reconcile_peer_with_actual_identity_replaces_endpoint_hash_key() {
|
||||
let mut state = SiteReplicationState::default();
|
||||
@@ -13450,6 +13831,7 @@ mod tests {
|
||||
assert_eq!(response.peer.deployment_id, "remote-deployment");
|
||||
assert_eq!(response.peer.endpoint, "https://remote.example.com");
|
||||
assert!(response.initial_sync_error_message.is_empty());
|
||||
assert!(!response.initial_sync_deferred);
|
||||
assert_eq!(
|
||||
response.applied, None,
|
||||
"a MinIO empty-body success reports nothing; it must not read as a no-op join"
|
||||
@@ -13459,6 +13841,7 @@ mod tests {
|
||||
let json = serde_json::to_vec(&SRPeerJoinResponse {
|
||||
peer: peer("actual", "https://actual.example.com"),
|
||||
initial_sync_error_message: "sync failed".to_string(),
|
||||
initial_sync_deferred: false,
|
||||
applied: Some(true),
|
||||
})
|
||||
.expect("serialize join response");
|
||||
@@ -14311,6 +14694,69 @@ mod tests {
|
||||
assert_eq!(value.get("deferSyncStateEnable"), Some(&Value::Bool(true)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_initial_sync_join_requires_the_committed_snapshot() {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let mut state = SiteReplicationState {
|
||||
updated_at: Some(now),
|
||||
service_account_access_key: "replicator".to_string(),
|
||||
peers: BTreeMap::from([
|
||||
("a".to_string(), peer("a", "https://a.example.com")),
|
||||
("b".to_string(), peer("b", "https://b.example.com")),
|
||||
]),
|
||||
..Default::default()
|
||||
};
|
||||
let mut request = SRPeerJoinReq {
|
||||
updated_at: Some(now),
|
||||
svc_acct_access_key: "replicator".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
ensure_initial_sync_join_current(&state, &request).expect("same committed join");
|
||||
for timestamp in [None, Some(now - time::Duration::SECOND), Some(now + time::Duration::SECOND)] {
|
||||
request.updated_at = timestamp;
|
||||
assert!(ensure_initial_sync_join_current(&state, &request).is_err());
|
||||
}
|
||||
request.updated_at = Some(now);
|
||||
request.svc_acct_access_key = "another-replicator".to_string();
|
||||
assert!(ensure_initial_sync_join_current(&state, &request).is_err());
|
||||
request.svc_acct_access_key.clone_from(&state.service_account_access_key);
|
||||
state.pending_remove = Some(PendingRemove::default());
|
||||
assert!(ensure_initial_sync_join_current(&state, &request).is_err());
|
||||
state.pending_remove = None;
|
||||
state.pending_rotation = Some(PendingRotation::default());
|
||||
assert!(ensure_initial_sync_join_current(&state, &request).is_err());
|
||||
state.pending_rotation = None;
|
||||
state.pending_endpoint_refresh = Some(PendingEndpointRefresh::default());
|
||||
assert!(ensure_initial_sync_join_current(&state, &request).is_err());
|
||||
state.pending_endpoint_refresh = None;
|
||||
state.service_account_access_key.clear();
|
||||
request.svc_acct_access_key.clear();
|
||||
assert!(ensure_initial_sync_join_current(&state, &request).is_err());
|
||||
state.service_account_access_key = "replicator".to_string();
|
||||
request.svc_acct_access_key.clone_from(&state.service_account_access_key);
|
||||
state.peers.clear();
|
||||
assert!(ensure_initial_sync_join_current(&state, &request).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_join_initial_sync_deferral_preserves_legacy_requests() {
|
||||
let legacy: SRPeerJoinEnvelope = serde_json::from_str("{}").expect("legacy join");
|
||||
assert!(!legacy.defer_initial_sync);
|
||||
assert!(serde_json::to_value(&legacy).unwrap().get("deferInitialSync").is_none());
|
||||
let deferred: SRPeerJoinEnvelope = serde_json::from_str(r#"{"deferInitialSync":true}"#).expect("deferred join");
|
||||
assert!(deferred.defer_initial_sync);
|
||||
assert_eq!(serde_json::to_value(deferred).unwrap()["deferInitialSync"], true);
|
||||
let mut response = applied_join_response(peer("b", "https://b.example.com"), String::new());
|
||||
assert!(serde_json::to_value(&response).unwrap().get("initialSyncDeferred").is_none());
|
||||
response.initial_sync_deferred = true;
|
||||
let wire = serde_json::to_vec(&response).unwrap();
|
||||
assert!(
|
||||
parse_peer_join_response(&wire, PeerInfo::default())
|
||||
.unwrap()
|
||||
.initial_sync_deferred
|
||||
);
|
||||
}
|
||||
|
||||
// BUG2: pre-existing-bucket back-fill failures must be surfaced in the add response's
|
||||
// initial_sync_error_message, not swallowed behind an unqualified success.
|
||||
#[test]
|
||||
@@ -14363,6 +14809,7 @@ mod tests {
|
||||
let value = serde_json::to_value(SRPeerJoinResponse {
|
||||
peer: peer("remote", "https://remote.example.com"),
|
||||
initial_sync_error_message: "bucket setup failed".to_string(),
|
||||
initial_sync_deferred: false,
|
||||
applied: Some(true),
|
||||
})
|
||||
.expect("serialize peer join response");
|
||||
@@ -14374,6 +14821,7 @@ mod tests {
|
||||
let value = serde_json::to_value(SRPeerJoinResponse {
|
||||
peer: peer("remote", "https://remote.example.com"),
|
||||
initial_sync_error_message: String::new(),
|
||||
initial_sync_deferred: false,
|
||||
applied: None,
|
||||
})
|
||||
.expect("serialize peer join response");
|
||||
@@ -15434,4 +15882,54 @@ mod tests {
|
||||
"no replicated config write may bypass the source stamp"
|
||||
);
|
||||
}
|
||||
|
||||
/// backlog#2367 A-4: `remove --all` notified "the peer" at the site's own
|
||||
/// registered endpoint. The tick-driven local peer carries the node's
|
||||
/// listen address, so an endpoint-only self check let the loop dial the
|
||||
/// site itself and report `Partial: failed to notify 1 peer(s)`.
|
||||
#[test]
|
||||
fn pending_notifications_skip_the_local_site_by_deployment_id() {
|
||||
let local_registered = PeerInfo {
|
||||
deployment_id: "site-b".to_string(),
|
||||
..peer("site-b", "http://site-b.example.com:9000")
|
||||
};
|
||||
let remote = PeerInfo {
|
||||
deployment_id: "site-a".to_string(),
|
||||
..peer("site-a", "http://site-a.example.com:9000")
|
||||
};
|
||||
let acked = PeerInfo {
|
||||
deployment_id: "site-c".to_string(),
|
||||
..peer("site-c", "http://site-c.example.com:9000")
|
||||
};
|
||||
let peers = BTreeMap::from([
|
||||
(local_registered.deployment_id.clone(), local_registered.clone()),
|
||||
(remote.deployment_id.clone(), remote),
|
||||
(acked.deployment_id.clone(), acked.clone()),
|
||||
]);
|
||||
let acked_ids = BTreeSet::from([acked.deployment_id]);
|
||||
|
||||
// The tick resolves the local peer from its own listen address.
|
||||
let local_from_tick = PeerInfo {
|
||||
deployment_id: "site-b".to_string(),
|
||||
..peer("site-b", "http://127.0.0.1:9000")
|
||||
};
|
||||
let to_notify: Vec<&str> = pending_peers_awaiting_notification(&peers, &local_from_tick, &acked_ids)
|
||||
.iter()
|
||||
.map(|peer| peer.deployment_id.as_str())
|
||||
.collect();
|
||||
assert_eq!(to_notify, vec!["site-a"], "the local site and the acked peer are never dialed");
|
||||
|
||||
// Identity stays consistent with what finalization waits for.
|
||||
assert_eq!(
|
||||
pending_remote_peer_ids(&peers, &local_from_tick),
|
||||
BTreeSet::from(["site-a".to_string(), "site-c".to_string()])
|
||||
);
|
||||
|
||||
// A handler-resolved local peer (registered endpoint) agrees.
|
||||
let to_notify: Vec<&str> = pending_peers_awaiting_notification(&peers, &local_registered, &acked_ids)
|
||||
.iter()
|
||||
.map(|peer| peer.deployment_id.as_str())
|
||||
.collect();
|
||||
assert_eq!(to_notify, vec!["site-a"]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -73,6 +73,7 @@ const SITE_REPLICATION_RESYNC_ROUTE: &str = "/rustfs/admin/v3/site-replication/r
|
||||
const SITE_REPLICATION_REPAIR_ROUTE: &str = "/rustfs/admin/v3/site-replication/repair";
|
||||
const SITE_REPLICATION_REPAIR_STATUS_ROUTE: &str = "/rustfs/admin/v3/site-replication/repair/status";
|
||||
const IAM_POLICY_ATTACH_ROUTE: &str = "/rustfs/admin/v3/idp/builtin/policy/attach";
|
||||
const DATA_USAGE_INFO_ROUTE: &str = "/rustfs/admin/v3/datausageinfo";
|
||||
const IAM_POLICY_DETACH_ROUTE: &str = "/rustfs/admin/v3/idp/builtin/policy/detach";
|
||||
const IAM_POLICY_ENTITIES_ROUTE: &str = "/rustfs/admin/v3/idp/builtin/policy-entities";
|
||||
const IAM_ACCESS_KEYS_BULK_ROUTE: &str = "/rustfs/admin/v3/list-access-keys-bulk";
|
||||
@@ -1077,12 +1078,25 @@ fn advertised_admin_capabilities() -> Vec<AdvertisedAdminCapability> {
|
||||
("admin.account.mfa", HttpMethod::Get, ACCOUNT_MFA_ROUTE),
|
||||
("admin.mfa.challenge", HttpMethod::Get, MFA_CHALLENGE_ROUTE),
|
||||
("admin.user.mfa", HttpMethod::Get, USER_MFA_ROUTE),
|
||||
// `rc du` is gated on this name. Before it was advertised the client
|
||||
// inferred it from a `1.0.0-rc.` version prefix, which no longer
|
||||
// matches once the server reports `1.0.0` (backlog#2367 E-2).
|
||||
("admin.data-usage", HttpMethod::Get, DATA_USAGE_INFO_ROUTE),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|(name, method, route)| AdvertisedAdminCapability {
|
||||
name,
|
||||
status: admin_route_capability(method, route),
|
||||
})
|
||||
.chain(std::iter::once(AdvertisedAdminCapability {
|
||||
// `rc watch` streams `GET /{bucket}?events=`, a misc extension route
|
||||
// dispatched by `admin::router` rather than an admin policy route,
|
||||
// so its status is not an inventory lookup (same version-prefix
|
||||
// inference on the client as `admin.data-usage`).
|
||||
name: "listen_notification",
|
||||
status: CapabilityStatus::supported()
|
||||
.with_reason("bucket listen notification (?events=) is dispatched by the admin router"),
|
||||
}))
|
||||
.collect()
|
||||
}
|
||||
|
||||
@@ -1258,6 +1272,9 @@ mod tests {
|
||||
"admin.iam.access-keys-bulk",
|
||||
"admin.iam.access-keys-bulk.ldap",
|
||||
"admin.iam.access-keys-bulk.openid",
|
||||
// rc pinned these two by version prefix until 1.0.0 (backlog#2367 E-2).
|
||||
"admin.data-usage",
|
||||
"listen_notification",
|
||||
];
|
||||
for name in expected_supported {
|
||||
let entry = response
|
||||
|
||||
@@ -346,7 +346,7 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
|
||||
admin(HttpMethod::Post, "/rustfs/admin/v3/rebalance/stop", REBALANCE, RouteRiskLevel::High),
|
||||
admin(HttpMethod::Post, "/rustfs/admin/v3/heal/", HEAL, RouteRiskLevel::High),
|
||||
admin(HttpMethod::Post, "/rustfs/admin/v3/heal/{bucket}", HEAL, RouteRiskLevel::High),
|
||||
admin(HttpMethod::Post, "/rustfs/admin/v3/heal/{bucket}/{*prefix}", HEAL, RouteRiskLevel::High),
|
||||
admin(HttpMethod::Post, "/rustfs/admin/v3/heal/{bucket}/{prefix}", HEAL, RouteRiskLevel::High),
|
||||
admin(HttpMethod::Post, "/rustfs/admin/v3/background-heal/status", HEAL, RouteRiskLevel::High),
|
||||
admin(
|
||||
HttpMethod::Get,
|
||||
|
||||
@@ -198,7 +198,7 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
|
||||
admin_route(Method::POST, "/v3/rebalance/stop"),
|
||||
admin_route(Method::POST, "/v3/heal/"),
|
||||
admin_route_sample(Method::POST, "/v3/heal/{bucket}", "/v3/heal/test-bucket"),
|
||||
admin_route_sample(Method::POST, "/v3/heal/{bucket}/{*prefix}", "/v3/heal/test-bucket/prefix"),
|
||||
admin_route_sample(Method::POST, "/v3/heal/{bucket}/{prefix}", "/v3/heal/test-bucket/prefix"),
|
||||
admin_route(Method::POST, "/v3/background-heal/status"),
|
||||
admin_route(Method::GET, "/v4/heal/replacement-recovery"),
|
||||
admin_route(Method::GET, "/v3/tier"),
|
||||
|
||||
@@ -33,6 +33,7 @@ use rustfs_credentials::Credentials;
|
||||
use rustfs_iam::federation::OIDC_VIRTUAL_PARENT_CLAIM;
|
||||
use rustfs_iam::sys::is_rustfs_oidc_claims;
|
||||
use rustfs_madmin::account::{AccountMutability, CredentialsSource, IdentityType};
|
||||
use serde_json::Value;
|
||||
|
||||
/// Claim written by the Keystone middleware onto its synthesized credentials.
|
||||
const KEYSTONE_ROLES_CLAIM: &str = "keystone_roles";
|
||||
@@ -54,6 +55,23 @@ pub(crate) fn session_parent_identity(credentials: &Credentials) -> Option<&str>
|
||||
.and_then(|value| value.as_str())
|
||||
}
|
||||
|
||||
/// Human-readable OIDC identity metadata for self-service responses. These
|
||||
/// values never replace the issuer-scoped virtual parent used for authorization.
|
||||
pub(crate) fn oidc_profile_fields(credentials: &Credentials) -> (Option<String>, Option<String>) {
|
||||
let Some(claims) = credentials.claims.as_ref().filter(|claims| is_rustfs_oidc_claims(claims)) else {
|
||||
return (None, None);
|
||||
};
|
||||
let string_claim = |name| {
|
||||
claims
|
||||
.get(name)
|
||||
.and_then(Value::as_str)
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
.map(ToOwned::to_owned)
|
||||
};
|
||||
|
||||
(string_claim("preferred_username"), string_claim("email"))
|
||||
}
|
||||
|
||||
/// Why a credential may not change its own authentication material.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum CredentialMutationDenial {
|
||||
@@ -398,6 +416,48 @@ mod tests {
|
||||
assert!(!caller.mutability().password);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oidc_profile_fields_return_normalized_display_claims() {
|
||||
let mut credentials = sts_session("TEMPKEY", "oidc-parent");
|
||||
credentials.claims = Some(HashMap::from([
|
||||
("iss".to_string(), Value::String("rustfs-oidc".to_string())),
|
||||
("oidc_provider".to_string(), Value::String("entraid".to_string())),
|
||||
("sub".to_string(), Value::String("subject-123".to_string())),
|
||||
("preferred_username".to_string(), Value::String("j.bruijns@pay.nl".to_string())),
|
||||
("email".to_string(), Value::String("fallback@pay.nl".to_string())),
|
||||
]));
|
||||
|
||||
assert_eq!(
|
||||
oidc_profile_fields(&credentials),
|
||||
(Some("j.bruijns@pay.nl".to_string()), Some("fallback@pay.nl".to_string()))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oidc_profile_fields_omit_missing_blank_and_non_string_values() {
|
||||
let mut credentials = sts_session("TEMPKEY", "oidc-parent");
|
||||
credentials.claims = Some(HashMap::from([
|
||||
("iss".to_string(), Value::String("rustfs-oidc".to_string())),
|
||||
("oidc_provider".to_string(), Value::String("keycloak".to_string())),
|
||||
("sub".to_string(), Value::String("subject-123".to_string())),
|
||||
("preferred_username".to_string(), Value::String(" ".to_string())),
|
||||
("email".to_string(), Value::Array(vec![Value::String("user@example.test".to_string())])),
|
||||
]));
|
||||
|
||||
assert_eq!(oidc_profile_fields(&credentials), (None, None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oidc_profile_fields_ignore_non_oidc_claim_shapes() {
|
||||
let mut credentials = sts_session("TEMPKEY", "ordinary-parent");
|
||||
credentials.claims = Some(HashMap::from([
|
||||
("preferred_username".to_string(), Value::String("attacker".to_string())),
|
||||
("email".to_string(), Value::String("attacker@example.test".to_string())),
|
||||
]));
|
||||
|
||||
assert_eq!(oidc_profile_fields(&credentials), (None, None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keystone_session_is_reported_as_federated() {
|
||||
let mut credentials = sts_session("TEMPKEY", "keystone-parent");
|
||||
|
||||
@@ -69,7 +69,7 @@ use super::storage_api::multipart_usecase::{
|
||||
};
|
||||
use crate::app::object::{
|
||||
ConcurrencyManager, ForegroundWriteAdmission, get_concurrency_manager, guard_put_object_body_read_timeout,
|
||||
put_object_body_read_timeout,
|
||||
put_object_body_read_timeout, reject_oversize_single_upload,
|
||||
};
|
||||
use crate::app::object_data_cache::{
|
||||
ObjectDataCacheAdapter, invalidate_object_data_cache_after_complete_multipart_success,
|
||||
@@ -90,6 +90,7 @@ use crate::auth::{
|
||||
use crate::capacity::record_capacity_write;
|
||||
use crate::error::ApiError;
|
||||
use crate::table_catalog;
|
||||
#[cfg(test)]
|
||||
use bytes::Bytes;
|
||||
use futures::StreamExt;
|
||||
use http::{HeaderMap, HeaderValue, Uri};
|
||||
@@ -104,7 +105,7 @@ use rustfs_utils::http::{
|
||||
SUFFIX_MAX_TOTAL_OBJECT_SIZE, SUFFIX_PLAINTEXT_CHECKSUM, SUFFIX_REPLICATION_GENERATION,
|
||||
SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP,
|
||||
SUFFIX_SOURCE_REPLICATION_REQUEST, contains_key_str, get_consistent_str, get_header, get_source_scheme,
|
||||
headers::{AMZ_CHECKSUM_TYPE, AMZ_DECODED_CONTENT_LENGTH, AMZ_OBJECT_TAGGING, AMZ_STORAGE_CLASS},
|
||||
headers::{AMZ_CHECKSUM_TYPE, AMZ_OBJECT_TAGGING, AMZ_STORAGE_CLASS},
|
||||
insert_str,
|
||||
};
|
||||
use s3s::dto::{
|
||||
@@ -114,6 +115,7 @@ use s3s::dto::{
|
||||
ServerSideEncryption, StreamingBlob, Timestamp, UploadPartCopyInput, UploadPartCopyOutput, UploadPartInput, UploadPartOutput,
|
||||
};
|
||||
use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE};
|
||||
use s3s::stream::ByteStream;
|
||||
use s3s::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::str::FromStr;
|
||||
@@ -377,42 +379,31 @@ fn extract_request_host(headers: &HeaderMap, uri: &Uri) -> Option<String> {
|
||||
.or_else(|| uri.authority().map(|authority| authority.as_str().to_string()))
|
||||
}
|
||||
|
||||
fn decoded_content_length_from_headers(headers: &HeaderMap) -> S3Result<Option<i64>> {
|
||||
let Some(val) = headers.get(AMZ_DECODED_CONTENT_LENGTH) else {
|
||||
return Ok(None);
|
||||
};
|
||||
fn resolve_upload_part_size(content_length: Option<i64>, body: Option<&StreamingBlob>) -> S3Result<Option<i64>> {
|
||||
if let Some(length) = content_length {
|
||||
return Ok(Some(length));
|
||||
}
|
||||
body.and_then(|body| body.remaining_length().exact())
|
||||
.map(i64::try_from)
|
||||
.transpose()
|
||||
.map_err(|_| s3_error!(UnexpectedContent))
|
||||
}
|
||||
|
||||
match atoi::atoi::<i64>(val.as_bytes()) {
|
||||
Some(x) => Ok(Some(x)),
|
||||
None => Err(s3_error!(UnexpectedContent)),
|
||||
fn require_upload_part_size(size: Option<i64>, capped: bool) -> S3Result<i64> {
|
||||
match size {
|
||||
Some(size) if size >= 0 => Ok(size),
|
||||
Some(_) => Err(s3_error!(UnexpectedContent)),
|
||||
None if capped => Err(s3_error!(UnexpectedContent)),
|
||||
None => Err(s3_error!(MissingContentLength)),
|
||||
}
|
||||
}
|
||||
|
||||
fn request_uses_aws_chunked(headers: &HeaderMap) -> bool {
|
||||
let has_aws_chunked = |header_name: &str| {
|
||||
headers
|
||||
.get(header_name)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.is_some_and(|value| value.split(',').any(|part| part.trim().eq_ignore_ascii_case("aws-chunked")))
|
||||
};
|
||||
|
||||
has_aws_chunked("content-encoding") || has_aws_chunked("transfer-encoding")
|
||||
}
|
||||
|
||||
fn resolve_upload_part_size(headers: &HeaderMap, content_length: Option<i64>) -> S3Result<Option<i64>> {
|
||||
let decoded_content_length = decoded_content_length_from_headers(headers)?;
|
||||
let size = match (request_uses_aws_chunked(headers), decoded_content_length, content_length) {
|
||||
(true, Some(decoded), _) => Some(decoded),
|
||||
(_, _, Some(length)) => Some(length),
|
||||
(_, Some(decoded), None) => Some(decoded),
|
||||
_ => None,
|
||||
};
|
||||
|
||||
if size == Some(-1) {
|
||||
return Err(s3_error!(UnexpectedContent));
|
||||
fn upload_part_body_read_timeout(configured: Duration, capped: bool) -> Duration {
|
||||
if capped {
|
||||
configured.max(Duration::from_secs(rustfs_config::DEFAULT_HTTP_REQUEST_BODY_READ_TIMEOUT))
|
||||
} else {
|
||||
configured
|
||||
}
|
||||
|
||||
Ok(size)
|
||||
}
|
||||
|
||||
fn build_complete_multipart_location(headers: &HeaderMap, uri: &Uri, bucket: &str, key: &str) -> String {
|
||||
@@ -1168,7 +1159,10 @@ impl DefaultMultipartUsecase {
|
||||
|
||||
validate_table_catalog_object_mutation(&bucket, &key).await?;
|
||||
|
||||
let mut size = resolve_upload_part_size(&req.headers, content_length)?;
|
||||
let size = resolve_upload_part_size(content_length, body.as_ref())?;
|
||||
if let Some(size) = size {
|
||||
reject_oversize_single_upload(size)?;
|
||||
}
|
||||
let mut body_stream = body.ok_or_else(|| s3_error!(IncompleteBody))?;
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
@@ -1178,20 +1172,15 @@ impl DefaultMultipartUsecase {
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
let max_total_object_size = multipart_max_total_object_size(&fi.user_defined)?;
|
||||
if max_total_object_size.is_some() && size.is_some_and(|size| size < 0) {
|
||||
return Err(S3Error::new(S3ErrorCode::UnexpectedContent));
|
||||
}
|
||||
if max_total_object_size.is_some() && size.is_none() {
|
||||
return Err(S3Error::new(S3ErrorCode::UnexpectedContent));
|
||||
}
|
||||
if let (Some(limit), Some(size)) = (max_total_object_size, size)
|
||||
let mut size = require_upload_part_size(size, max_total_object_size.is_some())?;
|
||||
if let Some(limit) = max_total_object_size
|
||||
&& u64::try_from(size).is_ok_and(|size| size > limit)
|
||||
{
|
||||
return Err(S3Error::new(S3ErrorCode::EntityTooLarge));
|
||||
}
|
||||
let upload_part_admission = match self
|
||||
.concurrency_manager()
|
||||
.admit_multipart_part(size.unwrap_or(-1))
|
||||
.admit_multipart_part(size)
|
||||
.await
|
||||
.map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "foreground write admission closed"))?
|
||||
{
|
||||
@@ -1208,42 +1197,30 @@ impl DefaultMultipartUsecase {
|
||||
));
|
||||
}
|
||||
};
|
||||
if max_total_object_size.is_some() {
|
||||
let request_id = req
|
||||
.extensions
|
||||
.get::<super::storage_api::multipart_usecase::request_context::RequestContext>()
|
||||
.map(|ctx| ctx.request_id.clone())
|
||||
.unwrap_or_default();
|
||||
body_stream = guard_put_object_body_read_timeout(
|
||||
body_stream,
|
||||
let request_id = req
|
||||
.extensions
|
||||
.get::<super::storage_api::multipart_usecase::request_context::RequestContext>()
|
||||
.map(|ctx| ctx.request_id.as_str())
|
||||
.unwrap_or_default();
|
||||
let timeout = upload_part_body_read_timeout(put_object_body_read_timeout(), max_total_object_size.is_some());
|
||||
let raw_control = req.extensions.get::<super::object::request_body::BodyReadControl>().cloned();
|
||||
let observe_read_demand = if let Some(control) = &raw_control {
|
||||
control.activate(
|
||||
timeout,
|
||||
&bucket,
|
||||
&key,
|
||||
&request_id,
|
||||
content_length,
|
||||
put_object_body_read_timeout().max(Duration::from_secs(rustfs_config::DEFAULT_HTTP_REQUEST_BODY_READ_TIMEOUT)),
|
||||
);
|
||||
}
|
||||
|
||||
if size.is_none() {
|
||||
let mut total = 0i64;
|
||||
let mut buffer = bytes::BytesMut::new();
|
||||
while let Some(chunk) = body_stream.next().await {
|
||||
let chunk = chunk.map_err(|e| ApiError::from(s3s_body_error_to_io(e)))?;
|
||||
total += chunk.len() as i64;
|
||||
buffer.extend_from_slice(&chunk);
|
||||
request_id,
|
||||
u64::try_from(size).map_err(|_| s3_error!(UnexpectedContent))?,
|
||||
)
|
||||
} else {
|
||||
// Direct protocol callers have no raw HTTP body. Retain their
|
||||
// existing capped-session guard without inventing a client cause.
|
||||
if max_total_object_size.is_some() {
|
||||
body_stream = guard_put_object_body_read_timeout(body_stream, &bucket, &key, request_id, Some(size), timeout);
|
||||
}
|
||||
false
|
||||
};
|
||||
|
||||
if total <= 0 {
|
||||
return Err(s3_error!(UnexpectedContent));
|
||||
}
|
||||
|
||||
size = Some(total);
|
||||
let combined = buffer.freeze();
|
||||
let stream = futures::stream::once(async move { Ok::<Bytes, std::io::Error>(combined) });
|
||||
body_stream = StreamingBlob::wrap(stream);
|
||||
}
|
||||
|
||||
let mut size = size.ok_or_else(|| s3_error!(UnexpectedContent))?;
|
||||
let ingress_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(std::time::Instant::now);
|
||||
|
||||
// Apply adaptive buffer sizing based on part size for optimal streaming performance.
|
||||
@@ -1388,6 +1365,11 @@ impl DefaultMultipartUsecase {
|
||||
};
|
||||
|
||||
reader = write_plan.apply(reader, actual_size).map_err(ApiError::from)?;
|
||||
if observe_read_demand && let Some(control) = raw_control {
|
||||
use rustfs_rio::HashReaderMut;
|
||||
let inner = reader.take_inner();
|
||||
reader.inner = rustfs_rio::boxed_reader(super::object::request_body::DemandReader::new(inner, control));
|
||||
}
|
||||
|
||||
let mut reader = PutObjReader::new(reader);
|
||||
|
||||
@@ -1932,6 +1914,8 @@ fn passthrough_part_actual_size(headers: &HeaderMap) -> Option<i64> {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
mod body_read_tests;
|
||||
use http::{Extensions, HeaderMap, Method, Uri, header::HeaderValue};
|
||||
use rustfs_filemeta::ObjectPartInfo;
|
||||
use rustfs_utils::http::{
|
||||
@@ -2136,23 +2120,45 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_upload_part_size_uses_decoded_length_for_aws_chunked() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("content-encoding", HeaderValue::from_static("aws-chunked"));
|
||||
headers.insert(AMZ_DECODED_CONTENT_LENGTH, HeaderValue::from_static("5242880"));
|
||||
|
||||
let size = resolve_upload_part_size(&headers, Some(5242962)).expect("decoded size should parse");
|
||||
|
||||
assert_eq!(size, Some(5242880));
|
||||
fn resolve_upload_part_size_uses_normalized_logical_length() {
|
||||
assert_eq!(resolve_upload_part_size(Some(5242880), None).expect("DTO length"), Some(5242880));
|
||||
assert_eq!(resolve_upload_part_size(None, None).expect("unknown length"), None);
|
||||
let body = StreamingBlob::from(Bytes::from_static(b"abc"));
|
||||
assert_eq!(resolve_upload_part_size(None, Some(&body)).expect("exact bytes"), Some(3));
|
||||
assert_eq!(resolve_upload_part_size(Some(0), Some(&body)).expect("explicit length wins"), Some(0));
|
||||
let body = StreamingBlob::wrap(futures::stream::iter([Ok::<_, std::io::Error>(Bytes::from_static(b"abc"))]));
|
||||
assert_eq!(futures::Stream::size_hint(&body), (1, Some(1)), "the stream knows its item count");
|
||||
assert_eq!(resolve_upload_part_size(None, Some(&body)).expect("unknown byte length"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_upload_part_size_preserves_regular_content_length() {
|
||||
let headers = HeaderMap::new();
|
||||
fn upload_part_length_contract_rejects_unknown_and_all_negative_lengths() {
|
||||
assert_eq!(
|
||||
require_upload_part_size(None, false).expect_err("ordinary unknown").code(),
|
||||
&S3ErrorCode::MissingContentLength
|
||||
);
|
||||
assert_eq!(
|
||||
require_upload_part_size(None, true).expect_err("capped unknown").code(),
|
||||
&S3ErrorCode::UnexpectedContent
|
||||
);
|
||||
for capped in [false, true] {
|
||||
for size in [-1, -2, i64::MIN] {
|
||||
assert_eq!(
|
||||
require_upload_part_size(Some(size), capped).expect_err("negative").code(),
|
||||
&S3ErrorCode::UnexpectedContent
|
||||
);
|
||||
}
|
||||
assert_eq!(require_upload_part_size(Some(0), capped).expect("zero is valid"), 0);
|
||||
}
|
||||
}
|
||||
|
||||
let size = resolve_upload_part_size(&headers, Some(5242880)).expect("regular size should parse");
|
||||
|
||||
assert_eq!(size, Some(5242880));
|
||||
#[test]
|
||||
fn upload_part_timeout_policy_preserves_disabled_and_capped_floor() {
|
||||
for seconds in [0, 1, 299, 300, 601] {
|
||||
let configured = Duration::from_secs(seconds);
|
||||
assert_eq!(upload_part_body_read_timeout(configured, false), configured);
|
||||
assert_eq!(upload_part_body_read_timeout(configured, true), Duration::from_secs(seconds.max(300)));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -3209,6 +3215,36 @@ mod tests {
|
||||
assert_eq!(err.code(), &S3ErrorCode::IncompleteBody);
|
||||
}
|
||||
|
||||
/// issue #7596: a part whose declared length exceeds the 5 GiB
|
||||
/// single-request ceiling is rejected before the body is polled or the
|
||||
/// store is consulted. Exact-cap and zero-length parts pass admission.
|
||||
#[tokio::test]
|
||||
async fn execute_upload_part_rejects_oversize_declared_part_before_reading_the_body() {
|
||||
let ceiling = i64::try_from(rustfs_config::MAX_SINGLE_PUT_OBJECT_SIZE).expect("ceiling fits i64");
|
||||
|
||||
for (declared, expect_too_large) in [(ceiling + 1, true), (ceiling, false), (0, false)] {
|
||||
let (body, polls) = crate::app::object::PollCountingBody::streaming_blob();
|
||||
let input = UploadPartInput::builder()
|
||||
.bucket("bucket".to_string())
|
||||
.key("object".to_string())
|
||||
.upload_id("upload-id".to_string())
|
||||
.part_number(1)
|
||||
.body(Some(body))
|
||||
.content_length(Some(declared))
|
||||
.build()
|
||||
.unwrap();
|
||||
let req = build_request(input, Method::PUT);
|
||||
|
||||
let err = make_usecase().execute_upload_part(req).await.unwrap_err();
|
||||
if expect_too_large {
|
||||
assert_eq!(err.code(), &S3ErrorCode::EntityTooLarge, "declared {declared}");
|
||||
assert_eq!(polls.load(std::sync::atomic::Ordering::SeqCst), 0, "body must not be polled");
|
||||
} else {
|
||||
assert_ne!(err.code(), &S3ErrorCode::EntityTooLarge, "declared {declared} must pass admission");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_upload_part_rejects_invalid_part_number_before_body_lookup() {
|
||||
for part_number in [-1, 0, 10001] {
|
||||
@@ -3227,6 +3263,85 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn execute_upload_part_rejects_unknown_length_before_admission_or_body_polling() {
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations, MakeBucketOptions};
|
||||
|
||||
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
|
||||
let ambient = crate::app::gating_test_env::shared_gating_ambient().await;
|
||||
let context = Arc::new(AppContext::new(Arc::clone(&store), ambient.iam(), ambient.kms()));
|
||||
let bucket = format!("upload-part-length-{}", Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket");
|
||||
let concurrency_manager = Arc::new(ConcurrencyManager::with_large_put_admission_for_test(
|
||||
true,
|
||||
1,
|
||||
rustfs_config::DEFAULT_PUT_LARGE_FOREGROUND_ADMISSION_MIN_SIZE_BYTES,
|
||||
Duration::ZERO,
|
||||
));
|
||||
let held = concurrency_manager
|
||||
.admit_multipart_part(1024)
|
||||
.await
|
||||
.expect("hold the only permit");
|
||||
let usecase = DefaultMultipartUsecase::with_context_and_concurrency_manager(Some(context), concurrency_manager);
|
||||
|
||||
for capped in [false, true] {
|
||||
let mut options = ObjectOptions::default();
|
||||
if capped {
|
||||
insert_str(&mut options.user_defined, SUFFIX_MAX_TOTAL_OBJECT_SIZE, "1024".to_owned());
|
||||
}
|
||||
let upload = store
|
||||
.new_multipart_upload(&bucket, "object", &options)
|
||||
.await
|
||||
.expect("upload session");
|
||||
for content_length in [None, Some(-1)] {
|
||||
for declared_chunk_encoding in [false, true] {
|
||||
let (body, polls) = crate::app::object::PollCountingBody::streaming_blob();
|
||||
let input = UploadPartInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key("object".to_owned())
|
||||
.upload_id(upload.upload_id.clone())
|
||||
.part_number(1)
|
||||
.content_length(content_length)
|
||||
.body(Some(body))
|
||||
.build()
|
||||
.expect("part request");
|
||||
let mut request = build_request(input, Method::PUT);
|
||||
request
|
||||
.headers
|
||||
.insert("x-amz-decoded-content-length", HeaderValue::from_static("1024"));
|
||||
if declared_chunk_encoding {
|
||||
request
|
||||
.headers
|
||||
.insert("content-encoding", HeaderValue::from_static("aws-chunked"));
|
||||
}
|
||||
let error = usecase
|
||||
.execute_upload_part(request)
|
||||
.await
|
||||
.expect_err("unknown or negative logical size");
|
||||
assert_eq!(
|
||||
error.code(),
|
||||
&if capped || content_length.is_some() {
|
||||
S3ErrorCode::UnexpectedContent
|
||||
} else {
|
||||
S3ErrorCode::MissingContentLength
|
||||
}
|
||||
);
|
||||
assert_eq!(polls.load(std::sync::atomic::Ordering::Relaxed), 0);
|
||||
}
|
||||
}
|
||||
let parts = store
|
||||
.list_object_parts(&bucket, "object", &upload.upload_id, None, 1000, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("list rejected session");
|
||||
assert!(parts.parts.is_empty());
|
||||
}
|
||||
drop(held);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn execute_upload_part_rejects_when_foreground_write_admission_is_full() {
|
||||
|
||||
@@ -0,0 +1,309 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use crate::app::object::request_body::{BodyReadControl, ObservedBody};
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations, MakeBucketOptions};
|
||||
use crate::app::storage_api::test::contract::object::ObjectIO;
|
||||
use http_body::Frame;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use tokio::sync::{mpsc, oneshot};
|
||||
|
||||
fn part_request(bucket: &str, upload: &str, body: StreamingBlob, size: i64) -> S3Request<UploadPartInput> {
|
||||
build_request(
|
||||
UploadPartInput::builder()
|
||||
.bucket(bucket.to_owned())
|
||||
.key("object".to_owned())
|
||||
.upload_id(upload.to_owned())
|
||||
.part_number(1)
|
||||
.content_length(Some(size))
|
||||
.body(Some(body))
|
||||
.build()
|
||||
.expect("part input"),
|
||||
Method::PUT,
|
||||
)
|
||||
}
|
||||
|
||||
type BodySender = mpsc::UnboundedSender<Result<Frame<Bytes>, std::io::Error>>;
|
||||
|
||||
fn observed_request(
|
||||
bucket: &str,
|
||||
upload: &str,
|
||||
size: usize,
|
||||
) -> (S3Request<UploadPartInput>, BodySender, oneshot::Receiver<()>, Arc<AtomicUsize>) {
|
||||
let (sender, mut receiver) = mpsc::unbounded_channel();
|
||||
let (started, waiting) = oneshot::channel();
|
||||
let mut started = Some(started);
|
||||
let polls = Arc::new(AtomicUsize::new(0));
|
||||
let body_polls = Arc::clone(&polls);
|
||||
let stream = futures::stream::poll_fn(move |cx| {
|
||||
body_polls.fetch_add(1, Ordering::Relaxed);
|
||||
let result = receiver.poll_recv(cx);
|
||||
if result.is_pending()
|
||||
&& let Some(started) = started.take()
|
||||
{
|
||||
let _ = started.send(());
|
||||
}
|
||||
result
|
||||
});
|
||||
let control = BodyReadControl::default();
|
||||
let body = ObservedBody::new(http_body_util::StreamBody::new(stream), control.clone());
|
||||
let mut request = part_request(bucket, upload, StreamingBlob::from(s3s::Body::http_body_unsync(body)), size as i64);
|
||||
request.extensions.insert(control);
|
||||
(request, sender, waiting, polls)
|
||||
}
|
||||
|
||||
async fn temporary_entries(disks: &[std::path::PathBuf]) -> std::collections::BTreeSet<std::path::PathBuf> {
|
||||
let mut entries = std::collections::BTreeSet::new();
|
||||
for disk in disks {
|
||||
let path = disk.join(".rustfs.sys/tmp");
|
||||
let mut directory = match tokio::fs::read_dir(path).await {
|
||||
Ok(directory) => directory,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
|
||||
Err(error) => panic!("temporary directory: {error}"),
|
||||
};
|
||||
while let Some(entry) = directory.next_entry().await.expect("temporary entry") {
|
||||
if entry.file_name() != ".trash" {
|
||||
entries.insert(entry.path());
|
||||
}
|
||||
}
|
||||
}
|
||||
entries
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn upload_part_body_timeout_cleans_storage_preserves_old_part_and_releases_permits() {
|
||||
crate::app::gating_test_env::run_large_stack_test("upload-part-timeout-direct", || async {
|
||||
assert_body_timeout_storage_lifecycle(4096, 512).await;
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn upload_part_body_timeout_pipeline_storage_lifecycle() {
|
||||
// Fresh processes with the existing ingest and batching settings cover
|
||||
// Vec, BytesMut, and batched pipelines independently.
|
||||
crate::app::gating_test_env::run_large_stack_test("upload-part-timeout-pipeline", || async {
|
||||
assert_body_timeout_storage_lifecycle(3 * 1024 * 1024, 2 * 1024 * 1024 + 512).await;
|
||||
});
|
||||
}
|
||||
|
||||
async fn assert_body_timeout_storage_lifecycle(part_size: usize, partial_size: usize) {
|
||||
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
|
||||
|
||||
struct RestoreMetrics(bool);
|
||||
impl Drop for RestoreMetrics {
|
||||
fn drop(&mut self) {
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(self.0);
|
||||
}
|
||||
}
|
||||
let _restore = RestoreMetrics(rustfs_io_metrics::put_stage_metrics_enabled());
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
let _recorder = metrics::set_default_local_recorder(&recorder);
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(true);
|
||||
let path = if part_size == 4096 {
|
||||
"multipart_write_single_block_non_inline"
|
||||
} else if part_size >= rustfs_utils::get_env_usize("RUSTFS_MULTIPART_PUT_LARGE_BATCH_MIN_SIZE_BYTES", 128 * 1024 * 1024) {
|
||||
"multipart_write_pipeline_batched_large"
|
||||
} else {
|
||||
"multipart_write_pipeline"
|
||||
};
|
||||
let path_count = || {
|
||||
snapshotter
|
||||
.snapshot()
|
||||
.into_vec()
|
||||
.into_iter()
|
||||
.filter_map(|(key, _, _, value)| {
|
||||
if key.key().name() == "rustfs_s3_put_object_path_total"
|
||||
&& key.key().labels().any(|label| label.key() == "path" && label.value() == path)
|
||||
&& let DebugValue::Counter(count) = value
|
||||
{
|
||||
Some(count)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.sum::<u64>()
|
||||
};
|
||||
let (disks, store) = crate::app::gating_test_env::shared_gating_ecstore_and_disk_paths().await;
|
||||
let ambient = crate::app::gating_test_env::shared_gating_ambient().await;
|
||||
let context = Arc::new(AppContext::new(Arc::clone(&store), ambient.iam(), ambient.kms()));
|
||||
let manager = Arc::new(ConcurrencyManager::with_large_put_admission_for_test(
|
||||
true,
|
||||
1,
|
||||
rustfs_config::DEFAULT_PUT_LARGE_FOREGROUND_ADMISSION_MIN_SIZE_BYTES,
|
||||
Duration::ZERO,
|
||||
));
|
||||
let usecase = Arc::new(DefaultMultipartUsecase::with_context_and_concurrency_manager(
|
||||
Some(context),
|
||||
Arc::clone(&manager),
|
||||
));
|
||||
let bucket = format!("body-stall-{}", Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket");
|
||||
|
||||
for capped in [false, true] {
|
||||
let mut options = ObjectOptions::default();
|
||||
if capped {
|
||||
insert_str(&mut options.user_defined, SUFFIX_MAX_TOTAL_OBJECT_SIZE, (2 * part_size).to_string());
|
||||
}
|
||||
let upload = store
|
||||
.new_multipart_upload(&bucket, "object", &options)
|
||||
.await
|
||||
.expect("upload session");
|
||||
let mut old_etag = None;
|
||||
for (replacement, retry_after_failure) in [(false, true), (true, true), (true, false)] {
|
||||
let baseline = temporary_entries(&disks).await;
|
||||
let _ = path_count();
|
||||
let (request, sender, waiting, _) = observed_request(&bucket, &upload.upload_id, part_size);
|
||||
sender
|
||||
.send(Ok(Frame::data(Bytes::from(vec![7; partial_size]))))
|
||||
.expect("partial body");
|
||||
let mut upload_future = Box::pin(usecase.execute_upload_part(request));
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
tokio::select! {
|
||||
result = &mut upload_future => panic!("upload finished before storage requested raw input: {:?}", result.err().map(|error| error.code().clone())),
|
||||
result = waiting => result.expect("raw reader"),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("storage must request raw input");
|
||||
// Advance only after actual storage demand; filesystem setup and
|
||||
// cleanup run on a real clock and cannot race auto-advance.
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(300)).await;
|
||||
tokio::time::resume();
|
||||
let error = tokio::time::timeout(Duration::from_secs(10), upload_future)
|
||||
.await
|
||||
.expect("inline cleanup must complete")
|
||||
.expect_err("stalled part");
|
||||
assert_eq!(error.code(), &S3ErrorCode::RequestTimeout);
|
||||
assert_eq!(path_count(), 1, "failure must exercise {path}");
|
||||
assert!(sender.is_closed(), "producer must release the failed raw body");
|
||||
assert_eq!(
|
||||
temporary_entries(&disks).await,
|
||||
baseline,
|
||||
"failed part must clean temporary shards inline"
|
||||
);
|
||||
let parts = store
|
||||
.list_object_parts(&bucket, "object", &upload.upload_id, None, 1000, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("list parts after error");
|
||||
assert_eq!(parts.parts.len(), usize::from(replacement));
|
||||
if replacement {
|
||||
assert_eq!(parts.parts[0].etag, old_etag, "failed overwrite must preserve the committed part");
|
||||
}
|
||||
if !retry_after_failure {
|
||||
continue;
|
||||
}
|
||||
let payload = vec![if replacement { 9 } else { 8 }; part_size];
|
||||
let request = part_request(&bucket, &upload.upload_id, StreamingBlob::from(Bytes::from(payload)), part_size as i64);
|
||||
let retry = tokio::time::timeout(Duration::from_secs(10), usecase.execute_upload_part(request))
|
||||
.await
|
||||
.expect("foreground and capped staging permits must be released")
|
||||
.expect("same-number retry");
|
||||
old_etag = retry.output.e_tag.map(|etag| etag.value().to_owned());
|
||||
}
|
||||
let parts = store
|
||||
.list_object_parts(&bucket, "object", &upload.upload_id, None, 1000, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("successful retry");
|
||||
assert_eq!(parts.parts.len(), 1);
|
||||
assert_eq!(parts.parts[0].etag, old_etag);
|
||||
assert_eq!(parts.parts[0].size, part_size);
|
||||
store
|
||||
.clone()
|
||||
.complete_multipart_upload(
|
||||
&bucket,
|
||||
"object",
|
||||
&upload.upload_id,
|
||||
vec![CompletePart {
|
||||
part_num: 1,
|
||||
etag: old_etag,
|
||||
..CompletePart::default()
|
||||
}],
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("failed replacement must leave the prior part completable");
|
||||
let mut object = store
|
||||
.get_object_reader(&bucket, "object", None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("completed old part remains readable");
|
||||
let mut restored = Vec::new();
|
||||
object.stream.read_to_end(&mut restored).await.expect("read all old bytes");
|
||||
assert_eq!(restored, vec![9; part_size]);
|
||||
}
|
||||
eprintln!(
|
||||
"verified storage lifecycle: path={path}, bytesmut={}",
|
||||
rustfs_utils::get_env_bool("RUSTFS_ERASURE_ENCODE_BYTESMUT_INGEST", true)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn upload_part_foreground_queue_does_not_consume_body_timeout() {
|
||||
crate::app::gating_test_env::run_large_stack_test("upload-part-timeout-queue", assert_foreground_queue);
|
||||
}
|
||||
|
||||
async fn assert_foreground_queue() {
|
||||
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
|
||||
let ambient = crate::app::gating_test_env::shared_gating_ambient().await;
|
||||
let context = Arc::new(AppContext::new(Arc::clone(&store), ambient.iam(), ambient.kms()));
|
||||
let manager = Arc::new(ConcurrencyManager::with_multipart_admission_queue_for_test(
|
||||
1,
|
||||
Duration::from_secs(1200),
|
||||
1,
|
||||
));
|
||||
let held = manager.admit_multipart_part(4096).await.expect("hold foreground permit");
|
||||
let usecase = DefaultMultipartUsecase::with_context_and_concurrency_manager(Some(context), Arc::clone(&manager));
|
||||
let bucket = format!("body-queue-{}", Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket");
|
||||
let upload = store
|
||||
.new_multipart_upload(&bucket, "object", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("upload session");
|
||||
let (request, sender, waiting, polls) = observed_request(&bucket, &upload.upload_id, 4096);
|
||||
let task = tokio::spawn(async move { usecase.execute_upload_part(request).await });
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
while manager.put_object_admission_snapshot().queued != Some(1) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("request must enter the actual foreground queue");
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(600)).await;
|
||||
tokio::time::resume();
|
||||
assert_eq!(polls.load(Ordering::Relaxed), 0, "queued requests must not poll the raw body");
|
||||
assert!(!task.is_finished());
|
||||
drop(held);
|
||||
waiting.await.expect("read begins after admission");
|
||||
sender
|
||||
.send(Ok(Frame::data(Bytes::from(vec![7; 4096]))))
|
||||
.expect("body after admission");
|
||||
drop(sender);
|
||||
tokio::time::timeout(Duration::from_secs(10), task)
|
||||
.await
|
||||
.expect("queued upload completes")
|
||||
.expect("upload task")
|
||||
.expect("queue time is not client inactivity");
|
||||
}
|
||||
@@ -211,6 +211,7 @@ mod head;
|
||||
mod internal_put;
|
||||
mod on_demand_migration_put;
|
||||
mod put;
|
||||
pub(crate) mod request_body;
|
||||
mod restore;
|
||||
pub(crate) mod shared;
|
||||
#[cfg(test)]
|
||||
@@ -223,8 +224,10 @@ pub(crate) use self::extract::*;
|
||||
pub(crate) use self::get::*;
|
||||
pub(crate) use self::internal_put::*;
|
||||
pub(crate) use self::on_demand_migration_put::*;
|
||||
#[cfg(test)]
|
||||
pub(crate) use self::put::PollCountingBody;
|
||||
use self::put::*;
|
||||
pub(crate) use self::put::{guard_put_object_body_read_timeout, put_object_body_read_timeout};
|
||||
pub(crate) use self::put::{guard_put_object_body_read_timeout, put_object_body_read_timeout, reject_oversize_single_upload};
|
||||
#[cfg(test)]
|
||||
pub(crate) use self::restore::RestoreStatusCommitBarrier;
|
||||
pub(crate) use self::shared::*;
|
||||
|
||||
@@ -109,6 +109,21 @@ fn resolve_put_object_authoritative_size(headers: &HeaderMap, content_length: Op
|
||||
Ok(size)
|
||||
}
|
||||
|
||||
/// Reject a declared upload length above the single-request ceiling
|
||||
/// ([`rustfs_config::MAX_SINGLE_PUT_OBJECT_SIZE`]) with `EntityTooLarge`.
|
||||
///
|
||||
/// Applies to `PutObject` and `UploadPart`. A negative or unknown length is
|
||||
/// left to the caller's existing validation.
|
||||
pub(crate) fn reject_oversize_single_upload(size: i64) -> S3Result<()> {
|
||||
if u64::try_from(size).is_ok_and(|size| size > rustfs_config::MAX_SINGLE_PUT_OBJECT_SIZE) {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::EntityTooLarge,
|
||||
ApiError::error_code_to_message(&S3ErrorCode::EntityTooLarge),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Resolve the S3 request-body inter-chunk read timeout from the environment.
|
||||
///
|
||||
/// Returns `Duration::ZERO` when disabled (`RUSTFS_HTTP_REQUEST_BODY_READ_TIMEOUT=0`),
|
||||
@@ -1287,6 +1302,12 @@ impl DefaultObjectUsecase {
|
||||
// Resolve the authoritative decoded/plain object length (rejecting negative/unknown) before anything else consumes it.
|
||||
let size = resolve_put_object_authoritative_size(&req.headers, content_length)?;
|
||||
|
||||
// The streaming-body limit (s3s `put_object_max_size`) only fires once the
|
||||
// client has already streamed 5 GiB. The declared length is authoritative,
|
||||
// so reject an oversize single PUT here, before any body byte is read
|
||||
// (issue #7596).
|
||||
reject_oversize_single_upload(size)?;
|
||||
|
||||
if let Some(limit) = max_content_length
|
||||
&& u64::try_from(size).is_ok_and(|size| size > limit)
|
||||
{
|
||||
@@ -3318,6 +3339,77 @@ mod tests {
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidStorageClass);
|
||||
}
|
||||
|
||||
/// issue #7596: a single PUT whose declared length exceeds the 5 GiB
|
||||
/// ceiling must be rejected from the headers, before any body byte is
|
||||
/// requested.
|
||||
#[tokio::test]
|
||||
async fn execute_put_object_rejects_oversize_content_length_before_reading_the_body() {
|
||||
let ceiling = i64::try_from(rustfs_config::MAX_SINGLE_PUT_OBJECT_SIZE).expect("ceiling fits i64");
|
||||
let (body, polls) = PollCountingBody::streaming_blob();
|
||||
let input = PutObjectInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("huge.bin".to_string())
|
||||
.body(Some(body))
|
||||
.content_length(Some(ceiling + 1))
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let req = build_request(input, Method::PUT);
|
||||
let usecase = DefaultObjectUsecase::without_context();
|
||||
let fs = FS::new();
|
||||
|
||||
let err = Box::pin(usecase.execute_put_object(&fs, req)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::EntityTooLarge);
|
||||
assert_eq!(polls.load(std::sync::atomic::Ordering::SeqCst), 0, "body must not be polled");
|
||||
}
|
||||
|
||||
/// Admission uses the logical object size, not the wire length: a signed
|
||||
/// aws-chunked request whose framed `Content-Length` exceeds the cap but
|
||||
/// whose decoded length is within it must not be rejected as oversize,
|
||||
/// while a decoded length above the cap must be.
|
||||
#[tokio::test]
|
||||
async fn execute_put_object_oversize_admission_uses_decoded_length_for_aws_chunked() {
|
||||
let ceiling = i64::try_from(rustfs_config::MAX_SINGLE_PUT_OBJECT_SIZE).expect("ceiling fits i64");
|
||||
let framing_overhead = 1_000_000;
|
||||
|
||||
for (decoded, expect_too_large) in [(ceiling, false), (ceiling + 1, true)] {
|
||||
let (body, polls) = PollCountingBody::streaming_blob();
|
||||
let input = PutObjectInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("huge.bin".to_string())
|
||||
.body(Some(body))
|
||||
.content_length(Some(decoded + framing_overhead))
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let mut req = build_request(input, Method::PUT);
|
||||
req.headers
|
||||
.insert(http::header::CONTENT_ENCODING, HeaderValue::from_static("aws-chunked"));
|
||||
req.headers.insert(
|
||||
HeaderName::from_static("x-amz-content-sha256"),
|
||||
HeaderValue::from_static("STREAMING-AWS4-HMAC-SHA256-PAYLOAD"),
|
||||
);
|
||||
req.headers.insert(
|
||||
HeaderName::from_static("x-amz-decoded-content-length"),
|
||||
HeaderValue::from_str(&decoded.to_string()).unwrap(),
|
||||
);
|
||||
let usecase = DefaultObjectUsecase::without_context();
|
||||
let fs = FS::new();
|
||||
|
||||
let err = Box::pin(usecase.execute_put_object(&fs, req)).await.unwrap_err();
|
||||
if expect_too_large {
|
||||
assert_eq!(err.code(), &S3ErrorCode::EntityTooLarge, "decoded {decoded}");
|
||||
assert_eq!(polls.load(std::sync::atomic::Ordering::SeqCst), 0, "body must not be polled");
|
||||
} else {
|
||||
assert_ne!(
|
||||
err.code(),
|
||||
&S3ErrorCode::EntityTooLarge,
|
||||
"framed wire length above the cap must not reject a decoded length at the cap"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_put_object_rejects_post_object_sse_kms_from_headers() {
|
||||
let input = PutObjectInput::builder()
|
||||
@@ -4184,3 +4276,55 @@ mod tests {
|
||||
assert!(is_err_object_not_found(&lookup_err), "{lookup_err}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Test-only request body that records how often it is polled, so admission
|
||||
/// tests can prove a rejection happened before any body byte was requested.
|
||||
#[cfg(test)]
|
||||
pub(crate) struct PollCountingBody {
|
||||
pub(crate) polls: std::sync::Arc<std::sync::atomic::AtomicUsize>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl PollCountingBody {
|
||||
pub(crate) fn streaming_blob() -> (StreamingBlob, std::sync::Arc<std::sync::atomic::AtomicUsize>) {
|
||||
let polls = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let body = StreamingBlob::new(Self {
|
||||
polls: std::sync::Arc::clone(&polls),
|
||||
});
|
||||
(body, polls)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Stream for PollCountingBody {
|
||||
type Item = Result<Bytes, StdError>;
|
||||
|
||||
fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||
self.polls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
Poll::Ready(Some(Ok(Bytes::from_static(b"x"))))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl ByteStream for PollCountingBody {}
|
||||
|
||||
#[cfg(test)]
|
||||
mod oversize_single_upload_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn reject_oversize_single_upload_enforces_the_single_request_ceiling() {
|
||||
let ceiling = i64::try_from(rustfs_config::MAX_SINGLE_PUT_OBJECT_SIZE).expect("ceiling fits i64");
|
||||
|
||||
assert!(reject_oversize_single_upload(0).is_ok());
|
||||
assert!(reject_oversize_single_upload(ceiling).is_ok(), "exact ceiling is allowed");
|
||||
assert!(reject_oversize_single_upload(-1).is_ok(), "unknown length is left to later validation");
|
||||
|
||||
let err = reject_oversize_single_upload(ceiling + 1).expect_err("one byte over must be rejected");
|
||||
assert_eq!(*err.code(), S3ErrorCode::EntityTooLarge);
|
||||
assert_eq!(
|
||||
err.message(),
|
||||
Some(ApiError::error_code_to_message(&S3ErrorCode::EntityTooLarge).as_str())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,364 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Client inactivity is observed before decoding, but charged only while the
|
||||
//! final, transformed reader is waiting. Compression can return buffered output
|
||||
//! after its input returned Pending, so the transport cannot infer read demand.
|
||||
|
||||
use super::{LOG_COMPONENT_APP, LOG_SUBSYSTEM_OBJECT};
|
||||
use crate::error::ClientBodyReadTimeout;
|
||||
use bytes::Bytes;
|
||||
use http_body::{Body, Frame, SizeHint};
|
||||
use parking_lot::Mutex;
|
||||
use rustfs_rio::{DynReader, EtagResolvable, HashReaderDetector, HashReaderMut, Index, TryGetIndex};
|
||||
use std::error::Error;
|
||||
use std::fmt;
|
||||
use std::future::Future;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::task::{Context, Poll};
|
||||
use std::time::Duration;
|
||||
use tokio::io::{AsyncRead, ReadBuf};
|
||||
use tokio::time::{Instant, Sleep};
|
||||
|
||||
const EVENT_UPLOAD_PART_BODY_READ_STALLED: &str = "upload_part_body_read_stalled";
|
||||
|
||||
struct ReadPolicy {
|
||||
timeout: Duration,
|
||||
bucket: String,
|
||||
key: String,
|
||||
request_id: String,
|
||||
expected_decoded_bytes: u64,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct ReadBudget {
|
||||
policy: Option<ReadPolicy>,
|
||||
demand: bool,
|
||||
waiting_since: Option<Instant>,
|
||||
waited: Duration,
|
||||
raw_bytes_received: u64,
|
||||
finished: bool,
|
||||
}
|
||||
|
||||
impl ReadBudget {
|
||||
fn pause(&mut self) {
|
||||
if let Some(start) = self.waiting_since.take() {
|
||||
self.waited = self.waited.saturating_add(start.elapsed());
|
||||
}
|
||||
self.demand = false;
|
||||
}
|
||||
}
|
||||
|
||||
/// Server-owned extension shared by the raw Body and the storage-facing reader.
|
||||
#[derive(Clone, Default)]
|
||||
pub(crate) struct BodyReadControl(Arc<SharedBudget>);
|
||||
|
||||
#[derive(Default)]
|
||||
struct SharedBudget {
|
||||
active: AtomicBool,
|
||||
budget: Mutex<ReadBudget>,
|
||||
}
|
||||
|
||||
impl BodyReadControl {
|
||||
pub(crate) fn activate(
|
||||
&self,
|
||||
timeout: Duration,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
request_id: &str,
|
||||
expected_decoded_bytes: u64,
|
||||
) -> bool {
|
||||
if timeout.is_zero() {
|
||||
return false;
|
||||
}
|
||||
let mut state = self.0.budget.lock();
|
||||
if state.finished {
|
||||
return false;
|
||||
}
|
||||
state.policy = Some(ReadPolicy {
|
||||
timeout,
|
||||
bucket: bucket.to_owned(),
|
||||
key: key.to_owned(),
|
||||
request_id: request_id.to_owned(),
|
||||
expected_decoded_bytes,
|
||||
});
|
||||
self.0.active.store(true, Ordering::Release);
|
||||
true
|
||||
}
|
||||
|
||||
fn begin_read(&self) {
|
||||
let mut state = self.0.budget.lock();
|
||||
if !state.finished {
|
||||
state.demand = true;
|
||||
}
|
||||
}
|
||||
|
||||
fn pause_read(&self) {
|
||||
self.0.budget.lock().pause();
|
||||
}
|
||||
|
||||
fn progress(&self, bytes: usize) {
|
||||
// Other HTTP operations never activate this UploadPart policy.
|
||||
if !self.0.active.load(Ordering::Acquire) {
|
||||
return;
|
||||
}
|
||||
let mut state = self.0.budget.lock();
|
||||
state.raw_bytes_received = state
|
||||
.raw_bytes_received
|
||||
.saturating_add(u64::try_from(bytes).unwrap_or(u64::MAX));
|
||||
state.waited = Duration::ZERO;
|
||||
state.waiting_since = None;
|
||||
}
|
||||
|
||||
fn finish(&self) {
|
||||
let mut state = self.0.budget.lock();
|
||||
self.0.active.store(false, Ordering::Release);
|
||||
state.finished = true;
|
||||
state.policy = None;
|
||||
state.waiting_since = None;
|
||||
state.demand = false;
|
||||
}
|
||||
|
||||
fn waiting_deadline(&self) -> Option<Instant> {
|
||||
if !self.0.active.load(Ordering::Acquire) {
|
||||
return None;
|
||||
}
|
||||
let mut state = self.0.budget.lock();
|
||||
let timeout = state.policy.as_ref()?.timeout;
|
||||
if !state.demand || state.finished {
|
||||
return None;
|
||||
}
|
||||
let remaining = timeout.saturating_sub(state.waited);
|
||||
let start = *state.waiting_since.get_or_insert_with(Instant::now);
|
||||
// A timeout beyond the clock's representable range cannot elapse.
|
||||
start.checked_add(remaining)
|
||||
}
|
||||
|
||||
fn expire(&self) -> Option<ClientBodyReadTimeout> {
|
||||
let (policy, raw_bytes_received) = {
|
||||
let mut state = self.0.budget.lock();
|
||||
let timeout = state.policy.as_ref()?.timeout;
|
||||
if !state.demand || state.waited.saturating_add(state.waiting_since?.elapsed()) < timeout {
|
||||
return None;
|
||||
}
|
||||
state.finished = true;
|
||||
self.0.active.store(false, Ordering::Release);
|
||||
state.demand = false;
|
||||
state.waiting_since = None;
|
||||
(state.policy.take()?, state.raw_bytes_received)
|
||||
};
|
||||
tracing::error!(
|
||||
event = EVENT_UPLOAD_PART_BODY_READ_STALLED,
|
||||
component = LOG_COMPONENT_APP,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT,
|
||||
state = "stall_timeout",
|
||||
operation = "UploadPart",
|
||||
request_id = %policy.request_id,
|
||||
bucket = %policy.bucket,
|
||||
key = %policy.key,
|
||||
raw_bytes_received,
|
||||
expected_decoded_bytes = policy.expected_decoded_bytes,
|
||||
timeout_secs = policy.timeout.as_secs(),
|
||||
"UploadPart request body read stalled"
|
||||
);
|
||||
Some(ClientBodyReadTimeout {
|
||||
timeout: policy.timeout,
|
||||
raw_bytes_received,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) enum ObservedBodyError<E> {
|
||||
Transport(E),
|
||||
Inactivity(ClientBodyReadTimeout),
|
||||
}
|
||||
|
||||
impl<E: fmt::Display> fmt::Display for ObservedBodyError<E> {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::Transport(error) => error.fmt(f),
|
||||
Self::Inactivity(error) => error.fmt(f),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<E: Error + 'static> Error for ObservedBodyError<E> {
|
||||
fn source(&self) -> Option<&(dyn Error + 'static)> {
|
||||
Some(match self {
|
||||
Self::Transport(error) => error,
|
||||
Self::Inactivity(error) => error,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Wraps the retained raw HTTP body, so a synthesized error never marks the
|
||||
/// underlying transport complete or prevents HTTP/1 early-response draining.
|
||||
pub(crate) struct ObservedBody<B> {
|
||||
inner: B,
|
||||
control: BodyReadControl,
|
||||
timer: Option<Pin<Box<Sleep>>>,
|
||||
ended: bool,
|
||||
}
|
||||
|
||||
impl<B> ObservedBody<B> {
|
||||
pub(crate) fn new(inner: B, control: BodyReadControl) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
control,
|
||||
timer: None,
|
||||
ended: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_timeout(&mut self, cx: &mut Context<'_>) -> Option<ClientBodyReadTimeout> {
|
||||
let deadline = self.control.waiting_deadline()?;
|
||||
let timer = self.timer.get_or_insert_with(|| Box::pin(tokio::time::sleep_until(deadline)));
|
||||
if timer.deadline() != deadline {
|
||||
timer.as_mut().reset(deadline);
|
||||
}
|
||||
if timer.as_mut().poll(cx).is_ready() {
|
||||
return self.control.expire();
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
impl<B: Body<Data = Bytes> + Unpin> Body for ObservedBody<B> {
|
||||
type Data = Bytes;
|
||||
type Error = ObservedBodyError<B::Error>;
|
||||
|
||||
fn poll_frame(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Result<Frame<Bytes>, Self::Error>>> {
|
||||
if self.ended {
|
||||
return Poll::Ready(None);
|
||||
}
|
||||
// Ignore empty data without resetting the budget. Bound work per poll
|
||||
// even if a body repeatedly returns immediately-ready empty frames.
|
||||
for _ in 0..32 {
|
||||
match Pin::new(&mut self.inner).poll_frame(cx) {
|
||||
Poll::Ready(Some(Ok(frame))) => {
|
||||
if let Some(data) = frame.data_ref() {
|
||||
if data.is_empty() {
|
||||
if let Some(error) = self.poll_timeout(cx) {
|
||||
self.ended = true;
|
||||
return Poll::Ready(Some(Err(ObservedBodyError::Inactivity(error))));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
self.control.progress(data.len());
|
||||
}
|
||||
return Poll::Ready(Some(Ok(frame)));
|
||||
}
|
||||
Poll::Ready(Some(Err(error))) => {
|
||||
self.ended = true;
|
||||
self.control.finish();
|
||||
return Poll::Ready(Some(Err(ObservedBodyError::Transport(error))));
|
||||
}
|
||||
Poll::Ready(None) => {
|
||||
self.ended = true;
|
||||
self.control.finish();
|
||||
return Poll::Ready(None);
|
||||
}
|
||||
Poll::Pending => {
|
||||
if let Some(error) = self.poll_timeout(cx) {
|
||||
self.ended = true;
|
||||
return Poll::Ready(Some(Err(ObservedBodyError::Inactivity(error))));
|
||||
}
|
||||
return Poll::Pending;
|
||||
}
|
||||
}
|
||||
}
|
||||
cx.waker().wake_by_ref();
|
||||
Poll::Pending
|
||||
}
|
||||
|
||||
fn is_end_stream(&self) -> bool {
|
||||
self.ended || self.inner.is_end_stream()
|
||||
}
|
||||
|
||||
fn size_hint(&self) -> SizeHint {
|
||||
if self.ended {
|
||||
SizeHint::with_exact(0)
|
||||
} else {
|
||||
self.inner.size_hint()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<B> Drop for ObservedBody<B> {
|
||||
fn drop(&mut self) {
|
||||
self.control.finish();
|
||||
}
|
||||
}
|
||||
|
||||
/// Must wrap the final HashReader's inner reader after all write transforms.
|
||||
/// Current erasure readers are owned by their read future/producer: canceling
|
||||
/// that owner drops this reader. A future retained-reader cancellation path
|
||||
/// must explicitly pause its read demand before retaining the reader.
|
||||
pub(crate) struct DemandReader {
|
||||
inner: DynReader,
|
||||
control: BodyReadControl,
|
||||
}
|
||||
|
||||
impl DemandReader {
|
||||
pub(crate) fn new(inner: DynReader, control: BodyReadControl) -> Self {
|
||||
Self { inner, control }
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncRead for DemandReader {
|
||||
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
|
||||
self.control.begin_read();
|
||||
let result = Pin::new(&mut self.inner).poll_read(cx, buf);
|
||||
if result.is_ready() {
|
||||
self.control.pause_read();
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for DemandReader {
|
||||
fn drop(&mut self) {
|
||||
self.control.finish();
|
||||
}
|
||||
}
|
||||
|
||||
impl EtagResolvable for DemandReader {
|
||||
fn is_etag_reader(&self) -> bool {
|
||||
self.inner.is_etag_reader()
|
||||
}
|
||||
fn try_resolve_etag(&mut self) -> Option<String> {
|
||||
self.inner.try_resolve_etag()
|
||||
}
|
||||
}
|
||||
|
||||
impl HashReaderDetector for DemandReader {
|
||||
fn is_hash_reader(&self) -> bool {
|
||||
self.inner.is_hash_reader()
|
||||
}
|
||||
fn as_hash_reader_mut(&mut self) -> Option<&mut dyn HashReaderMut> {
|
||||
self.inner.as_hash_reader_mut()
|
||||
}
|
||||
}
|
||||
|
||||
impl TryGetIndex for DemandReader {
|
||||
fn try_get_index(&self) -> Option<&Index> {
|
||||
self.inner.try_get_index()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,267 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use crate::error::ApiError;
|
||||
use futures::{StreamExt, poll};
|
||||
use http_body_util::StreamBody;
|
||||
use s3s::S3ErrorCode;
|
||||
use s3s::dto::StreamingBlob;
|
||||
use std::io;
|
||||
use tokio::io::AsyncReadExt;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_stream::wrappers::UnboundedReceiverStream;
|
||||
use tokio_util::io::StreamReader;
|
||||
|
||||
mod protocol;
|
||||
|
||||
type FrameSender = mpsc::UnboundedSender<Result<Frame<Bytes>, io::Error>>;
|
||||
|
||||
fn raw_reader(timeout: Duration) -> (FrameSender, DynReader, BodyReadControl) {
|
||||
let (sender, receiver) = mpsc::unbounded_channel();
|
||||
let control = BodyReadControl::default();
|
||||
control.activate(timeout, "bucket", "object", "request", 65536);
|
||||
let body = ObservedBody::new(StreamBody::new(UnboundedReceiverStream::new(receiver)), control.clone());
|
||||
let stream = StreamingBlob::from(s3s::Body::http_body_unsync(body));
|
||||
let reader = rustfs_rio::wrap_reader(StreamReader::new(stream.map(|item| item.map_err(io::Error::other))));
|
||||
(sender, reader, control)
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn body_stall_survives_s3s_and_io_wrapping() {
|
||||
let (_sender, inner, control) = raw_reader(Duration::from_secs(300));
|
||||
let mut reader = DemandReader::new(inner, control);
|
||||
let mut output = Vec::new();
|
||||
let mut read = Box::pin(reader.read_to_end(&mut output));
|
||||
assert!(poll!(read.as_mut()).is_pending());
|
||||
tokio::time::advance(Duration::from_secs(299)).await;
|
||||
assert!(poll!(read.as_mut()).is_pending());
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
let error = read.await.expect_err("a body that remains open must time out");
|
||||
let api = ApiError::from(error);
|
||||
assert_eq!(api.code, S3ErrorCode::RequestTimeout);
|
||||
let s3_error = s3s::S3Error::from(api);
|
||||
assert_eq!(s3_error.status_code(), Some(http::StatusCode::BAD_REQUEST));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn positive_raw_progress_can_outlast_the_inactivity_timeout() {
|
||||
let (sender, inner, control) = raw_reader(Duration::from_secs(300));
|
||||
let mut reader = DemandReader::new(inner, control);
|
||||
let mut output = Vec::new();
|
||||
let mut read = Box::pin(reader.read_to_end(&mut output));
|
||||
assert!(poll!(read.as_mut()).is_pending());
|
||||
let start = Instant::now();
|
||||
for _ in 0..8 {
|
||||
tokio::time::advance(Duration::from_secs(60)).await;
|
||||
sender
|
||||
.send(Ok(Frame::data(Bytes::from(vec![7; 8192]))))
|
||||
.expect("body receiver");
|
||||
assert!(poll!(read.as_mut()).is_pending());
|
||||
}
|
||||
drop(sender);
|
||||
assert_eq!(read.await.expect("progressing upload"), 65536);
|
||||
assert_eq!(start.elapsed(), Duration::from_secs(480));
|
||||
assert_eq!(output, vec![7; 65536]);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn empty_frames_do_not_extend_the_inactivity_budget() {
|
||||
let (sender, inner, control) = raw_reader(Duration::from_secs(300));
|
||||
let mut reader = DemandReader::new(inner, control);
|
||||
let mut output = Vec::new();
|
||||
let mut read = Box::pin(reader.read_to_end(&mut output));
|
||||
assert!(poll!(read.as_mut()).is_pending());
|
||||
for _ in 0..4 {
|
||||
tokio::time::advance(Duration::from_secs(60)).await;
|
||||
sender.send(Ok(Frame::data(Bytes::new()))).expect("body receiver");
|
||||
assert!(poll!(read.as_mut()).is_pending());
|
||||
}
|
||||
tokio::time::advance(Duration::from_secs(60)).await;
|
||||
sender.send(Ok(Frame::data(Bytes::new()))).expect("body receiver");
|
||||
assert_eq!(
|
||||
ApiError::from(read.await.expect_err("empty frames are not progress")).code,
|
||||
S3ErrorCode::RequestTimeout
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn disabled_or_not_yet_read_body_has_no_inactivity_deadline() {
|
||||
for timeout in [Duration::ZERO, Duration::from_secs(300)] {
|
||||
let (sender, inner, control) = raw_reader(timeout);
|
||||
// Covers both foreground admission and staging admission, before the
|
||||
// owner ever asks its final reader for input.
|
||||
tokio::time::advance(Duration::from_secs(1000)).await;
|
||||
let mut reader = DemandReader::new(inner, control);
|
||||
let mut output = Vec::new();
|
||||
let mut read = Box::pin(reader.read_to_end(&mut output));
|
||||
assert!(poll!(read.as_mut()).is_pending());
|
||||
if timeout.is_zero() {
|
||||
tokio::time::advance(Duration::from_secs(1000)).await;
|
||||
assert!(poll!(read.as_mut()).is_pending());
|
||||
}
|
||||
sender
|
||||
.send(Ok(Frame::data(Bytes::from_static(b"ok"))))
|
||||
.expect("body receiver");
|
||||
drop(sender);
|
||||
assert_eq!(read.await.expect("queued or disabled body"), 2);
|
||||
assert_eq!(output, b"ok");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn compressed_output_pauses_raw_wait_during_storage_backpressure() {
|
||||
use rustfs_utils::compress::CompressionAlgorithm;
|
||||
|
||||
let (sender, inner, control) = raw_reader(Duration::from_secs(300));
|
||||
let compressed = rustfs_rio::CompressReader::with_block_size(inner, 8192, CompressionAlgorithm::default());
|
||||
let mut reader = DemandReader::new(rustfs_rio::boxed_reader(compressed), control);
|
||||
sender
|
||||
.send(Ok(Frame::data(Bytes::from(vec![3; 1024]))))
|
||||
.expect("body receiver");
|
||||
let mut buffer = vec![0; 16384];
|
||||
// CompressReader sees the partial input and then Pending, yet can return
|
||||
// a complete compressed block to the storage writer.
|
||||
let first = reader.read(&mut buffer).await.expect("buffered compressed block");
|
||||
assert!(first > 0);
|
||||
let mut compressed_bytes = buffer[..first].to_vec();
|
||||
tokio::time::advance(Duration::from_secs(600)).await;
|
||||
|
||||
let mut read = Box::pin(reader.read(&mut buffer));
|
||||
assert!(poll!(read.as_mut()).is_pending(), "storage backpressure is not a client stall");
|
||||
tokio::time::advance(Duration::from_secs(299)).await;
|
||||
assert!(poll!(read.as_mut()).is_pending());
|
||||
sender
|
||||
.send(Ok(Frame::data(Bytes::from(vec![4; 1024]))))
|
||||
.expect("body receiver");
|
||||
drop(sender);
|
||||
let next = read.await.expect("input after storage backpressure");
|
||||
compressed_bytes.extend_from_slice(&buffer[..next]);
|
||||
reader
|
||||
.read_to_end(&mut compressed_bytes)
|
||||
.await
|
||||
.expect("remaining compressed data");
|
||||
|
||||
let mut restored = Vec::new();
|
||||
rustfs_rio::DecompressReader::new(std::io::Cursor::new(compressed_bytes), CompressionAlgorithm::default())
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("roundtrip after backpressure");
|
||||
assert_eq!(restored, [vec![3; 1024], vec![4; 1024]].concat());
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn read_owner_cancellation_releases_the_raw_body() {
|
||||
let (sender, inner, control) = raw_reader(Duration::from_secs(300));
|
||||
let mut reader = DemandReader::new(inner, control);
|
||||
let owner = tokio::spawn(async move { reader.read_to_end(&mut Vec::new()).await });
|
||||
tokio::task::yield_now().await;
|
||||
owner.abort();
|
||||
assert!(owner.await.expect_err("read owner was canceled").is_cancelled());
|
||||
assert!(sender.is_closed(), "the canceled producer must release the body receiver");
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn buffered_output_pauses_but_does_not_reset_elapsed_inactivity() {
|
||||
struct BufferedOutput {
|
||||
inner: DynReader,
|
||||
ready: Arc<AtomicBool>,
|
||||
}
|
||||
impl AsyncRead for BufferedOutput {
|
||||
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
|
||||
if self.ready.swap(false, Ordering::AcqRel) {
|
||||
buf.put_slice(b"x");
|
||||
return Poll::Ready(Ok(()));
|
||||
}
|
||||
Pin::new(&mut self.inner).poll_read(cx, buf)
|
||||
}
|
||||
}
|
||||
let (_sender, inner, control) = raw_reader(Duration::from_secs(300));
|
||||
let ready = Arc::new(AtomicBool::new(false));
|
||||
let transform = BufferedOutput {
|
||||
inner,
|
||||
ready: Arc::clone(&ready),
|
||||
};
|
||||
let mut reader = DemandReader::new(rustfs_rio::wrap_reader(transform), control);
|
||||
let mut buffer = [0; 1];
|
||||
let mut read = Box::pin(reader.read(&mut buffer));
|
||||
assert!(poll!(read.as_mut()).is_pending());
|
||||
tokio::time::advance(Duration::from_secs(200)).await;
|
||||
assert!(poll!(read.as_mut()).is_pending());
|
||||
ready.store(true, Ordering::Release);
|
||||
assert_eq!(read.await.expect("transform releases buffered output"), 1);
|
||||
tokio::time::advance(Duration::from_secs(600)).await;
|
||||
let mut read = Box::pin(reader.read(&mut buffer));
|
||||
assert!(poll!(read.as_mut()).is_pending());
|
||||
tokio::time::advance(Duration::from_secs(99)).await;
|
||||
assert!(poll!(read.as_mut()).is_pending());
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
let result = poll!(read.as_mut());
|
||||
let Poll::Ready(Err(error)) = result else {
|
||||
panic!("remaining inactivity budget must expire immediately at 100 seconds");
|
||||
};
|
||||
assert_eq!(ApiError::from(error).code, S3ErrorCode::RequestTimeout);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn final_write_reader_preserves_checksums_through_compression_and_sse() {
|
||||
use crate::app::storage_api::multipart_usecase::io::{HashReader, WriteEncryption, WritePlan};
|
||||
use rustfs_rio::{Checksum, ChecksumType};
|
||||
use rustfs_utils::CompressionAlgorithm;
|
||||
|
||||
let payload = vec![7; 65536];
|
||||
for plan in [
|
||||
WritePlan::new().with_compression(CompressionAlgorithm::default()),
|
||||
WritePlan::new().with_encryption(WriteEncryption::multipart([5; 32], [9; 12], 1)),
|
||||
WritePlan::new()
|
||||
.with_compression(CompressionAlgorithm::default())
|
||||
.with_encryption(WriteEncryption::multipart([5; 32], [9; 12], 1)),
|
||||
] {
|
||||
let (sender, inner, control) = raw_reader(Duration::from_secs(300));
|
||||
let checksum = Checksum::new_from_data(ChecksumType::CRC32, &payload).expect("plaintext checksum");
|
||||
let mut plaintext = HashReader::from_reader(inner, 65536, 65536, None, None, false).expect("plaintext reader");
|
||||
plaintext
|
||||
.add_non_trailing_checksum(Some(checksum.clone()), false)
|
||||
.expect("attach checksum");
|
||||
let mut reader = plan.apply(plaintext, 65536).expect("write plan");
|
||||
let inner = reader.take_inner();
|
||||
reader.inner = rustfs_rio::boxed_reader(DemandReader::new(inner, control));
|
||||
let mut output = Vec::new();
|
||||
let mut read = Box::pin(reader.read_to_end(&mut output));
|
||||
assert!(poll!(read.as_mut()).is_pending());
|
||||
for chunk in payload.chunks(8192) {
|
||||
tokio::time::advance(Duration::from_secs(60)).await;
|
||||
sender
|
||||
.send(Ok(Frame::data(Bytes::copy_from_slice(chunk))))
|
||||
.expect("raw body progress");
|
||||
assert!(poll!(read.as_mut()).is_pending());
|
||||
}
|
||||
drop(sender);
|
||||
read.await.expect("transformed reader completes beyond one inactivity period");
|
||||
assert!(!output.is_empty());
|
||||
assert_eq!(reader.content_crc_type(), Some(ChecksumType::CRC32));
|
||||
assert_eq!(reader.content_crc().get("CRC32"), Some(&checksum.encoded));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn native_body_errors_preserve_their_original_source() {
|
||||
let (sender, inner, control) = raw_reader(Duration::from_secs(300));
|
||||
let mut reader = DemandReader::new(inner, control);
|
||||
sender
|
||||
.send(Err(io::Error::new(io::ErrorKind::TimedOut, "disk timeout")))
|
||||
.expect("body receiver");
|
||||
let error = reader.read_to_end(&mut Vec::new()).await.expect_err("native error");
|
||||
assert_eq!(ApiError::from(error).code, S3ErrorCode::InternalError);
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use http_body_util::BodyExt;
|
||||
use s3s::config::{S3Config, StaticConfigProvider};
|
||||
use s3s::dto::{UploadPartInput, UploadPartOutput};
|
||||
use s3s::service::{S3Service, S3ServiceBuilder};
|
||||
use s3s::{S3, S3Request, S3Response, S3Result};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct Consumer {
|
||||
received: Arc<AtomicUsize>,
|
||||
committed: Arc<Mutex<Option<Vec<u8>>>>,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl S3 for Consumer {
|
||||
async fn upload_part(&self, req: S3Request<UploadPartInput>) -> S3Result<S3Response<UploadPartOutput>> {
|
||||
let expected = req.input.content_length.expect("S3S must normalize the decoded length");
|
||||
let control = req
|
||||
.extensions
|
||||
.get::<BodyReadControl>()
|
||||
.expect("HTTP control extension")
|
||||
.clone();
|
||||
control.activate(Duration::from_secs(300), "test-bucket", "test-key", "request", expected as u64);
|
||||
let stream = req.input.body.expect("body");
|
||||
let inner = rustfs_rio::wrap_reader(StreamReader::new(stream.map(|item| item.map_err(io::Error::other))));
|
||||
let mut reader =
|
||||
rustfs_rio::HashReader::from_stream(DemandReader::new(inner, control), expected, expected, None, None, false)
|
||||
.expect("logical body reader");
|
||||
reader
|
||||
.add_checksum_from_s3s(&req.headers, req.trailing_headers, false)
|
||||
.expect("request checksum context");
|
||||
let mut output = Vec::new();
|
||||
let mut buffer = [0; 8192];
|
||||
loop {
|
||||
let count = reader
|
||||
.read(&mut buffer)
|
||||
.await
|
||||
.map_err(|error| s3s::S3Error::from(ApiError::from(error)))?;
|
||||
if count == 0 {
|
||||
break;
|
||||
}
|
||||
self.received.fetch_add(count, Ordering::Relaxed);
|
||||
output.extend_from_slice(&buffer[..count]);
|
||||
}
|
||||
assert_eq!(output.len() as i64, expected, "wire length must not reach the business DTO");
|
||||
*self.committed.lock() = Some(output);
|
||||
Ok(S3Response::new(UploadPartOutput {
|
||||
checksum_crc32: reader.content_crc().get("CRC32").cloned(),
|
||||
..UploadPartOutput::default()
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
fn service(consumer: Consumer) -> S3Service {
|
||||
let mut builder = S3ServiceBuilder::new(consumer);
|
||||
builder.set_auth(s3s::auth::SimpleAuth::from_single("test-access", "test-secret"));
|
||||
let mut config = S3Config::default();
|
||||
config.presigned_url_max_skew_time_secs = u32::MAX;
|
||||
builder.set_config(Arc::new(StaticConfigProvider::new(Arc::new(config))));
|
||||
builder.build()
|
||||
}
|
||||
|
||||
struct SignedRequest {
|
||||
request: http::Request<s3s::Body>,
|
||||
sender: FrameSender,
|
||||
prefix: Bytes,
|
||||
suffix: Bytes,
|
||||
}
|
||||
|
||||
/// Constructs a real SigV4 fixture using the production crypto primitives.
|
||||
/// S3S verifies both the request authorization and every signed chunk.
|
||||
fn signed_request(payload: &[u8], unsigned_trailer: bool) -> SignedRequest {
|
||||
use rustfs_utils::{hex_sha256, hmac_sha256};
|
||||
|
||||
const DATE: &str = "20130524T000000Z";
|
||||
const SCOPE: &str = "20130524/us-east-1/s3/aws4_request";
|
||||
let mode = if unsigned_trailer {
|
||||
"STREAMING-UNSIGNED-PAYLOAD-TRAILER"
|
||||
} else {
|
||||
"STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
|
||||
};
|
||||
let signed_headers = "host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length";
|
||||
let headers = format!(
|
||||
"host:s3.amazonaws.com\nx-amz-content-sha256:{mode}\nx-amz-date:{DATE}\nx-amz-decoded-content-length:{}\n",
|
||||
payload.len()
|
||||
);
|
||||
let canonical = format!("PUT\n/test-bucket/test-key\npartNumber=1&uploadId=test-upload\n{headers}\n{signed_headers}\n{mode}");
|
||||
let key = hmac_sha256("AWS4test-secret", "20130524");
|
||||
let key = hmac_sha256(key, "us-east-1");
|
||||
let key = hmac_sha256(key, "s3");
|
||||
let key = hmac_sha256(key, "aws4_request");
|
||||
let digest = |data: &[u8]| hex_sha256(data, str::to_owned);
|
||||
let encode = |data: [u8; 32]| hex_simd::encode_to_string(data, hex_simd::AsciiCase::Lower);
|
||||
let seed = encode(hmac_sha256(
|
||||
key,
|
||||
format!("AWS4-HMAC-SHA256\n{DATE}\n{SCOPE}\n{}", digest(canonical.as_bytes())),
|
||||
));
|
||||
let chunk_signature = |previous: &str, data: &[u8]| {
|
||||
encode(hmac_sha256(
|
||||
key,
|
||||
format!("AWS4-HMAC-SHA256-PAYLOAD\n{DATE}\n{SCOPE}\n{previous}\n{}\n{}", digest(b""), digest(data)),
|
||||
))
|
||||
};
|
||||
let (prefix, suffix) = if unsigned_trailer {
|
||||
(
|
||||
format!("{:x}\r\n", payload.len()),
|
||||
"\r\n0\r\nx-amz-checksum-crc32:y/Q5Jg==\r\n\r\n".to_owned(),
|
||||
)
|
||||
} else {
|
||||
let signature = chunk_signature(&seed, payload);
|
||||
(
|
||||
format!("{:x};chunk-signature={signature}\r\n", payload.len()),
|
||||
format!("\r\n0;chunk-signature={}\r\n\r\n", chunk_signature(&signature, b"")),
|
||||
)
|
||||
};
|
||||
let (sender, receiver) = mpsc::unbounded_channel();
|
||||
let control = BodyReadControl::default();
|
||||
let body = ObservedBody::new(StreamBody::new(UnboundedReceiverStream::new(receiver)), control.clone());
|
||||
let mut builder = http::Request::builder()
|
||||
.method("PUT")
|
||||
.uri("https://s3.amazonaws.com/test-bucket/test-key?partNumber=1&uploadId=test-upload")
|
||||
.header("host", "s3.amazonaws.com")
|
||||
.header("content-encoding", "aws-chunked")
|
||||
.header("content-length", prefix.len() + payload.len() + suffix.len())
|
||||
.header("x-amz-content-sha256", mode)
|
||||
.header("x-amz-date", DATE)
|
||||
.header("x-amz-decoded-content-length", payload.len())
|
||||
.header(
|
||||
"authorization",
|
||||
format!("AWS4-HMAC-SHA256 Credential=test-access/{SCOPE}, SignedHeaders={signed_headers}, Signature={seed}"),
|
||||
);
|
||||
if unsigned_trailer {
|
||||
builder = builder.header("x-amz-trailer", "x-amz-checksum-crc32");
|
||||
}
|
||||
let mut request = builder.body(s3s::Body::http_body_unsync(body)).expect("signed request");
|
||||
request.extensions_mut().insert(control);
|
||||
SignedRequest {
|
||||
request,
|
||||
sender,
|
||||
prefix: Bytes::from(prefix),
|
||||
suffix: Bytes::from(suffix),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn signed_chunk_with_raw_progress_survives_eight_minutes_without_decoded_output() {
|
||||
let payload = vec![7; 65536];
|
||||
let SignedRequest {
|
||||
request,
|
||||
sender,
|
||||
prefix,
|
||||
suffix,
|
||||
} = signed_request(&payload, false);
|
||||
let consumer = Consumer::default();
|
||||
let service = service(consumer.clone());
|
||||
let mut call = Box::pin(service.call(request));
|
||||
sender.send(Ok(Frame::data(prefix))).expect("body receiver");
|
||||
assert!(poll!(call.as_mut()).is_pending());
|
||||
let start = Instant::now();
|
||||
for chunk in payload.chunks(8192) {
|
||||
assert_eq!(
|
||||
consumer.received.load(Ordering::Relaxed),
|
||||
0,
|
||||
"incomplete chunks must not escape signature validation"
|
||||
);
|
||||
tokio::time::advance(Duration::from_secs(60)).await;
|
||||
sender
|
||||
.send(Ok(Frame::data(Bytes::copy_from_slice(chunk))))
|
||||
.expect("body receiver");
|
||||
assert!(poll!(call.as_mut()).is_pending());
|
||||
}
|
||||
sender.send(Ok(Frame::data(suffix))).expect("body receiver");
|
||||
drop(sender);
|
||||
let response = call.await.expect("S3 response");
|
||||
assert_eq!(response.status(), http::StatusCode::OK);
|
||||
assert_eq!(start.elapsed(), Duration::from_secs(480));
|
||||
assert_eq!(*consumer.committed.lock(), Some(payload));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn decoded_length_does_not_end_waiting_for_terminator_trailer_or_raw_eof() {
|
||||
for (unsigned_trailer, send_suffix) in [(false, false), (false, true), (true, false), (true, true)] {
|
||||
let payload = b"123456789";
|
||||
let SignedRequest {
|
||||
request,
|
||||
sender,
|
||||
prefix,
|
||||
suffix,
|
||||
} = signed_request(payload, unsigned_trailer);
|
||||
let consumer = Consumer::default();
|
||||
let service = service(consumer.clone());
|
||||
sender.send(Ok(Frame::data(prefix))).expect("prefix");
|
||||
sender.send(Ok(Frame::data(Bytes::from_static(payload)))).expect("payload");
|
||||
sender
|
||||
.send(Ok(Frame::data(if send_suffix { suffix } else { Bytes::from_static(b"\r\n") })))
|
||||
.expect("suffix");
|
||||
let mut call = Box::pin(service.call(request));
|
||||
assert!(poll!(call.as_mut()).is_pending());
|
||||
assert_eq!(consumer.received.load(Ordering::Relaxed), payload.len());
|
||||
tokio::time::advance(Duration::from_secs(300)).await;
|
||||
let response = call.await.expect("S3 error response");
|
||||
assert_eq!(response.status(), http::StatusCode::BAD_REQUEST);
|
||||
let xml = BodyExt::collect(response.into_body()).await.expect("error XML").to_bytes();
|
||||
assert!(String::from_utf8_lossy(&xml).contains("<Code>RequestTimeout</Code>"));
|
||||
assert!(consumer.committed.lock().is_none());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unsigned_trailer_normalizes_length_and_signed_corruption_cannot_commit() {
|
||||
for corrupt in [false, true] {
|
||||
let payload = b"123456789";
|
||||
let SignedRequest {
|
||||
request,
|
||||
sender,
|
||||
prefix,
|
||||
suffix,
|
||||
} = signed_request(payload, !corrupt);
|
||||
let consumer = Consumer::default();
|
||||
let service = service(consumer.clone());
|
||||
sender.send(Ok(Frame::data(prefix))).expect("prefix");
|
||||
sender
|
||||
.send(Ok(Frame::data(Bytes::from_static(if corrupt { b"923456789" } else { payload }))))
|
||||
.expect("payload");
|
||||
sender.send(Ok(Frame::data(suffix))).expect("suffix");
|
||||
drop(sender);
|
||||
let response = service.call(request).await.expect("S3 response");
|
||||
if corrupt {
|
||||
assert_ne!(response.status(), http::StatusCode::OK);
|
||||
assert_eq!(consumer.received.load(Ordering::Relaxed), 0);
|
||||
assert!(consumer.committed.lock().is_none());
|
||||
} else {
|
||||
assert_eq!(response.status(), http::StatusCode::OK);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get("x-amz-checksum-crc32")
|
||||
.expect("validated response checksum"),
|
||||
"y/Q5Jg=="
|
||||
);
|
||||
assert_eq!(*consumer.committed.lock(), Some(payload.to_vec()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unsigned_trailer_with_wrong_checksum_cannot_commit() {
|
||||
let SignedRequest {
|
||||
request, sender, prefix, ..
|
||||
} = signed_request(b"123456789", true);
|
||||
let consumer = Consumer::default();
|
||||
let service = service(consumer.clone());
|
||||
sender.send(Ok(Frame::data(prefix))).expect("prefix");
|
||||
sender
|
||||
.send(Ok(Frame::data(Bytes::from_static(
|
||||
b"123456789\r\n0\r\nx-amz-checksum-crc32:AAAAAA==\r\n\r\n",
|
||||
))))
|
||||
.expect("invalid trailer");
|
||||
drop(sender);
|
||||
let response = service.call(request).await.expect("S3 response");
|
||||
assert_eq!(response.status(), http::StatusCode::BAD_REQUEST);
|
||||
let xml = BodyExt::collect(response.into_body()).await.expect("error XML").to_bytes();
|
||||
assert!(String::from_utf8_lossy(&xml).contains("<Code>BadDigest</Code>"));
|
||||
assert!(consumer.committed.lock().is_none());
|
||||
}
|
||||
+279
-39
@@ -126,6 +126,26 @@ impl std::fmt::Display for UploadLimitExceeded {
|
||||
|
||||
impl std::error::Error for UploadLimitExceeded {}
|
||||
|
||||
/// Identifies inactivity of an external client body, rather than a storage timeout.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub(crate) struct ClientBodyReadTimeout {
|
||||
pub timeout: std::time::Duration,
|
||||
pub raw_bytes_received: u64,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for ClientBodyReadTimeout {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(
|
||||
f,
|
||||
"request body made no progress for {} seconds after {} raw bytes",
|
||||
self.timeout.as_secs(),
|
||||
self.raw_bytes_received
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for ClientBodyReadTimeout {}
|
||||
|
||||
/// Marks a server-side object/source reader failure that must not be reported as
|
||||
/// a malformed client request body.
|
||||
#[derive(Debug)]
|
||||
@@ -412,25 +432,7 @@ fn error_chain_has_type<T>(err: &(dyn std::error::Error + 'static)) -> bool
|
||||
where
|
||||
T: std::error::Error + 'static,
|
||||
{
|
||||
if err.downcast_ref::<T>().is_some() {
|
||||
return true;
|
||||
}
|
||||
|
||||
if let Some(io_err) = err.downcast_ref::<std::io::Error>()
|
||||
&& let Some(inner) = io_err.get_ref()
|
||||
&& error_chain_has_type::<T>(inner)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
let mut current = Some(err);
|
||||
while let Some(err) = current {
|
||||
if err.downcast_ref::<T>().is_some() {
|
||||
return true;
|
||||
}
|
||||
current = err.source();
|
||||
}
|
||||
false
|
||||
error_chain_find(err, |error| error.is::<T>().then_some(())).is_some()
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
@@ -451,33 +453,50 @@ fn classify_s3s_body_stream_error_display(err: &(dyn std::error::Error + 'static
|
||||
}
|
||||
|
||||
fn error_chain_s3s_body_stream_error(err: &(dyn std::error::Error + 'static)) -> Option<S3sBodyStreamError> {
|
||||
if let Some(classified) = classify_s3s_body_stream_error_display(err) {
|
||||
return Some(classified);
|
||||
}
|
||||
error_chain_find(err, classify_s3s_body_stream_error_display)
|
||||
}
|
||||
|
||||
if let Some(io_err) = err.downcast_ref::<std::io::Error>()
|
||||
&& let Some(inner) = io_err.get_ref()
|
||||
&& let Some(classified) = error_chain_s3s_body_stream_error(inner)
|
||||
{
|
||||
return Some(classified);
|
||||
}
|
||||
|
||||
let mut current = err.source();
|
||||
while let Some(err) = current {
|
||||
if let Some(classified) = classify_s3s_body_stream_error_display(err) {
|
||||
/// `io::Error::source` skips its custom payload itself. Visit that payload
|
||||
/// explicitly at every level, then follow its source exactly once. The bound
|
||||
/// also makes cyclic or excessively deep foreign error chains safe.
|
||||
fn error_chain_find<T>(
|
||||
err: &(dyn std::error::Error + 'static),
|
||||
mut classify: impl FnMut(&(dyn std::error::Error + 'static)) -> Option<T>,
|
||||
) -> Option<T> {
|
||||
let mut current = Some(err);
|
||||
for _ in 0..64 {
|
||||
let error = current?;
|
||||
if let Some(classified) = classify(error) {
|
||||
return Some(classified);
|
||||
}
|
||||
if let Some(io_err) = err.downcast_ref::<std::io::Error>()
|
||||
&& let Some(inner) = io_err.get_ref()
|
||||
&& let Some(classified) = error_chain_s3s_body_stream_error(inner)
|
||||
{
|
||||
return Some(classified);
|
||||
}
|
||||
current = err.source();
|
||||
current = error
|
||||
.downcast_ref::<std::io::Error>()
|
||||
.and_then(std::io::Error::get_ref)
|
||||
.map(|inner| inner as &(dyn std::error::Error + 'static))
|
||||
.or_else(|| error.source());
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn error_chain_has_body_size_limit_exceeded(err: &(dyn std::error::Error + 'static)) -> bool {
|
||||
error_chain_has_type::<s3s::BodySizeLimitExceeded>(err)
|
||||
}
|
||||
|
||||
/// hyper reports a request body whose connection hit EOF before
|
||||
/// `Content-Length` bytes arrived as a `Kind::Body` error carrying an
|
||||
/// `UnexpectedEof` `io::Error` (its `IncompleteBody` marker is private).
|
||||
/// That is a client-side short body, not a server fault.
|
||||
fn is_hyper_body_eof(err: &(dyn std::error::Error + 'static)) -> bool {
|
||||
err.downcast_ref::<hyper::Error>()
|
||||
.and_then(|hyper_err| std::error::Error::source(hyper_err))
|
||||
.and_then(|cause| cause.downcast_ref::<std::io::Error>())
|
||||
.is_some_and(|io_err| io_err.kind() == std::io::ErrorKind::UnexpectedEof)
|
||||
}
|
||||
|
||||
fn error_chain_has_hyper_body_eof(err: &(dyn std::error::Error + 'static)) -> bool {
|
||||
error_chain_find(err, |error| is_hyper_body_eof(error).then_some(())).is_some()
|
||||
}
|
||||
|
||||
impl From<ApiError> for S3Error {
|
||||
fn from(err: ApiError) -> Self {
|
||||
let status = custom_error_status(&err.code);
|
||||
@@ -535,6 +554,30 @@ impl From<StorageError> for ApiError {
|
||||
};
|
||||
}
|
||||
|
||||
if error_chain_has_type::<ClientBodyReadTimeout>(inner) {
|
||||
return ApiError {
|
||||
code: S3ErrorCode::RequestTimeout,
|
||||
message: ApiError::error_code_to_message(&S3ErrorCode::RequestTimeout),
|
||||
source: Some(Box::new(err)),
|
||||
};
|
||||
}
|
||||
|
||||
if error_chain_has_body_size_limit_exceeded(inner) {
|
||||
return ApiError {
|
||||
code: S3ErrorCode::EntityTooLarge,
|
||||
message: ApiError::error_code_to_message(&S3ErrorCode::EntityTooLarge),
|
||||
source: Some(Box::new(err)),
|
||||
};
|
||||
}
|
||||
|
||||
if error_chain_has_hyper_body_eof(inner) {
|
||||
return ApiError {
|
||||
code: S3ErrorCode::IncompleteBody,
|
||||
message: ApiError::error_code_to_message(&S3ErrorCode::IncompleteBody),
|
||||
source: Some(Box::new(err)),
|
||||
};
|
||||
}
|
||||
|
||||
if matches!(s3s_body_stream_error, Some(S3sBodyStreamError::IncompleteBody)) {
|
||||
return ApiError {
|
||||
code: S3ErrorCode::IncompleteBody,
|
||||
@@ -678,6 +721,30 @@ impl From<std::io::Error> for ApiError {
|
||||
source: Some(Box::new(err)),
|
||||
};
|
||||
}
|
||||
if error_chain_has_type::<ClientBodyReadTimeout>(inner) {
|
||||
return ApiError {
|
||||
code: S3ErrorCode::RequestTimeout,
|
||||
message: ApiError::error_code_to_message(&S3ErrorCode::RequestTimeout),
|
||||
source: Some(Box::new(err)),
|
||||
};
|
||||
}
|
||||
|
||||
if error_chain_has_body_size_limit_exceeded(inner) {
|
||||
return ApiError {
|
||||
code: S3ErrorCode::EntityTooLarge,
|
||||
message: ApiError::error_code_to_message(&S3ErrorCode::EntityTooLarge),
|
||||
source: Some(Box::new(err)),
|
||||
};
|
||||
}
|
||||
|
||||
if error_chain_has_hyper_body_eof(inner) {
|
||||
return ApiError {
|
||||
code: S3ErrorCode::IncompleteBody,
|
||||
message: ApiError::error_code_to_message(&S3ErrorCode::IncompleteBody),
|
||||
source: Some(Box::new(err)),
|
||||
};
|
||||
}
|
||||
|
||||
if matches!(s3s_body_stream_error, Some(S3sBodyStreamError::IncompleteBody)) {
|
||||
return ApiError {
|
||||
code: S3ErrorCode::IncompleteBody,
|
||||
@@ -950,6 +1017,179 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn body_size_limit_exceeded_maps_to_entity_too_large_across_io_boundaries() {
|
||||
// Shape observed in production (issue #7596):
|
||||
// Custom { UnexpectedEof, Custom { Other, BodySizeLimitExceeded { size, limit } } }
|
||||
let nested = || {
|
||||
IoError::new(
|
||||
ErrorKind::UnexpectedEof,
|
||||
IoError::other(s3s::BodySizeLimitExceeded {
|
||||
size: 16384,
|
||||
limit: 6389,
|
||||
}),
|
||||
)
|
||||
};
|
||||
|
||||
let direct: ApiError = nested().into();
|
||||
assert_eq!(direct.code, S3ErrorCode::EntityTooLarge);
|
||||
assert_eq!(direct.message, ApiError::error_code_to_message(&S3ErrorCode::EntityTooLarge));
|
||||
|
||||
let storage: ApiError = StorageError::Io(nested()).into();
|
||||
assert_eq!(storage.code, S3ErrorCode::EntityTooLarge);
|
||||
assert!(storage.source.is_some());
|
||||
|
||||
// An unrelated message that merely mentions a limit stays internal.
|
||||
let other: ApiError = IoError::other(MockS3sBodyStreamError("limit exceeded for something else")).into();
|
||||
assert_eq!(other.code, S3ErrorCode::InternalError);
|
||||
let impostor: ApiError = IoError::other(MockS3sBodyStreamError("body size 16384 exceeds limit 6389")).into();
|
||||
assert_eq!(impostor.code, S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn client_body_timeout_survives_intermediate_io_and_storage_errors() {
|
||||
#[derive(Debug)]
|
||||
struct DecoderError(IoError);
|
||||
impl std::fmt::Display for DecoderError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str("decoder source failed")
|
||||
}
|
||||
}
|
||||
impl std::error::Error for DecoderError {
|
||||
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
|
||||
Some(&self.0)
|
||||
}
|
||||
}
|
||||
let nested = || {
|
||||
IoError::other(DecoderError(IoError::other(IoError::new(
|
||||
ErrorKind::TimedOut,
|
||||
ClientBodyReadTimeout {
|
||||
timeout: std::time::Duration::from_secs(300),
|
||||
raw_bytes_received: 8192,
|
||||
},
|
||||
))))
|
||||
};
|
||||
for error in [ApiError::from(nested()), ApiError::from(StorageError::Io(nested()))] {
|
||||
assert_eq!(error.code, S3ErrorCode::RequestTimeout);
|
||||
assert!(error_chain_has_type::<ClientBodyReadTimeout>(&error));
|
||||
let s3_error = S3Error::from(error);
|
||||
assert_eq!(s3_error.status_code(), Some(StatusCode::BAD_REQUEST));
|
||||
}
|
||||
for error in [
|
||||
ApiError::from(IoError::new(ErrorKind::TimedOut, "disk read timeout")),
|
||||
ApiError::from(StorageError::Io(IoError::new(ErrorKind::TimedOut, "peer timeout"))),
|
||||
] {
|
||||
assert_eq!(error.code, S3ErrorCode::InternalError);
|
||||
}
|
||||
// A server-side source wrapper has precedence even if a remote source
|
||||
// has carried its own client-body marker across an I/O boundary.
|
||||
let source = ServerSideSourceReadError::new("CopyObject", nested());
|
||||
assert_eq!(ApiError::from(IoError::other(source)).code, S3ErrorCode::ServiceUnavailable);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn body_error_classification_bounds_cyclic_source_chains() {
|
||||
#[derive(Debug)]
|
||||
struct Cycle;
|
||||
impl std::fmt::Display for Cycle {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str("cycle")
|
||||
}
|
||||
}
|
||||
impl std::error::Error for Cycle {
|
||||
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
|
||||
Some(self)
|
||||
}
|
||||
}
|
||||
assert!(!error_chain_has_type::<ClientBodyReadTimeout>(&Cycle));
|
||||
assert_eq!(ApiError::from(IoError::other(Cycle)).code, S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
/// Exercise the public error type through the actual body budget.
|
||||
#[tokio::test]
|
||||
async fn real_s3s_body_size_limit_error_maps_to_entity_too_large() {
|
||||
use futures::StreamExt;
|
||||
|
||||
let real_error = || async {
|
||||
let mut body = s3s::Body::from(bytes::Bytes::from_static(b"hello"));
|
||||
body.set_limit(Some(4));
|
||||
body.next()
|
||||
.await
|
||||
.expect("one frame")
|
||||
.expect_err("five bytes must exceed a four-byte budget")
|
||||
};
|
||||
|
||||
let err = real_error().await;
|
||||
assert!(err.is::<s3s::BodySizeLimitExceeded>(), "unexpected body error: {err}");
|
||||
|
||||
let err = real_error().await;
|
||||
let storage: ApiError = StorageError::Io(IoError::new(ErrorKind::UnexpectedEof, IoError::other(err))).into();
|
||||
assert_eq!(storage.code, S3ErrorCode::EntityTooLarge);
|
||||
assert_eq!(storage.message, ApiError::error_code_to_message(&S3ErrorCode::EntityTooLarge));
|
||||
|
||||
let err = real_error().await;
|
||||
let direct: ApiError = IoError::other(err).into();
|
||||
assert_eq!(direct.code, S3ErrorCode::EntityTooLarge);
|
||||
}
|
||||
|
||||
/// Drive a real hyper HTTP/1 server so the test sees hyper's own body EOF
|
||||
/// error (`hyper::Error(Body, UnexpectedEof, IncompleteBody)`), which has no
|
||||
/// public constructor.
|
||||
async fn capture_hyper_body_eof_error() -> hyper::Error {
|
||||
use http_body_util::BodyExt;
|
||||
use hyper::service::service_fn;
|
||||
use hyper_util::rt::TokioIo;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("bind");
|
||||
let addr = listener.local_addr().expect("local addr");
|
||||
let captured: Arc<Mutex<Option<hyper::Error>>> = Arc::new(Mutex::new(None));
|
||||
let server_slot = Arc::clone(&captured);
|
||||
let server = tokio::spawn(async move {
|
||||
let (stream, _) = listener.accept().await.expect("accept");
|
||||
let slot = server_slot;
|
||||
let service = service_fn(move |req: hyper::Request<hyper::body::Incoming>| {
|
||||
let slot = Arc::clone(&slot);
|
||||
async move {
|
||||
let err = req.into_body().collect().await.expect_err("short body must fail");
|
||||
*slot.lock().expect("slot") = Some(err);
|
||||
Ok::<_, std::convert::Infallible>(hyper::Response::new(String::new()))
|
||||
}
|
||||
});
|
||||
let _ = hyper::server::conn::http1::Builder::new()
|
||||
.serve_connection(TokioIo::new(stream), service)
|
||||
.await;
|
||||
});
|
||||
|
||||
let mut client = tokio::net::TcpStream::connect(addr).await.expect("connect");
|
||||
client
|
||||
.write_all(b"PUT /bucket/key HTTP/1.1\r\nHost: localhost\r\nContent-Length: 100\r\n\r\nabc")
|
||||
.await
|
||||
.expect("write partial body");
|
||||
client.shutdown().await.expect("shutdown write side");
|
||||
let _ = tokio::time::timeout(std::time::Duration::from_secs(10), server).await;
|
||||
let captured = captured.lock().expect("slot").take();
|
||||
captured.expect("hyper body error captured")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn hyper_body_eof_maps_to_incomplete_body_across_io_boundaries() {
|
||||
let hyper_err = capture_hyper_body_eof_error().await;
|
||||
assert!(is_hyper_body_eof(&hyper_err), "unexpected hyper error shape: {hyper_err:?}");
|
||||
|
||||
// Shape observed in production (issue #7596):
|
||||
// Custom { UnexpectedEof, Custom { Other, hyper::Error(Body, UnexpectedEof, IncompleteBody) } }
|
||||
let nested = IoError::new(ErrorKind::UnexpectedEof, IoError::other(hyper_err));
|
||||
let storage: ApiError = StorageError::Io(nested).into();
|
||||
assert_eq!(storage.code, S3ErrorCode::IncompleteBody);
|
||||
assert_eq!(storage.message, ApiError::error_code_to_message(&S3ErrorCode::IncompleteBody));
|
||||
|
||||
let hyper_err = capture_hyper_body_eof_error().await;
|
||||
let direct: ApiError = IoError::other(hyper_err).into();
|
||||
assert_eq!(direct.code, S3ErrorCode::IncompleteBody);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn server_side_source_read_error_maps_to_service_unavailable_before_incomplete_body() {
|
||||
let short_source = IoError::new(ErrorKind::UnexpectedEof, rustfs_rio::IncompleteBody { remaining: 17 });
|
||||
|
||||
@@ -418,7 +418,7 @@ impl PathCategory {
|
||||
PathCategory::InternodeRpc
|
||||
} else if path.starts_with("/rustfs/admin/") || path.starts_with("/minio/admin/") {
|
||||
PathCategory::AdminApi
|
||||
} else if path.starts_with("/rustfs/console") {
|
||||
} else if crate::server::has_path_prefix(path, crate::server::console_prefix()) {
|
||||
PathCategory::Console
|
||||
} else if path == "/health"
|
||||
|| path.starts_with("/health/")
|
||||
@@ -766,8 +766,10 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_path_category_classify_console() {
|
||||
assert_eq!(PathCategory::classify("/rustfs/console/index.html"), PathCategory::Console);
|
||||
assert_eq!(PathCategory::classify("/rustfs/console"), PathCategory::Console);
|
||||
let prefix = crate::server::console_prefix();
|
||||
assert_eq!(PathCategory::classify(&format!("{prefix}/index.html")), PathCategory::Console);
|
||||
assert_eq!(PathCategory::classify(prefix), PathCategory::Console);
|
||||
assert_eq!(PathCategory::classify(&format!("{prefix}-other/index.html")), PathCategory::S3DataPlane);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
+173
-20
@@ -14,6 +14,7 @@
|
||||
|
||||
// Import HTTP server components and compression configuration
|
||||
use crate::admin;
|
||||
use crate::app::object::request_body::{BodyReadControl, ObservedBody};
|
||||
use crate::auth::IAMAuth;
|
||||
use crate::auth_keystone;
|
||||
use crate::config;
|
||||
@@ -158,13 +159,11 @@ static HTTP_STATUS_CLASS_METRICS: std::sync::LazyLock<[HttpStatusClassMetrics; 6
|
||||
static HTTP_TRANSPORT_FAILURES_COUNTER: std::sync::LazyLock<metrics::Counter> =
|
||||
std::sync::LazyLock::new(|| counter!(METRIC_HTTP_SERVER_FAILURES_TOTAL, LABEL_HTTP_STATUS_CLASS => "transport"));
|
||||
|
||||
const RUSTFS_S3_PUT_OBJECT_MAX_SIZE: u64 = 5 * 1024 * 1024 * 1024;
|
||||
|
||||
fn rustfs_s3_config() -> S3Config {
|
||||
let mut s3_config = S3Config::default();
|
||||
s3_config.normalize_forward_slash_path = true;
|
||||
s3_config.enable_sig_v2 = true;
|
||||
s3_config.put_object_max_size = Some(RUSTFS_S3_PUT_OBJECT_MAX_SIZE);
|
||||
s3_config.put_object_max_size = Some(rustfs_config::MAX_SINGLE_PUT_OBJECT_SIZE);
|
||||
s3_config.sig_v4_allowed_services.push("s3tables".to_string());
|
||||
s3_config
|
||||
}
|
||||
@@ -827,13 +826,11 @@ where
|
||||
|
||||
impl<S, B, ResBody, ServiceError> Service<HttpRequest<B>> for EarlyResponseBodyService<S>
|
||||
where
|
||||
S: Service<HttpRequest<B>, Response = Response<ResBody>, Error = ServiceError>
|
||||
+ Service<HttpRequest<EarlyResponseBody<B>>, Response = Response<ResBody>, Error = ServiceError>
|
||||
S: Service<HttpRequest<ObservedBody<EarlyResponseBody<B>>>, Response = Response<ResBody>, Error = ServiceError>
|
||||
+ Clone
|
||||
+ Send
|
||||
+ 'static,
|
||||
<S as Service<HttpRequest<B>>>::Future: Send + 'static,
|
||||
<S as Service<HttpRequest<EarlyResponseBody<B>>>>::Future: Send + 'static,
|
||||
<S as Service<HttpRequest<ObservedBody<EarlyResponseBody<B>>>>>::Future: Send + 'static,
|
||||
B: http_body::Body<Data = Bytes> + Send + Unpin + 'static,
|
||||
B::Error: std::error::Error + Send + Sync + 'static,
|
||||
ResBody: Send + 'static,
|
||||
@@ -844,32 +841,32 @@ where
|
||||
type Future = Pin<Box<dyn Future<Output = std::result::Result<Self::Response, Self::Error>> + Send>>;
|
||||
|
||||
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<std::result::Result<(), Self::Error>> {
|
||||
match <S as Service<HttpRequest<B>>>::poll_ready(&mut self.inner, cx)? {
|
||||
Poll::Ready(()) => <S as Service<HttpRequest<EarlyResponseBody<B>>>>::poll_ready(&mut self.inner, cx),
|
||||
Poll::Pending => Poll::Pending,
|
||||
}
|
||||
self.inner.poll_ready(cx)
|
||||
}
|
||||
|
||||
fn call(&mut self, req: HttpRequest<B>) -> Self::Future {
|
||||
let version = req.version();
|
||||
let preserve_on_drop = matches!(version, Version::HTTP_10 | Version::HTTP_11) && !req.body().is_end_stream();
|
||||
let mut inner = self.inner.clone();
|
||||
if !preserve_on_drop {
|
||||
return Box::pin(async move { <S as Service<HttpRequest<B>>>::call(&mut inner, req).await });
|
||||
}
|
||||
let mut req = req;
|
||||
let control = BodyReadControl::default();
|
||||
req.extensions_mut().insert(control.clone());
|
||||
|
||||
let mut drain_context = EarlyResponseBodyDrainContext::from_request(&req, self.idle_timeout);
|
||||
let state = Arc::new(EarlyResponseBodyState::default());
|
||||
let guarded_req = req.map({
|
||||
let state = Arc::clone(&state);
|
||||
move |body| EarlyResponseBody::new(body, state)
|
||||
move |body| ObservedBody::new(EarlyResponseBody::new(body, state), control)
|
||||
});
|
||||
|
||||
Box::pin(async move {
|
||||
let result = <S as Service<HttpRequest<EarlyResponseBody<B>>>>::call(&mut inner, guarded_req).await;
|
||||
let result = inner.call(guarded_req).await;
|
||||
let Some(abandoned) = state.take_abandoned() else {
|
||||
return result;
|
||||
};
|
||||
if !preserve_on_drop {
|
||||
return result;
|
||||
}
|
||||
|
||||
match result {
|
||||
Ok(mut response) => {
|
||||
@@ -966,6 +963,7 @@ pub async fn start_http_server(
|
||||
readiness: Arc<GlobalReadiness>,
|
||||
server_ctx: Arc<ServerContextSlot>,
|
||||
) -> Result<(ShutdownHandle, SocketAddr)> {
|
||||
crate::server::init_console_prefix()?;
|
||||
let server_addr = parse_and_resolve_address(config.address.as_str()).map_err(Error::other)?;
|
||||
|
||||
// The listening address and port are obtained from the parameters
|
||||
@@ -1213,6 +1211,7 @@ pub async fn start_http_server(
|
||||
let now_time = jiff::Zoned::now().strftime("%Y-%m-%d %H:%M:%S").to_string();
|
||||
if config.console_enable {
|
||||
admin::console::init_console_cfg(local_ip, local_port);
|
||||
let console_prefix = crate::server::console_prefix();
|
||||
|
||||
info!(
|
||||
target: "rustfs::console::startup",
|
||||
@@ -1220,7 +1219,7 @@ pub async fn start_http_server(
|
||||
component = LOG_COMPONENT_SERVER,
|
||||
subsystem = LOG_SUBSYSTEM_STARTUP,
|
||||
service = "console",
|
||||
endpoint = %format!("{protocol}://{local_ip_str}:{local_port}/rustfs/console/index.html"),
|
||||
endpoint = %format!("{protocol}://{local_ip_str}:{local_port}{console_prefix}/index.html"),
|
||||
"Startup endpoint available"
|
||||
);
|
||||
info!(
|
||||
@@ -1229,7 +1228,7 @@ pub async fn start_http_server(
|
||||
component = LOG_COMPONENT_SERVER,
|
||||
subsystem = LOG_SUBSYSTEM_STARTUP,
|
||||
service = "console_localhost",
|
||||
endpoint = %format!("{protocol}://127.0.0.1:{local_port}/rustfs/console/index.html"),
|
||||
endpoint = %format!("{protocol}://127.0.0.1:{local_port}{console_prefix}/index.html"),
|
||||
"Startup endpoint available"
|
||||
);
|
||||
} else {
|
||||
@@ -2458,9 +2457,10 @@ mod tests {
|
||||
use crate::storage_api::server::http::ScannerScopedDirtyUsageAckEntry;
|
||||
use bytes::Bytes;
|
||||
use http::Request as HttpRequest;
|
||||
use http::header::CONTENT_LENGTH;
|
||||
use http::{HeaderMap, StatusCode};
|
||||
use http_body::Frame;
|
||||
use http_body_util::{Empty, Full};
|
||||
use http_body_util::{BodyExt, Empty, Full};
|
||||
use metrics::with_local_recorder;
|
||||
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
|
||||
use opentelemetry::propagation::Extractor;
|
||||
@@ -2887,6 +2887,159 @@ mod tests {
|
||||
assert_eq!(bytes_polled.load(Ordering::Relaxed), 0);
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct UploadPartTimeoutS3 {
|
||||
timeout: Duration,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl s3s::S3 for UploadPartTimeoutS3 {
|
||||
async fn upload_part(
|
||||
&self,
|
||||
req: s3s::S3Request<s3s::dto::UploadPartInput>,
|
||||
) -> s3s::S3Result<s3s::S3Response<s3s::dto::UploadPartOutput>> {
|
||||
use futures::StreamExt;
|
||||
use tokio_util::io::StreamReader;
|
||||
|
||||
let control = req
|
||||
.extensions
|
||||
.get::<BodyReadControl>()
|
||||
.expect("HTTP route must install the control")
|
||||
.clone();
|
||||
control.activate(self.timeout, "bucket", "object", "request", 1024);
|
||||
let body = req.input.body.expect("upload body");
|
||||
let inner = rustfs_rio::wrap_reader(StreamReader::new(body.map(|item| item.map_err(std::io::Error::other))));
|
||||
let mut reader = crate::app::object::request_body::DemandReader::new(inner, control);
|
||||
reader
|
||||
.read_to_end(&mut Vec::new())
|
||||
.await
|
||||
.map_err(|error| s3s::S3Error::from(crate::error::ApiError::from(error)))?;
|
||||
Ok(s3s::S3Response::new(s3s::dto::UploadPartOutput::default()))
|
||||
}
|
||||
|
||||
async fn head_bucket(
|
||||
&self,
|
||||
_req: s3s::S3Request<s3s::dto::HeadBucketInput>,
|
||||
) -> s3s::S3Result<s3s::S3Response<s3s::dto::HeadBucketOutput>> {
|
||||
Ok(s3s::S3Response::new(s3s::dto::HeadBucketOutput::default()))
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn upload_part_timeout_preserves_http1_drain_and_http2_stream_drop() {
|
||||
for version in [Version::HTTP_11, Version::HTTP_2] {
|
||||
let (sender, body, bytes_polled, dropped) = tracked_request_body();
|
||||
let request = HttpRequest::builder()
|
||||
.version(version)
|
||||
.method(Method::PUT)
|
||||
.uri("/bucket/object?partNumber=1&uploadId=upload")
|
||||
.header(CONTENT_LENGTH, "1024")
|
||||
.body(body)
|
||||
.expect("upload request");
|
||||
let inner = s3s::service::S3ServiceBuilder::new(UploadPartTimeoutS3 {
|
||||
timeout: Duration::from_secs(300),
|
||||
})
|
||||
.build();
|
||||
let mut service = EarlyResponseBodyService::new(inner, Duration::from_secs(30));
|
||||
let mut call = Box::pin(service.call(request));
|
||||
assert!(futures::poll!(call.as_mut()).is_pending());
|
||||
tokio::time::advance(Duration::from_secs(300)).await;
|
||||
let response = call.await.expect("timeout response");
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
assert_eq!(response.headers().get(CONNECTION).is_some(), version == Version::HTTP_11);
|
||||
let xml = response.into_body().collect().await.expect("timeout XML").to_bytes();
|
||||
assert!(String::from_utf8_lossy(&xml).contains("<Code>RequestTimeout</Code>"));
|
||||
if version == Version::HTTP_11 {
|
||||
assert!(
|
||||
!dropped.load(Ordering::Acquire),
|
||||
"synthetic errors must retain the unfinished raw transport"
|
||||
);
|
||||
sender
|
||||
.send(Bytes::from_static(b"late payload"))
|
||||
.expect("native drain receiver");
|
||||
drop(sender);
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(bytes_polled.load(Ordering::Relaxed), 12);
|
||||
} else {
|
||||
assert!(sender.is_closed(), "HTTP/2 must drop only the failed body");
|
||||
assert_eq!(bytes_polled.load(Ordering::Relaxed), 0);
|
||||
}
|
||||
assert!(dropped.load(Ordering::Acquire));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn upload_part_timeout_leaves_other_http2_streams_usable() {
|
||||
let (client_io, server_io) = tokio::io::duplex(64 * 1024);
|
||||
let inner = s3s::service::S3ServiceBuilder::new(UploadPartTimeoutS3 {
|
||||
timeout: Duration::from_millis(500),
|
||||
})
|
||||
.build();
|
||||
let service = EarlyResponseBodyService::new(inner, Duration::from_secs(1));
|
||||
let server = tokio::spawn(async move {
|
||||
hyper::server::conn::http2::Builder::new(hyper_util::rt::TokioExecutor::new())
|
||||
.serve_connection(TokioIo::new(server_io), TowerToHyperService::new(service))
|
||||
.await
|
||||
});
|
||||
let (mut client, connection) = hyper::client::conn::http2::handshake::<_, _, TrackedRequestBody>(
|
||||
hyper_util::rt::TokioExecutor::new(),
|
||||
TokioIo::new(client_io),
|
||||
)
|
||||
.await
|
||||
.expect("HTTP/2 handshake");
|
||||
let connection = tokio::spawn(connection);
|
||||
let (_sender, body, _, _) = tracked_request_body();
|
||||
let stalled = client.send_request(
|
||||
HttpRequest::builder()
|
||||
.method(Method::PUT)
|
||||
.uri("http://localhost/bucket/object?partNumber=1&uploadId=upload")
|
||||
.header(CONTENT_LENGTH, "1024")
|
||||
.body(body)
|
||||
.expect("stalled request"),
|
||||
);
|
||||
|
||||
client.ready().await.expect("same connection remains ready");
|
||||
let (sender, body, _, _) = tracked_request_body();
|
||||
drop(sender);
|
||||
let response = client
|
||||
.send_request(
|
||||
HttpRequest::builder()
|
||||
.method(Method::HEAD)
|
||||
.uri("http://localhost/bucket")
|
||||
.body(body)
|
||||
.expect("healthy stream"),
|
||||
)
|
||||
.await
|
||||
.expect("healthy response");
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert!(!response.headers().contains_key(CONNECTION));
|
||||
response.into_body().collect().await.expect("healthy stream completes");
|
||||
let response = tokio::time::timeout(Duration::from_secs(5), stalled)
|
||||
.await
|
||||
.expect("stream timeout")
|
||||
.expect("S3 response");
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
let xml = response.into_body().collect().await.expect("timeout body").to_bytes();
|
||||
assert!(String::from_utf8_lossy(&xml).contains("<Code>RequestTimeout</Code>"));
|
||||
client.ready().await.expect("connection after failed stream");
|
||||
let (sender, body, _, _) = tracked_request_body();
|
||||
drop(sender);
|
||||
let response = client
|
||||
.send_request(
|
||||
HttpRequest::builder()
|
||||
.method(Method::HEAD)
|
||||
.uri("http://localhost/bucket")
|
||||
.body(body)
|
||||
.expect("later stream"),
|
||||
)
|
||||
.await
|
||||
.expect("later response");
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
drop(client);
|
||||
connection.abort();
|
||||
server.abort();
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn early_response_body_drain_releases_stalled_body_after_idle_timeout() {
|
||||
let (_sender, body, _bytes_polled, dropped) = tracked_request_body();
|
||||
@@ -3049,7 +3202,7 @@ mod tests {
|
||||
assert!(s3_config.normalize_forward_slash_path);
|
||||
assert!(s3_config.normalize_content_length);
|
||||
assert!(s3_config.enable_sig_v2);
|
||||
assert_eq!(s3_config.put_object_max_size, Some(RUSTFS_S3_PUT_OBJECT_MAX_SIZE));
|
||||
assert_eq!(s3_config.put_object_max_size, Some(rustfs_config::MAX_SINGLE_PUT_OBJECT_SIZE));
|
||||
assert!(s3_config.sig_v4_allowed_services.iter().any(|service| service == "s3"));
|
||||
assert!(s3_config.sig_v4_allowed_services.iter().any(|service| service == "sts"));
|
||||
assert!(s3_config.sig_v4_allowed_services.iter().any(|service| service == "s3tables"));
|
||||
|
||||
@@ -20,10 +20,10 @@ use crate::server::RemoteAddr;
|
||||
use crate::server::cors;
|
||||
use crate::server::hybrid::{HybridBody, is_grpc_request};
|
||||
use crate::server::{
|
||||
ADMIN_PREFIX, CONSOLE_PREFIX, HEALTH_COMPAT_LIVE_PATH, HEALTH_PREFIX, HEALTH_READY_PATH, HealthProbe, MINIO_ADMIN_PREFIX,
|
||||
ADMIN_PREFIX, HEALTH_COMPAT_LIVE_PATH, HEALTH_PREFIX, HEALTH_READY_PATH, HealthProbe, MINIO_ADMIN_PREFIX,
|
||||
MINIO_ADMIN_V3_PREFIX, MINIO_HEALTH_CLUSTER_PATH, MINIO_HEALTH_CLUSTER_READ_PATH, MINIO_HEALTH_LIVE_PATH,
|
||||
MINIO_HEALTH_READY_PATH, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH, RPC_PREFIX, RUSTFS_ADMIN_PREFIX, active_http_requests,
|
||||
build_health_response_parts, collect_probe_readiness, has_path_prefix, is_admin_path, is_table_catalog_path,
|
||||
build_health_response_parts, collect_probe_readiness, console_prefix, has_path_prefix, is_admin_path, is_table_catalog_path,
|
||||
kms_probe_staleness_limit, kms_ready_from_probe,
|
||||
};
|
||||
use crate::shared_types::ReadinessDegradedReason;
|
||||
@@ -625,7 +625,7 @@ where
|
||||
// Create redirect response
|
||||
let redirect_response = Response::builder()
|
||||
.status(StatusCode::FOUND)
|
||||
.header(http::header::LOCATION, "/rustfs/console/")
|
||||
.header(http::header::LOCATION, format!("{}/", console_prefix()))
|
||||
.body(HybridBody::Rest {
|
||||
rest_body: RestBody::default(),
|
||||
})
|
||||
@@ -1861,7 +1861,7 @@ fn is_object_attributes_request<B>(req: &HttpRequest<B>) -> bool {
|
||||
|| has_path_prefix(path, RUSTFS_ADMIN_PREFIX)
|
||||
|| has_path_prefix(path, MINIO_ADMIN_V3_PREFIX)
|
||||
|| is_table_catalog_path(path)
|
||||
|| has_path_prefix(path, CONSOLE_PREFIX)
|
||||
|| has_path_prefix(path, console_prefix())
|
||||
|| has_path_prefix(path, RPC_PREFIX)
|
||||
{
|
||||
return false;
|
||||
@@ -2242,7 +2242,74 @@ fn rewrite_double_slash_root(uri: &Uri) -> Option<Uri> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[tokio::test]
|
||||
async fn console_prefix_process_case_browser_redirect() {
|
||||
if std::env::var_os("RUSTFS_TEST_CONSOLE_PREFIX_PROCESS").is_none() {
|
||||
return;
|
||||
}
|
||||
crate::server::init_console_prefix().expect("initialize console prefix");
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("redirect listener");
|
||||
let addr = listener.local_addr().expect("redirect listener address");
|
||||
let server = tokio::spawn(async move {
|
||||
let (stream, _) = listener.accept().await.expect("redirect client");
|
||||
let inner = tower::service_fn(|_request: Request<Incoming>| async {
|
||||
Ok::<_, Infallible>(Response::new(HybridBody::<Empty<Bytes>, Empty<Bytes>>::Rest { rest_body: Empty::new() }))
|
||||
});
|
||||
let service = RedirectLayer.layer(inner);
|
||||
hyper::server::conn::http1::Builder::new()
|
||||
.serve_connection(
|
||||
hyper_util::rt::TokioIo::new(stream),
|
||||
hyper_util::service::TowerToHyperService::new(service),
|
||||
)
|
||||
.await
|
||||
.expect("redirect connection");
|
||||
});
|
||||
let client = reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.http1_only()
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.timeout(Duration::from_secs(5))
|
||||
.build()
|
||||
.expect("redirect client");
|
||||
let response = client
|
||||
.get(format!("http://{addr}/"))
|
||||
.header(http::header::USER_AGENT, "Mozilla/5.0")
|
||||
.header(http::header::CONNECTION, "close")
|
||||
.send()
|
||||
.await
|
||||
.expect("browser response");
|
||||
assert_eq!(response.status(), StatusCode::FOUND);
|
||||
assert_eq!(response.headers()[http::header::LOCATION], format!("{}/", console_prefix()));
|
||||
response.bytes().await.expect("redirect body");
|
||||
tokio::time::timeout(Duration::from_secs(5), server)
|
||||
.await
|
||||
.expect("bounded redirect server shutdown")
|
||||
.expect("redirect task");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn console_prefix_process_case_classification() {
|
||||
if std::env::var_os("RUSTFS_TEST_CONSOLE_PREFIX_PROCESS").is_none() {
|
||||
return;
|
||||
}
|
||||
crate::server::init_console_prefix().expect("initialize console prefix");
|
||||
let prefix = crate::server::console_prefix();
|
||||
let console_uri = format!("{prefix}/index.html").parse().expect("console URI");
|
||||
assert!(is_empty_body_console_path(&Method::GET, &console_uri));
|
||||
let request = HttpRequest::builder()
|
||||
.uri(format!("{prefix}/index.html?attributes"))
|
||||
.body(())
|
||||
.expect("console attributes request");
|
||||
assert!(!is_object_attributes_request(&request));
|
||||
let s3_request = HttpRequest::builder()
|
||||
.uri("/bucket/object?attributes")
|
||||
.body(())
|
||||
.expect("S3 attributes request");
|
||||
assert!(is_object_attributes_request(&s3_request));
|
||||
}
|
||||
|
||||
use super::*;
|
||||
use crate::server::CONSOLE_PREFIX;
|
||||
use crate::server::compress::{HttpCompressionConfig, PathAwareHttpCompressionPredicate, PathCategoryInjectionLayer};
|
||||
use crate::server::{FAVICON_PATH, LICENSE, RemoteAddr, VERSION};
|
||||
use futures::future::{Ready, ready};
|
||||
@@ -2333,7 +2400,7 @@ mod tests {
|
||||
for path in [
|
||||
"/rustfs/admin/v3/metrics",
|
||||
"/minio/admin/v3/storageinfo",
|
||||
"/rustfs/console/",
|
||||
CONSOLE_PREFIX,
|
||||
"/rustfs/rpc/test",
|
||||
"/health/ready",
|
||||
"/_iceberg/v1/config",
|
||||
@@ -2624,7 +2691,7 @@ mod tests {
|
||||
for path in [
|
||||
"/rustfs/admin/v3/info",
|
||||
"/minio/admin/v3/info",
|
||||
"/rustfs/console/",
|
||||
CONSOLE_PREFIX,
|
||||
HEALTH_PREFIX,
|
||||
"/iceberg/v1/config",
|
||||
"/rustfs/rpc/v1/read-file",
|
||||
@@ -3983,7 +4050,7 @@ mod tests {
|
||||
"/minio/admin/v3/pools/cancel?versionId=unused",
|
||||
"/rustfs/admin/v3/pools/cancel?versionId=unused",
|
||||
"/rustfs/rpc/read_file_stream?versionId=unused",
|
||||
"/rustfs/console/index.html?versionId=unused",
|
||||
&format!("{CONSOLE_PREFIX}/index.html?versionId=unused"),
|
||||
"/health?versionId=unused",
|
||||
"/health/ready?versionId=unused",
|
||||
"/profile/cpu?versionId=unused",
|
||||
|
||||
@@ -72,7 +72,7 @@ pub(crate) use prefix::{
|
||||
HEALTH_COMPAT_LIVE_PATH, HEALTH_PREFIX, HEALTH_READY_PATH, LICENSE, MINIO_ADMIN_PREFIX, MINIO_ADMIN_V3_PREFIX,
|
||||
MINIO_HEALTH_CLUSTER_PATH, MINIO_HEALTH_CLUSTER_READ_PATH, MINIO_HEALTH_LIVE_PATH, MINIO_HEALTH_READY_PATH, PROFILE_CPU_PATH,
|
||||
PROFILE_MEMORY_PATH, RPC_PREFIX, RUSTFS_ADMIN_PREFIX, TABLE_CATALOG_COMPAT_PREFIX, TABLE_CATALOG_PREFIX, TONIC_PREFIX,
|
||||
VERSION, has_path_prefix, is_admin_path, is_table_catalog_path,
|
||||
VERSION, console_prefix, has_path_prefix, init_console_prefix, is_admin_path, is_table_catalog_path,
|
||||
};
|
||||
pub(crate) use readiness::ReadinessDegradedReason;
|
||||
pub(crate) use readiness::ReadinessGateLayer;
|
||||
|
||||
+219
-4
@@ -83,10 +83,81 @@ pub(crate) const RUSTFS_ADMIN_PREFIX: &str = "/rustfs/admin/v3";
|
||||
/// MinIO-compatible admin API prefix accepted by RustFS.
|
||||
pub(crate) const MINIO_ADMIN_V3_PREFIX: &str = "/minio/admin/v3";
|
||||
|
||||
/// Predefined console prefix for RustFS server routes.
|
||||
/// This prefix is used for endpoints that handle console-related tasks
|
||||
/// such as user interface and management.
|
||||
pub(crate) const CONSOLE_PREFIX: &str = "/rustfs/console";
|
||||
/// Console asset base path embedded at build time and used as the startup default.
|
||||
/// It must match NEXT_PUBLIC_BASE_PATH when building the bundled frontend.
|
||||
pub(crate) const CONSOLE_PREFIX: &str = match option_env!("RUSTFS_CONSOLE_BASE_PATH") {
|
||||
Some(path) if !path.is_empty() => path,
|
||||
_ => rustfs_config::DEFAULT_CONSOLE_PREFIX,
|
||||
};
|
||||
|
||||
static CONFIGURED_CONSOLE_PREFIX: std::sync::OnceLock<String> = std::sync::OnceLock::new();
|
||||
|
||||
/// The prefix is fixed before listeners start; request handling never reads the environment.
|
||||
pub(crate) fn console_prefix() -> &'static str {
|
||||
CONFIGURED_CONSOLE_PREFIX.get().map(String::as_str).unwrap_or(CONSOLE_PREFIX)
|
||||
}
|
||||
|
||||
pub(crate) fn init_console_prefix() -> std::io::Result<()> {
|
||||
let raw = match std::env::var(rustfs_config::ENV_RUSTFS_CONSOLE_PREFIX) {
|
||||
Ok(value) => value,
|
||||
Err(std::env::VarError::NotPresent) => CONSOLE_PREFIX.to_string(),
|
||||
Err(err) => return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, err)),
|
||||
};
|
||||
let prefix = validate_console_prefix(&raw)?;
|
||||
if CONFIGURED_CONSOLE_PREFIX.get_or_init(|| prefix.clone()) != &prefix {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
"RUSTFS_CONSOLE_PREFIX cannot change after server initialization",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_console_prefix(raw: &str) -> std::io::Result<String> {
|
||||
let prefix = raw.strip_suffix('/').unwrap_or(raw);
|
||||
// Keep the value safe in HTTP headers, Axum routes, and embedded HTML/JS.
|
||||
if !prefix.starts_with('/')
|
||||
|| prefix.len() > 256
|
||||
|| prefix[1..].split('/').any(|segment| {
|
||||
segment.is_empty()
|
||||
|| matches!(segment, "." | "..")
|
||||
|| !segment
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~'))
|
||||
})
|
||||
{
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
"RUSTFS_CONSOLE_PREFIX must be a non-root absolute path of at most 256 bytes with nonempty URL-safe segments",
|
||||
));
|
||||
}
|
||||
let reserved = [
|
||||
ADMIN_PREFIX,
|
||||
MINIO_ADMIN_PREFIX,
|
||||
TABLE_CATALOG_PREFIX,
|
||||
TABLE_CATALOG_COMPAT_PREFIX,
|
||||
RPC_PREFIX,
|
||||
TONIC_PREFIX,
|
||||
"/rustfs/peer",
|
||||
HEALTH_PREFIX,
|
||||
"/minio/health",
|
||||
"/profile",
|
||||
"/index.html",
|
||||
FAVICON_PATH,
|
||||
APPLE_TOUCH_ICON_PATH,
|
||||
APPLE_TOUCH_ICON_PRECOMPOSED_PATH,
|
||||
];
|
||||
if reserved
|
||||
.iter()
|
||||
.any(|path| has_path_prefix(prefix, path) || has_path_prefix(path, prefix))
|
||||
{
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
"RUSTFS_CONSOLE_PREFIX overlaps a reserved server route",
|
||||
));
|
||||
}
|
||||
Ok(prefix.to_string())
|
||||
}
|
||||
|
||||
/// Predefined RPC prefix for RustFS server routes.
|
||||
/// This prefix is used for endpoints that handle remote procedure calls (RPC).
|
||||
@@ -111,3 +182,147 @@ pub const LOGO: &str = r#"
|
||||
░▀░▀░▀▀▀░▀▀▀░░▀░░▀░░░▀▀▀
|
||||
|
||||
"#;
|
||||
|
||||
#[cfg(test)]
|
||||
mod console_prefix_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn console_prefix_validation() {
|
||||
for (raw, expected) in [
|
||||
(CONSOLE_PREFIX, CONSOLE_PREFIX),
|
||||
("/console", "/console"),
|
||||
("/management/console/", "/management/console"),
|
||||
("/health-dashboard", "/health-dashboard"),
|
||||
] {
|
||||
assert_eq!(validate_console_prefix(raw).expect("valid console prefix"), expected);
|
||||
}
|
||||
for raw in [
|
||||
"",
|
||||
"/",
|
||||
"console",
|
||||
"//console",
|
||||
"/console//",
|
||||
"/a//b",
|
||||
"/a/../b",
|
||||
"/a/./b",
|
||||
"/%2e%2e",
|
||||
"/console?x=1",
|
||||
"/console#x",
|
||||
"/console\\x",
|
||||
"/a\n",
|
||||
"/{param}",
|
||||
"/<script>",
|
||||
"/控制台",
|
||||
"/rustfs",
|
||||
"/rustfs/admin",
|
||||
"/rustfs/admin/v3/ui",
|
||||
"/minio",
|
||||
"/minio/admin",
|
||||
"/health",
|
||||
"/health/ui",
|
||||
"/iceberg",
|
||||
"/_iceberg/v1",
|
||||
"/rustfs/rpc",
|
||||
"/rustfs/peer",
|
||||
"/node_service.NodeService",
|
||||
"/profile",
|
||||
"/index.html",
|
||||
"/favicon.ico",
|
||||
"/index.html",
|
||||
] {
|
||||
assert_eq!(validate_console_prefix(raw).expect_err(raw).kind(), std::io::ErrorKind::InvalidInput);
|
||||
}
|
||||
assert!(validate_console_prefix(&format!("/{}", "a".repeat(255))).is_ok());
|
||||
assert!(validate_console_prefix(&format!("/{}", "a".repeat(256))).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn configured_console_prefix_subprocesses() {
|
||||
// Startup configuration is process-wide; isolate each value from other unit tests.
|
||||
for prefix in [None, Some("/console"), Some("/management/console/")] {
|
||||
let mut command = std::process::Command::new(std::env::current_exe().expect("test executable"));
|
||||
command
|
||||
.args(["console_prefix_process_case", "--test-threads=1"])
|
||||
.env("RUSTFS_TEST_CONSOLE_PREFIX_PROCESS", "1")
|
||||
.env("RUSTFS_CONSOLE_BASE_PATH", "/runtime-ignored/console")
|
||||
.env("RUSTFS_BROWSER_REDIRECT_URL", "https://console.example.com")
|
||||
.env("RUSTFS_HEALTH_ENDPOINT_ENABLE", "true")
|
||||
.env("RUSTFS_CONSOLE_RATE_LIMIT_ENABLE", "false");
|
||||
if let Some(prefix) = prefix {
|
||||
command.env(rustfs_config::ENV_RUSTFS_CONSOLE_PREFIX, prefix);
|
||||
} else {
|
||||
command.env_remove(rustfs_config::ENV_RUSTFS_CONSOLE_PREFIX);
|
||||
}
|
||||
let output = command.output().expect("run isolated console tests");
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"prefix {prefix:?}: {}{}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn console_prefix_process_case_routes() {
|
||||
if std::env::var_os("RUSTFS_TEST_CONSOLE_PREFIX_PROCESS").is_none() {
|
||||
return;
|
||||
}
|
||||
use axum::body::Body;
|
||||
use http::{Request, StatusCode};
|
||||
use tower::ServiceExt;
|
||||
init_console_prefix().expect("initialize configured console prefix");
|
||||
let expected = std::env::var(rustfs_config::ENV_RUSTFS_CONSOLE_PREFIX).unwrap_or_else(|_| CONSOLE_PREFIX.to_string());
|
||||
let prefix = expected.trim_end_matches('/');
|
||||
assert_eq!(console_prefix(), prefix);
|
||||
assert!(crate::admin::console::is_console_path(&format!("{prefix}/index.html")));
|
||||
assert!(!crate::admin::console::is_console_path(&format!("{prefix}-other/index.html")));
|
||||
for path in ["/rustfs/admin/v3/info", "/minio/admin/v3/info", "/health", "/bucket/object"] {
|
||||
assert!(!crate::admin::console::is_console_path(path), "reserved or S3 path {path}");
|
||||
}
|
||||
assert_eq!(
|
||||
crate::server::compress::PathCategory::classify(&format!("{prefix}/asset.js")),
|
||||
crate::server::compress::PathCategory::Console
|
||||
);
|
||||
if prefix != CONSOLE_PREFIX {
|
||||
assert!(!crate::admin::console::is_console_path(CONSOLE_PREFIX));
|
||||
}
|
||||
crate::admin::console::init_console_cfg(std::net::Ipv4Addr::LOCALHOST.into(), 9001);
|
||||
let router = crate::admin::console::make_console_server();
|
||||
for (suffix, expected_status, expected_ready) in [
|
||||
("/health", StatusCode::OK, None),
|
||||
("/health/live", StatusCode::OK, None),
|
||||
("/health/ready", StatusCode::SERVICE_UNAVAILABLE, Some(false)),
|
||||
] {
|
||||
let response = router
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri(format!("{prefix}{suffix}"))
|
||||
.body(Body::empty())
|
||||
.expect("health request"),
|
||||
)
|
||||
.await
|
||||
.expect("health response");
|
||||
assert_eq!(response.status(), expected_status, "{suffix}");
|
||||
let body = axum::body::to_bytes(response.into_body(), 65536).await.expect("health body");
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("health JSON");
|
||||
assert_eq!(payload.get("ready").and_then(serde_json::Value::as_bool), expected_ready, "{suffix}");
|
||||
}
|
||||
for suffix in ["/version", "/license"] {
|
||||
let response = router
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri(format!("{prefix}{suffix}"))
|
||||
.body(Body::empty())
|
||||
.expect("console request"),
|
||||
)
|
||||
.await
|
||||
.expect("console response");
|
||||
assert_eq!(response.status(), StatusCode::OK, "{suffix}");
|
||||
assert_eq!(response.headers()[http::header::CONTENT_TYPE], "application/json");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,9 +54,10 @@
|
||||
//! dimension, whose key space (bucket names) is attacker-chosen.
|
||||
|
||||
use crate::server::{
|
||||
CONSOLE_PREFIX, FAVICON_PATH, HEALTH_COMPAT_LIVE_PATH, HEALTH_PREFIX, HEALTH_READY_PATH, MINIO_HEALTH_CLUSTER_PATH,
|
||||
FAVICON_PATH, HEALTH_COMPAT_LIVE_PATH, HEALTH_PREFIX, HEALTH_READY_PATH, MINIO_HEALTH_CLUSTER_PATH,
|
||||
MINIO_HEALTH_CLUSTER_READ_PATH, MINIO_HEALTH_LIVE_PATH, MINIO_HEALTH_READY_PATH, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH,
|
||||
RPC_PREFIX, RemoteAddr, TONIC_PREFIX, has_path_prefix, is_admin_path, is_table_catalog_path, strip_valid_port_suffix,
|
||||
RPC_PREFIX, RemoteAddr, TONIC_PREFIX, console_prefix, has_path_prefix, is_admin_path, is_table_catalog_path,
|
||||
strip_valid_port_suffix,
|
||||
};
|
||||
use crate::storage_api::server::layer::request_context::RequestContext;
|
||||
use bytes::Bytes;
|
||||
@@ -405,7 +406,7 @@ fn is_rate_limit_exempt_path(path: &str) -> bool {
|
||||
| FAVICON_PATH
|
||||
) || has_path_prefix(path, RPC_PREFIX)
|
||||
|| has_path_prefix(path, TONIC_PREFIX)
|
||||
|| has_path_prefix(path, CONSOLE_PREFIX)
|
||||
|| has_path_prefix(path, console_prefix())
|
||||
}
|
||||
|
||||
/// Apply the standard throttling headers shared by every rate-limited scope.
|
||||
@@ -630,6 +631,18 @@ where
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn console_prefix_process_case_classification() {
|
||||
if std::env::var_os("RUSTFS_TEST_CONSOLE_PREFIX_PROCESS").is_none() {
|
||||
return;
|
||||
}
|
||||
crate::server::init_console_prefix().expect("initialize console prefix");
|
||||
let prefix = crate::server::console_prefix();
|
||||
assert!(is_rate_limit_exempt_path(&format!("{prefix}/version")));
|
||||
assert!(!is_rate_limit_exempt_path(&format!("{prefix}-other/version")));
|
||||
assert!(!is_rate_limit_exempt_path("/bucket/object"));
|
||||
}
|
||||
|
||||
use super::*;
|
||||
use http_body_util::BodyExt;
|
||||
use serial_test::serial;
|
||||
@@ -779,7 +792,7 @@ mod tests {
|
||||
"/favicon.ico",
|
||||
"/rustfs/rpc/anything",
|
||||
"/node_service.NodeService/Ping",
|
||||
"/rustfs/console/index.html",
|
||||
&format!("{}/index.html", console_prefix()),
|
||||
] {
|
||||
assert!(is_rate_limit_exempt_path(path), "{path} must be exempt");
|
||||
}
|
||||
|
||||
@@ -137,7 +137,7 @@ fn is_probe_path(path: &str) -> bool {
|
||||
let is_prefix_probe = has_path_prefix(path, crate::server::RUSTFS_ADMIN_PREFIX)
|
||||
|| has_path_prefix(path, crate::server::MINIO_ADMIN_V3_PREFIX)
|
||||
|| is_table_catalog_path(path)
|
||||
|| has_path_prefix(path, crate::server::CONSOLE_PREFIX)
|
||||
|| has_path_prefix(path, crate::server::console_prefix())
|
||||
|| has_path_prefix(path, crate::server::RPC_PREFIX)
|
||||
|| has_path_prefix(path, crate::server::ADMIN_PREFIX)
|
||||
|| has_path_prefix(path, crate::server::MINIO_ADMIN_PREFIX)
|
||||
@@ -1158,6 +1158,18 @@ where
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn console_prefix_process_case_classification() {
|
||||
if std::env::var_os("RUSTFS_TEST_CONSOLE_PREFIX_PROCESS").is_none() {
|
||||
return;
|
||||
}
|
||||
crate::server::init_console_prefix().expect("initialize console prefix");
|
||||
let prefix = crate::server::console_prefix();
|
||||
assert!(is_probe_path(&format!("{prefix}/index.html")));
|
||||
assert!(!is_probe_path(&format!("{prefix}-other/index.html")));
|
||||
assert!(!is_probe_path("/bucket/object"));
|
||||
}
|
||||
|
||||
use super::*;
|
||||
use crate::storage_api::server::readiness::{DiskOption, new_disk};
|
||||
use rustfs_madmin::{BackendInfo, Disk};
|
||||
@@ -1624,7 +1636,7 @@ mod tests {
|
||||
assert!(is_probe_path("/rustfs/admin/v3/info"));
|
||||
assert!(is_probe_path(&format!("{}/config", crate::server::TABLE_CATALOG_PREFIX)));
|
||||
assert!(is_probe_path("/_iceberg/v1/config"));
|
||||
assert!(is_probe_path("/rustfs/console/"));
|
||||
assert!(is_probe_path(&format!("{}/", crate::server::console_prefix())));
|
||||
assert!(!is_probe_path("/minio/adminx/object"));
|
||||
assert!(!is_probe_path("/rustfs/adminx/object"));
|
||||
assert!(!is_probe_path("/bucket/object"));
|
||||
|
||||
@@ -1630,6 +1630,44 @@ pub(crate) fn build_site_replication_config(
|
||||
}
|
||||
}
|
||||
|
||||
/// Reload `bucket`'s metadata on every other node of this site after a
|
||||
/// site-replication write. Every S3 bucket-config write does this
|
||||
/// (`app::bucket_usecase::notify_bucket_metadata_reload`); the
|
||||
/// site-replication writers did not, so on a multi-node site a node other
|
||||
/// than the one that applied the write served the previous targets and
|
||||
/// rules for up to the 15-minute refresh — a `resync start` routed to such a
|
||||
/// node reported every freshly wired bucket as `Config not found` or
|
||||
/// `recorded remote target no longer exists` (backlog#2367 A-5, backlog#2195
|
||||
/// item 2). Best effort like the S3 path: the write is durable and the
|
||||
/// refresh loop is the fallback, so an unreachable node must not fail the
|
||||
/// operation that already committed.
|
||||
pub(crate) async fn reload_bucket_metadata_on_peers(bucket: &str, operation: &'static str, scanner_maintenance_change: bool) {
|
||||
if scanner_maintenance_change {
|
||||
rustfs_scanner::record_scanner_maintenance_change(bucket);
|
||||
}
|
||||
let Some(notification_sys) = crate::admin::runtime_sources::current_notification_system() else {
|
||||
return;
|
||||
};
|
||||
let result = if scanner_maintenance_change {
|
||||
notification_sys.load_bucket_metadata_for_scanner_maintenance(bucket).await
|
||||
} else {
|
||||
notification_sys.load_bucket_metadata(bucket).await
|
||||
};
|
||||
if let Err(err) = result {
|
||||
warn!(
|
||||
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
|
||||
bucket = %bucket,
|
||||
operation,
|
||||
result = "peer_metadata_reload_failed",
|
||||
error = %err,
|
||||
"admin site replication state"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns whether the bucket targets were rewritten.
|
||||
pub(crate) async fn ensure_site_replication_bucket_targets_with_runtime(
|
||||
bucket: &str,
|
||||
state: &SiteReplicationState,
|
||||
@@ -1637,7 +1675,7 @@ pub(crate) async fn ensure_site_replication_bucket_targets_with_runtime(
|
||||
config: Option<&ReplicationConfiguration>,
|
||||
service_account_secret_key: &str,
|
||||
expected_incarnation_id: Uuid,
|
||||
) -> S3Result<()> {
|
||||
) -> S3Result<bool> {
|
||||
let existing = match metadata_sys::list_bucket_targets(bucket).await {
|
||||
Ok(targets) => targets,
|
||||
Err(StorageError::ConfigNotFound) => BucketTargets::default(),
|
||||
@@ -1649,7 +1687,7 @@ pub(crate) async fn ensure_site_replication_bucket_targets_with_runtime(
|
||||
let updated =
|
||||
reconcile_site_replication_bucket_targets(existing, bucket, state, local_peer, config, service_account_secret_key)?;
|
||||
if updated.targets.is_empty() {
|
||||
return Ok(());
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let json_targets = serde_json::to_vec(&updated)
|
||||
@@ -1658,12 +1696,12 @@ pub(crate) async fn ensure_site_replication_bucket_targets_with_runtime(
|
||||
// client — noticeable now that startup reconciles all buckets, not just the one bucket
|
||||
// an operation touched.
|
||||
if json_targets == existing_json {
|
||||
return Ok(());
|
||||
return Ok(false);
|
||||
}
|
||||
metadata_sys::update_if_incarnation(bucket, BUCKET_TARGETS_FILE, json_targets, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
Ok(())
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
pub(crate) async fn bucket_replication_config_for_target_refresh(bucket: &str) -> S3Result<Option<ReplicationConfiguration>> {
|
||||
@@ -1674,13 +1712,14 @@ pub(crate) async fn bucket_replication_config_for_target_refresh(bucket: &str) -
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns whether the replication configuration was rewritten.
|
||||
pub(crate) async fn ensure_site_replication_bucket_replication_config_with_runtime(
|
||||
bucket: &str,
|
||||
state: &SiteReplicationState,
|
||||
local_peer: &PeerInfo,
|
||||
service_account_secret_key: &str,
|
||||
expected_incarnation_id: Uuid,
|
||||
) -> S3Result<()> {
|
||||
) -> S3Result<bool> {
|
||||
let existing = match metadata_sys::get_replication_config(bucket).await {
|
||||
Ok((existing, _)) => Some(existing),
|
||||
Err(StorageError::ConfigNotFound) => None,
|
||||
@@ -1689,7 +1728,7 @@ pub(crate) async fn ensure_site_replication_bucket_replication_config_with_runti
|
||||
|
||||
let Some(desired) = build_site_replication_config(bucket, state, local_peer, service_account_secret_key, existing.as_ref())?
|
||||
else {
|
||||
return Ok(());
|
||||
return Ok(false);
|
||||
};
|
||||
|
||||
// Derived rules are state owned by this site: rebuild them from the current peer
|
||||
@@ -1721,7 +1760,7 @@ pub(crate) async fn ensure_site_replication_bucket_replication_config_with_runti
|
||||
};
|
||||
|
||||
if rules == existing_rules && role == existing_role {
|
||||
return Ok(());
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let data = serialize(&ReplicationConfiguration { role, rules })
|
||||
@@ -1730,7 +1769,7 @@ pub(crate) async fn ensure_site_replication_bucket_replication_config_with_runti
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
Ok(())
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
pub(crate) async fn ensure_site_replication_bucket_setup_with_runtime(
|
||||
@@ -1748,9 +1787,9 @@ pub(crate) async fn ensure_site_replication_bucket_setup_with_runtime_for_incarn
|
||||
runtime: &SiteReplicationRuntime,
|
||||
expected_incarnation_id: Uuid,
|
||||
) -> S3Result<()> {
|
||||
let _targets_guard = lock_bucket_targets_metadata(bucket).await;
|
||||
let targets_guard = lock_bucket_targets_metadata(bucket).await;
|
||||
let config = bucket_replication_config_for_target_refresh(bucket).await?;
|
||||
ensure_site_replication_bucket_targets_with_runtime(
|
||||
let targets_written = ensure_site_replication_bucket_targets_with_runtime(
|
||||
bucket,
|
||||
&runtime.state,
|
||||
&runtime.local_peer,
|
||||
@@ -1759,7 +1798,7 @@ pub(crate) async fn ensure_site_replication_bucket_setup_with_runtime_for_incarn
|
||||
expected_incarnation_id,
|
||||
)
|
||||
.await?;
|
||||
ensure_site_replication_bucket_replication_config_with_runtime(
|
||||
let config_written = ensure_site_replication_bucket_replication_config_with_runtime(
|
||||
bucket,
|
||||
&runtime.state,
|
||||
&runtime.local_peer,
|
||||
@@ -1767,6 +1806,10 @@ pub(crate) async fn ensure_site_replication_bucket_setup_with_runtime_for_incarn
|
||||
expected_incarnation_id,
|
||||
)
|
||||
.await?;
|
||||
drop(targets_guard);
|
||||
if targets_written || config_written {
|
||||
reload_bucket_metadata_on_peers(bucket, "site_replication_bucket_setup", config_written).await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1791,6 +1834,7 @@ pub(crate) async fn ensure_site_replication_bucket_versioning(bucket: &str) -> S
|
||||
metadata_sys::update_if_incarnation(bucket, BUCKET_VERSIONING_CONFIG, bucket_versioning_xml()?, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
reload_bucket_metadata_on_peers(bucket, "site_replication_bucket_versioning", false).await;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -52,7 +52,11 @@ pub(crate) struct SiteReplicationRetryEvent {
|
||||
/// deletion body (if it was a deletion) recorded in
|
||||
/// [`SiteReplicationState::iam_deletion_replays`]. Only then may a
|
||||
/// successful deletion replay plus a stable snapshot resend settle the
|
||||
/// entry; a legacy entry (or one degraded by record overflow) keeps the
|
||||
/// entry. Every entry this binary creates starts recorded: the IAM
|
||||
/// change hook records deletion bodies, and the other creators (the add
|
||||
/// bootstrap's snapshot send, the drain's own replay) never carry a
|
||||
/// deletion. A legacy entry persisted by a binary that predates recording
|
||||
/// (serde default `false`), or one degraded by record overflow, keeps the
|
||||
/// escalation semantics because an unrecorded deletion may hide in it.
|
||||
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
|
||||
pub(crate) deletions_recorded: bool,
|
||||
@@ -308,7 +312,12 @@ fn push_site_replication_retry_event(
|
||||
updated_at: Some(OffsetDateTime::now_utc()),
|
||||
edit_generation: generation,
|
||||
peer_unreachable,
|
||||
deletions_recorded: false,
|
||||
// See the field doc: only a row persisted by an older binary is
|
||||
// unrecorded. Stamping at creation is what lets an entry first
|
||||
// created by the bootstrap snapshot send settle after a later
|
||||
// deletion is replayed, instead of escalating forever
|
||||
// (backlog#2367 A-3).
|
||||
deletions_recorded: true,
|
||||
});
|
||||
Ok(evicted)
|
||||
}
|
||||
@@ -598,22 +607,7 @@ pub(crate) fn record_failed_iam_delivery(
|
||||
item: &SRIAMItem,
|
||||
error: &str,
|
||||
) -> S3Result<()> {
|
||||
let existed = state
|
||||
.retry_queue
|
||||
.iter()
|
||||
.any(|event| retry_event_matches(event, peer, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH));
|
||||
upsert_site_replication_retry_event(&mut state.retry_queue, peer, SITE_REPLICATION_PEER_IAM_ITEM_WIRE_PATH, error, None)?;
|
||||
if !existed
|
||||
&& let Some(event) = state
|
||||
.retry_queue
|
||||
.iter_mut()
|
||||
.find(|event| retry_event_matches(event, peer, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH))
|
||||
{
|
||||
// Fresh entry: every failure it will ever collapse goes through this
|
||||
// recording path, so a deletion replay plus a stable snapshot resend
|
||||
// can later settle it instead of escalating.
|
||||
event.deletions_recorded = true;
|
||||
}
|
||||
|
||||
let Some(entity) = iam_item_deletion_entity(item) else {
|
||||
return Ok(());
|
||||
@@ -1418,6 +1412,33 @@ pub(crate) fn site_replication_retry_backoff_elapsed(event: &SiteReplicationRetr
|
||||
now.unix_timestamp().saturating_sub(updated_at.unix_timestamp()) >= delay
|
||||
}
|
||||
|
||||
/// Backoff evaluation time for the heavyweight tick: halfway to the next
|
||||
/// tick. Backoffs are multiples of the tick interval, so an entry stamped δ
|
||||
/// seconds after a tick is `600 − δ` old at the next one and slipped a whole
|
||||
/// extra interval for every δ > 0 — a first replay landed at T+1200 rather
|
||||
/// than T+600 (backlog#2367 A-1). Evaluating at the midpoint bounds the slip
|
||||
/// to half an interval either way; timestamps written back stay real time.
|
||||
pub(crate) fn heavyweight_retry_drain_horizon(now: OffsetDateTime) -> OffsetDateTime {
|
||||
let half_interval = crate::site_replication_reconcile::RECONCILE_INTERVAL / 2;
|
||||
now + time::Duration::seconds(i64::try_from(half_interval.as_secs()).unwrap_or(i64::MAX))
|
||||
}
|
||||
|
||||
/// What the lightweight 30-second pass may act on. It replays bounded bucket
|
||||
/// ops only, but probes every backed-off class: promotion is a state flip
|
||||
/// the heavyweight tick then replays, so an IAM or bucket-metadata snapshot
|
||||
/// owed to a peer that came back is resent at the next tick instead of
|
||||
/// after its own backoff has fully elapsed (backlog#2367 A-1).
|
||||
pub(crate) fn lightweight_retry_drain_partition(
|
||||
state: &SiteReplicationState,
|
||||
now: OffsetDateTime,
|
||||
) -> (Vec<SiteReplicationRetryEvent>, Vec<SiteReplicationRetryEvent>) {
|
||||
let mut actionable = actionable_site_replication_retry_events(state, now);
|
||||
actionable.retain(|event| {
|
||||
classify_site_replication_retry_event(event).is_some_and(|action| is_lightweight_retry_drain_action(&action))
|
||||
});
|
||||
(actionable, deferred_site_replication_retry_events(state, now))
|
||||
}
|
||||
|
||||
/// The subset of the retry queue the background drain is allowed to touch.
|
||||
pub(crate) fn actionable_site_replication_retry_events(
|
||||
state: &SiteReplicationState,
|
||||
@@ -1666,14 +1687,7 @@ async fn drain_site_replication_retry_queue_lightweight_inner() -> S3Result<()>
|
||||
return Ok(());
|
||||
}
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let mut actionable = actionable_site_replication_retry_events(&runtime.state, now);
|
||||
let mut deferred = deferred_site_replication_retry_events(&runtime.state, now);
|
||||
actionable.retain(|event| {
|
||||
classify_site_replication_retry_event(event).is_some_and(|action| is_lightweight_retry_drain_action(&action))
|
||||
});
|
||||
deferred.retain(|event| {
|
||||
classify_site_replication_retry_event(event).is_some_and(|action| is_lightweight_retry_drain_action(&action))
|
||||
});
|
||||
let (actionable, deferred) = lightweight_retry_drain_partition(&runtime.state, now);
|
||||
if actionable.is_empty() && deferred.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
@@ -1697,10 +1711,7 @@ async fn drain_site_replication_retry_queue_lightweight_inner() -> S3Result<()>
|
||||
return Ok(());
|
||||
}
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let mut actionable = actionable_site_replication_retry_events(&runtime.state, now);
|
||||
actionable.retain(|event| {
|
||||
classify_site_replication_retry_event(event).is_some_and(|action| is_lightweight_retry_drain_action(&action))
|
||||
});
|
||||
let (actionable, _) = lightweight_retry_drain_partition(&runtime.state, now);
|
||||
if actionable.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
@@ -1717,9 +1728,9 @@ pub(crate) async fn drain_site_replication_retry_queue_inner() -> S3Result<()> {
|
||||
// The alert must fire even when nothing is drainable this tick —
|
||||
// escalated markers are exactly the entries the drain skips.
|
||||
log_site_replication_retry_liabilities(&runtime.state);
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let actionable = actionable_site_replication_retry_events(&runtime.state, now);
|
||||
let deferred = deferred_site_replication_retry_events(&runtime.state, now);
|
||||
let horizon = heavyweight_retry_drain_horizon(OffsetDateTime::now_utc());
|
||||
let actionable = actionable_site_replication_retry_events(&runtime.state, horizon);
|
||||
let deferred = deferred_site_replication_retry_events(&runtime.state, horizon);
|
||||
if actionable.is_empty() && deferred.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
@@ -1764,8 +1775,8 @@ pub(crate) async fn drain_site_replication_retry_queue_inner() -> S3Result<()> {
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let actionable = actionable_site_replication_retry_events(&runtime.state, now);
|
||||
let horizon = heavyweight_retry_drain_horizon(OffsetDateTime::now_utc());
|
||||
let actionable = actionable_site_replication_retry_events(&runtime.state, horizon);
|
||||
if actionable.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -98,11 +98,46 @@ async fn spawn_test_tls_server_with_response(response: &'static [u8]) -> (String
|
||||
break;
|
||||
}
|
||||
}
|
||||
stream.write_all(response).await.is_ok()
|
||||
// Flush buffered TLS records and send close_notify before dropping the socket.
|
||||
stream.write_all(response).await.is_ok() && stream.shutdown().await.is_ok()
|
||||
});
|
||||
(endpoint, ca_pem, task)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tls_test_server_delivers_response_and_closes_cleanly() {
|
||||
use rustls_pki_types::pem::PemObject;
|
||||
|
||||
let (endpoint, ca_pem, server) = spawn_test_tls_server().await;
|
||||
let mut roots = rustls::RootCertStore::empty();
|
||||
roots
|
||||
.add(rustls_pki_types::CertificateDer::from_pem_slice(ca_pem.as_bytes()).expect("parse test CA"))
|
||||
.expect("trust test CA");
|
||||
let config = rustls::ClientConfig::builder()
|
||||
.with_root_certificates(roots)
|
||||
.with_no_client_auth();
|
||||
let connector = tokio_rustls::TlsConnector::from(Arc::new(config));
|
||||
let socket = tokio::net::TcpStream::connect(endpoint.strip_prefix("https://").expect("TLS endpoint"))
|
||||
.await
|
||||
.expect("connect to TLS test server");
|
||||
let mut stream = connector
|
||||
.connect(rustls_pki_types::ServerName::try_from("127.0.0.1").expect("test server name"), socket)
|
||||
.await
|
||||
.expect("trust TLS test server");
|
||||
stream
|
||||
.write_all(b"GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n")
|
||||
.await
|
||||
.expect("write test request");
|
||||
stream.flush().await.expect("flush test request");
|
||||
let mut response = Vec::new();
|
||||
tokio::time::timeout(Duration::from_secs(5), stream.read_to_end(&mut response))
|
||||
.await
|
||||
.expect("TLS response must finish")
|
||||
.expect("TLS test server must send close_notify before closing");
|
||||
assert!(response.ends_with(b"\r\n\r\nok"));
|
||||
assert!(server.await.expect("TLS test server task"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_connection_validation_accepts_supported_combinations() {
|
||||
let ca = valid_test_ca_pem("peer.example.com");
|
||||
@@ -845,7 +880,8 @@ fn test_record_failed_iam_delivery_records_deletions_and_flags_entry() {
|
||||
record_failed_iam_delivery(&mut state, &target, &policy_delete_item("readonly"), "peer offline").expect("record failure");
|
||||
assert_eq!(state.iam_deletion_replays.len(), 2);
|
||||
|
||||
// A legacy entry (created without recording) is never stamped.
|
||||
// A legacy entry (persisted by a binary that predates recording, so it
|
||||
// deserialized with the `false` default) is never stamped.
|
||||
let legacy = PeerInfo {
|
||||
deployment_id: "legacy-dep".to_string(),
|
||||
..peer("legacy", "https://legacy.example.com")
|
||||
@@ -859,6 +895,12 @@ fn test_record_failed_iam_delivery_records_deletions_and_flags_entry() {
|
||||
None,
|
||||
)
|
||||
.expect("upsert retry event");
|
||||
state
|
||||
.retry_queue
|
||||
.iter_mut()
|
||||
.find(|event| event.peer_deployment_id == legacy.deployment_id)
|
||||
.expect("legacy entry")
|
||||
.deletions_recorded = false;
|
||||
record_failed_iam_delivery(&mut state, &legacy, &user_delete_item("bob"), "peer offline").expect("record failure");
|
||||
let legacy_event = state
|
||||
.retry_queue
|
||||
@@ -871,6 +913,43 @@ fn test_record_failed_iam_delivery_records_deletions_and_flags_entry() {
|
||||
);
|
||||
}
|
||||
|
||||
/// backlog#2367 A-3: an entry first created by a non-deletion failure — the
|
||||
/// add bootstrap's snapshot send, or the drain's own replay — hides no
|
||||
/// unrecorded deletion, so a deletion recorded later plus a stable snapshot
|
||||
/// resend must settle it instead of escalating it to the permanent marker
|
||||
/// that only `replicate repair` clears.
|
||||
#[test]
|
||||
fn test_bootstrap_created_iam_entry_settles_after_deletion_replay() {
|
||||
let target = PeerInfo {
|
||||
deployment_id: "remote-dep".to_string(),
|
||||
..peer("remote", "https://remote.example.com")
|
||||
};
|
||||
let mut state = deletion_replay_state(&target);
|
||||
upsert_site_replication_retry_event(
|
||||
&mut state.retry_queue,
|
||||
&target,
|
||||
SITE_REPLICATION_PEER_IAM_ITEM_WIRE_PATH,
|
||||
"peer request to https://remote.example.com failed (connect): connection refused",
|
||||
None,
|
||||
)
|
||||
.expect("bootstrap send failure");
|
||||
assert!(state.retry_queue[0].deletions_recorded, "a fresh entry carries no unrecorded deletion");
|
||||
|
||||
record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "peer offline").expect("record failure");
|
||||
assert_eq!(state.retry_queue.len(), 1, "the hook failure collapses into the bootstrap entry");
|
||||
assert!(state.retry_queue[0].deletions_recorded);
|
||||
assert_eq!(state.iam_deletion_replays.len(), 1);
|
||||
|
||||
let observed = state.retry_queue[0].clone();
|
||||
let replayed: Vec<String> = state.iam_deletion_replays.iter().map(|record| record.id.clone()).collect();
|
||||
assert!(
|
||||
settle_replayed_iam_retry_events(&mut state, &target, &observed, &replayed),
|
||||
"the replayed deletion plus the snapshot resend settle the entry"
|
||||
);
|
||||
assert!(state.retry_queue.is_empty(), "no escalation marker may remain: {:?}", state.retry_queue);
|
||||
assert!(state.iam_deletion_replays.is_empty());
|
||||
}
|
||||
|
||||
/// Overflowing the per-peer record cap degrades the entry back to the
|
||||
/// escalation semantics: the record set is no longer complete, so a replay
|
||||
/// can no longer prove the peer converged.
|
||||
@@ -1626,6 +1705,74 @@ fn test_deferred_retry_events_do_not_probe_fresh_application_failures() {
|
||||
assert!(actionable_site_replication_retry_events(&state, now).is_empty());
|
||||
}
|
||||
|
||||
/// backlog#2367 A-1: the lightweight pass replays bucket ops only, but
|
||||
/// probes every backed-off class so a recovered peer's IAM snapshot is
|
||||
/// promoted within 30 seconds instead of waiting for the heavyweight tick
|
||||
/// to notice it.
|
||||
#[test]
|
||||
fn test_lightweight_partition_probes_snapshot_entries_but_replays_bucket_ops_only() {
|
||||
let now = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp");
|
||||
let mut state = SiteReplicationState::default();
|
||||
state
|
||||
.peers
|
||||
.insert("remote".to_string(), peer("remote", "https://remote.example.com"));
|
||||
|
||||
let bucket_make = "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning";
|
||||
let mut iam_unreachable = drain_event(
|
||||
"remote",
|
||||
SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH,
|
||||
3,
|
||||
Some(now - time::Duration::seconds(30)),
|
||||
);
|
||||
iam_unreachable.peer_unreachable = true;
|
||||
let mut bucket_unreachable = drain_event("remote", bucket_make, 3, Some(now - time::Duration::seconds(30)));
|
||||
bucket_unreachable.peer_unreachable = true;
|
||||
state.retry_queue = vec![
|
||||
iam_unreachable,
|
||||
bucket_unreachable,
|
||||
// Already promoted (or never stamped): due now.
|
||||
drain_event("remote", SITE_REPLICATION_RETRY_BUCKET_METADATA_SNAPSHOT_PATH, 1, None),
|
||||
drain_event("remote", bucket_make, 1, None),
|
||||
];
|
||||
|
||||
let (actionable, deferred) = lightweight_retry_drain_partition(&state, now);
|
||||
let deferred_paths: Vec<&str> = deferred.iter().map(|event| event.path.as_str()).collect();
|
||||
assert!(
|
||||
deferred_paths.contains(&SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH),
|
||||
"the backed-off IAM snapshot must be probed by the lightweight pass: {deferred_paths:?}"
|
||||
);
|
||||
assert!(deferred_paths.contains(&bucket_make));
|
||||
assert_eq!(
|
||||
actionable.iter().map(|event| event.path.as_str()).collect::<Vec<_>>(),
|
||||
vec![bucket_make],
|
||||
"only the bounded bucket op is replayed by the lightweight pass"
|
||||
);
|
||||
}
|
||||
|
||||
/// backlog#2367 A-1: the heavyweight tick evaluates backoff halfway to its
|
||||
/// next tick. A first failure stamped one second after a tick is 599 s old
|
||||
/// at the next tick; without the horizon it slipped to the tick after.
|
||||
#[test]
|
||||
fn test_heavyweight_horizon_absorbs_tick_phase() {
|
||||
let now = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp");
|
||||
let horizon = heavyweight_retry_drain_horizon(now);
|
||||
assert_eq!(horizon - now, time::Duration::seconds(300));
|
||||
|
||||
let elapsed_at_horizon = |secs_ago: i64| {
|
||||
site_replication_retry_backoff_elapsed(
|
||||
&drain_event("remote", "/p", 1, Some(now - time::Duration::seconds(secs_ago))),
|
||||
horizon,
|
||||
)
|
||||
};
|
||||
// Stamped just after the previous tick: due at this tick, not the next.
|
||||
assert!(elapsed_at_horizon(599));
|
||||
// Due before the next tick's midpoint: drained now rather than a whole
|
||||
// interval late.
|
||||
assert!(elapsed_at_horizon(301));
|
||||
// Due after the midpoint: waits for the next tick.
|
||||
assert!(!elapsed_at_horizon(299));
|
||||
}
|
||||
|
||||
/// The drain settles a peer-edit success under a freshly allocated
|
||||
/// generation; legacy queue entries carry `edit_generation: None` and
|
||||
/// must be cleared by that generation-scoped settlement (`(Some, None)`
|
||||
|
||||
@@ -32,7 +32,7 @@ use tokio::time::Instant;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::warn;
|
||||
|
||||
const RECONCILE_INTERVAL: Duration = Duration::from_secs(600);
|
||||
pub(crate) const RECONCILE_INTERVAL: Duration = Duration::from_secs(600);
|
||||
pub(crate) const RETRY_DRAIN_INTERVAL: Duration = Duration::from_secs(30);
|
||||
|
||||
/// A reconciler reports its own failures; the outcome carries no value because neither
|
||||
|
||||
@@ -62,10 +62,10 @@ pub(crate) async fn init_embedded_bucket_metadata_runtime(store: Arc<ECStore>, c
|
||||
|
||||
let buckets: Vec<String> = buckets_list.into_iter().map(|v| v.name).collect();
|
||||
|
||||
try_migrate_bucket_metadata(store.clone()).await;
|
||||
try_migrate_bucket_metadata(store.clone()).await?;
|
||||
init_on_demand_migration_runtime();
|
||||
init_bucket_metadata_sys(store.clone(), buckets.clone()).await;
|
||||
try_migrate_iam_config(store).await;
|
||||
try_migrate_iam_config(store).await?;
|
||||
spawn_bucket_resync_startup_reconcile(buckets.clone(), ctx.clone(), false);
|
||||
|
||||
Ok(buckets)
|
||||
@@ -82,9 +82,9 @@ pub(crate) async fn init_bucket_metadata_runtime(store: Arc<ECStore>, ctx: Cance
|
||||
|
||||
let buckets: Vec<String> = buckets_list.into_iter().map(|v| v.name).collect();
|
||||
|
||||
try_migrate_bucket_metadata(store.clone()).await;
|
||||
try_migrate_bucket_metadata(store.clone()).await?;
|
||||
|
||||
try_migrate_iam_config(store.clone()).await;
|
||||
try_migrate_iam_config(store.clone()).await?;
|
||||
init_on_demand_migration_runtime();
|
||||
init_bucket_metadata_sys(store, buckets.clone()).await;
|
||||
spawn_bucket_resync_startup_reconcile(buckets.clone(), ctx, true);
|
||||
|
||||
@@ -81,6 +81,7 @@ pub(crate) async fn init_startup_listen_context(
|
||||
config: &Config,
|
||||
instance_ctx: &Arc<InstanceContext>,
|
||||
) -> Result<StartupListenContext> {
|
||||
crate::server::init_console_prefix()?;
|
||||
log_sanitized_server_config(config);
|
||||
let readiness = Arc::new(GlobalReadiness::new());
|
||||
|
||||
|
||||
@@ -377,10 +377,6 @@ impl FS {
|
||||
|
||||
pub(crate) fn parse_object_version_id(version_id: Option<String>) -> S3Result<Option<Uuid>> {
|
||||
if let Some(vid) = version_id {
|
||||
if vid == "null" {
|
||||
// A nil UUID selects the stored null version; None selects latest.
|
||||
return Ok(Some(Uuid::nil()));
|
||||
}
|
||||
let uuid = Uuid::parse_str(&vid).map_err(|e| {
|
||||
error!("Invalid version ID: {}", e);
|
||||
s3_error!(InvalidArgument, "Invalid version ID")
|
||||
@@ -1187,11 +1183,7 @@ impl S3 for FS {
|
||||
error = %e,
|
||||
"Object tags not found"
|
||||
);
|
||||
return Err(S3Error::new(if opts.version_id.is_some() {
|
||||
S3ErrorCode::NoSuchVersion
|
||||
} else {
|
||||
S3ErrorCode::NoSuchKey
|
||||
}));
|
||||
return Err(s3_error!(NoSuchKey));
|
||||
}
|
||||
error!(
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
|
||||
@@ -17,9 +17,7 @@ mod tests {
|
||||
use crate::config::WorkloadProfile;
|
||||
use crate::server::cors;
|
||||
use crate::storage::StorageError;
|
||||
use crate::storage::ecfs::{
|
||||
FS, parse_object_version_id, propagate_object_lock_peer_reload, validate_object_lock_configuration_input,
|
||||
};
|
||||
use crate::storage::ecfs::{FS, propagate_object_lock_peer_reload, validate_object_lock_configuration_input};
|
||||
use crate::storage::ecfs_extend::{apply_bucket_default_lock_retention, map_bucket_object_lock_config_state};
|
||||
use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner};
|
||||
use crate::storage::storage_api::ecstore_bucket::metadata_sys::ObjectLockConfigState;
|
||||
@@ -643,36 +641,6 @@ mod tests {
|
||||
assert_eq!(metadata.get("content-type"), Some(&"application/octet-stream".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tagging_version_id_preserves_explicit_null_and_latest_selection() {
|
||||
assert_eq!(parse_object_version_id(None).expect("latest version selector"), None);
|
||||
assert_eq!(
|
||||
parse_object_version_id(Some("null".to_owned())).expect("explicit null version selector"),
|
||||
Some(uuid::Uuid::nil())
|
||||
);
|
||||
for version in [uuid::Uuid::nil(), uuid::Uuid::new_v4()] {
|
||||
assert_eq!(
|
||||
parse_object_version_id(Some(version.to_string())).expect("UUID version selector"),
|
||||
Some(version)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tagging_version_id_rejects_invalid_values_instead_of_selecting_latest() {
|
||||
for version in [
|
||||
"",
|
||||
"NULL",
|
||||
" null ",
|
||||
"not-a-version",
|
||||
"null/other",
|
||||
"00000000-0000-0000-0000-00000000000g",
|
||||
] {
|
||||
let err = parse_object_version_id(Some(version.to_owned())).expect_err("invalid version must fail closed");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument, "version: {version:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_object_tagging_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
|
||||
@@ -1185,17 +1185,21 @@ pub(crate) fn get_global_transition_state() -> Arc<TransitionState> {
|
||||
ecstore_bucket::lifecycle::bucket_lifecycle_ops::get_global_transition_state()
|
||||
}
|
||||
|
||||
pub(crate) async fn try_migrate_bucket_metadata(store: Arc<ECStore>) {
|
||||
ecstore_bucket::migration::try_migrate_bucket_metadata(store).await;
|
||||
pub(crate) async fn try_migrate_bucket_metadata(store: Arc<ECStore>) -> std::io::Result<()> {
|
||||
ecstore_bucket::migration::try_migrate_bucket_metadata(store)
|
||||
.await
|
||||
.map_err(ecstore_bucket::migration::migration_startup_error)
|
||||
}
|
||||
|
||||
pub(crate) async fn try_migrate_iam_config(store: Arc<ECStore>) {
|
||||
pub(crate) async fn try_migrate_iam_config(store: Arc<ECStore>) -> std::io::Result<()> {
|
||||
// MinIO encrypts IAM identity/service-account files at rest with a key derived
|
||||
// from the root credentials. Inject the IAM crate's decryption so those blobs
|
||||
// are decrypted before normalization instead of being skipped as "incompatible".
|
||||
let decrypt_fn: ecstore_bucket::migration::LegacyBlobDecryptFn =
|
||||
Arc::new(|data: &[u8]| rustfs_iam::try_decrypt_iam_blob(data));
|
||||
ecstore_bucket::migration::try_migrate_iam_config(store, Some(decrypt_fn)).await;
|
||||
ecstore_bucket::migration::try_migrate_iam_config(store, Some(decrypt_fn))
|
||||
.await
|
||||
.map_err(ecstore_bucket::migration::migration_startup_error)
|
||||
}
|
||||
|
||||
pub(crate) fn init_ecstore_config() {
|
||||
|
||||
@@ -0,0 +1,326 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#![recursion_limit = "256"]
|
||||
|
||||
use reqwest::StatusCode;
|
||||
use rustfs::embedded::{RustFSServerBuilder, find_available_port};
|
||||
use rustfs_ecstore::api::config::com::{delete_config, read_config};
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Stdio;
|
||||
use std::time::Duration;
|
||||
use tokio::process::Command;
|
||||
|
||||
mod common;
|
||||
|
||||
const TEST_NAME: &str = "native_migration_failure_blocks_server_startup_and_repair_preserves_records";
|
||||
const STAGE_ENV: &str = "RUSTFS_NATIVE_MIGRATION_TEST_STAGE";
|
||||
const ROOT_ENV: &str = "RUSTFS_NATIVE_MIGRATION_TEST_ROOT";
|
||||
const ADDRESS_ENV: &str = "RUSTFS_NATIVE_MIGRATION_TEST_ADDRESS";
|
||||
const FAILURE_ENV: &str = "RUSTFS_NATIVE_MIGRATION_TEST_FAILURE";
|
||||
const STOP_ENV: &str = "RUSTFS_NATIVE_MIGRATION_TEST_STOP";
|
||||
const ACCESS_KEY: &str = "native-migration-root";
|
||||
const SECRET_KEY: &str = "native-migration-root-secret";
|
||||
const LEGACY_BUCKET: &str = ".minio.sys";
|
||||
const TARGET_BUCKET: &str = ".rustfs.sys";
|
||||
const BUCKET_METADATA: &str = "buckets/interop/.metadata.bin";
|
||||
const IAM_RECORD: &str = "config/iam/groups/migration-group/members.json";
|
||||
const IAM_FORMAT: &str = "config/iam/format.json";
|
||||
const EXISTING_FORMAT: &[u8] = br#"{"version":1}"#;
|
||||
const STARTUP_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
enum StartupMode {
|
||||
Server,
|
||||
Embedded,
|
||||
}
|
||||
|
||||
fn volumes(root: &Path) -> Vec<PathBuf> {
|
||||
(1..=4).map(|index| root.join(format!("disk{index}"))).collect()
|
||||
}
|
||||
|
||||
fn minio_bucket_metadata() -> Vec<u8> {
|
||||
let hex: String = include_str!("../../crates/ecstore/tests/fixtures/minio/bucket_metadata.blob.hex")
|
||||
.chars()
|
||||
.filter(|ch| !ch.is_whitespace())
|
||||
.collect();
|
||||
hex.as_bytes()
|
||||
.as_chunks::<2>()
|
||||
.0
|
||||
.iter()
|
||||
.map(|pair| {
|
||||
u8::from_str_radix(std::str::from_utf8(pair).expect("fixture hex is UTF-8"), 16).expect("valid MinIO fixture hex")
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn prepare_or_verify_fixture(root: &Path, seed: bool) {
|
||||
let env = rustfs_test_utils::TestECStoreEnv::builder()
|
||||
.base_dir(root)
|
||||
.disk_count(4)
|
||||
.build()
|
||||
.await;
|
||||
if seed {
|
||||
env.make_bucket("interop", false).await;
|
||||
env.make_bucket(LEGACY_BUCKET, false).await;
|
||||
env.put_object_bytes(LEGACY_BUCKET, BUCKET_METADATA, minio_bucket_metadata())
|
||||
.await;
|
||||
env.put_object_bytes(
|
||||
LEGACY_BUCKET,
|
||||
IAM_RECORD,
|
||||
br#"{"version":1,"status":"enabled","members":[],"updatedAt":"2026-09-10T00:00:00Z"}"#.to_vec(),
|
||||
)
|
||||
.await;
|
||||
env.put_object_bytes(TARGET_BUCKET, IAM_FORMAT, EXISTING_FORMAT.to_vec())
|
||||
.await;
|
||||
// A completed record must be skipped before reading even a broken old copy.
|
||||
env.put_object_bytes(LEGACY_BUCKET, IAM_FORMAT, b"do not overwrite the existing target".to_vec())
|
||||
.await;
|
||||
delete_config(env.ecstore.clone(), BUCKET_METADATA)
|
||||
.await
|
||||
.expect("leave bucket metadata pending migration");
|
||||
} else {
|
||||
assert_eq!(
|
||||
read_config(env.ecstore.clone(), BUCKET_METADATA)
|
||||
.await
|
||||
.expect("migrated bucket metadata"),
|
||||
minio_bucket_metadata(),
|
||||
"migration must preserve the MinIO bucket settings"
|
||||
);
|
||||
let group: serde_json::Value = serde_json::from_slice(
|
||||
&read_config(env.ecstore.clone(), IAM_RECORD)
|
||||
.await
|
||||
.expect("migrated IAM group"),
|
||||
)
|
||||
.expect("valid migrated IAM JSON");
|
||||
assert_eq!(group["status"], "enabled");
|
||||
assert_eq!(group["members"], serde_json::json!([]));
|
||||
}
|
||||
assert_eq!(
|
||||
read_config(env.ecstore.clone(), IAM_FORMAT)
|
||||
.await
|
||||
.expect("existing IAM format"),
|
||||
EXISTING_FORMAT,
|
||||
"retry must not overwrite records already migrated"
|
||||
);
|
||||
}
|
||||
|
||||
async fn run_embedded_child(root: &Path) {
|
||||
let address = std::env::var(ADDRESS_ENV).expect("embedded child address");
|
||||
let result = RustFSServerBuilder::new()
|
||||
.address(address)
|
||||
.access_key(ACCESS_KEY)
|
||||
.secret_key(SECRET_KEY)
|
||||
.volumes(volumes(root).iter().map(|path| path.to_string_lossy().into_owned()).collect())
|
||||
.build()
|
||||
.await;
|
||||
match result {
|
||||
Ok(server) => {
|
||||
let stop = PathBuf::from(std::env::var_os(STOP_ENV).expect("embedded stop path"));
|
||||
while !stop.exists() {
|
||||
tokio::time::sleep(Duration::from_millis(25)).await;
|
||||
}
|
||||
server.shutdown().await;
|
||||
}
|
||||
Err(error) => {
|
||||
fs::write(std::env::var_os(FAILURE_ENV).expect("embedded failure path"), error.to_string())
|
||||
.expect("record the actual embedded startup error");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn child_command(root: &Path, stage: &str, log: &Path) -> Command {
|
||||
let mut command = Command::new(std::env::current_exe().expect("integration test executable"));
|
||||
command
|
||||
.args(["--exact", TEST_NAME, "--nocapture"])
|
||||
.env(STAGE_ENV, stage)
|
||||
.env(ROOT_ENV, root);
|
||||
configure_process(&mut command, log);
|
||||
command
|
||||
}
|
||||
|
||||
fn configure_process(command: &mut Command, log: &Path) {
|
||||
let output = fs::File::create(log).expect("create isolated process log");
|
||||
command
|
||||
// These disposable erasure volumes intentionally share the test runner's disk.
|
||||
.env("RUSTFS_UNSAFE_BYPASS_DISK_CHECK", "true")
|
||||
.env("RUSTFS_CONSOLE_ENABLE", "false")
|
||||
.env("NO_PROXY", "localhost,127.0.0.1,::1")
|
||||
.env("no_proxy", "localhost,127.0.0.1,::1")
|
||||
.env("RUST_LOG", "warn")
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::from(output.try_clone().expect("clone process log")))
|
||||
.stderr(Stdio::from(output))
|
||||
.kill_on_drop(true);
|
||||
}
|
||||
|
||||
async fn fixture_process(root: &Path, stage: &str) {
|
||||
let log = root.join(format!("{stage}.log"));
|
||||
let status = tokio::time::timeout(STARTUP_TIMEOUT, child_command(root, stage, &log).status())
|
||||
.await
|
||||
.expect("fixture process must finish")
|
||||
.expect("run fixture process");
|
||||
assert!(status.success(), "{stage} failed: {}", fs::read_to_string(log).expect("fixture log"));
|
||||
}
|
||||
|
||||
async fn check_startup(root: &Path, mode: StartupMode, failure_record: Option<&str>, label: &str) {
|
||||
let ready = failure_record.is_none();
|
||||
let address = format!("127.0.0.1:{}", find_available_port().expect("free startup probe port"));
|
||||
let log = root.join(format!("{label}.log"));
|
||||
let failure = root.join(format!("{label}.failure"));
|
||||
let stop = root.join(format!("{label}.stop"));
|
||||
let mut command = match mode {
|
||||
StartupMode::Server => {
|
||||
let mut command = Command::new(env!("CARGO_BIN_EXE_rustfs"));
|
||||
command
|
||||
.args(["--address", &address, "--access-key", ACCESS_KEY, "--secret-key", SECRET_KEY])
|
||||
.args(volumes(root));
|
||||
configure_process(&mut command, &log);
|
||||
command
|
||||
}
|
||||
StartupMode::Embedded => {
|
||||
let mut command = child_command(root, "embedded", &log);
|
||||
command
|
||||
.env(ADDRESS_ENV, &address)
|
||||
.env(FAILURE_ENV, &failure)
|
||||
.env(STOP_ENV, &stop);
|
||||
command
|
||||
}
|
||||
};
|
||||
let mut child = command.spawn().expect("start isolated server process");
|
||||
let http = reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.timeout(Duration::from_millis(500))
|
||||
.build()
|
||||
.expect("local readiness client");
|
||||
let result = tokio::time::timeout(STARTUP_TIMEOUT, async {
|
||||
loop {
|
||||
if let Ok(response) = http.get(format!("http://{address}/health/ready")).send().await
|
||||
&& response.status() == StatusCode::OK
|
||||
{
|
||||
assert!(ready, "{mode:?} published Ready after a migration I/O failure");
|
||||
return;
|
||||
}
|
||||
if let Some(status) = child.try_wait().expect("poll server process") {
|
||||
let details = fs::read_to_string(&log).expect("startup log");
|
||||
assert!(!ready, "{mode:?} exited before Ready ({status}): {details}");
|
||||
let record = failure_record.expect("failed startup has an obstructed record");
|
||||
match mode {
|
||||
StartupMode::Server => {
|
||||
assert_eq!(status.code(), Some(1), "startup must fail: {details}");
|
||||
assert_migration_io_error(&details, record);
|
||||
}
|
||||
StartupMode::Embedded => {
|
||||
assert!(status.success(), "embedded test process failed unexpectedly: {details}");
|
||||
let error = fs::read_to_string(&failure).expect("embedded startup returned an error");
|
||||
assert_migration_io_error(&error, record);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(25)).await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
result.is_ok(),
|
||||
"{mode:?} did not reach the expected startup outcome: {}",
|
||||
fs::read_to_string(&log).expect("startup diagnostics")
|
||||
);
|
||||
if ready {
|
||||
match mode {
|
||||
StartupMode::Embedded => {
|
||||
fs::write(stop, b"stop").expect("request embedded shutdown");
|
||||
assert!(
|
||||
tokio::time::timeout(STARTUP_TIMEOUT, child.wait())
|
||||
.await
|
||||
.expect("embedded shutdown completes")
|
||||
.expect("wait for embedded shutdown")
|
||||
.success()
|
||||
);
|
||||
}
|
||||
StartupMode::Server => child.kill().await.expect("stop the isolated server"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_migration_io_error(error: &str, record: &str) {
|
||||
let lower = error.to_ascii_lowercase();
|
||||
assert!(
|
||||
(lower.contains("access denied")
|
||||
|| lower.contains("access is denied")
|
||||
|| lower.contains("not a directory")
|
||||
|| lower.contains("not regular"))
|
||||
&& error.contains(&format!("{TARGET_BUCKET}/{record}")),
|
||||
"startup must fail because of the obstructed metadata record, not an unrelated initialization error: {error}"
|
||||
);
|
||||
}
|
||||
|
||||
async fn run_startup_cases(mode: StartupMode) {
|
||||
let ordinary = tempfile::TempDir::with_prefix("rustfs-no-legacy-").expect("ordinary store");
|
||||
for volume in volumes(ordinary.path()) {
|
||||
fs::create_dir_all(volume).expect("ordinary volume");
|
||||
}
|
||||
check_startup(ordinary.path(), mode, None, "ordinary").await;
|
||||
|
||||
let control = tempfile::TempDir::with_prefix("rustfs-migration-control-").expect("control fixture");
|
||||
fixture_process(control.path(), "seed").await;
|
||||
check_startup(control.path(), mode, None, "control").await;
|
||||
fixture_process(control.path(), "verify").await;
|
||||
|
||||
for record in [BUCKET_METADATA, IAM_RECORD] {
|
||||
let target = tempfile::TempDir::with_prefix("rustfs-migration-failure-").expect("disposable migration target");
|
||||
fixture_process(target.path(), "seed").await;
|
||||
let blockers: Vec<_> = volumes(target.path())
|
||||
.iter()
|
||||
.map(|volume| volume.join(TARGET_BUCKET).join(record))
|
||||
.collect();
|
||||
for blocker in &blockers {
|
||||
fs::create_dir_all(blocker.parent().expect("record parent")).expect("create target parent");
|
||||
assert!(!blocker.exists(), "the record must still need migration");
|
||||
// A non-directory target causes real filesystem I/O errors even when tests run as root.
|
||||
fs::write(blocker, b"blocked migration target").expect("block only the destination record");
|
||||
}
|
||||
check_startup(target.path(), mode, Some(record), "blocked").await;
|
||||
for blocker in blockers {
|
||||
fs::remove_file(blocker).expect("repair the same partially migrated target");
|
||||
}
|
||||
check_startup(target.path(), mode, None, "repaired").await;
|
||||
fixture_process(target.path(), "verify").await;
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_migration_failure_blocks_server_startup_and_repair_preserves_records() {
|
||||
// Cold processes keep failed initialization and cached metadata out of subsequent restart attempts.
|
||||
common::run_embedded_test(|| async {
|
||||
match std::env::var(STAGE_ENV).ok().as_deref() {
|
||||
Some("seed") => {
|
||||
prepare_or_verify_fixture(&PathBuf::from(std::env::var_os(ROOT_ENV).expect("fixture root")), true).await
|
||||
}
|
||||
Some("verify") => {
|
||||
prepare_or_verify_fixture(&PathBuf::from(std::env::var_os(ROOT_ENV).expect("fixture root")), false).await
|
||||
}
|
||||
Some("embedded") => run_embedded_child(&PathBuf::from(std::env::var_os(ROOT_ENV).expect("fixture root"))).await,
|
||||
None => run_startup_cases(StartupMode::Server).await,
|
||||
Some(stage) => panic!("unknown native migration test stage: {stage}"),
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_migration_failure_blocks_embedded_startup_and_repair_preserves_records() {
|
||||
common::run_embedded_test(|| run_startup_cases(StartupMode::Embedded));
|
||||
}
|
||||
@@ -58,10 +58,8 @@ cd "$(dirname "$0")/.."
|
||||
# 1589 -> 1588 on 2026-09-08: the GA blocker set (rustfs/backlog#2366) added
|
||||
# three invocation lines to the endpoint-refresh paths and folded the five
|
||||
# copies of the concurrent-change error into one constructor, netting -1.
|
||||
# 1588 -> 1586 on 2026-09-10: the release merge no longer introduces direct
|
||||
# s3_error! constructors for heal percent-decoding or tagging not-found errors.
|
||||
S3S_IMPORT_FILES_BASELINE=213
|
||||
S3_ERROR_LINES_BASELINE=1586
|
||||
S3_ERROR_LINES_BASELINE=1588
|
||||
# ecstore-scoped ratchet (rustfs/backlog#1842): the storage engine must not
|
||||
# know S3 wire/DTO types (ARCHITECTURE.md invariant 4). The S3-*consuming*
|
||||
# client was extracted to crates/s3-client, where s3s usage is legitimate;
|
||||
|
||||
@@ -15,6 +15,7 @@ ROOT = Path(__file__).resolve().parent.parent
|
||||
ALWAYS_JOBS = ("classify-changes", "typos", "quick-checks")
|
||||
CODE_JOBS = (
|
||||
"test-and-lint", "test-ilm-integration-serial", "test-and-lint-rio-v2",
|
||||
"offline-enrollment-root-boundary",
|
||||
"connect-short-credential-boundary", "test-and-lint-protocols",
|
||||
"build-rustfs-debug-binary", "uring-integration", "e2e-tests",
|
||||
"s3-implemented-tests", "s3-lifecycle-behavior-tests",
|
||||
|
||||
@@ -83,12 +83,9 @@ runtime_profile_for() {
|
||||
background-target-crash|background-target-restart)
|
||||
echo "background-4x1"
|
||||
;;
|
||||
background-target-crash-ec8-4|background-target-restart-ec8-4)
|
||||
background-target-crash-ec8-4|background-target-restart-ec8-4|background-target-restart-ec8-4-multi-set)
|
||||
echo "background-ec8-4"
|
||||
;;
|
||||
background-target-restart-ec8-4-multi-set)
|
||||
echo "background-ec8-4-multi-set"
|
||||
;;
|
||||
background-target-crash-ec8-4-multi-pool)
|
||||
echo "background-ec8-4-multi-pool"
|
||||
;;
|
||||
@@ -114,11 +111,6 @@ apply_runtime_profile() {
|
||||
export RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES="${RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES:-8388608}"
|
||||
export RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS="${RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS:-180}"
|
||||
;;
|
||||
background-ec8-4-multi-set)
|
||||
export RUSTFS_HEAL_CHAOS_OBJECT_COUNT="${RUSTFS_HEAL_CHAOS_OBJECT_COUNT:-64}"
|
||||
export RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES="${RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES:-16777216}"
|
||||
export RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS="${RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS:-240}"
|
||||
;;
|
||||
background-ec8-4-multi-pool)
|
||||
export RUSTFS_HEAL_CHAOS_OBJECT_COUNT="${RUSTFS_HEAL_CHAOS_OBJECT_COUNT:-64}"
|
||||
export RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES="${RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES:-4194304}"
|
||||
@@ -173,40 +165,6 @@ if not status.get("pending_gates"):
|
||||
PY
|
||||
}
|
||||
|
||||
nextest_junit_candidates() {
|
||||
local target_dir="$1"
|
||||
local profile="$2"
|
||||
local primary="$target_dir/nextest/$profile/junit.xml"
|
||||
local fallback="$ROOT/target/nextest/$profile/junit.xml"
|
||||
printf '%s\n' "$primary"
|
||||
if [[ "$fallback" != "$primary" ]]; then
|
||||
printf '%s\n' "$fallback"
|
||||
fi
|
||||
}
|
||||
|
||||
remove_nextest_junit_candidates() {
|
||||
local target_dir="$1"
|
||||
local profile="$2"
|
||||
local candidate
|
||||
while IFS= read -r candidate; do
|
||||
rm -f "$candidate"
|
||||
done < <(nextest_junit_candidates "$target_dir" "$profile")
|
||||
}
|
||||
|
||||
copy_nextest_junit() {
|
||||
local target_dir="$1"
|
||||
local profile="$2"
|
||||
local run_dir="$3"
|
||||
local candidate
|
||||
while IFS= read -r candidate; do
|
||||
if [[ -f "$candidate" ]]; then
|
||||
cp "$candidate" "$run_dir/junit.xml"
|
||||
return 0
|
||||
fi
|
||||
done < <(nextest_junit_candidates "$target_dir" "$profile")
|
||||
return 1
|
||||
}
|
||||
|
||||
run_self_test() {
|
||||
if "$0" --case release --plan-only >/dev/null 2>&1; then
|
||||
echo "self-test failed: release pseudo-case must not be runnable" >&2
|
||||
@@ -228,24 +186,6 @@ run_self_test() {
|
||||
return 1
|
||||
fi
|
||||
done < <(case_ids)
|
||||
|
||||
local junit_profile="scanner-heal-junit-self-test-$$"
|
||||
local junit_run_dir="$ROOT/target/scanner-heal-junit-self-test-$$"
|
||||
local junit_target_dir="$ROOT/target/scanner-heal-junit-custom-target-$$"
|
||||
local junit_fallback="$ROOT/target/nextest/$junit_profile/junit.xml"
|
||||
mkdir -p "$(dirname "$junit_fallback")" "$junit_run_dir"
|
||||
printf '<testsuites />\n' >"$junit_fallback"
|
||||
if ! copy_nextest_junit "$junit_target_dir" "$junit_profile" "$junit_run_dir"; then
|
||||
echo "self-test failed: nextest junit fallback was not copied" >&2
|
||||
rm -rf "$junit_run_dir" "$junit_target_dir" "$(dirname "$junit_fallback")"
|
||||
return 1
|
||||
fi
|
||||
if ! cmp -s "$junit_fallback" "$junit_run_dir/junit.xml"; then
|
||||
echo "self-test failed: copied nextest junit fallback changed content" >&2
|
||||
rm -rf "$junit_run_dir" "$junit_target_dir" "$(dirname "$junit_fallback")"
|
||||
return 1
|
||||
fi
|
||||
rm -rf "$junit_run_dir" "$junit_target_dir" "$(dirname "$junit_fallback")"
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
@@ -327,25 +267,9 @@ if [[ "$NOFILE_SOFT" =~ ^[0-9]+$ && "$NOFILE_HARD" =~ ^[0-9]+$ && "$NOFILE_SOFT"
|
||||
ulimit -n "$NOFILE_HARD" || true
|
||||
fi
|
||||
fi
|
||||
RUN_PARENT="$(dirname "$RUN_DIR")"
|
||||
mkdir -p "$RUN_PARENT"
|
||||
RUN_TMP_ROOT_CREATED=0
|
||||
if [[ -z "${TMPDIR:-}" ]]; then
|
||||
RUN_TMP_ROOT="$RUN_PARENT/.tmp-$(basename "$RUN_DIR")"
|
||||
mkdir -p "$RUN_TMP_ROOT"
|
||||
export TMPDIR="$RUN_TMP_ROOT"
|
||||
RUN_TMP_ROOT_CREATED=1
|
||||
else
|
||||
RUN_TMP_ROOT=""
|
||||
fi
|
||||
TMP_DIR="$(mktemp -d "${TMPDIR%/}/rustfs-scanner-heal-evidence.XXXXXX")"
|
||||
cleanup_tmp() {
|
||||
rm -rf "$TMP_DIR"
|
||||
if [[ "$RUN_TMP_ROOT_CREATED" == 1 ]]; then
|
||||
rm -rf "$RUN_TMP_ROOT"
|
||||
fi
|
||||
}
|
||||
trap cleanup_tmp EXIT
|
||||
mkdir -p "$(dirname "$RUN_DIR")"
|
||||
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/rustfs-scanner-heal-evidence.XXXXXX")"
|
||||
trap 'rm -rf "$TMP_DIR"' EXIT
|
||||
|
||||
BUILD_FEATURES="${RUSTFS_BUILD_FEATURES:-}"
|
||||
TARGET_DIR="${CARGO_TARGET_DIR:-$ROOT/target}"
|
||||
@@ -376,7 +300,8 @@ export RUSTFS_E2E_LOG_DIR="${RUSTFS_E2E_LOG_DIR:-$RUN_DIR/e2e-logs}"
|
||||
export RUSTFS_HEAL_CHAOS_LOG_DIR="${RUSTFS_HEAL_CHAOS_LOG_DIR:-$RUSTFS_E2E_LOG_DIR}"
|
||||
mkdir -p "$RUSTFS_E2E_LOG_DIR"
|
||||
|
||||
remove_nextest_junit_candidates "$TARGET_DIR" "$PROFILE"
|
||||
JUNIT_PATH="$TARGET_DIR/nextest/$PROFILE/junit.xml"
|
||||
rm -f "$JUNIT_PATH"
|
||||
set +e
|
||||
NO_PROXY="${NO_PROXY:-127.0.0.1,localhost}" \
|
||||
HTTP_PROXY= \
|
||||
@@ -386,7 +311,9 @@ cargo nextest run --profile "$PROFILE" -p e2e_test -E "$TEST_FILTER" --no-tests=
|
||||
STATUS=$?
|
||||
set -e
|
||||
|
||||
copy_nextest_junit "$TARGET_DIR" "$PROFILE" "$RUN_DIR" || true
|
||||
if [[ -f "$JUNIT_PATH" ]]; then
|
||||
cp "$JUNIT_PATH" "$RUN_DIR/junit.xml"
|
||||
fi
|
||||
"$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --finish-scanner-heal "$RUN_DIR" "$STATUS"
|
||||
|
||||
if [[ "$STATUS" -ne 0 ]]; then
|
||||
|
||||
Reference in New Issue
Block a user