mirror of
https://github.com/rustfs/rustfs.git
synced 2026-10-28 06:47:21 +00:00
Compare commits
282 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| d47f54bfb2 | |||
| 9cc9c48677 | |||
| 40b2aadb93 | |||
| 3d3debf938 | |||
| a708820cda | |||
| 5fdc16fec7 | |||
| a9596debb9 | |||
| 5249a90540 | |||
| 8c5ce4c9bc | |||
| 08a6f3ad42 | |||
| 2bdec2fdcd | |||
| c5a327df45 | |||
| 3774b75b93 | |||
| 3b7d11037d | |||
| e1e3cf4620 | |||
| 3999544ece | |||
| 6a2dc68130 | |||
| 230f31beff | |||
| 646c7c087f | |||
| 756968d735 | |||
| 9e84ede33f | |||
| 343cd86551 | |||
| 2822cb408a | |||
| 63fec31f72 | |||
| 14a464a313 | |||
| a7deb27bcf | |||
| 3dc4d9e40f | |||
| 1c06aa29a7 | |||
| b1b79d3c54 | |||
| 002b000055 | |||
| 12261efd1d | |||
| e97dff1201 | |||
| 1c47ca22eb | |||
| c9270c48a2 | |||
| 18547b1bb8 | |||
| 1eb14a8874 | |||
| 3e69654f91 | |||
| de60f6ad6e | |||
| 397ee852da | |||
| 6fc748d6de | |||
| de0a7f1090 | |||
| 7192eecbfb | |||
| 201e314dd9 | |||
| 28db20f7c4 | |||
| c020372948 | |||
| e40ed3c9e3 | |||
| 9fbe8079af | |||
| 89fd23fcae | |||
| 8c3b5afbea | |||
| 31d64c8ef2 | |||
| 203a9e25ed | |||
| 34d0b1b5bb | |||
| 7abb0f5f1d | |||
| 55f7514f90 | |||
| 703b1a5c9c | |||
| b61dc6519f | |||
| 95e90dbbc9 | |||
| e4031940e2 | |||
| 73552e40bc | |||
| cb3efc4a98 | |||
| 5dd92bd783 | |||
| 4e16705873 | |||
| 8982b4a3d2 | |||
| 568e8cbe0c | |||
| ed7baf534b | |||
| 0900d78877 | |||
| 8f6d06db21 | |||
| 3e156ee61e | |||
| 3ca3e26cec | |||
| 13093c5afc | |||
| 41770983d6 | |||
| 244e7dfb99 | |||
| 17ddecb075 | |||
| d0c7aec0b4 | |||
| 39ccd3abb0 | |||
| f21b06dfd2 | |||
| 71c002d467 | |||
| 9b835bcca5 | |||
| 7a7b080265 | |||
| cfe1d4d456 | |||
| ae8d079658 | |||
| 2cfefcfd65 | |||
| 7776004977 | |||
| 19c7ed68be | |||
| 455c36a3fd | |||
| 684900fbb9 | |||
| 023a674cd4 | |||
| adfde0110b | |||
| bcf5184a85 | |||
| f90d4071a5 | |||
| e3ecbf5db9 | |||
| 59f1c68759 | |||
| 4f0e050b39 | |||
| 741cd0ec3b | |||
| 824231ff1a | |||
| 66026d7d7b | |||
| 82a4083981 | |||
| 9f5ff23fd8 | |||
| 5fc92cdd27 | |||
| 7bd09a00b0 | |||
| 729cff5f32 | |||
| c478a392e7 | |||
| c8ccc1e198 | |||
| 74ba5c205c | |||
| 9fcd54734b | |||
| 6eb200022e | |||
| 92ce782999 | |||
| bf425ca32c | |||
| 0ae38aabbc | |||
| 77ba4f1b2e | |||
| 7b6e45d372 | |||
| 09c85a5f29 | |||
| 00aeb12914 | |||
| ffb18979f8 | |||
| 97c7b451d2 | |||
| 9ba95cac37 | |||
| cec796328c | |||
| 1aa4fa145f | |||
| d286f3d06c | |||
| 3391528025 | |||
| 0d1490da3c | |||
| 6421a7ff60 | |||
| 27141f6312 | |||
| 940e221988 | |||
| 8b66e9b62a | |||
| 6a879be1fb | |||
| 0b40d47a8a | |||
| e292637ee0 | |||
| 8b6b1e53a3 | |||
| 358bf6832d | |||
| 70fb8bf504 | |||
| 110f630a5c | |||
| 54bd3c8e24 | |||
| 1643cb29f1 | |||
| bf8d5a32b6 | |||
| 230eeb5fb5 | |||
| 2c6f5f22c0 | |||
| d7b6a8c10d | |||
| e8ffd72575 | |||
| 0b03d535c3 | |||
| 3afb389247 | |||
| 86b6569071 | |||
| 0940fbe1b2 | |||
| 9e7c5dc932 | |||
| 45806bf295 | |||
| 42f00d8532 | |||
| 3e46d61a91 | |||
| 88183daf10 | |||
| 8be1e9b2c1 | |||
| 9b4b366209 | |||
| 43436ad5a7 | |||
| 0c7b6188b2 | |||
| 3ffc3704af | |||
| de3ac27a6a | |||
| 2d0e82e9f7 | |||
| c92fabea3a | |||
| b3b0d892bb | |||
| d9ae1654b6 | |||
| 15949180c9 | |||
| f65305e97f | |||
| 28ac3ab4e7 | |||
| cf41b1d3cf | |||
| ca876930b3 | |||
| c05b3ee01e | |||
| 1076e2fb4f | |||
| 51e6733a9c | |||
| ed2b2cdd19 | |||
| 084477e079 | |||
| bb1b5dea16 | |||
| 1c87413788 | |||
| e1fee0569a | |||
| a6e5bdcbd2 | |||
| 923f51cf12 | |||
| e16274ab14 | |||
| 5eecd657e7 | |||
| 929d906131 | |||
| 6b802a91eb | |||
| 14b3cffe8f | |||
| c62384e58d | |||
| 3f30f6c139 | |||
| 584a52ce3d | |||
| 76e025a50c | |||
| b40eb193df | |||
| 023c987397 | |||
| a4b265bf77 | |||
| c176220a34 | |||
| 82ac2dff05 | |||
| e24eae9eaa | |||
| 8617f2701b | |||
| 8d5592f83e | |||
| b5c9229e77 | |||
| a974e50b1d | |||
| 081910e825 | |||
| 7f7e4fe40b | |||
| 684e6f313a | |||
| b3a02f305e | |||
| 0ea9c19569 | |||
| ae662da256 | |||
| b5d33a1f4e | |||
| d288496752 | |||
| 40baedba4f | |||
| 0c3fd48c22 | |||
| 3e8b0e7c83 | |||
| dd2af9ff88 | |||
| 5db0e14d04 | |||
| 9429225cf4 | |||
| 857584b3c2 | |||
| cfda9b451c | |||
| 354e49de4c | |||
| 8e987ce0a6 | |||
| 929f836e40 | |||
| c7dec044eb | |||
| 14a59f7770 | |||
| 084e9c4b82 | |||
| 921cc7ad94 | |||
| bfa8df00e7 | |||
| 33a8469e27 | |||
| 128080ceb9 | |||
| ae9fe62fb1 | |||
| 929a9f27c4 | |||
| 15376a7fa6 | |||
| 5d789006dd | |||
| f4bd53a63c | |||
| 21fee7f7ff | |||
| 23ed00fc85 | |||
| f49ffa2fec | |||
| ee5f287324 | |||
| 158c613d0e | |||
| a950f914ca | |||
| 1de9b40a30 | |||
| 831468b2e9 | |||
| 600d037b25 | |||
| 3fa2b334be | |||
| 1b549d5907 | |||
| 11c4ce96eb | |||
| 4508a0985d | |||
| ed1b9f25d6 | |||
| 274c2bf402 | |||
| 9aebcefa9c | |||
| 1748814bbf | |||
| ee5f76c180 | |||
| 4d68c32b75 | |||
| 8c6689ff13 | |||
| 595f9f662d | |||
| 4a2b15cb82 | |||
| 2ba7f95547 | |||
| 7b3dad6bae | |||
| c0754f5b1c | |||
| 5dc0e3b402 | |||
| cc5487e7de | |||
| 0b05b6c6ff | |||
| c507da8f75 | |||
| 550dabeffd | |||
| 4d7f0344d3 | |||
| 084338add6 | |||
| 8d339da706 | |||
| 817ad0a682 | |||
| 80321e5bb4 | |||
| e7475cfa4d | |||
| 8c15025a5a | |||
| 6f2ec66263 | |||
| 62542ddc57 | |||
| a8bb53218d | |||
| 5355d9f8f8 | |||
| 50d7a049ee | |||
| 3149c87cf2 | |||
| b7fa6a4615 | |||
| 6fe83f87a4 | |||
| df6981d88e | |||
| 590adad5ae | |||
| be5d14c985 | |||
| 703086d71d | |||
| a159f312f0 | |||
| 2f6f095298 | |||
| efd8ef005f | |||
| 60fa33773a | |||
| ee752b0b03 | |||
| 99c1f4418b | |||
| 7ce0ac72cf | |||
| 944e26d432 | |||
| f0b0a99260 | |||
| 33ddc10ffd |
@@ -101,7 +101,6 @@ Update this file only when an advisory adds or changes a reusable lesson, affect
|
||||
### SSE and on-disk storage invariants
|
||||
|
||||
- `GHSA-xrrf-67jm-3c2r`: SSE metadata reported encryption while reader composition bypassed `EncryptReader` and stored plaintext. Lesson: test actual bytes on disk and wrapper order, not only API metadata.
|
||||
- `GHSA-wqmc-vjgv-jrpw`: SSE-C persisted and listed the plaintext MD5 as the ETag, allowing list-only users to confirm or brute-force low-entropy content and correlate equal plaintext across keys. Lesson: encrypted-object ETags must not expose deterministic plaintext fingerprints; protect them with per-object key material and keep listing, conditional, copy, and multipart semantics consistent.
|
||||
|
||||
### Object Lock and retention invariants
|
||||
|
||||
@@ -127,7 +126,7 @@ rg -n "normalize_extract_entry_key|Snowball|auto-extract|PathBuf::join|canonical
|
||||
rg -n "DEFAULT_SECRET|DEFAULT_ACCESS|TEST_PRIVATE_KEY|rustfs rpc|RUSTFS_RPC_SECRET" rustfs crates
|
||||
rg -n "TONIC_RPC_PREFIX|verify_rpc_signature|check_auth|NodeServiceServer|x-rustfs-signature" rustfs crates
|
||||
rg -n "debug!|trace!|info!|error!|\\?resp|\\?merged_config|session_token|secret_key" rustfs crates
|
||||
rg -n "HashReader|EncryptReader|try_resolve_etag|ETag|SSE|server-side encryption|Access-Control-Allow-Credentials|Origin" rustfs crates
|
||||
rg -n "HashReader|EncryptReader|SSE|server-side encryption|Access-Control-Allow-Credentials|Origin" rustfs crates
|
||||
rg -n "ObjectLock|object_lock|retention|COMPLIANCE|GOVERNANCE|delete_prefix|lifecycle|scanner" rustfs crates
|
||||
rg -n "deny_unknown_fields|serde.default|as u32|as usize|as i32" rustfs crates
|
||||
```
|
||||
@@ -146,5 +145,5 @@ rg -n "deny_unknown_fields|serde.default|as u32|as usize|as i32" rustfs crates
|
||||
- IAM export fixes: assert exported archives omit plaintext user and service-account secrets unless the format deliberately encrypts or seals them.
|
||||
- RPC auth fixes: include captured metadata replay across two concrete methods, stale timestamps, wrong path, wrong method surrogate, wrong secret, and valid same-method calls.
|
||||
- Browser/CORS fixes: assert no credentials on reflected/default origins, correct behavior for explicit allowlists, and no same-origin script execution for previewed object content.
|
||||
- SSE fixes: inspect stored bytes and verify API metadata, read-back behavior, and on-disk ciphertext together; for ETags, upload equal plaintext under different keys and assert list-only callers cannot derive or correlate the plaintext digest, including multipart objects.
|
||||
- SSE fixes: inspect stored bytes and verify API metadata, read-back behavior, and on-disk ciphertext together.
|
||||
- Object Lock fixes: include unreadable metadata, fabricated metadata defaults, unparsable config, confirmed absent config, COMPLIANCE/GOVERNANCE retention, lifecycle expiry, scanner sweeps, and force-delete paths.
|
||||
|
||||
@@ -29,7 +29,6 @@ script-tests: ## Run shell script tests
|
||||
./scripts/test_entrypoint_credentials.sh
|
||||
./scripts/test_internode_grpc_ab_bench.sh
|
||||
./scripts/test_object_batch_bench_enhanced.sh
|
||||
./scripts/test_package_service_scripts.sh
|
||||
./scripts/test_hotpath_warp_ab_gate.sh
|
||||
./scripts/test_hotpath_warp_abba.sh
|
||||
./scripts/test_scanner_validation_harness.sh
|
||||
|
||||
@@ -132,12 +132,6 @@ test-group = 'ecstore-serial-flaky'
|
||||
filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnostic_capability_e2e))'
|
||||
test-group = 'embedded-test-ports'
|
||||
|
||||
# The real object probe includes bucket creation and cleanup in its short
|
||||
# measurement budget. Keep competing storage fixtures outside that budget.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs) & binary(connect_perf_object) & test(=real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put)'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# Serialize the durable manual-transition checkpoint test across nextest's
|
||||
# process boundary; it mutates bucket lifecycle metadata and is not quarantined.
|
||||
[[profile.default.overrides]]
|
||||
@@ -333,10 +327,6 @@ test-group = 'ecstore-serial-flaky'
|
||||
filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnostic_capability_e2e))'
|
||||
test-group = 'embedded-test-ports'
|
||||
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs) & binary(connect_perf_object) & test(=real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put)'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# Serialize the durable manual-transition checkpoint test under the ci profile
|
||||
# too. No retries: failures stay visible.
|
||||
[[profile.ci.overrides]]
|
||||
|
||||
@@ -24,9 +24,7 @@ on:
|
||||
- '.github/actions/**'
|
||||
- '.github/workflows/**'
|
||||
- 'scripts/release/create_or_update_release.sh'
|
||||
- 'scripts/release/package_service_scripts.sh'
|
||||
- 'scripts/release/package_versions.sh'
|
||||
- 'scripts/test_package_service_scripts.sh'
|
||||
- 'scripts/test_package_versions.sh'
|
||||
- 'scripts/security/check_performance_ab_workflow.sh'
|
||||
- 'scripts/security/check_preview_release_workflow.sh'
|
||||
@@ -42,9 +40,7 @@ on:
|
||||
- '.github/actions/**'
|
||||
- '.github/workflows/**'
|
||||
- 'scripts/release/create_or_update_release.sh'
|
||||
- 'scripts/release/package_service_scripts.sh'
|
||||
- 'scripts/release/package_versions.sh'
|
||||
- 'scripts/test_package_service_scripts.sh'
|
||||
- 'scripts/test_package_versions.sh'
|
||||
- 'scripts/security/check_performance_ab_workflow.sh'
|
||||
- 'scripts/security/check_preview_release_workflow.sh'
|
||||
@@ -162,9 +158,6 @@ jobs:
|
||||
- name: Check package version contract
|
||||
run: ./scripts/test_package_versions.sh
|
||||
|
||||
- name: Check package service lifecycle contract
|
||||
run: ./scripts/test_package_service_scripts.sh
|
||||
|
||||
dependency-review:
|
||||
name: Dependency Review
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -16,7 +16,7 @@ name: Continuous Integration
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main ]
|
||||
branches: [ main, release ]
|
||||
paths-ignore:
|
||||
- "**.md"
|
||||
- "docs/**"
|
||||
@@ -36,7 +36,7 @@ on:
|
||||
- "flake.lock"
|
||||
pull_request:
|
||||
types: [ opened, synchronize, reopened, closed ]
|
||||
branches: [ main ]
|
||||
branches: [ main, release ]
|
||||
merge_group:
|
||||
types: [ checks_requested ]
|
||||
schedule:
|
||||
@@ -867,13 +867,14 @@ jobs:
|
||||
# Merge gate only (backlog#1149 ci-5): the never-automated user-visible
|
||||
# suites — KMS, object_lock, multipart_auth, quota, checksum, encryption,
|
||||
# security-boundary, ... — via the e2e-full nextest profile. Too heavy for
|
||||
# every PR, so it is gated to main pushes, the merge queue, and manual
|
||||
# every PR, so it is gated to main/release pushes, the merge queue, and manual
|
||||
# dispatch. protocols / the 7 cluster suites / replication / #[ignore] are
|
||||
# owned by other lanes (see .config/nextest.toml profile.e2e-full).
|
||||
if: >-
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
github.event_name == 'merge_group' ||
|
||||
(github.event_name == 'push' && github.ref == 'refs/heads/main')
|
||||
(github.event_name == 'push' &&
|
||||
(github.ref == 'refs/heads/main' || github.ref == 'refs/heads/release'))
|
||||
needs: [ build-rustfs-debug-binary ]
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 55
|
||||
|
||||
@@ -542,54 +542,6 @@ jobs:
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
|
||||
# Docker Hub's overview is a separate `full_description` field that
|
||||
# `docker push` never touches, so it drifted from README.md and kept
|
||||
# linking to pages that no longer exist (#7995). Republish it from the
|
||||
# commit the images were built from, only when images were pushed.
|
||||
sync-dockerhub-description:
|
||||
name: Sync Docker Hub Description
|
||||
needs: [ build-check, build-docker ]
|
||||
if: needs.build-check.outputs.should_build == 'true' && needs.build-check.outputs.should_push == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ needs.build-check.outputs.source_ref }}
|
||||
sparse-checkout: README.md
|
||||
sparse-checkout-cone-mode: false
|
||||
|
||||
- name: Sync Docker Hub description
|
||||
uses: peter-evans/dockerhub-description@1b9a80c056b620d92cedb9d9b5a223409c68ddfa # v5.0.0
|
||||
with:
|
||||
username: ${{ env.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
repository: ${{ env.REGISTRY_DOCKERHUB }}
|
||||
readme-filepath: ./README.md
|
||||
# README.md links to docs/ and CONTRIBUTING.md by relative path;
|
||||
# rewrite those to github.com URLs so they resolve on Docker Hub.
|
||||
enable-url-completion: true
|
||||
|
||||
# Docker Hub caps `full_description` at 25,000 bytes and the action
|
||||
# truncates to fit with only a warning, which would publish a README
|
||||
# cut off mid-sentence. README.md is already near the cap, so read the
|
||||
# published overview back and fail if it hit the limit.
|
||||
- name: Check published description was not truncated
|
||||
env:
|
||||
REPOSITORY: ${{ env.REGISTRY_DOCKERHUB }}
|
||||
run: |
|
||||
limit=25000
|
||||
published=$(curl -sSf "https://hub.docker.com/v2/repositories/${REPOSITORY}/" \
|
||||
| jq -j '.full_description' | wc -c | tr -d ' ')
|
||||
echo "Published overview: ${published} bytes (Docker Hub limit: ${limit})"
|
||||
# Truncation lands within one UTF-8 character of the cap.
|
||||
if [ "$published" -ge $((limit - 4)) ]; then
|
||||
echo "::error file=README.md::Docker Hub overview was truncated at ${limit} bytes; trim README.md before the next release."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Docker build summary
|
||||
docker-summary:
|
||||
name: Docker Build Summary
|
||||
|
||||
@@ -117,15 +117,10 @@ jobs:
|
||||
|
||||
cat > "${PKG_DIR}/etc/default/rustfs" << 'ENVEOF'
|
||||
# RustFS Environment Configuration
|
||||
# See https://docs.rustfs.com for more information
|
||||
# RUSTFS_ACCESS_KEY=<changeme>
|
||||
# RUSTFS_SECRET_KEY=<changeme>
|
||||
# See https://rustfs.com/docs/ for more information
|
||||
# RUSTFS_VOLUMES=""
|
||||
# RUSTFS_ADDRESS=":9000"
|
||||
# RUSTFS_CONSOLE_ADDRESS=":9001"
|
||||
# RUSTFS_CONSOLE_ENABLE=true
|
||||
# RUSTFS_OBS_LOGGER_LEVEL=error
|
||||
# RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/"
|
||||
# RUSTFS_ROOT_USER=""
|
||||
# RUSTFS_ROOT_PASSWORD=""
|
||||
ENVEOF
|
||||
|
||||
# dpkg versions must start with a digit and cannot contain hyphens;
|
||||
@@ -151,13 +146,37 @@ jobs:
|
||||
/etc/default/rustfs
|
||||
CONFFILES
|
||||
|
||||
./scripts/release/package_service_scripts.sh after-install > "${PKG_DIR}/DEBIAN/postinst"
|
||||
cat > "${PKG_DIR}/DEBIAN/postinst" << 'POSTINST'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if ! getent passwd rustfs > /dev/null 2>&1; then
|
||||
useradd -r -s /bin/false -d /opt/rustfs rustfs
|
||||
fi
|
||||
mkdir -p /opt/rustfs /data/rustfs /var/log/rustfs
|
||||
chown rustfs:rustfs /opt/rustfs /data/rustfs /var/log/rustfs
|
||||
if [ -d /run/systemd/system ]; then
|
||||
systemctl daemon-reload
|
||||
fi
|
||||
echo "RustFS installed. Configure /etc/default/rustfs then: systemctl start rustfs"
|
||||
POSTINST
|
||||
chmod 755 "${PKG_DIR}/DEBIAN/postinst"
|
||||
|
||||
./scripts/release/package_service_scripts.sh before-remove > "${PKG_DIR}/DEBIAN/prerm"
|
||||
cat > "${PKG_DIR}/DEBIAN/prerm" << 'PRERM'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if [ -d /run/systemd/system ] && systemctl is-active --quiet rustfs; then
|
||||
systemctl stop rustfs
|
||||
fi
|
||||
PRERM
|
||||
chmod 755 "${PKG_DIR}/DEBIAN/prerm"
|
||||
|
||||
./scripts/release/package_service_scripts.sh after-remove > "${PKG_DIR}/DEBIAN/postrm"
|
||||
cat > "${PKG_DIR}/DEBIAN/postrm" << 'POSTRM'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if [ -d /run/systemd/system ]; then
|
||||
systemctl daemon-reload
|
||||
fi
|
||||
POSTRM
|
||||
chmod 755 "${PKG_DIR}/DEBIAN/postrm"
|
||||
|
||||
cp LICENSE "${PKG_DIR}/usr/share/doc/rustfs/"
|
||||
@@ -195,15 +214,10 @@ jobs:
|
||||
mkdir -p ./tmp-pkg/etc/default
|
||||
cat > ./tmp-pkg/etc/default/rustfs << 'ENVEOF'
|
||||
# RustFS Environment Configuration
|
||||
# See https://docs.rustfs.com for more information
|
||||
# RUSTFS_ACCESS_KEY=<changeme>
|
||||
# RUSTFS_SECRET_KEY=<changeme>
|
||||
# See https://rustfs.com/docs/ for more information
|
||||
# RUSTFS_VOLUMES=""
|
||||
# RUSTFS_ADDRESS=":9000"
|
||||
# RUSTFS_CONSOLE_ADDRESS=":9001"
|
||||
# RUSTFS_CONSOLE_ENABLE=true
|
||||
# RUSTFS_OBS_LOGGER_LEVEL=error
|
||||
# RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/"
|
||||
# RUSTFS_ROOT_USER=""
|
||||
# RUSTFS_ROOT_PASSWORD=""
|
||||
ENVEOF
|
||||
|
||||
fpm -s dir -t rpm \
|
||||
@@ -217,13 +231,35 @@ jobs:
|
||||
--description "High-performance distributed object storage" \
|
||||
--url "https://rustfs.com" \
|
||||
--license "Apache-2.0" \
|
||||
--before-install <(./scripts/release/package_service_scripts.sh rpm-before-install) \
|
||||
--before-upgrade <(./scripts/release/package_service_scripts.sh rpm-before-upgrade) \
|
||||
--after-install <(./scripts/release/package_service_scripts.sh after-install) \
|
||||
--after-upgrade <(./scripts/release/package_service_scripts.sh after-install) \
|
||||
--before-remove <(./scripts/release/package_service_scripts.sh before-remove) \
|
||||
--after-remove <(./scripts/release/package_service_scripts.sh after-remove) \
|
||||
--rpm-posttrans <(./scripts/release/package_service_scripts.sh rpm-posttrans) \
|
||||
--after-install <(cat << 'POSTINST'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if ! getent passwd rustfs > /dev/null 2>&1; then
|
||||
useradd -r -s /bin/false -d /opt/rustfs rustfs
|
||||
fi
|
||||
mkdir -p /opt/rustfs /data/rustfs /var/log/rustfs
|
||||
chown rustfs:rustfs /opt/rustfs /data/rustfs /var/log/rustfs
|
||||
if [ -d /run/systemd/system ]; then
|
||||
systemctl daemon-reload
|
||||
fi
|
||||
POSTINST
|
||||
) \
|
||||
--before-remove <(cat << 'PRERM'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if [ -d /run/systemd/system ] && systemctl is-active --quiet rustfs; then
|
||||
systemctl stop rustfs
|
||||
fi
|
||||
PRERM
|
||||
) \
|
||||
--after-remove <(cat << 'POSTRM'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if [ -d /run/systemd/system ]; then
|
||||
systemctl daemon-reload
|
||||
fi
|
||||
POSTRM
|
||||
) \
|
||||
--config-files /etc/default/rustfs \
|
||||
"rustfs-nightly-${DEB_DATE}/usr/bin/rustfs=/usr/bin/rustfs" \
|
||||
./tmp-pkg/etc/default/rustfs=/etc/default/rustfs \
|
||||
|
||||
@@ -328,15 +328,10 @@ jobs:
|
||||
|
||||
cat > "${PKG_DIR}/etc/default/rustfs" << 'ENVEOF'
|
||||
# RustFS Environment Configuration
|
||||
# See https://docs.rustfs.com for more information
|
||||
# RUSTFS_ACCESS_KEY=<changeme>
|
||||
# RUSTFS_SECRET_KEY=<changeme>
|
||||
# See https://rustfs.com/docs/ for more information
|
||||
# RUSTFS_VOLUMES=""
|
||||
# RUSTFS_ADDRESS=":9000"
|
||||
# RUSTFS_CONSOLE_ADDRESS=":9001"
|
||||
# RUSTFS_CONSOLE_ENABLE=true
|
||||
# RUSTFS_OBS_LOGGER_LEVEL=error
|
||||
# RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/"
|
||||
# RUSTFS_ROOT_USER=""
|
||||
# RUSTFS_ROOT_PASSWORD=""
|
||||
ENVEOF
|
||||
|
||||
cat > "${PKG_DIR}/DEBIAN/control" << EOF
|
||||
@@ -359,13 +354,37 @@ jobs:
|
||||
/etc/default/rustfs
|
||||
CONFFILES
|
||||
|
||||
./scripts/release/package_service_scripts.sh after-install > "${PKG_DIR}/DEBIAN/postinst"
|
||||
cat > "${PKG_DIR}/DEBIAN/postinst" << 'POSTINST'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if ! getent passwd rustfs > /dev/null 2>&1; then
|
||||
useradd -r -s /bin/false -d /opt/rustfs rustfs
|
||||
fi
|
||||
mkdir -p /opt/rustfs /data/rustfs /var/log/rustfs
|
||||
chown rustfs:rustfs /opt/rustfs /data/rustfs /var/log/rustfs
|
||||
if [ -d /run/systemd/system ]; then
|
||||
systemctl daemon-reload
|
||||
fi
|
||||
echo "RustFS installed. Configure /etc/default/rustfs then: systemctl start rustfs"
|
||||
POSTINST
|
||||
chmod 755 "${PKG_DIR}/DEBIAN/postinst"
|
||||
|
||||
./scripts/release/package_service_scripts.sh before-remove > "${PKG_DIR}/DEBIAN/prerm"
|
||||
cat > "${PKG_DIR}/DEBIAN/prerm" << 'PRERM'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if [ -d /run/systemd/system ] && systemctl is-active --quiet rustfs; then
|
||||
systemctl stop rustfs
|
||||
fi
|
||||
PRERM
|
||||
chmod 755 "${PKG_DIR}/DEBIAN/prerm"
|
||||
|
||||
./scripts/release/package_service_scripts.sh after-remove > "${PKG_DIR}/DEBIAN/postrm"
|
||||
cat > "${PKG_DIR}/DEBIAN/postrm" << 'POSTRM'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if [ -d /run/systemd/system ]; then
|
||||
systemctl daemon-reload
|
||||
fi
|
||||
POSTRM
|
||||
chmod 755 "${PKG_DIR}/DEBIAN/postrm"
|
||||
|
||||
cp LICENSE "${PKG_DIR}/usr/share/doc/rustfs/"
|
||||
@@ -403,15 +422,10 @@ jobs:
|
||||
mkdir -p ./tmp-pkg/etc/default
|
||||
cat > ./tmp-pkg/etc/default/rustfs << 'ENVEOF'
|
||||
# RustFS Environment Configuration
|
||||
# See https://docs.rustfs.com for more information
|
||||
# RUSTFS_ACCESS_KEY=<changeme>
|
||||
# RUSTFS_SECRET_KEY=<changeme>
|
||||
# See https://rustfs.com/docs/ for more information
|
||||
# RUSTFS_VOLUMES=""
|
||||
# RUSTFS_ADDRESS=":9000"
|
||||
# RUSTFS_CONSOLE_ADDRESS=":9001"
|
||||
# RUSTFS_CONSOLE_ENABLE=true
|
||||
# RUSTFS_OBS_LOGGER_LEVEL=error
|
||||
# RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/"
|
||||
# RUSTFS_ROOT_USER=""
|
||||
# RUSTFS_ROOT_PASSWORD=""
|
||||
ENVEOF
|
||||
|
||||
fpm -s dir -t rpm \
|
||||
@@ -425,13 +439,35 @@ jobs:
|
||||
--description "High-performance distributed object storage" \
|
||||
--url "https://rustfs.com" \
|
||||
--license "Apache-2.0" \
|
||||
--before-install <(./scripts/release/package_service_scripts.sh rpm-before-install) \
|
||||
--before-upgrade <(./scripts/release/package_service_scripts.sh rpm-before-upgrade) \
|
||||
--after-install <(./scripts/release/package_service_scripts.sh after-install) \
|
||||
--after-upgrade <(./scripts/release/package_service_scripts.sh after-install) \
|
||||
--before-remove <(./scripts/release/package_service_scripts.sh before-remove) \
|
||||
--after-remove <(./scripts/release/package_service_scripts.sh after-remove) \
|
||||
--rpm-posttrans <(./scripts/release/package_service_scripts.sh rpm-posttrans) \
|
||||
--after-install <(cat <<'POSTINST'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if ! getent passwd rustfs > /dev/null 2>&1; then
|
||||
useradd -r -s /bin/false -d /opt/rustfs rustfs
|
||||
fi
|
||||
mkdir -p /opt/rustfs /data/rustfs /var/log/rustfs
|
||||
chown rustfs:rustfs /opt/rustfs /data/rustfs /var/log/rustfs
|
||||
if [ -d /run/systemd/system ]; then
|
||||
systemctl daemon-reload
|
||||
fi
|
||||
POSTINST
|
||||
) \
|
||||
--before-remove <(cat <<'PRERM'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if [ -d /run/systemd/system ] && systemctl is-active --quiet rustfs; then
|
||||
systemctl stop rustfs
|
||||
fi
|
||||
PRERM
|
||||
) \
|
||||
--after-remove <(cat <<'POSTRM'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if [ -d /run/systemd/system ]; then
|
||||
systemctl daemon-reload
|
||||
fi
|
||||
POSTRM
|
||||
) \
|
||||
--config-files /etc/default/rustfs \
|
||||
./bin/rustfs=/usr/bin/rustfs \
|
||||
./tmp-pkg/etc/default/rustfs=/etc/default/rustfs \
|
||||
|
||||
@@ -4,14 +4,13 @@
|
||||
# behavior under drive and node loss against the erasure-coding contract and
|
||||
# snapshots health-endpoint responses at every tier.
|
||||
#
|
||||
# A single-node 4 drives (SNMD): revoke access to 1/2/3 drives, restore
|
||||
# A single-node 4 drives (SNMD): hide 1/2/3 drives, restore
|
||||
# B multi-node 4x1 (one drive per node): stop 1/2/3 nodes, restore
|
||||
# C multi-node 4x4 (16 drives, EC:4): stop 1 node (read-quorum boundary),
|
||||
# stop 2 nodes, restore
|
||||
# C2 multi-node 4x4 with EC:8: 2 nodes down puts 8 drives online -- reads
|
||||
# satisfy the EC read quorum while the lock majority is broken (the
|
||||
# reported divergence window: reads 503 with lock_quorum_unavailable)
|
||||
# D multi-node 4x1: stop 1/2 nodes, then restore
|
||||
#
|
||||
# Expectations come from product source (default_parity_count, erasure set
|
||||
# sizing). By default a "reads refused although the read quorum is met"
|
||||
@@ -33,11 +32,6 @@ on:
|
||||
description: 'Direct .deb URL. Required unless the nightly default is wanted.'
|
||||
required: false
|
||||
type: string
|
||||
auto_testing_ref:
|
||||
description: 'auto-testing branch, tag, or SHA (defaults to main)'
|
||||
required: false
|
||||
default: main
|
||||
type: string
|
||||
strict:
|
||||
description: 'Fail the suite when reads are refused despite a met read quorum'
|
||||
type: boolean
|
||||
@@ -58,7 +52,7 @@ on:
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# The suite stops services and revokes drive access on the shared fleet; only one
|
||||
# The suite stops services and hides drive dirs on the shared fleet; only one
|
||||
# functional suite may touch the environment at a time.
|
||||
concurrency:
|
||||
group: rustfs-shared-functional-tests-v2
|
||||
@@ -79,7 +73,7 @@ env:
|
||||
jobs:
|
||||
fault-tolerance-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: 480
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout repository (for report parser)
|
||||
@@ -92,15 +86,13 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-fault-tolerance-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch" "${FUNCTIONAL_ARTIFACTS_DIR}/evidence"
|
||||
FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-ft-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}/evidence"
|
||||
{
|
||||
printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'RESULTS_FILE=%s/results.json\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'EVIDENCE_DIR=%s/evidence\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
|
||||
# Pin the private suite checkout when the chain supplies a candidate.
|
||||
@@ -112,32 +104,22 @@ jobs:
|
||||
run: python3 scripts/functional_chain_evidence.py consume
|
||||
|
||||
- name: Checkout auto-testing scripts
|
||||
timeout-minutes: 5
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
repository: rustfs/auto-testing
|
||||
ref: ${{ steps.chain.outputs.testing_sha || inputs.auto_testing_ref || 'main' }}
|
||||
ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
path: auto-testing
|
||||
persist-credentials: false
|
||||
|
||||
- name: Record auto-testing revision
|
||||
run: |
|
||||
set -euo pipefail
|
||||
AUTO_TESTING_SHA="$(git -C auto-testing rev-parse HEAD)"
|
||||
printf 'AUTO_TESTING_SHA=%s\n' "${AUTO_TESTING_SHA}" >> "${GITHUB_ENV}"
|
||||
echo "auto-testing revision: ${AUTO_TESTING_SHA}"
|
||||
|
||||
- name: Show environment
|
||||
run: |
|
||||
uname -a
|
||||
jq --version
|
||||
aws --version
|
||||
echo "auto-testing: ${AUTO_TESTING_SHA}"
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
if: ${{ inputs.cleanup_before != 'false' }}
|
||||
run: |
|
||||
./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}"
|
||||
@@ -149,21 +131,19 @@ jobs:
|
||||
CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
|
||||
run: python3 scripts/prepare_functional_package.py prepare
|
||||
|
||||
- name: Run fault-tolerance scenarios (A, B, C, C2, D, E)
|
||||
timeout-minutes: 45
|
||||
- name: Run fault-tolerance scenarios (A, B, C, C2)
|
||||
id: test
|
||||
# Standalone case failures are reported by the backlog manager.
|
||||
# Chain evidence separately requires complete passing cases.
|
||||
continue-on-error: true
|
||||
run: |
|
||||
ARGS=(--all -y --package-url "${{ steps.chain_package.outputs.package_url || inputs.package_url || env.RUSTFS_NIGHTLY_PACKAGE_URL }}" --log-file "${LOG_FILE}" --results-file "${RESULTS_FILE}")
|
||||
ARGS=(--all -y --package-url "${{ steps.chain_package.outputs.package_url || inputs.package_url || env.RUSTFS_NIGHTLY_PACKAGE_URL }}" --log-file "${LOG_FILE}")
|
||||
if [ "${{ inputs.strict }}" = "true" ]; then
|
||||
ARGS+=(--strict)
|
||||
fi
|
||||
./auto-testing/rustfs-fault-tolerance-test.sh "${ARGS[@]}"
|
||||
|
||||
- name: Generate report
|
||||
timeout-minutes: 2
|
||||
id: chain_report
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
run: |
|
||||
@@ -173,13 +153,12 @@ jobs:
|
||||
echo ""
|
||||
echo "- Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo "- Package: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
|
||||
echo "- auto-testing: \`${AUTO_TESTING_SHA}\`"
|
||||
echo "- Strict mode: ${{ inputs.strict || 'false' }}"
|
||||
echo ""
|
||||
echo "## Per-probe results"
|
||||
echo ""
|
||||
echo '```'
|
||||
grep -E '^FT-(CASE|COMPLETE|SUMMARY|REPORT)' "${LOG_FILE}" || echo "(no FT-CASE lines found)"
|
||||
grep -E '^FT-(CASE|SUMMARY|REPORT)' "${LOG_FILE}" || echo "(no FT-CASE lines found)"
|
||||
echo '```'
|
||||
echo ""
|
||||
echo "## Health snapshots"
|
||||
@@ -190,40 +169,28 @@ jobs:
|
||||
done
|
||||
} > "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
printf 'complete=false\n' >> "${GITHUB_OUTPUT}"
|
||||
jq -e --arg testing_sha "${AUTO_TESTING_SHA}" '
|
||||
.schema_version == 1 and
|
||||
.auto_testing_sha == $testing_sha and
|
||||
.selected_scenarios == ["A", "B", "C", "C2", "D", "E"] and
|
||||
.complete == true and
|
||||
.counts.expected == 44 and
|
||||
.counts.seen == 44 and
|
||||
.counts.harness_errors == 0 and
|
||||
(.expected_cases | length) == 44 and
|
||||
(.expected_cases | unique | length) == 44 and
|
||||
(.seen_cases | length) == 44 and
|
||||
(.seen_cases | unique | length) == 44 and
|
||||
(.seen_cases | sort) == (.expected_cases | sort) and
|
||||
(.missing_cases | length) == 0 and
|
||||
(.harness_errors | length) == 0 and
|
||||
(.cases | length) == 44 and
|
||||
([.cases[].case_id] | sort) == (.expected_cases | sort) and
|
||||
([.cases[].case_id] | unique | length) == 44 and
|
||||
all(.cases[]; .verdict == "pass" or .verdict == "known-divergence" or .verdict == "UNEXPECTED") and
|
||||
.counts.unexpected == ([.cases[] | select(.verdict == "UNEXPECTED")] | length) and
|
||||
.counts.known_divergence == ([.cases[] | select(.verdict == "known-divergence")] | length)
|
||||
' "${RESULTS_FILE}" >/dev/null
|
||||
printf 'complete=true\n' >> "${GITHUB_OUTPUT}"
|
||||
# Product gate: failing FT cases keep the run green — the report above and
|
||||
# the backlog issue manager carry the signal. Only harness/environment
|
||||
# breakdowns (no case verdicts at all) turn the workflow red.
|
||||
FT_CASES="$(grep -cE '^FT-CASE:' "${LOG_FILE}" 2>/dev/null || true)"
|
||||
FT_FAILED="$(grep -cE '^FT-CASE: .* verdict=UNEXPECTED' "${LOG_FILE}" 2>/dev/null || true)"
|
||||
FT_PASSED=$(( ${FT_CASES:-0} - ${FT_FAILED:-0} ))
|
||||
HARNESS_OK=0
|
||||
if [ "${{ steps.test.outcome }}" = "success" ]; then
|
||||
HARNESS_OK=1
|
||||
elif [ "${{ steps.test.outcome }}" = "failure" ] && [ "${FT_CASES}" -gt 0 ] && [ "${FT_PASSED}" -ge 1 ]; then
|
||||
HARNESS_OK=1
|
||||
fi
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
|
||||
- name: Manage backlog issues (dedup / label / auto-close)
|
||||
timeout-minutes: 2
|
||||
# Signal-based lifecycle: dedups against open issues by label
|
||||
# (fault-tolerance + FT case), labels new issues (functional-test,
|
||||
# category, case IDs, env), and closes fixed issues after a fully
|
||||
# green run. Never acts on cancelled runs. Logic lives in
|
||||
# auto-testing/scripts/issue_manager.py, which parses the
|
||||
# FT-CASE verdict lines from the suite log.
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' && steps.chain_report.outputs.complete == 'true' }}
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
@@ -261,51 +228,18 @@ jobs:
|
||||
--package-source "${PACKAGE_SOURCE}" \
|
||||
--date "$(date -u +%Y-%m-%d)"
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
REPORT_FILE: ${{ env.REPORT_FILE }}
|
||||
SUITE: fault-tolerance
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
|
||||
exit 0
|
||||
fi
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
||||
B64_FILE="$(mktemp)"
|
||||
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n"))}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
fi
|
||||
echo "dashboard updated: ${REPORT_PATH}"
|
||||
|
||||
- name: Upload test logs & evidence
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: rustfs-fault-tolerance-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results.json
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/evidence/
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && inputs.cleanup_after != 'false' }}
|
||||
run: |
|
||||
./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}" || true
|
||||
|
||||
@@ -46,7 +46,6 @@ jobs:
|
||||
timeout-minutes: 10
|
||||
outputs:
|
||||
manifest: ${{ steps.candidate.outputs.manifest }}
|
||||
performance_ready: ${{ steps.perf_probe.outputs.performance_ready }}
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
@@ -54,10 +53,7 @@ jobs:
|
||||
- name: Resolve published candidate
|
||||
id: candidate
|
||||
env:
|
||||
# The staleness probe in resolve_functional_candidate.py reads the
|
||||
# private rustfs/auto-testing repo; github.token cannot, and a probe
|
||||
# failure silently keeps a stale testing_sha pinned for the chain.
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
BUILD_RUN_ID: ${{ inputs.build_run_id }}
|
||||
BUILD_RUN_ATTEMPT: ${{ inputs.build_run_attempt }}
|
||||
CHAIN_OUTPUT: ${{ runner.temp }}/chain-candidate.json
|
||||
@@ -69,25 +65,10 @@ jobs:
|
||||
path: ${{ runner.temp }}/chain-candidate.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Check shared functional fleet runner before scheduling suites
|
||||
# The eleven functional suites run on the smoke-testing fleet; a hard
|
||||
# requirement. The performance lane runs on pf-testing, which is a
|
||||
# separate fleet - probe it without failing the whole chain so an
|
||||
# offline performance runner skips only the performance lane.
|
||||
- name: Check shared functional runners before scheduling suites
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: python3 scripts/check_functional_runners.py smoke-testing
|
||||
|
||||
- name: Probe performance fleet runner
|
||||
id: perf_probe
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
if python3 scripts/check_functional_runners.py pf-testing; then
|
||||
echo "performance_ready=online" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "performance_ready=offline" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
run: python3 scripts/check_functional_runners.py smoke-testing pf-testing
|
||||
|
||||
upgrade:
|
||||
needs: [prepare]
|
||||
@@ -179,7 +160,7 @@ jobs:
|
||||
|
||||
performance:
|
||||
needs: [prepare, table]
|
||||
if: ${{ always() && needs.prepare.result == 'success' && needs.prepare.outputs.performance_ready == 'online' }}
|
||||
if: ${{ always() && needs.prepare.result == 'success' }}
|
||||
uses: ./.github/workflows/rustfs-performance-test.yml
|
||||
with:
|
||||
chain_manifest: ${{ needs.prepare.outputs.manifest }}
|
||||
@@ -227,7 +208,6 @@ jobs:
|
||||
CHAIN_NEEDS: ${{ toJSON(needs) }}
|
||||
run: >-
|
||||
python3 scripts/functional_chain_evidence.py aggregate
|
||||
--allow-skipped "${{ needs.prepare.outputs.performance_ready == 'online' && '' || 'performance' }}"
|
||||
--directory "${RUNNER_TEMP}/chain-evidence"
|
||||
--output "${RUNNER_TEMP}/chain-complete.json"
|
||||
- name: Upload complete-chain evidence
|
||||
|
||||
@@ -60,7 +60,7 @@ env:
|
||||
jobs:
|
||||
heal-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: 480
|
||||
# Standalone manual run, or one link of the nightly functional chain
|
||||
# (storage -> heal -> pool). Pool expansion no longer re-runs heal.
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
@@ -110,7 +110,6 @@ jobs:
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
if: ${{ inputs.cleanup_before != 'false' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -130,7 +129,6 @@ jobs:
|
||||
done
|
||||
|
||||
- name: Install RustFS package & start cluster
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
ARGS=(--steps "1,2" -y --endpoint "${{ env.RUSTFS_API_ENDPOINT }}")
|
||||
if [ -n "${{ inputs.package_url }}" ]; then
|
||||
@@ -141,7 +139,6 @@ jobs:
|
||||
./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
|
||||
|
||||
- name: Preflight checks
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
ARGS=(--preflight --endpoint "${{ env.RUSTFS_API_ENDPOINT }}")
|
||||
if [ -n "${{ inputs.package_url }}" ]; then
|
||||
@@ -152,7 +149,6 @@ jobs:
|
||||
./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
|
||||
|
||||
- name: Run heal test (write -> outage -> heal -> verify)
|
||||
timeout-minutes: 45
|
||||
id: test
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
@@ -167,7 +163,6 @@ jobs:
|
||||
|
||||
- name: Generate report
|
||||
id: chain_report
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -285,7 +280,6 @@ jobs:
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
@@ -317,7 +311,6 @@ jobs:
|
||||
rm -f "${B64_FILE}"
|
||||
|
||||
- name: Manage backlog issues (dedup / label / auto-close)
|
||||
timeout-minutes: 2
|
||||
# Replaces the old per-run failure filing. One entry point that:
|
||||
# - dedups by signal: failing cases are matched against open backlog
|
||||
# issues by label (category + case ID); covered cases become a
|
||||
@@ -364,7 +357,6 @@ jobs:
|
||||
--date "$(date -u +%Y-%m-%d)"
|
||||
|
||||
- name: Upload test logs
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
@@ -377,7 +369,6 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && inputs.cleanup_after != 'false' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
@@ -55,7 +55,7 @@ env:
|
||||
jobs:
|
||||
kms-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: 420
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout repository (for report parser)
|
||||
@@ -101,7 +101,6 @@ jobs:
|
||||
docker --version || true
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
@@ -130,7 +129,6 @@ jobs:
|
||||
|
||||
- name: Run KMS suite
|
||||
id: test
|
||||
timeout-minutes: 45
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
continue-on-error: true
|
||||
@@ -164,7 +162,6 @@ jobs:
|
||||
|
||||
- name: Generate report
|
||||
id: chain_report
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -230,7 +227,6 @@ jobs:
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
@@ -262,7 +258,6 @@ jobs:
|
||||
rm -f "${B64_FILE}"
|
||||
|
||||
- name: Manage backlog issues (dedup / label / auto-close)
|
||||
timeout-minutes: 2
|
||||
# Replaces the old per-run failure filing. One entry point that:
|
||||
# - dedups by signal: failing cases are matched against open backlog
|
||||
# issues by label (category + case ID); covered cases become a
|
||||
@@ -313,7 +308,6 @@ jobs:
|
||||
--date "$(date -u +%Y-%m-%d)"
|
||||
|
||||
- name: Upload report and logs
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
@@ -325,7 +319,6 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
timeout-minutes: 5
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
@@ -61,13 +61,10 @@ on:
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# Workflow-level so the lock is held through cleanup as well.
|
||||
# The default performance nodes overlap the other suites' remote VMs, even
|
||||
# though the runner differs. Hold the shared lock through cleanup as well.
|
||||
concurrency:
|
||||
# Own group: the performance suite runs on the pf-testing fleet - a fully
|
||||
# separate environment from the smoke-testing fleet the functional suites
|
||||
# use. No shared node, no shared runner: it must never queue behind them
|
||||
# (or block them).
|
||||
group: rustfs-performance-suite
|
||||
group: rustfs-shared-functional-tests-v2
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
|
||||
@@ -82,7 +82,7 @@ jobs:
|
||||
pool-expansion-test:
|
||||
name: Pool expansion / decommission test
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: 360
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
env:
|
||||
RUSTFS_POOL_ADMIN_ENDPOINT: ${{ secrets.RUSTFS_POOL_ADMIN_ENDPOINT || vars.RUSTFS_POOL_ADMIN_ENDPOINT || 'http://rustfs-node1:9000' }}
|
||||
@@ -140,7 +140,6 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
if: ${{ inputs.cleanup_before != 'false' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -160,7 +159,6 @@ jobs:
|
||||
done
|
||||
|
||||
- name: Install RustFS package & start cluster
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
ARGS=(--steps "1,2,3" -y \
|
||||
--admin-endpoint "${RUSTFS_POOL_ADMIN_ENDPOINT}" \
|
||||
@@ -177,7 +175,6 @@ jobs:
|
||||
./auto-testing/rustfs_pool_expand.sh "${ARGS[@]}"
|
||||
|
||||
- name: Preflight checks
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
ARGS=(--preflight \
|
||||
--admin-endpoint "${RUSTFS_POOL_ADMIN_ENDPOINT}" \
|
||||
@@ -194,7 +191,6 @@ jobs:
|
||||
./auto-testing/rustfs_pool_expand.sh "${ARGS[@]}"
|
||||
|
||||
- name: Reset dedicated pool proxy
|
||||
timeout-minutes: 2
|
||||
run: |
|
||||
set -euo pipefail
|
||||
RUSTFS_POOL_NGINX_CONFIG_PATH=/etc/nginx/conf.d/rustfs-pool-test.conf \
|
||||
@@ -204,7 +200,6 @@ jobs:
|
||||
./auto-testing/rustfs_pool_nginx_stage.sh cleanup
|
||||
|
||||
- name: Capture pool test baseline
|
||||
timeout-minutes: 2
|
||||
run: |
|
||||
set -uo pipefail
|
||||
BASELINE_FILE="${POOL_ARTIFACT_DIR}/pool-baseline.log"
|
||||
@@ -251,7 +246,6 @@ jobs:
|
||||
|
||||
- name: Run pool expansion & decommission test
|
||||
id: pool_test
|
||||
timeout-minutes: 45
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
continue-on-error: true
|
||||
@@ -291,7 +285,6 @@ jobs:
|
||||
./auto-testing/rustfs_pool_expand.sh "${ARGS[@]}"
|
||||
|
||||
- name: Collect pool test diagnostics
|
||||
timeout-minutes: 2
|
||||
if: always()
|
||||
run: |
|
||||
set -uo pipefail
|
||||
@@ -393,7 +386,6 @@ jobs:
|
||||
|
||||
- name: Generate report
|
||||
id: chain_report
|
||||
timeout-minutes: 2
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -481,7 +473,6 @@ jobs:
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Validate pool diagnostic completeness
|
||||
timeout-minutes: 2
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -571,7 +562,6 @@ jobs:
|
||||
[ "${failed}" -eq 0 ] || exit 1
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
timeout-minutes: 2
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
env:
|
||||
@@ -604,7 +594,6 @@ jobs:
|
||||
rm -f "${B64_FILE}"
|
||||
|
||||
- name: Manage backlog issues (dedup / label / auto-close)
|
||||
timeout-minutes: 2
|
||||
# Replaces the old per-run failure filing. One entry point that:
|
||||
# - dedups by signal: failing cases are matched against open backlog
|
||||
# issues by label (category + case ID); covered cases become a
|
||||
@@ -654,7 +643,6 @@ jobs:
|
||||
--date "$(date -u +%Y-%m-%d)"
|
||||
|
||||
- name: Upload test logs
|
||||
timeout-minutes: 2
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
@@ -663,7 +651,6 @@ jobs:
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Restore dedicated pool proxy
|
||||
timeout-minutes: 2
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -674,7 +661,6 @@ jobs:
|
||||
./auto-testing/rustfs_pool_nginx_stage.sh cleanup
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && steps.topology.outcome == 'success' && inputs.cleanup_after != 'false' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
@@ -67,7 +67,7 @@ env:
|
||||
jobs:
|
||||
replication-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: 360
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout repository (for report parser)
|
||||
@@ -114,7 +114,6 @@ jobs:
|
||||
df -h /data | tail -1 || true
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
@@ -134,7 +133,6 @@ jobs:
|
||||
|
||||
- name: Run replication suite
|
||||
id: test
|
||||
timeout-minutes: 45
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
continue-on-error: true
|
||||
@@ -161,7 +159,6 @@ jobs:
|
||||
|
||||
- name: Generate report
|
||||
id: chain_report
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -238,7 +235,6 @@ jobs:
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
@@ -270,7 +266,6 @@ jobs:
|
||||
rm -f "${B64_FILE}"
|
||||
|
||||
- name: Manage backlog issues (dedup / label / auto-close)
|
||||
timeout-minutes: 2
|
||||
# Replaces the old per-run failure filing. One entry point that:
|
||||
# - dedups by signal: failing cases are matched against open backlog
|
||||
# issues by label (category + case ID); covered cases become a
|
||||
@@ -321,7 +316,6 @@ jobs:
|
||||
--date "$(date -u +%Y-%m-%d)"
|
||||
|
||||
- name: Upload report and logs
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
@@ -333,7 +327,6 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
timeout-minutes: 5
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
@@ -43,7 +43,7 @@ env:
|
||||
jobs:
|
||||
s3-compat-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: 360
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout repository (for report parser)
|
||||
@@ -89,7 +89,6 @@ jobs:
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
@@ -109,7 +108,6 @@ jobs:
|
||||
|
||||
- name: Run S3 compatibility suite
|
||||
id: test
|
||||
timeout-minutes: 45
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
continue-on-error: true
|
||||
@@ -130,7 +128,6 @@ jobs:
|
||||
|
||||
- name: Generate report
|
||||
id: chain_report
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -207,7 +204,6 @@ jobs:
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
@@ -239,7 +235,6 @@ jobs:
|
||||
rm -f "${B64_FILE}"
|
||||
|
||||
- name: Manage backlog issues (dedup / label / auto-close)
|
||||
timeout-minutes: 2
|
||||
# Replaces the old per-run failure filing. One entry point that:
|
||||
# - dedups by signal: failing cases are matched against open backlog
|
||||
# issues by label (category + case ID); covered cases become a
|
||||
@@ -290,7 +285,6 @@ jobs:
|
||||
--date "$(date -u +%Y-%m-%d)"
|
||||
|
||||
- name: Upload report and logs
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
@@ -302,7 +296,6 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
timeout-minutes: 5
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
@@ -80,7 +80,7 @@ env:
|
||||
jobs:
|
||||
security-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: 360
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
# Checkout the repository into its own subdirectory. Checking out at
|
||||
@@ -127,7 +127,6 @@ jobs:
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
if: ${{ inputs.cleanup_before != 'false' || github.event_name != 'workflow_dispatch' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -148,7 +147,6 @@ jobs:
|
||||
|
||||
- name: Run security suite
|
||||
id: test
|
||||
timeout-minutes: 45
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
continue-on-error: true
|
||||
@@ -182,7 +180,6 @@ jobs:
|
||||
|
||||
- name: Generate report
|
||||
id: report
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
env:
|
||||
TEST_OUTCOME: ${{ steps.test.outcome }}
|
||||
@@ -240,7 +237,6 @@ jobs:
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
@@ -273,7 +269,6 @@ jobs:
|
||||
rm -f "${B64_FILE}"
|
||||
|
||||
- name: Manage backlog issues (dedup / label / auto-close)
|
||||
timeout-minutes: 2
|
||||
# Replaces the old per-run failure filing. One entry point that:
|
||||
# - dedups by signal: failing cases are matched against open backlog
|
||||
# issues by label (category + case ID); covered cases become a
|
||||
@@ -323,7 +318,6 @@ jobs:
|
||||
--date "$(date -u +%Y-%m-%d)"
|
||||
|
||||
- name: Upload report and logs
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
@@ -336,7 +330,6 @@ jobs:
|
||||
retention-days: 3
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
@@ -52,7 +52,7 @@ env:
|
||||
jobs:
|
||||
storage-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: 360
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout repository (for report parser)
|
||||
@@ -98,7 +98,6 @@ jobs:
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
@@ -118,7 +117,6 @@ jobs:
|
||||
|
||||
- name: Run storage engine suite
|
||||
id: test
|
||||
timeout-minutes: 45
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
continue-on-error: true
|
||||
@@ -145,7 +143,6 @@ jobs:
|
||||
|
||||
- name: Generate report
|
||||
id: chain_report
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -222,7 +219,6 @@ jobs:
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
@@ -254,7 +250,6 @@ jobs:
|
||||
rm -f "${B64_FILE}"
|
||||
|
||||
- name: Manage backlog issues (dedup / label / auto-close)
|
||||
timeout-minutes: 2
|
||||
# Replaces the old per-run failure filing. One entry point that:
|
||||
# - dedups by signal: failing cases are matched against open backlog
|
||||
# issues by label (category + case ID); covered cases become a
|
||||
@@ -305,7 +300,6 @@ jobs:
|
||||
--date "$(date -u +%Y-%m-%d)"
|
||||
|
||||
- name: Upload report and logs
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
@@ -317,7 +311,6 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
timeout-minutes: 5
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
@@ -43,7 +43,7 @@ jobs:
|
||||
table-test:
|
||||
name: table-test
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: 90
|
||||
steps:
|
||||
- name: Checkout repository (for report parser)
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
@@ -72,7 +72,6 @@ jobs:
|
||||
run: python3 scripts/functional_chain_evidence.py consume
|
||||
|
||||
- name: Checkout auto-testing scripts
|
||||
timeout-minutes: 5
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
repository: rustfs/auto-testing
|
||||
@@ -90,7 +89,6 @@ jobs:
|
||||
python3 --version
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
continue-on-error: true
|
||||
run: |
|
||||
set -uo pipefail
|
||||
@@ -116,7 +114,6 @@ jobs:
|
||||
|
||||
- name: Run table suite
|
||||
id: test
|
||||
timeout-minutes: 45
|
||||
# Standalone case failures are reported by the backlog manager.
|
||||
# Chain evidence separately requires complete passing cases.
|
||||
continue-on-error: true
|
||||
@@ -140,7 +137,6 @@ jobs:
|
||||
./auto-testing/rustfs-table-test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
|
||||
|
||||
- name: Generate report
|
||||
timeout-minutes: 2
|
||||
id: chain_report
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
run: |
|
||||
@@ -207,7 +203,6 @@ jobs:
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
@@ -237,7 +232,6 @@ jobs:
|
||||
echo "dashboard updated: ${REPORT_PATH}"
|
||||
|
||||
- name: Manage backlog issues (dedup / label / auto-close)
|
||||
timeout-minutes: 2
|
||||
# Replaces the old per-run failure filing. One entry point that:
|
||||
# - dedups by signal: failing cases are matched against open backlog
|
||||
# issues by label (category + case ID); covered cases become a
|
||||
@@ -279,7 +273,6 @@ jobs:
|
||||
--date "$(date -u +%Y-%m-%d)"
|
||||
|
||||
- name: Upload report and logs
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
@@ -292,7 +285,6 @@ jobs:
|
||||
retention-days: 30
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
timeout-minutes: 5
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
run: |
|
||||
|
||||
@@ -67,7 +67,7 @@ env:
|
||||
jobs:
|
||||
tier-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: 420
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout chain tooling
|
||||
@@ -104,7 +104,6 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Prepare pinned RustFS CLI
|
||||
timeout-minutes: 5
|
||||
id: rc
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -217,7 +216,6 @@ jobs:
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sudo docker rm -f rustfs-test-mqtt >/dev/null 2>&1 || true
|
||||
@@ -238,7 +236,6 @@ jobs:
|
||||
done
|
||||
|
||||
- name: Ensure MQTT broker + clients
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if ! command -v mosquitto_sub >/dev/null 2>&1; then
|
||||
@@ -268,7 +265,6 @@ jobs:
|
||||
|
||||
- name: Run tier suite
|
||||
id: test
|
||||
timeout-minutes: 45
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
continue-on-error: true
|
||||
@@ -313,7 +309,6 @@ jobs:
|
||||
|
||||
- name: Generate report
|
||||
id: chain_report
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
env:
|
||||
PACKAGE_URL_INPUT: ${{ inputs.package_url }}
|
||||
@@ -394,7 +389,6 @@ jobs:
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
@@ -427,7 +421,6 @@ jobs:
|
||||
rm -f "${B64_FILE}"
|
||||
|
||||
- name: Verify required tier evidence
|
||||
timeout-minutes: 2
|
||||
id: evidence_verify
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
run: |
|
||||
@@ -458,7 +451,6 @@ jobs:
|
||||
[ "${failed}" -eq 0 ]
|
||||
|
||||
- name: Upload report and logs
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
@@ -467,7 +459,6 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
timeout-minutes: 5
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -489,7 +480,6 @@ jobs:
|
||||
done
|
||||
|
||||
- name: Cleanup pinned RustFS CLI
|
||||
timeout-minutes: 2
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -508,103 +498,23 @@ jobs:
|
||||
GATE_RC_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier-gate.rc
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Product gate: failing cases keep the run green (the structured gate
|
||||
# rc and the report carry the per-case signal, and the backlog issue
|
||||
# manager tracks failures). Only harness/environment breakdowns —
|
||||
# evidence initialization or a missing structured gate result — turn
|
||||
# the workflow red.
|
||||
failed=0
|
||||
if [ "${EVIDENCE_OUTCOME}" != "success" ]; then
|
||||
echo "tier evidence directory initialization is ${EVIDENCE_OUTCOME}, expected success" >&2
|
||||
failed=1
|
||||
fi
|
||||
if [ "${TEST_OUTCOME}" != "success" ]; then
|
||||
echo "tier suite step outcome is ${TEST_OUTCOME}, expected success" >&2
|
||||
failed=1
|
||||
fi
|
||||
if [ "${EVIDENCE_OUTCOME}" != "success" ]; then
|
||||
echo "structured gate result is unavailable because evidence initialization failed" >&2
|
||||
elif [ ! -s "${GATE_RC_FILE}" ]; then
|
||||
echo "structured gate result is missing" >&2
|
||||
failed=1
|
||||
else
|
||||
GATE_RC="$(tr -d '[:space:]' < "${GATE_RC_FILE}")"
|
||||
if ! [[ "${GATE_RC}" =~ ^[0-9]+$ ]] || [ "${GATE_RC}" -ne 0 ]; then
|
||||
echo "structured 56-case gate failed with exit ${GATE_RC:-invalid}" >&2
|
||||
failed=1
|
||||
fi
|
||||
fi
|
||||
[ "${failed}" -eq 0 ]
|
||||
|
||||
- name: File failure issue in rustfs/backlog
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && (steps.evidence.outcome != 'success' || steps.test.outcome == 'cancelled' || steps.evidence_verify.outcome == 'failure' || steps.evidence_verify.outcome == 'cancelled' || (steps.gate.outcome == 'failure' && steps.test.outcome != 'failure') || steps.gate.outcome == 'cancelled') }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
SUITE: 'tier'
|
||||
SUITE_LABEL: 'Tier'
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
EVIDENCE_DIR: ${{ env.TIER_ARTIFACTS_DIR }}
|
||||
EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
|
||||
VERIFY_OUTCOME: ${{ steps.evidence_verify.outcome }}
|
||||
GATE_OUTCOME: ${{ steps.gate.outcome }}
|
||||
REPORT_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier-report.md
|
||||
LOG_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier.log
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
|
||||
exit 0
|
||||
fi
|
||||
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID}, attempt ${GITHUB_RUN_ATTEMPT})"
|
||||
EXISTING="$(gh issue list -R rustfs/backlog --state all \
|
||||
--search "in:title \"run ${GITHUB_RUN_ID}\"" \
|
||||
--json number,title --jq ".[] | select(.title == \"${TITLE}\") | .number")"
|
||||
if [ -n "${EXISTING}" ]; then
|
||||
echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
|
||||
exit 0
|
||||
fi
|
||||
redact() {
|
||||
sed -E \
|
||||
-e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
|
||||
-e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
|
||||
-e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
|
||||
-e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
|
||||
}
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The **${SUITE_LABEL}** functional suite failed."
|
||||
echo ""
|
||||
echo "- Suite: \`${SUITE}\`"
|
||||
echo "- Run: ${RUN_URL}"
|
||||
echo "- Trigger: ${GITHUB_EVENT_NAME}"
|
||||
echo "- Date: $(date -u +%Y-%m-%d)"
|
||||
echo "- Evidence initialization: ${EVIDENCE_OUTCOME}"
|
||||
echo "- Evidence verification: ${VERIFY_OUTCOME}"
|
||||
echo "- Final gate: ${GATE_OUTCOME}"
|
||||
echo ""
|
||||
echo "## Report (errors and symptoms)"
|
||||
echo ""
|
||||
if [ "${EVIDENCE_OUTCOME}" != "success" ]; then
|
||||
echo "(the run evidence directory was rejected; its contents were not read)"
|
||||
elif [ ! -d "${EVIDENCE_DIR}" ] || [ -L "${EVIDENCE_DIR}" ]; then
|
||||
echo "(the run evidence directory is missing or unsafe; its contents were not read)"
|
||||
elif [ -s "${REPORT_FILE}" ]; then
|
||||
redact < "${REPORT_FILE}"
|
||||
elif [ -s "${LOG_FILE:-}" ]; then
|
||||
echo "(report file missing; log tail below)"
|
||||
echo ""
|
||||
tail -n 200 "${LOG_FILE}" | redact
|
||||
else
|
||||
echo "(no report or log file was produced)"
|
||||
fi
|
||||
} | head -c 55000 > "${BODY_FILE}"
|
||||
gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
|
||||
if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test; then
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
|
||||
fi
|
||||
echo "filed backlog issue for suite ${SUITE}"
|
||||
|
||||
- name: Manage backlog issues (dedup / label / auto-close)
|
||||
timeout-minutes: 2
|
||||
# Case-level issue management complements the infrastructure failure report:
|
||||
# Replaces the old per-run failure filing. One entry point that:
|
||||
# - dedups by signal: failing cases are matched against open backlog
|
||||
# issues by label (category + case ID); covered cases become a
|
||||
# comment on the existing issue, only uncovered cases file a new one
|
||||
@@ -615,9 +525,6 @@ jobs:
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
TEST_OUTCOME: ${{ steps.test.outcome }}
|
||||
VERIFY_OUTCOME: ${{ steps.evidence_verify.outcome }}
|
||||
GATE_OUTCOME: ${{ steps.gate.outcome }}
|
||||
SUITE: 'tier'
|
||||
SUITE_LABEL: 'Tier'
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
@@ -631,14 +538,6 @@ jobs:
|
||||
echo "issue_manager.py not found in auto-testing checkout; skipping"
|
||||
exit 0
|
||||
fi
|
||||
if [ ! -d "${TIER_ARTIFACTS_DIR}" ] || [ -L "${TIER_ARTIFACTS_DIR}" ]; then
|
||||
echo "tier evidence directory is missing or unsafe; skipping backlog issue management" >&2
|
||||
exit 1
|
||||
fi
|
||||
OUTCOME="${TEST_OUTCOME}"
|
||||
if [ "${OUTCOME}" != "cancelled" ] && { [ "${VERIFY_OUTCOME}" != "success" ] || [ "${GATE_OUTCOME}" != "success" ]; }; then
|
||||
OUTCOME=failure
|
||||
fi
|
||||
PACKAGE_URL='${{ inputs.package_url }}'
|
||||
RUSTFS_VERSION='${{ inputs.rustfs_version }}'
|
||||
PACKAGE_SOURCE=""
|
||||
@@ -652,7 +551,7 @@ jobs:
|
||||
python3 auto-testing/scripts/issue_manager.py handle \
|
||||
--repo rustfs/backlog \
|
||||
--suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
|
||||
--outcome "${OUTCOME}" \
|
||||
--outcome "${{ steps.test.outcome }}" \
|
||||
--report "${TIER_ARTIFACTS_DIR}/rustfs-tier-cases.md" \
|
||||
--report-file "${TIER_ARTIFACTS_DIR}/rustfs-tier-report.md" \
|
||||
--log "${TIER_ARTIFACTS_DIR}/rustfs-tier.log" \
|
||||
|
||||
@@ -85,7 +85,7 @@ env:
|
||||
jobs:
|
||||
upgrade-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: 420
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout repository (for report parser)
|
||||
@@ -133,7 +133,6 @@ jobs:
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
if: ${{ inputs.cleanup_before != 'false' || github.event_name != 'workflow_dispatch' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -153,7 +152,6 @@ jobs:
|
||||
done
|
||||
|
||||
- name: Ensure docker (Vault container)
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
sudo apt-get update
|
||||
@@ -164,7 +162,6 @@ jobs:
|
||||
|
||||
- name: Run upgrade compatibility suite
|
||||
id: test
|
||||
timeout-minutes: 45
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
continue-on-error: true
|
||||
@@ -227,7 +224,6 @@ jobs:
|
||||
|
||||
- name: Generate report
|
||||
id: chain_report
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -308,7 +304,6 @@ jobs:
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
@@ -340,7 +335,6 @@ jobs:
|
||||
rm -f "${B64_FILE}"
|
||||
|
||||
- name: Manage backlog issues (dedup / label / auto-close)
|
||||
timeout-minutes: 2
|
||||
# Replaces the old per-run failure filing. One entry point that:
|
||||
# - dedups by signal: failing cases are matched against open backlog
|
||||
# issues by label (category + case ID); covered cases become a
|
||||
@@ -398,7 +392,6 @@ jobs:
|
||||
--date "$(date -u +%Y-%m-%d)"
|
||||
|
||||
- name: Upload report and logs
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
@@ -412,7 +405,6 @@ jobs:
|
||||
retention-days: 3
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
@@ -16,7 +16,7 @@ name: Windows Filesystem Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main ]
|
||||
branches: [ main, release ]
|
||||
paths:
|
||||
- "crates/ecstore/src/disk/**"
|
||||
- "crates/ecstore/src/store/init_format.rs"
|
||||
@@ -26,7 +26,7 @@ on:
|
||||
- ".github/actions/setup/**"
|
||||
- ".github/workflows/windows-filesystem.yml"
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
branches: [ main, release ]
|
||||
paths:
|
||||
- "crates/ecstore/src/disk/**"
|
||||
- "crates/ecstore/src/store/init_format.rs"
|
||||
|
||||
Generated
+204
-274
File diff suppressed because it is too large
Load Diff
+67
-68
@@ -73,7 +73,7 @@ edition = "2024"
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/rustfs/rustfs"
|
||||
rust-version = "1.98.1"
|
||||
version = "1.0.1"
|
||||
version = "1.0.0"
|
||||
homepage = "https://rustfs.com"
|
||||
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
||||
keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
|
||||
@@ -90,62 +90,62 @@ redundant_clone = "warn"
|
||||
|
||||
[workspace.dependencies]
|
||||
# RustFS Internal Crates
|
||||
rustfs = { path = "./rustfs", version = "1.0.1" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.1" }
|
||||
rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.1" }
|
||||
rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.1" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.1" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.1" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.1" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.1" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.1" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.1" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.1" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.1" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.1" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.1" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.1" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.1" }
|
||||
rustfs-license = { path = "crates/license", version = "1.0.1" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.1" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.1" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.1" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.1" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.1" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.1" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.1" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.1" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.1", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.1" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.1" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.1" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.1" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.1" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.1" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.1" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.1" }
|
||||
rustfs-s3-client = { path = "crates/s3-client", version = "1.0.1" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.1" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.1" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.1" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.1" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.1" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.1" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.1" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.1" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.1" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.1" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.1" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.1" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.1" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.1" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.1" }
|
||||
rustfs = { path = "./rustfs", version = "1.0.0" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0" }
|
||||
rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.0" }
|
||||
rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.0" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0" }
|
||||
rustfs-license = { path = "crates/license", version = "1.0.0" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0" }
|
||||
rustfs-s3-client = { path = "crates/s3-client", version = "1.0.0" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0" }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
async_zip = { default-features = false, version = "0.0.19" }
|
||||
mysql_async = { default-features = false, version = "0.37.1" }
|
||||
async-compression = { version = "0.4.48" }
|
||||
async-compression = { version = "0.4.47" }
|
||||
async-recursion = "1.1.1"
|
||||
async-trait = "0.1.92"
|
||||
async-nats = { version = "0.50.0", default-features = false }
|
||||
@@ -163,7 +163,6 @@ hyper-util = { version = "0.1.20" }
|
||||
http = "1.5.0"
|
||||
http-body = "1.1.0"
|
||||
http-body-util = "0.1.5"
|
||||
memchr = "2.8.3"
|
||||
minlz = "1.2.3"
|
||||
reqwest = "0.13.5"
|
||||
rustfs-kafka-async = { version = "1.3.1" }
|
||||
@@ -207,7 +206,7 @@ blake2 = "0.11.0"
|
||||
chacha20poly1305 = { version = "0.11.0" }
|
||||
crc-fast = "1.10.0"
|
||||
hmac = { version = "0.13.0" }
|
||||
jsonwebtoken = { version = "11.1.0" }
|
||||
jsonwebtoken = { version = "11.0.0" }
|
||||
openidconnect = { default-features = false, version = "4.0" }
|
||||
pbkdf2 = "0.13.0"
|
||||
p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
|
||||
@@ -244,13 +243,13 @@ atoi = "3.1.0"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.12.0" }
|
||||
aws-credential-types = { version = "1.3.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.120.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.148.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.116.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.118.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.146.1" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.114.0" }
|
||||
aws-smithy-async = { version = "1.3.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.4.2" }
|
||||
aws-smithy-runtime-api = { version = "1.16.2" }
|
||||
aws-smithy-types = { version = "1.6.4" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.4.0" }
|
||||
aws-smithy-runtime-api = { version = "1.16.0" }
|
||||
aws-smithy-types = { version = "1.6.3" }
|
||||
base64-simd = "0.8.0"
|
||||
brotli = "9.0.0"
|
||||
clap = { version = "4.6.7" }
|
||||
@@ -298,23 +297,23 @@ percent-encoding = "2.3.2"
|
||||
# Server-side QR rendering for TOTP enrollment, so neither the console nor the
|
||||
# CLI needs its own QR encoder. No default features: the image/render backends
|
||||
# pull in an image stack this only needs SVG and text output from.
|
||||
qrcode-rs = { version = "2.1.1", default-features = false, features = ["std", "svg"] }
|
||||
qrcode-rs = { version = "2.1.0", default-features = false, features = ["std", "svg"] }
|
||||
pin-project-lite = "0.2.17"
|
||||
pretty_assertions = "1.4.1"
|
||||
rand = { version = "0.10.2" }
|
||||
ratelimit = "2.0.0"
|
||||
rayon = "1.12.0"
|
||||
rustfs-erasure-codec = { version = "9.0.0" }
|
||||
rustfs-erasure-codec = { version = "8.0.2" }
|
||||
reed-solomon-simd = "3.1.0"
|
||||
regex = { version = "1.13.1" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.34.0" }
|
||||
redis = { version = "1.7.0" }
|
||||
rustify = { version = "0.7", default-features = false }
|
||||
rustix = { version = "1.1.5" }
|
||||
rustix = { version = "1.1.4" }
|
||||
rust-embed = { version = "8.12.0" }
|
||||
rustc-hash = { version = "2.1.3" }
|
||||
s3s = { git = "https://github.com/s3s-project/s3s.git", rev = "50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6", features = ["minio"] }
|
||||
s3s-sigv4 = { git = "https://github.com/s3s-project/s3s.git", rev = "50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6" }
|
||||
s3s = { version = "0.16.0", features = ["minio"] }
|
||||
s3s-sigv4 = { version = "0.16.0" }
|
||||
serial_test = "4.0.1"
|
||||
shadow-rs = { default-features = false, version = "2.0.0" }
|
||||
siphasher = "1.0.3"
|
||||
@@ -351,7 +350,7 @@ zstd = "0.14.0"
|
||||
# Observability and Metrics
|
||||
metrics = "0.24.6"
|
||||
metrics-util = "0.20"
|
||||
dial9 = { version = "0.5.1", default-features = false }
|
||||
dial9-tokio-telemetry = "0.5.0"
|
||||
opentelemetry = { version = "0.32.0" }
|
||||
opentelemetry-appender-tracing = { version = "0.32.0" }
|
||||
opentelemetry-otlp = { version = "0.32.0" }
|
||||
@@ -373,9 +372,9 @@ russh-sftp = "3.0.0"
|
||||
dav-server = "0.11.0"
|
||||
|
||||
# Performance Analysis and Memory Profiling
|
||||
rustfs-mimalloc = { version = "0.5.4" }
|
||||
rustfs-mimalloc = { version = "0.5.3" }
|
||||
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
|
||||
hotpath = { version = "0.26.0", default-features = false }
|
||||
hotpath = { version = "0.25.1", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.48" }
|
||||
|
||||
|
||||
@@ -141,7 +141,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# Using specific version
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.1
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0
|
||||
```
|
||||
|
||||
If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
|
||||
|
||||
+1
-1
@@ -138,7 +138,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# 使用指定版本运行
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.1
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0
|
||||
```
|
||||
|
||||
如果您通过绑定挂载启用 TLS 证书目录,也请用同样方式准备该目录:
|
||||
|
||||
@@ -663,15 +663,6 @@ pub const ENV_OBJECT_LOCK_RPC_DETACHED_LIMIT: &str = "RUSTFS_OBJECT_LOCK_RPC_DET
|
||||
/// Default per-peer budget of detached (timed-out but still running) lock RPCs: 256.
|
||||
pub const DEFAULT_OBJECT_LOCK_RPC_DETACHED_LIMIT: usize = 256;
|
||||
|
||||
/// Environment variable for the maximum number of in-flight lock acquisition
|
||||
/// RPCs admitted to one peer. Requests beyond this bound fail fast as
|
||||
/// retryable contention so a slow endpoint cannot accumulate an unbounded
|
||||
/// queue while its channel remains healthy.
|
||||
pub const ENV_OBJECT_LOCK_RPC_REQUEST_LIMIT: &str = "RUSTFS_OBJECT_LOCK_RPC_REQUEST_LIMIT";
|
||||
|
||||
/// Default per-peer in-flight lock acquisition RPC admission limit.
|
||||
pub const DEFAULT_OBJECT_LOCK_RPC_REQUEST_LIMIT: usize = 128;
|
||||
|
||||
/// Environment variable to enable object namespace lock diagnostics.
|
||||
///
|
||||
/// When enabled, RustFS emits slow lock acquisition and long lock hold
|
||||
|
||||
@@ -39,7 +39,7 @@ pub const ENV_RUNTIME_DIAL9_S3_BUCKET: &str = "RUSTFS_RUNTIME_DIAL9_S3_BUCKET";
|
||||
/// Accepted but not honoured; see [`ENV_RUNTIME_DIAL9_S3_BUCKET`].
|
||||
pub const ENV_RUNTIME_DIAL9_S3_PREFIX: &str = "RUSTFS_RUNTIME_DIAL9_S3_PREFIX";
|
||||
// Note: there are deliberately no task-dump knobs. dial9 only captures task
|
||||
// dumps for futures spawned through `dial9::spawn`, and RustFS
|
||||
// dumps for futures spawned through `dial9_tokio_telemetry::spawn`, and RustFS
|
||||
// spawns with `tokio::spawn` throughout, so the switch could never do anything.
|
||||
// Measured: 0 dumps via tokio::spawn vs 14709 via dial9::spawn on an identical
|
||||
// workload. See rustfs/backlog#1157 (D9-16).
|
||||
|
||||
@@ -55,13 +55,6 @@ pub const ENV_OBS_LOG_FILENAME: &str = "RUSTFS_OBS_LOG_FILENAME";
|
||||
pub const ENV_OBS_LOG_ROTATION_TIME: &str = "RUSTFS_OBS_LOG_ROTATION_TIME";
|
||||
pub const ENV_OBS_LOG_KEEP_FILES: &str = "RUSTFS_OBS_LOG_KEEP_FILES";
|
||||
|
||||
/// Span lifecycle events emitted by the JSON log layers (`none`|`close`|`full`).
|
||||
///
|
||||
/// Defaults to `none`: a record per span lifecycle multiplies log volume on
|
||||
/// object hot paths, and emitting the ancestor span list on every record
|
||||
/// produced multi-megabyte lines that journald truncated (backlog#2642).
|
||||
pub const ENV_OBS_LOG_SPAN_EVENTS: &str = "RUSTFS_OBS_LOG_SPAN_EVENTS";
|
||||
|
||||
/// Log cleanup related configurations
|
||||
pub const ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES: &str = "RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES";
|
||||
pub const ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES: &str = "RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES";
|
||||
@@ -106,8 +99,6 @@ pub const DEFAULT_OBS_LOG_DRY_RUN: bool = false;
|
||||
pub const DEFAULT_OBS_LOG_MATCH_MODE_PREFIX: &str = "prefix";
|
||||
pub const DEFAULT_OBS_LOG_MATCH_MODE: &str = "suffix";
|
||||
|
||||
pub const DEFAULT_OBS_LOG_SPAN_EVENTS: &str = "none";
|
||||
|
||||
/// Default values for observability configuration
|
||||
// ### Supported Environment Values
|
||||
// - `production` - Secure file-only logging
|
||||
@@ -150,7 +141,6 @@ mod tests {
|
||||
assert_eq!(ENV_OBS_LOG_FILENAME, "RUSTFS_OBS_LOG_FILENAME");
|
||||
assert_eq!(ENV_OBS_LOG_ROTATION_TIME, "RUSTFS_OBS_LOG_ROTATION_TIME");
|
||||
assert_eq!(ENV_OBS_LOG_KEEP_FILES, "RUSTFS_OBS_LOG_KEEP_FILES");
|
||||
assert_eq!(ENV_OBS_LOG_SPAN_EVENTS, "RUSTFS_OBS_LOG_SPAN_EVENTS");
|
||||
assert_eq!(ENV_OBS_TRACES_EXPORT_ENABLED, "RUSTFS_OBS_TRACES_EXPORT_ENABLED");
|
||||
assert_eq!(ENV_OBS_METRICS_EXPORT_ENABLED, "RUSTFS_OBS_METRICS_EXPORT_ENABLED");
|
||||
assert_eq!(ENV_OBS_LOGS_EXPORT_ENABLED, "RUSTFS_OBS_LOGS_EXPORT_ENABLED");
|
||||
@@ -188,7 +178,6 @@ mod tests {
|
||||
assert_eq!(DEFAULT_OBS_LOG_COMPRESSION_ALGORITHM_GZIP, "gzip");
|
||||
assert_eq!(DEFAULT_OBS_LOG_COMPRESSION_ALGORITHM_ZSTD, "zstd");
|
||||
assert_eq!(DEFAULT_OBS_LOG_MATCH_MODE_PREFIX, "prefix");
|
||||
assert_eq!(DEFAULT_OBS_LOG_SPAN_EVENTS, "none");
|
||||
assert_eq!(DEFAULT_OBS_LOG_MATCH_MODE, "suffix");
|
||||
assert_eq!(DEFAULT_OBS_LOG_COMPRESSION_ALGORITHM, "zstd");
|
||||
}
|
||||
|
||||
@@ -152,29 +152,9 @@ async fn test_compression_roundtrip() -> Result<(), Box<dyn std::error::Error +
|
||||
let content_length = head_response.content_length().unwrap_or(0);
|
||||
assert_eq!(content_length as usize, original_size, "Content-Length should be original size");
|
||||
|
||||
// A 5 KiB compressed object sits inside the inline budget, so its shard is
|
||||
// embedded in xl.meta rather than written as a part file; the stored size
|
||||
// recorded in the metadata is the compressed length.
|
||||
let part_files = find_part_files(&env.temp_dir, COMPRESSION_TEST_BUCKET, object_key)?;
|
||||
assert!(
|
||||
part_files.is_empty(),
|
||||
"small compressed object must be stored inline, found part files {part_files:?}"
|
||||
);
|
||||
let xl_meta_path = PathBuf::from(&env.temp_dir)
|
||||
.join(COMPRESSION_TEST_BUCKET)
|
||||
.join(object_key)
|
||||
.join("xl.meta");
|
||||
let file_info = rustfs_filemeta::FileMeta::load(&fs::read(&xl_meta_path)?)?.into_fileinfo(
|
||||
COMPRESSION_TEST_BUCKET,
|
||||
object_key,
|
||||
"",
|
||||
true,
|
||||
false,
|
||||
true,
|
||||
)?;
|
||||
assert!(file_info.inline_data(), "xl.meta must carry the inline marker for the compressed object");
|
||||
assert!(file_info.is_compressed(), "xl.meta must carry the compression marker");
|
||||
let total_physical_size = u64::try_from(file_info.size)?;
|
||||
assert!(!part_files.is_empty(), "expected on-disk part files for the compressed object");
|
||||
let total_physical_size = part_files_total_size(&part_files)?;
|
||||
|
||||
assert!(
|
||||
total_physical_size < original_size as u64,
|
||||
|
||||
@@ -184,7 +184,7 @@ async fn four_node_health_inventory_metrics_and_audit_delivery_are_consistent()
|
||||
let (status, metrics_body) = admin_request(
|
||||
&node.url,
|
||||
Method::GET,
|
||||
"/rustfs/admin/v3/realtime?n=1&by-host=true&by-disk=true",
|
||||
"/rustfs/admin/v3/metrics?n=1&by-host=true&by-disk=true",
|
||||
None,
|
||||
&dist.cluster.access_key,
|
||||
&dist.cluster.secret_key,
|
||||
@@ -257,7 +257,7 @@ async fn node_admin_body(dist: &DistCluster, node: usize, path: &str) -> TestRes
|
||||
}
|
||||
|
||||
async fn http_put_counts(dist: &DistCluster, node: usize) -> TestResult<[u64; 2]> {
|
||||
let body = node_admin_body(dist, node, "/rustfs/admin/v3/realtime?types=512&by-host=true&n=1").await?;
|
||||
let body = node_admin_body(dist, node, "/rustfs/admin/v3/metrics?types=512&by-host=true&n=1").await?;
|
||||
let sample: RealtimeMetrics = serde_json::from_str(body.lines().next().ok_or("empty HTTP metrics stream")?)?;
|
||||
assert!(sample.errors.is_empty(), "HTTP metrics returned errors: {:?}", sample.errors);
|
||||
let http = sample.aggregated.http.ok_or("HTTP metrics missing at WARN log level")?;
|
||||
@@ -400,36 +400,17 @@ async fn verify_write_observations_during_peer_failure(dist: &DistCluster, bucke
|
||||
let observations = storage["info"]["observations"]
|
||||
.as_array()
|
||||
.ok_or("recovery omitted observations")?;
|
||||
let disks = storage["info"]["disks"].as_array().ok_or("recovery omitted disks")?;
|
||||
Ok(observations.len() == 4
|
||||
&& observations
|
||||
.iter()
|
||||
.all(|item| item["status"] == "succeeded" && item["cached"] == false)
|
||||
&& disks.len() == 16
|
||||
&& disks.iter().all(|disk| {
|
||||
disk["state"].as_str().is_some_and(|state| state.eq_ignore_ascii_case("ok"))
|
||||
&& disk["runtimeState"]
|
||||
.as_str()
|
||||
.is_some_and(|state| state.eq_ignore_ascii_case("online"))
|
||||
}))
|
||||
.all(|item| item["status"] == "succeeded" && item["cached"] == false))
|
||||
},
|
||||
"peer probes and remote disks recover to fresh successful observations",
|
||||
"peer probes recover to fresh successful observations",
|
||||
)
|
||||
.await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
let recovered_before = http_put_counts(dist, 0).await?;
|
||||
wait_until(
|
||||
Duration::from_secs(90),
|
||||
|| async { Ok(observed_put(dist, 0, bucket, "recovered").await?.is_success()) },
|
||||
"recovered PUT after remote disk and lock convergence",
|
||||
)
|
||||
.await?;
|
||||
let recovered_after = http_put_counts(dist, 0).await?;
|
||||
assert_eq!(recovered_after[0], recovered_before[0] + 1);
|
||||
assert!(
|
||||
recovered_after[1] >= baseline[0][1] + 2,
|
||||
"recovery retries must retain the two intentional sub-quorum failures: before={recovered_before:?}, after={recovered_after:?}"
|
||||
);
|
||||
assert!(observed_put(dist, 0, bucket, "recovered").await?.is_success());
|
||||
assert_eq!(http_put_counts(dist, 0).await?, [baseline[0][0] + 1, baseline[0][1] + 2]);
|
||||
for node in 0..dist.cluster.nodes.len() {
|
||||
assert_object_bytes(&dist.client(node)?, bucket, "healthy-0", b"write-observation").await?;
|
||||
assert_object_bytes(&dist.client(node)?, bucket, "recovered", b"write-observation").await?;
|
||||
|
||||
@@ -1794,8 +1794,6 @@ async fn four_node_inline_fallback_controls() -> TestResult {
|
||||
configure_reader_metric_cluster(&mut cluster, &collector);
|
||||
let sse_master_key = base64_simd::STANDARD.encode_to_string([0x42u8; 32]);
|
||||
cluster.set_env("RUSTFS_SSE_S3_MASTER_KEY", &sse_master_key);
|
||||
// Inspect every disk only after the PUT rename fanout has drained.
|
||||
cluster.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
|
||||
cluster.start().await?;
|
||||
|
||||
let bucket = "inline-fallback-controls";
|
||||
@@ -1847,10 +1845,6 @@ async fn four_node_inline_fallback_controls() -> TestResult {
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
// 16 KiB of plaintext is 8 KiB per data shard on EC 2+2, inside the inline
|
||||
// budget: an SSE-S3 PUT is admitted inline by its plaintext size even though
|
||||
// the ciphertext length is unknown up front (backlog#2393 STOR-115).
|
||||
assert_storage_layout(&cluster, bucket, encrypted_key, None, true)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1863,8 +1857,6 @@ async fn four_node_compressed_inline_fallback() -> TestResult {
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
|
||||
configure_reader_metric_cluster(&mut cluster, &collector);
|
||||
cluster.set_env("RUSTFS_COMPRESSION_ENABLED", "true");
|
||||
// Inspect every disk only after the PUT rename fanout has drained.
|
||||
cluster.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
|
||||
cluster.start().await?;
|
||||
|
||||
let bucket = "inline-compressed-fallback";
|
||||
@@ -1885,33 +1877,6 @@ async fn four_node_compressed_inline_fallback() -> TestResult {
|
||||
ReaderPathExpectation::for_class(ReaderObject::new(bucket, key, &body, put.e_tag(), None), LEGACY_DUPLEX, COMPRESSED),
|
||||
)
|
||||
.await?;
|
||||
// 64 KiB of plaintext is 32 KiB per data shard on EC 2+2, inside the inline
|
||||
// budget: a compressed PUT is admitted inline by its plaintext size even
|
||||
// though its stored size is unknown up front (backlog#2393 STOR-115).
|
||||
assert_storage_layout(&cluster, bucket, key, None, true)?;
|
||||
|
||||
// A server-side copy onto another compressible key re-compresses the
|
||||
// decompressed source stream and lands inline the same way (STOR-125).
|
||||
let copy_key = "compressed/copy.txt";
|
||||
let copy = client
|
||||
.copy_object()
|
||||
.bucket(bucket)
|
||||
.key(copy_key)
|
||||
.copy_source(format!("{bucket}/{key}"))
|
||||
.send()
|
||||
.await?;
|
||||
let copy_etag = copy.copy_object_result().and_then(|result| result.e_tag()).map(str::to_owned);
|
||||
assert_reader_path(
|
||||
&collector,
|
||||
&client,
|
||||
ReaderPathExpectation::for_class(
|
||||
ReaderObject::new(bucket, copy_key, &body, copy_etag.as_deref(), None),
|
||||
LEGACY_DUPLEX,
|
||||
COMPRESSED,
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
assert_storage_layout(&cluster, bucket, copy_key, None, true)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -391,9 +391,8 @@ pub mod data_usage {
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::data_movement::scanner_backlog::test_util::NativeScannerPauseBacklogWriteFault;
|
||||
pub use crate::data_movement::scanner_backlog::{
|
||||
MAX_SCANNER_PAUSE_BACKLOG_BYTES, ScannerPauseBacklogRetirementError, ScannerPauseBacklogRetirementPlan,
|
||||
ScannerPauseBacklogRetirementPlanner, ScannerPauseBacklogRetirementReplica,
|
||||
register_scanner_pause_backlog_retirement_planner,
|
||||
MAX_SCANNER_PAUSE_BACKLOG_BYTES, ScannerPauseBacklogRetirementPlan, ScannerPauseBacklogRetirementPlanner,
|
||||
ScannerPauseBacklogRetirementReplica, register_scanner_pause_backlog_retirement_planner,
|
||||
};
|
||||
pub use crate::data_usage::{
|
||||
DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
|
||||
|
||||
@@ -515,26 +515,20 @@ impl ExpiryStats {
|
||||
Self::add_nonnegative(&self.missed_tier_journal_tasks, 1);
|
||||
}
|
||||
|
||||
// The pending and active gauges are balanced by design: every increment
|
||||
// has exactly one matching decrement. They must not saturate at zero on
|
||||
// update, because a worker can dequeue (and decrement) before the
|
||||
// enqueuing side has recorded its increment. Clamping that transient -1
|
||||
// to 0 turns the later +1 into a phantom task that never drains
|
||||
// (rustfs#7921). Readers clamp negative snapshots instead.
|
||||
fn increment_pending_tasks(&self) {
|
||||
self.pending_tasks.fetch_add(1, Ordering::AcqRel);
|
||||
Self::add_nonnegative(&self.pending_tasks, 1);
|
||||
}
|
||||
|
||||
fn decrement_pending_tasks(&self) {
|
||||
self.pending_tasks.fetch_sub(1, Ordering::AcqRel);
|
||||
Self::add_nonnegative(&self.pending_tasks, -1);
|
||||
}
|
||||
|
||||
fn increment_active_tasks(&self) {
|
||||
self.active_tasks.fetch_add(1, Ordering::AcqRel);
|
||||
Self::add_nonnegative(&self.active_tasks, 1);
|
||||
}
|
||||
|
||||
fn decrement_active_tasks(&self) {
|
||||
self.active_tasks.fetch_sub(1, Ordering::AcqRel);
|
||||
Self::add_nonnegative(&self.active_tasks, -1);
|
||||
}
|
||||
|
||||
fn increment_workers(&self) {
|
||||
@@ -1080,13 +1074,9 @@ impl ExpiryState {
|
||||
}
|
||||
|
||||
fn send_expiry_task(&self, wrkr: Sender<Option<ExpiryOpType>>, task: ExpiryOpType) -> bool {
|
||||
// Account for the task before a worker can observe it. The worker
|
||||
// decrements on dequeue, so incrementing after `try_send` would let a
|
||||
// fast dequeue run the gauge through zero first.
|
||||
self.stats.increment_pending_tasks();
|
||||
let queued = wrkr.try_send(Some(task)).is_ok();
|
||||
if !queued {
|
||||
self.stats.decrement_pending_tasks();
|
||||
if queued {
|
||||
self.stats.increment_pending_tasks();
|
||||
}
|
||||
queued
|
||||
}
|
||||
@@ -1454,14 +1444,11 @@ async fn enqueue_recovered_free_version_with_state(state: &Arc<RwLock<ExpiryStat
|
||||
return false;
|
||||
};
|
||||
|
||||
// Same ordering rule as `ExpiryState::send_expiry_task`: count first, so
|
||||
// a worker that dequeues immediately cannot decrement before this
|
||||
// increment lands.
|
||||
stats.increment_pending_tasks();
|
||||
let queued = wrkr.try_send(Some(Box::new(task))).is_ok();
|
||||
if !queued {
|
||||
stats.decrement_pending_tasks();
|
||||
stats.increment_missed_freevers_tasks();
|
||||
} else {
|
||||
stats.increment_pending_tasks();
|
||||
}
|
||||
stats.record_scanner_expiry_state();
|
||||
queued
|
||||
@@ -3770,8 +3757,11 @@ pub async fn enqueue_transition_immediate(oi: &ObjectInfo, src: LcEventSrc) {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn enqueue_immediate_expiry(api: Arc<ECStore>, oi: &ObjectInfo, src: LcEventSrc, opts: &ObjectOptions) {
|
||||
let configs = match metadata_boundary::get_expiry_configs_for_options(&api, &oi.bucket, opts).await {
|
||||
pub async fn enqueue_immediate_expiry(oi: &ObjectInfo, src: LcEventSrc) {
|
||||
let Some(api) = runtime_sources::object_store_handle() else {
|
||||
return;
|
||||
};
|
||||
let configs = match metadata_boundary::get_expiry_configs(&api, &oi.bucket).await {
|
||||
Ok(configs) => configs,
|
||||
Err(err) => {
|
||||
observe_lifecycle_observability_event(EVENT_LIFECYCLE_EVALUATION_FAILED, "failed", Some("metadata_unavailable"));
|
||||
@@ -7582,65 +7572,6 @@ mod tests {
|
||||
assert!(recovery_notify.notified().now_or_never().is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn expiry_pending_gauge_survives_dequeue_landing_before_enqueue_accounting() {
|
||||
// A worker may dequeue and decrement before the enqueuing side records
|
||||
// its increment. The gauge must return to zero afterwards instead of
|
||||
// clamping the transient -1 away and reporting a phantom pending task
|
||||
// that no idle check can ever drain (rustfs#7921).
|
||||
let state = ExpiryState::new();
|
||||
let stats = Arc::clone(&state.read().await.stats);
|
||||
|
||||
stats.decrement_pending_tasks();
|
||||
stats.increment_pending_tasks();
|
||||
assert_eq!(stats.pending_tasks(), 0);
|
||||
assert_eq!(state.read().await.pending_tasks(), 0);
|
||||
|
||||
stats.decrement_active_tasks();
|
||||
stats.increment_active_tasks();
|
||||
assert_eq!(stats.active_tasks(), 0);
|
||||
assert_eq!(state.read().await.active_tasks(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn free_version_enqueue_rolls_back_pending_when_queue_full() {
|
||||
// Single-threaded, so this cannot observe the count-before-publish
|
||||
// ordering itself; it pins the rollback that ordering requires: a
|
||||
// rejected send must not leave its speculative increment behind.
|
||||
let state = ExpiryState::new_with_unconsumed_worker_channel(1);
|
||||
let oi = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
transitioned_object: TransitionedObject {
|
||||
name: "remote/object".to_string(),
|
||||
version_id: "remote-version".to_string(),
|
||||
tier: "WARM".to_string(),
|
||||
free_version: true,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(state.read().await.enqueue_free_version(oi.clone()));
|
||||
assert_eq!(state.read().await.stats.pending_tasks(), 1);
|
||||
|
||||
// The single-slot queue is full for both enqueue paths.
|
||||
assert!(!enqueue_recovered_free_version_with_state(&state, oi.clone()).await);
|
||||
assert_eq!(state.read().await.stats.pending_tasks(), 1);
|
||||
assert_eq!(state.read().await.stats.missed_free_vers_tasks(), 1);
|
||||
|
||||
assert!(!state.read().await.enqueue_free_version(oi));
|
||||
assert_eq!(state.read().await.stats.pending_tasks(), 1);
|
||||
assert_eq!(state.read().await.stats.missed_free_vers_tasks(), 2);
|
||||
|
||||
// Draining the one real task returns the gauge to zero.
|
||||
let receiver = state.read().await.tasks_rx[0].clone();
|
||||
let task = receiver.lock().await.recv().await.expect("queued task");
|
||||
assert!(task.is_some());
|
||||
state.read().await.stats.decrement_pending_tasks();
|
||||
assert_eq!(state.read().await.pending_tasks(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn enqueue_recovered_free_version_reports_false_without_worker_channel() {
|
||||
let state = ExpiryState::new();
|
||||
@@ -12772,8 +12703,7 @@ mod tests {
|
||||
.push((event, state, reason));
|
||||
});
|
||||
|
||||
super::enqueue_immediate_expiry(Arc::clone(&ecstore), &object_info, LcEventSrc::S3PutObject, &ObjectOptions::default())
|
||||
.await;
|
||||
super::enqueue_immediate_expiry(&object_info, LcEventSrc::S3PutObject).await;
|
||||
|
||||
assert!(
|
||||
observed.lock().expect("observed events should not poison").contains(&(
|
||||
|
||||
@@ -55,15 +55,7 @@ pub(crate) async fn lifecycle_expiry_allowed(
|
||||
}
|
||||
|
||||
pub(crate) async fn get_expiry_configs(api: &crate::store::ECStore, bucket: &str) -> Result<LifecycleExpiryConfigs> {
|
||||
get_expiry_configs_for_options(api, bucket, &crate::object_api::ObjectOptions::default()).await
|
||||
}
|
||||
|
||||
pub(crate) async fn get_expiry_configs_for_options(
|
||||
api: &crate::store::ECStore,
|
||||
bucket: &str,
|
||||
opts: &crate::object_api::ObjectOptions,
|
||||
) -> Result<LifecycleExpiryConfigs> {
|
||||
let bucket_incarnation_id = metadata_sys::get_bucket_incarnation_id_for_options_in(&api.ctx, bucket, opts).await?;
|
||||
let bucket_incarnation_id = api.bucket_incarnation_id_from_disk(bucket).await?;
|
||||
let metadata = get_authoritative_metadata(api, bucket, bucket_incarnation_id).await?;
|
||||
let table_bucket_enabled = metadata.table_bucket_enabled();
|
||||
|
||||
|
||||
@@ -35,7 +35,6 @@ use crate::bucket::lifecycle::tier_sweeper::{
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::{Error, Result as EcstoreResult};
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions};
|
||||
use crate::services::notification_sys::acquire_remote_version_state_writer_fleet_proof;
|
||||
use crate::services::tier::{tier::TierConfigMgr, warm_backend::TransitionCandidateProbe};
|
||||
use crate::storage_api_contracts::{
|
||||
list::ListOperations as _,
|
||||
@@ -1238,12 +1237,6 @@ pub async fn delete_transition_candidate_for_operator(
|
||||
lease
|
||||
.validate_remote_version_id(remote_version_id)
|
||||
.map_err(TransitionOperatorError::Remote)?;
|
||||
if remote_version_requires_fleet_proof(remote_version_id) && acquire_remote_version_state_writer_fleet_proof().is_none() {
|
||||
return Err(TransitionOperatorError::Remote(std::io::Error::new(
|
||||
std::io::ErrorKind::Unsupported,
|
||||
"opaque remote tier version cleanup requires the operator-attested live fleet capability gate",
|
||||
)));
|
||||
}
|
||||
let before_delete_probe = lease
|
||||
.probe_transition_candidate_for(&transaction.remote_object, transaction.transaction_id)
|
||||
.await
|
||||
@@ -1852,11 +1845,6 @@ async fn recover_unknown_upload_outcome(
|
||||
))
|
||||
}
|
||||
TransitionCandidateProbe::VersionedPresent(version_id) => {
|
||||
if remote_version_requires_fleet_proof(&version_id) && acquire_remote_version_state_writer_fleet_proof().is_none() {
|
||||
return Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(
|
||||
IlmRecoveryErrorCode::RemoteVersionUnknown,
|
||||
));
|
||||
}
|
||||
cleanup_recovered_unknown_upload_candidate(api, transaction, TransitionRemoteVersion::versioned(version_id)).await
|
||||
}
|
||||
TransitionCandidateProbe::Ambiguous => Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(
|
||||
@@ -1952,14 +1940,6 @@ fn transition_source_lookup_options(transaction: &TransitionTransaction) -> Obje
|
||||
async fn delete_transition_remote_candidate(api: Arc<ECStore>, transaction: &TransitionTransaction) -> EcstoreResult<()> {
|
||||
let version_id = transaction.remote_version.tier_delete_version_id().unwrap_or_default();
|
||||
let version_id_exact = transaction.remote_version.kind == TransitionRemoteVersionKind::Versioned;
|
||||
if version_id_exact
|
||||
&& remote_version_requires_fleet_proof(version_id)
|
||||
&& acquire_remote_version_state_writer_fleet_proof().is_none()
|
||||
{
|
||||
return Err(Error::other(
|
||||
"opaque remote tier version cleanup requires the operator-attested live fleet capability gate",
|
||||
));
|
||||
}
|
||||
delete_object_from_remote_tier_idempotent_with_manager_and_identity(
|
||||
&transaction.remote_object,
|
||||
version_id,
|
||||
@@ -1973,10 +1953,6 @@ async fn delete_transition_remote_candidate(api: Arc<ECStore>, transaction: &Tra
|
||||
.map_err(Error::other)
|
||||
}
|
||||
|
||||
fn remote_version_requires_fleet_proof(version_id: &str) -> bool {
|
||||
!version_id.is_empty() && Uuid::parse_str(version_id).is_err()
|
||||
}
|
||||
|
||||
pub async fn recover_transition_transaction_records(
|
||||
api: Arc<ECStore>,
|
||||
limit: usize,
|
||||
@@ -2463,14 +2439,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn opaque_remote_versions_require_the_fleet_proof_boundary() {
|
||||
assert!(!remote_version_requires_fleet_proof(""));
|
||||
assert!(!remote_version_requires_fleet_proof(&Uuid::new_v4().to_string()));
|
||||
assert!(remote_version_requires_fleet_proof("null"));
|
||||
assert!(remote_version_requires_fleet_proof("b2-opaque-version"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_distinguishes_unknown_unversioned_and_versioned() {
|
||||
assert_eq!(TransitionRemoteVersion::known_from_put_response("").tier_delete_version_id(), None);
|
||||
|
||||
@@ -1464,13 +1464,9 @@ pub(crate) async fn load_bucket_metadata_parse_with_presence(
|
||||
"bucket incarnation sidecar is missing for new-format metadata: {bucket}"
|
||||
)));
|
||||
}
|
||||
} else if incarnation.is_some() {
|
||||
return Err(Error::other("bucket incarnation sidecar exists without bucket metadata"));
|
||||
}
|
||||
// A sidecar without `.metadata.bin` is the window between the two legacy
|
||||
// migration writes (rustfs/rustfs#8003): the sidecar lands first, and a
|
||||
// crash or lost namespace lease before the metadata write leaves the
|
||||
// bucket in this state on every node. Report it as not persisted so the
|
||||
// migration runs again; the writers adopt the stored incarnation instead
|
||||
// of minting a new one, so the sidecar keeps its authority.
|
||||
|
||||
bm.default_timestamps();
|
||||
|
||||
|
||||
@@ -24,7 +24,6 @@ use crate::bucket::metadata::{load_bucket_metadata_parse, load_bucket_metadata_p
|
||||
use crate::bucket::utils::is_meta_bucketname;
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::{Error, Result, is_err_bucket_not_found, is_err_strict_volume_not_found};
|
||||
use crate::object_api::ObjectOptions;
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use crate::storage_api_contracts::heal::HealOperations as _;
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
@@ -491,18 +490,6 @@ pub(crate) async fn get_bucket_incarnation_id_in(ctx: &crate::runtime::instance:
|
||||
sys.get_bucket_incarnation_id_from_disk(bucket).await
|
||||
}
|
||||
|
||||
/// Validate against disk while retaining an already-held Object Lock fence.
|
||||
pub(crate) async fn get_bucket_incarnation_id_for_options_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<Uuid> {
|
||||
let sys = bucket_metadata_sys_of(ctx)?;
|
||||
let sys = sys.read().await.clone();
|
||||
let guard = acquire_bucket_metadata_transaction_read_lock_for_options_in(ctx, bucket, opts).await?;
|
||||
sys.get_bucket_incarnation_id_under_transaction_lock(bucket, &guard).await
|
||||
}
|
||||
|
||||
pub(crate) async fn get_cached_bucket_incarnation_id_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
@@ -1068,24 +1055,6 @@ pub(crate) async fn acquire_bucket_metadata_transaction_read_lock_in(
|
||||
Ok(lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?)
|
||||
}
|
||||
|
||||
/// Readers already holding an Object Lock snapshot must share its guard:
|
||||
/// a fresh read acquisition can queue behind a writer waiting for that snapshot.
|
||||
pub(crate) async fn acquire_bucket_metadata_transaction_read_lock_for_options_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<Arc<rustfs_lock::NamespaceLockGuard>> {
|
||||
if let Some(snapshot) = opts.object_lock_config_snapshot.as_ref() {
|
||||
let store = object_store_in(ctx).await?;
|
||||
return snapshot
|
||||
.metadata_transaction_guard_for(store.id, bucket, opts.expected_bucket_incarnation_id)
|
||||
.ok_or_else(|| {
|
||||
Error::other("Object Lock snapshot does not hold a valid metadata transaction fence for this bucket")
|
||||
});
|
||||
}
|
||||
Ok(Arc::new(acquire_bucket_metadata_transaction_read_lock_in(ctx, bucket).await?))
|
||||
}
|
||||
|
||||
async fn acquire_transaction_lock_with_sys(
|
||||
sys: &Arc<RwLock<BucketMetadataSys>>,
|
||||
bucket: &str,
|
||||
@@ -2388,17 +2357,8 @@ impl BucketMetadataSys {
|
||||
let _transaction_guard = transaction_lock
|
||||
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await?;
|
||||
self.get_bucket_incarnation_id_under_transaction_lock(bucket, &_transaction_guard)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get_bucket_incarnation_id_under_transaction_lock(
|
||||
&self,
|
||||
bucket: &str,
|
||||
transaction_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
) -> Result<Uuid> {
|
||||
let incarnation_id = load_bucket_incarnation(self.object_store(), bucket).await?;
|
||||
if transaction_guard.is_lock_lost() {
|
||||
if _transaction_guard.is_lock_lost() {
|
||||
return Err(Error::other(format!("bucket incarnation metadata transaction lock was lost: {bucket}")));
|
||||
}
|
||||
match incarnation_id {
|
||||
@@ -2504,16 +2464,6 @@ impl BucketMetadataSys {
|
||||
if !persisted {
|
||||
metadata = BucketMetadata::new(bucket);
|
||||
metadata.created = bucket_info.created.unwrap_or(OffsetDateTime::UNIX_EPOCH);
|
||||
// An interrupted migration may already have published this
|
||||
// bucket's incarnation; re-read it under the transaction lock so
|
||||
// the retry never replaces an identity other nodes fenced on. A
|
||||
// retired incarnation is residue of a deleted bucket and must not
|
||||
// come back, or heal would reclaim the bucket's new objects.
|
||||
if let Some(stored) = load_bucket_incarnation(self.object_store(), bucket).await?
|
||||
&& !crate::bucket::retirement::is_retired(self.object_store(), bucket, stored).await?
|
||||
{
|
||||
metadata.bucket_incarnation_id = stored;
|
||||
}
|
||||
} else if metadata.bucket_incarnation_id.is_nil() {
|
||||
metadata.bucket_incarnation_id = Uuid::new_v4();
|
||||
}
|
||||
@@ -3566,98 +3516,6 @@ mod tests {
|
||||
assert!(err.to_string().contains("sidecar is missing"));
|
||||
}
|
||||
|
||||
/// rustfs/rustfs#8003: the legacy migration writes the incarnation sidecar
|
||||
/// before `.metadata.bin`. A crash or lost namespace lease between the two
|
||||
/// left every pre-existing bucket answering 500 on every request, with no
|
||||
/// path back. The sidecar-only state must load as a legacy bucket, and the
|
||||
/// retried migration must keep the stored incarnation.
|
||||
#[tokio::test]
|
||||
async fn issue_8003_sidecar_without_metadata_loads_as_legacy_and_migrates_with_its_incarnation() {
|
||||
let (dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
let sys = BucketMetadataSys::new(ecstore.clone());
|
||||
let bucket = "issue-8003-sidecar-only";
|
||||
for dir in &dirs {
|
||||
std::fs::create_dir_all(dir.path().join(bucket)).unwrap();
|
||||
}
|
||||
let stored = Uuid::new_v4();
|
||||
save_bucket_incarnation(ecstore.clone(), bucket, stored)
|
||||
.await
|
||||
.expect("simulate the interrupted migration's first write");
|
||||
|
||||
let (fabricated, persisted) = load_bucket_metadata_parse_with_presence(ecstore.clone(), bucket, true)
|
||||
.await
|
||||
.expect("a sidecar without metadata must read as a legacy bucket, not fail closed");
|
||||
assert!(!persisted);
|
||||
assert!(!fabricated.bucket_incarnation_sidecar);
|
||||
|
||||
let (_, fabricated_read) = sys
|
||||
.get_config(bucket)
|
||||
.await
|
||||
.expect("request-path metadata reads must not fail closed on the sidecar-only state");
|
||||
assert!(fabricated_read);
|
||||
|
||||
let migrated = sys
|
||||
.get_authoritative_metadata(bucket)
|
||||
.await
|
||||
.expect("the retried legacy migration must complete");
|
||||
assert!(migrated.bucket_incarnation_sidecar);
|
||||
assert_eq!(
|
||||
migrated.bucket_incarnation_id, stored,
|
||||
"the retry must adopt the published incarnation instead of minting a new one"
|
||||
);
|
||||
assert_eq!(sys.get_bucket_incarnation_id(bucket).await.unwrap(), stored);
|
||||
|
||||
let on_disk = sys.get_config_from_disk(bucket).await.expect("metadata is now persisted");
|
||||
assert!(on_disk.bucket_incarnation_sidecar);
|
||||
assert_eq!(on_disk.bucket_incarnation_id, stored);
|
||||
assert_eq!(load_bucket_incarnation(ecstore, bucket).await.unwrap(), Some(stored));
|
||||
}
|
||||
|
||||
/// A sidecar left behind by a deleted bucket names a retired incarnation.
|
||||
/// The migration must mint a new identity for a same-name volume instead
|
||||
/// of re-publishing the retired one, or heal would treat the bucket's new
|
||||
/// objects as reclaimable residue.
|
||||
#[tokio::test]
|
||||
async fn issue_8003_sidecar_only_migration_does_not_adopt_a_retired_incarnation() {
|
||||
let (dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
let sys = BucketMetadataSys::new(ecstore.clone());
|
||||
let bucket = "issue-8003-retired-sidecar";
|
||||
for dir in &dirs {
|
||||
std::fs::create_dir_all(dir.path().join(bucket)).unwrap();
|
||||
}
|
||||
let retired = Uuid::new_v4();
|
||||
save_bucket_incarnation(ecstore.clone(), bucket, retired)
|
||||
.await
|
||||
.expect("residual sidecar");
|
||||
crate::bucket::retirement::commit_retirement(
|
||||
ecstore.clone(),
|
||||
bucket,
|
||||
retired,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("retirement record");
|
||||
|
||||
let migrated = sys
|
||||
.get_authoritative_metadata(bucket)
|
||||
.await
|
||||
.expect("the migration must still complete for the same-name volume");
|
||||
assert!(migrated.bucket_incarnation_sidecar);
|
||||
assert_ne!(
|
||||
migrated.bucket_incarnation_id, retired,
|
||||
"a retired incarnation must never be re-published"
|
||||
);
|
||||
assert!(!migrated.bucket_incarnation_id.is_nil());
|
||||
assert_eq!(
|
||||
load_bucket_incarnation(ecstore, bucket).await.unwrap(),
|
||||
Some(migrated.bucket_incarnation_id),
|
||||
"the sidecar must be rewritten to the new incarnation"
|
||||
);
|
||||
}
|
||||
|
||||
/// Concurrent cache misses for one bucket must collapse into a single disk
|
||||
/// load.
|
||||
///
|
||||
|
||||
@@ -38,7 +38,6 @@ const MAX_ORPHANS_REAPED_PER_WRITE: usize = 64;
|
||||
const MAX_ORPHAN_PROBES_PER_WRITE: usize = 128;
|
||||
const ORPHAN_PROBE_CONCURRENCY: usize = 32;
|
||||
const EVENT_QUOTA_LEDGER_SETTLEMENT: &str = "quota_ledger_settlement";
|
||||
const EVENT_QUOTA_ADMISSION: &str = "quota_admission";
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_QUOTA: &str = "quota";
|
||||
|
||||
@@ -47,8 +46,6 @@ static FAIL_NEXT_LEDGER_SAVE: std::sync::atomic::AtomicBool = std::sync::atomic:
|
||||
|
||||
// Lock order: caller-held destination object/upload, bucket metadata
|
||||
// transaction (read), operation reservation, then quota ledger.
|
||||
// When Object Lock already holds the metadata transaction read lock, reuse
|
||||
// that guard: reacquiring it behind a waiting metadata writer would deadlock.
|
||||
|
||||
#[cfg(not(any(test, feature = "test-util")))]
|
||||
const ORPHAN_MIN_AGE_SECONDS: i64 = 30;
|
||||
@@ -213,7 +210,7 @@ pub(crate) struct QuotaContext {
|
||||
capability_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
|
||||
snapshot_admission: Option<QuotaAdmission>,
|
||||
legacy_data_movement: bool,
|
||||
metadata_guard: Option<Arc<NamespaceLockGuard>>,
|
||||
metadata_guard: Option<NamespaceLockGuard>,
|
||||
pool_index: Option<usize>,
|
||||
set_index: Option<usize>,
|
||||
}
|
||||
@@ -298,7 +295,7 @@ impl QuotaContext {
|
||||
limit: quota_limit,
|
||||
});
|
||||
}
|
||||
if operation_guard.is_lock_lost() || metadata_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
|
||||
if operation_guard.is_lock_lost() || metadata_guard.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost) {
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "quota_reservation",
|
||||
bucket: ledger_data.bucket.clone(),
|
||||
@@ -335,7 +332,7 @@ struct LedgerReservationData {
|
||||
pub(crate) struct QuotaReservation {
|
||||
ledger: Option<LedgerReservationData>,
|
||||
operation_guard: Option<NamespaceLockGuard>,
|
||||
metadata_guard: Option<Arc<NamespaceLockGuard>>,
|
||||
metadata_guard: Option<NamespaceLockGuard>,
|
||||
capability_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
|
||||
state: ReservationState,
|
||||
}
|
||||
@@ -349,7 +346,7 @@ enum ReservationState {
|
||||
}
|
||||
|
||||
impl QuotaReservation {
|
||||
fn unlimited(metadata_guard: Option<Arc<NamespaceLockGuard>>) -> Self {
|
||||
fn unlimited(metadata_guard: Option<NamespaceLockGuard>) -> Self {
|
||||
Self {
|
||||
ledger: None,
|
||||
operation_guard: None,
|
||||
@@ -361,7 +358,7 @@ impl QuotaReservation {
|
||||
|
||||
pub(crate) fn is_lock_lost(&self) -> bool {
|
||||
self.operation_guard.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost)
|
||||
|| self.metadata_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| self.metadata_guard.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost)
|
||||
}
|
||||
|
||||
pub(crate) fn capability_proof_matches(&self) -> bool {
|
||||
@@ -441,12 +438,11 @@ pub(crate) async fn begin(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
snapshot_admission: Option<QuotaAdmission>,
|
||||
data_movement: bool,
|
||||
pool_index: usize,
|
||||
set_index: usize,
|
||||
) -> Result<QuotaContext> {
|
||||
let snapshot_admission = opts.quota_admission;
|
||||
let data_movement = opts.data_movement;
|
||||
if crate::bucket::utils::is_meta_bucketname(bucket) {
|
||||
return Ok(QuotaContext {
|
||||
store: None,
|
||||
@@ -501,7 +497,7 @@ pub(crate) async fn begin(
|
||||
});
|
||||
}
|
||||
|
||||
let metadata_guard = metadata_sys::acquire_bucket_metadata_transaction_read_lock_for_options_in(ctx, bucket, opts).await?;
|
||||
let metadata_guard = metadata_sys::acquire_bucket_metadata_transaction_read_lock_in(ctx, bucket).await?;
|
||||
let (quota, bucket_incarnation, quota_revision) =
|
||||
metadata_sys::get_quota_config_and_incarnation_from_disk_in(ctx, bucket).await?;
|
||||
if metadata_guard.is_lock_lost() {
|
||||
@@ -517,7 +513,6 @@ pub(crate) async fn begin(
|
||||
.as_ref()
|
||||
.is_some_and(|quota| quota.has_unsupported_reservation_protocol())
|
||||
{
|
||||
log_admission_rejected(bucket, object, "unsupported_reservation_protocol");
|
||||
return Err(StorageError::PartMissingOrCorrupt);
|
||||
}
|
||||
let durable_quota = quota.as_ref().filter(|quota| quota.uses_durable_reservations());
|
||||
@@ -534,16 +529,7 @@ pub(crate) async fn begin(
|
||||
Some(quota) => match (quota.quota, snapshot_admission) {
|
||||
(Some(limit), Some(admission)) if admission.quota_limit() == limit => Some(admission),
|
||||
(Some(_), None) if data_movement => None,
|
||||
(Some(_), _) => {
|
||||
// A snapshot-protocol quota requires the request handler's
|
||||
// admission on every write. Missing or mismatched admission
|
||||
// means a caller rebuilt `ObjectOptions` without carrying it
|
||||
// over (rustfs/rustfs#7674 lost it on CopyObject); fail closed
|
||||
// but leave a diagnosable trace, because the storage error is
|
||||
// the generic `PartMissingOrCorrupt`.
|
||||
log_admission_rejected(bucket, object, "snapshot_quota_admission_missing");
|
||||
return Err(StorageError::PartMissingOrCorrupt);
|
||||
}
|
||||
(Some(_), _) => return Err(StorageError::PartMissingOrCorrupt),
|
||||
(None, _) => None,
|
||||
},
|
||||
None => None,
|
||||
@@ -918,18 +904,6 @@ pub fn fail_next_quota_ledger_save_for_test() {
|
||||
FAIL_NEXT_LEDGER_SAVE.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
}
|
||||
|
||||
fn log_admission_rejected(bucket: &str, object: &str, state: &'static str) {
|
||||
warn!(
|
||||
event = EVENT_QUOTA_ADMISSION,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_QUOTA,
|
||||
state,
|
||||
bucket = %bucket,
|
||||
object = %object,
|
||||
"quota admission rejected the write before commit"
|
||||
);
|
||||
}
|
||||
|
||||
fn log_deferred_settlement(data: &LedgerReservationData, state: &'static str, err: &StorageError) {
|
||||
warn!(
|
||||
event = EVENT_QUOTA_LEDGER_SETTLEMENT,
|
||||
|
||||
@@ -199,38 +199,6 @@ fn has_raw_status(err: &SdkError<HeadObjectError>, status: u16) -> bool {
|
||||
err.raw_response().is_some_and(|r| r.status().as_u16() == status)
|
||||
}
|
||||
|
||||
fn raw_header_value(err: &SdkError<HeadObjectError>, name: &str) -> Option<String> {
|
||||
err.raw_response()
|
||||
.and_then(|response| response.headers().get(name))
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum DeleteMarkerHeadOutcome {
|
||||
Missing,
|
||||
Exists,
|
||||
Ambiguous,
|
||||
}
|
||||
|
||||
fn classify_delete_marker_head_error(err: &SdkError<HeadObjectError>) -> DeleteMarkerHeadOutcome {
|
||||
let (is_not_found, code) = err
|
||||
.as_service_error()
|
||||
.map(|service_err| (service_err.is_not_found(), service_err.code()))
|
||||
.unwrap_or((false, None));
|
||||
|
||||
// Prefer the raw HTTP status when present. HEAD responses commonly have no
|
||||
// body, and a contradictory XML code must not turn a 403/5xx into success.
|
||||
match err.raw_response().map(|response| response.status().as_u16()) {
|
||||
Some(404) => DeleteMarkerHeadOutcome::Missing,
|
||||
Some(405) => DeleteMarkerHeadOutcome::Exists,
|
||||
Some(_) => DeleteMarkerHeadOutcome::Ambiguous,
|
||||
None if is_not_found => DeleteMarkerHeadOutcome::Missing,
|
||||
None if !is_retryable_delete_replication_head_error(false, code) => DeleteMarkerHeadOutcome::Exists,
|
||||
None => DeleteMarkerHeadOutcome::Ambiguous,
|
||||
}
|
||||
}
|
||||
|
||||
fn metadata_requires_existing_target(op_type: ReplicationType, object_info: &ObjectInfo) -> bool {
|
||||
op_type == ReplicationType::Metadata
|
||||
&& object_info
|
||||
@@ -1841,14 +1809,32 @@ async fn verify_resync_head_result(
|
||||
(roi.size, None)
|
||||
}
|
||||
Err(err) if roi.delete_marker => {
|
||||
// A marker creation is verified by either absence (404) or
|
||||
// presence (405). A marker-version purge has the opposite goal:
|
||||
// only absence proves completion. All other outcomes remain
|
||||
// failed (backlog#862 / #799 B13).
|
||||
let retryable = if roi.version_purge_status.is_empty() {
|
||||
matches!(classify_delete_marker_head_error(&err), DeleteMarkerHeadOutcome::Ambiguous)
|
||||
} else {
|
||||
!matches!(classify_delete_marker_head_error(&err), DeleteMarkerHeadOutcome::Missing)
|
||||
// Verifying a replicated delete marker: only a
|
||||
// definitive 404/NoSuchKey or 405/MethodNotAllowed
|
||||
// confirms the marker propagated. Any other
|
||||
// (retryable/ambiguous) HEAD error leaves the outcome
|
||||
// unverified, so it must count as failed — not as a
|
||||
// blanket success (backlog#862 / #799 B13).
|
||||
// A HEAD carries no body, so the SDK only synthesizes an error
|
||||
// code for 404; a target answering the delete marker's version
|
||||
// with 405 (RustFS, MinIO, AWS) leaves `code()` empty and the
|
||||
// raw status is the only evidence (rustfs/backlog#2479).
|
||||
// A marker *version purge* (`version_purge_status` set) asks
|
||||
// for the opposite outcome: the marker must be gone from the
|
||||
// target, so only absence verifies it and a 405 proves the
|
||||
// marker is still there.
|
||||
let retryable = {
|
||||
let (is_not_found, code) = err
|
||||
.as_service_error()
|
||||
.map(|se| (se.is_not_found(), se.code()))
|
||||
.unwrap_or((false, None));
|
||||
let not_found = is_not_found || has_raw_status(&err, 404);
|
||||
if roi.version_purge_status.is_empty() {
|
||||
let code = code.or_else(|| has_raw_status(&err, 405).then_some("405"));
|
||||
is_retryable_delete_replication_head_error(not_found, code)
|
||||
} else {
|
||||
!not_found
|
||||
}
|
||||
};
|
||||
if retryable {
|
||||
st.failed_count += 1;
|
||||
@@ -3567,7 +3553,7 @@ async fn replicate_delete_to_target(
|
||||
}
|
||||
}
|
||||
|
||||
if !is_version_purge && dobj.delete_object.delete_marker && dobj.delete_object.delete_marker_version_id.is_some() {
|
||||
if dobj.delete_object.delete_marker && dobj.delete_object.delete_marker_version_id.is_some() {
|
||||
match head_object_for_worker(
|
||||
tgt_client.as_ref(),
|
||||
&tgt_client.bucket,
|
||||
@@ -3576,48 +3562,20 @@ async fn replicate_delete_to_target(
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(head) if head.delete_marker == Some(true) => {
|
||||
// Some targets expose an existing marker as a successful HEAD
|
||||
// with the marker bit instead of the usual 405. Treat that as
|
||||
// positive convergence evidence and retain the target version
|
||||
// id for any later marker purge.
|
||||
rinfo.target_delete_marker_version_id = head.version_id.filter(|version_id| !version_id.is_empty());
|
||||
rinfo.replication_status = ReplicationStatusType::Completed;
|
||||
return rinfo;
|
||||
}
|
||||
Ok(_) => {
|
||||
// A successful HEAD without the delete-marker bit proves that
|
||||
// the addressed version is a regular object (or that the
|
||||
// target omitted the marker contract). Creating a marker in
|
||||
// that state would hide a live version, so fail closed.
|
||||
rinfo.replication_status = ReplicationStatusType::Failed;
|
||||
rinfo.error = Some("target version is not a delete marker".to_string());
|
||||
return rinfo;
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
match classify_delete_marker_head_error(e.as_ref()) {
|
||||
DeleteMarkerHeadOutcome::Exists => {
|
||||
// An explicit HEAD of a delete-marker version is expected
|
||||
// to answer 405 (often without an SDK error code). That
|
||||
// is positive evidence the marker already exists, so the
|
||||
// creation is complete and must not issue another
|
||||
// versionless DELETE that would mint a second marker.
|
||||
if let Some(version_id) = raw_header_value(e.as_ref(), "x-amz-version-id") {
|
||||
rinfo.target_delete_marker_version_id = Some(version_id);
|
||||
}
|
||||
rinfo.replication_status = ReplicationStatusType::Completed;
|
||||
return rinfo;
|
||||
}
|
||||
DeleteMarkerHeadOutcome::Missing => {}
|
||||
DeleteMarkerHeadOutcome::Ambiguous => {
|
||||
// Only a definitive not-found means the marker is absent
|
||||
// and may be created. Keep transport and ambiguous
|
||||
// failures fail-closed instead of creating blindly.
|
||||
rinfo.replication_status = ReplicationStatusType::Failed;
|
||||
rinfo.error = Some(e.to_string());
|
||||
mark_replication_target_offline_if_needed(&tgt_client, &e).await;
|
||||
return rinfo;
|
||||
}
|
||||
let non_retryable = matches!(
|
||||
e.as_ref(),
|
||||
SdkError::ServiceError(service_err)
|
||||
if is_retryable_delete_replication_head_error(
|
||||
service_err.err().is_not_found(),
|
||||
service_err.err().code(),
|
||||
)
|
||||
);
|
||||
if non_retryable {
|
||||
rinfo.replication_status = ReplicationStatusType::Failed;
|
||||
rinfo.error = Some(e.to_string());
|
||||
return rinfo;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -6571,110 +6529,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn marker_creation_dobj(target_arn: &str) -> DeletedObjectReplicationInfo {
|
||||
let mut state = ReplicationState::default();
|
||||
state.targets.insert(target_arn.to_string(), ReplicationStatusType::Pending);
|
||||
DeletedObjectReplicationInfo {
|
||||
bucket: "source".to_string(),
|
||||
target_arn: target_arn.to_string(),
|
||||
op_type: ReplicationType::Heal,
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: "gone.txt".to_string(),
|
||||
delete_marker: true,
|
||||
delete_marker_version_id: Some(Uuid::new_v4()),
|
||||
replication_state: Some(state),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// A target that already has the marker answers a versioned HEAD with a
|
||||
/// bodiless 405. The live worker must settle the marker as completed and
|
||||
/// avoid minting another marker with a versionless DELETE.
|
||||
#[tokio::test]
|
||||
async fn live_delete_marker_worker_accepts_bodiless_405_without_delete() {
|
||||
let (endpoint, server) = spawn_scripted_target_server(1, |_| {
|
||||
"HTTP/1.1 405 Method Not Allowed\r\nX-Amz-Version-Id: target-marker\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
|
||||
.to_string()
|
||||
});
|
||||
let target = test_target_client(endpoint);
|
||||
register_test_target(&target).await;
|
||||
let dobj = marker_creation_dobj(&target.arn);
|
||||
|
||||
let rinfo = replicate_delete_to_target(&dobj, target.clone(), None).await;
|
||||
|
||||
assert_eq!(rinfo.replication_status, ReplicationStatusType::Completed, "{rinfo:?}");
|
||||
assert!(rinfo.error.is_none(), "{rinfo:?}");
|
||||
assert_eq!(rinfo.target_delete_marker_version_id.as_deref(), Some("target-marker"));
|
||||
let seen = server.join().expect("test HTTP server should finish");
|
||||
assert_eq!(seen.len(), 1, "an existing marker must not be re-created: {seen:?}");
|
||||
assert!(seen[0].starts_with("HEAD "), "{seen:?}");
|
||||
}
|
||||
|
||||
/// A successful HEAD that does not identify a delete marker is not
|
||||
/// convergence evidence: creating a marker would hide a live version.
|
||||
#[tokio::test]
|
||||
async fn live_delete_marker_worker_rejects_successful_regular_head_without_delete() {
|
||||
let (endpoint, server) = spawn_scripted_target_server(1, |_| empty_response("200 OK"));
|
||||
let target = test_target_client(endpoint);
|
||||
register_test_target(&target).await;
|
||||
let dobj = marker_creation_dobj(&target.arn);
|
||||
|
||||
let rinfo = replicate_delete_to_target(&dobj, target.clone(), None).await;
|
||||
|
||||
assert_eq!(rinfo.replication_status, ReplicationStatusType::Failed, "{rinfo:?}");
|
||||
assert_eq!(rinfo.error.as_deref(), Some("target version is not a delete marker"));
|
||||
let seen = server.join().expect("test HTTP server should finish");
|
||||
assert_eq!(seen.len(), 1, "a regular object must not be hidden by a new marker: {seen:?}");
|
||||
assert!(seen[0].starts_with("HEAD "), "{seen:?}");
|
||||
}
|
||||
|
||||
/// A definitive 404 means the marker is absent, so exactly one marker
|
||||
/// creation DELETE is still required.
|
||||
#[tokio::test]
|
||||
async fn live_delete_marker_worker_creates_marker_after_404() {
|
||||
let (endpoint, server) = spawn_scripted_target_server(2, |line| {
|
||||
if line.starts_with("HEAD ") {
|
||||
empty_response("404 Not Found")
|
||||
} else if line.starts_with("DELETE ") {
|
||||
empty_response("204 No Content")
|
||||
} else {
|
||||
empty_response("500 Unexpected")
|
||||
}
|
||||
});
|
||||
let target = test_target_client(endpoint);
|
||||
register_test_target(&target).await;
|
||||
let dobj = marker_creation_dobj(&target.arn);
|
||||
|
||||
let rinfo = replicate_delete_to_target(&dobj, target.clone(), None).await;
|
||||
|
||||
assert_eq!(rinfo.replication_status, ReplicationStatusType::Completed, "{rinfo:?}");
|
||||
let seen = server.join().expect("test HTTP server should finish");
|
||||
assert_eq!(seen.len(), 2, "a missing marker should be created once: {seen:?}");
|
||||
assert!(seen[0].starts_with("HEAD "), "{seen:?}");
|
||||
assert!(seen[1].starts_with("DELETE "), "{seen:?}");
|
||||
assert!(!seen[1].contains("versionId="), "marker creation must remain versionless: {seen:?}");
|
||||
}
|
||||
|
||||
/// Ambiguous service errors must remain failed and must not fall through
|
||||
/// to a blind marker-creation DELETE.
|
||||
#[tokio::test]
|
||||
async fn live_delete_marker_worker_keeps_ambiguous_head_failure_closed() {
|
||||
let (endpoint, server) = spawn_scripted_target_server(1, |_| empty_response("503 Service Unavailable"));
|
||||
let target = test_target_client(endpoint);
|
||||
register_test_target(&target).await;
|
||||
let dobj = marker_creation_dobj(&target.arn);
|
||||
|
||||
let rinfo = replicate_delete_to_target(&dobj, target.clone(), None).await;
|
||||
|
||||
assert_eq!(rinfo.replication_status, ReplicationStatusType::Failed, "{rinfo:?}");
|
||||
assert!(rinfo.error.is_some(), "{rinfo:?}");
|
||||
let seen = server.join().expect("test HTTP server should finish");
|
||||
assert_eq!(seen.len(), 1, "an ambiguous HEAD must not trigger DELETE: {seen:?}");
|
||||
assert!(seen[0].starts_with("HEAD "), "{seen:?}");
|
||||
}
|
||||
|
||||
/// A target answers `HEAD ?versionId=<delete marker>` with a bodiless
|
||||
/// 405, so the SDK reports no error code. That is the marker having
|
||||
/// propagated, not a failure (rustfs/backlog#2479).
|
||||
|
||||
@@ -23,7 +23,7 @@ use std::{
|
||||
future::Future,
|
||||
io::ErrorKind,
|
||||
pin::Pin,
|
||||
sync::{Arc, OnceLock, atomic::AtomicBool},
|
||||
sync::{Arc, OnceLock},
|
||||
task::{Context, Poll},
|
||||
time::Duration,
|
||||
};
|
||||
@@ -234,8 +234,6 @@ pub struct ListPathRawOptions {
|
||||
pub skip_walkdir_total_timeout: bool,
|
||||
pub walkdir_timeout: Option<Duration>,
|
||||
pub walkdir_stall_timeout: Option<Duration>,
|
||||
/// Shared terminal state for bounded producer walks.
|
||||
pub producer_limit_reached: Option<Arc<AtomicBool>>,
|
||||
pub agreed: Option<AgreedFn>,
|
||||
pub partial: Option<PartialFn>,
|
||||
pub finished: Option<FinishedFn>,
|
||||
@@ -269,7 +267,6 @@ impl Clone for ListPathRawOptions {
|
||||
skip_walkdir_total_timeout: self.skip_walkdir_total_timeout,
|
||||
walkdir_timeout: self.walkdir_timeout,
|
||||
walkdir_stall_timeout: self.walkdir_stall_timeout,
|
||||
producer_limit_reached: self.producer_limit_reached.clone(),
|
||||
#[cfg(test)]
|
||||
test_reader_behaviors: self.test_reader_behaviors.clone(),
|
||||
#[cfg(test)]
|
||||
@@ -295,7 +292,6 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
|
||||
skip_total_timeout: opts.skip_walkdir_total_timeout,
|
||||
timeout_ms: opts.walkdir_timeout.map(duration_millis),
|
||||
stall_timeout_ms: opts.walkdir_stall_timeout.map(duration_millis),
|
||||
producer_limit_reached: opts.producer_limit_reached.clone(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1170,11 +1170,6 @@ impl RemoteDisk {
|
||||
self.health.force_runtime_state_for_test(state);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn force_offline_for_test(&self) {
|
||||
self.health.force_offline_for_test();
|
||||
}
|
||||
|
||||
/// Same as [`DiskHealthTracker::reset_for_store_init_retry`]: undo a transient faulty mark before another format load attempt.
|
||||
pub fn reset_health_for_store_init_retry(&self) {
|
||||
self.health.reset_for_store_init_retry(&self.endpoint);
|
||||
@@ -2141,7 +2136,7 @@ impl RemoteDisk {
|
||||
let file_info_bin = encode_file_info_msgpack(fi)?;
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(RenameDataRequest {
|
||||
bucket_incarnation_id: crate::store::bucket_heal_scope_for_object(dst_volume, dst_path)
|
||||
bucket_incarnation_id: crate::store::bucket_heal_scope(dst_volume)
|
||||
.map(|scope| scope.incarnation.as_bytes().to_vec().into())
|
||||
.unwrap_or_default(),
|
||||
disk: self.endpoint.to_string(),
|
||||
|
||||
@@ -47,37 +47,10 @@ fn attach_lock_mutation_body_digest<T: rustfs_protos::CanonicalMutationBody>(req
|
||||
|
||||
/// Work to run if an RPC that already timed out for its caller completes later.
|
||||
type LateCompletion<T> = Option<Box<dyn FnOnce(T) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send>>;
|
||||
type LateFailure = LateCompletion<()>;
|
||||
|
||||
struct RpcCleanup<T> {
|
||||
late: LateCompletion<T>,
|
||||
late_failure: LateFailure,
|
||||
admission: Option<LockRequestAdmission>,
|
||||
}
|
||||
|
||||
impl<T> RpcCleanup<T> {
|
||||
fn new(late: LateCompletion<T>, late_failure: LateFailure, admission: Option<LockRequestAdmission>) -> Self {
|
||||
Self {
|
||||
late,
|
||||
late_failure,
|
||||
admission,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The liveness window is this many RPC deadlines: a peer that completed a
|
||||
/// lock RPC within it is slow, not gone, and keeps its channel on a timeout.
|
||||
const LOCK_RPC_LIVENESS_WINDOW_DEADLINES: u32 = 2;
|
||||
const LOCK_RPC_REQUEST_BACKOFF_THRESHOLD: u32 = 3;
|
||||
const LOCK_RPC_REQUEST_BACKOFF_MAX: Duration = Duration::from_secs(30);
|
||||
const LATE_RELEASE_RETRY_DELAYS: [Duration; 3] = [Duration::from_millis(100), Duration::from_millis(500), Duration::from_secs(1)];
|
||||
const LATE_RELEASE_IN_FLIGHT_LIMIT: usize = 32;
|
||||
const TIMEOUT_LOG_STATE_MAX: usize = 1024;
|
||||
const TIMEOUT_LOG_STATE_RETENTION: Duration = Duration::from_secs(600);
|
||||
|
||||
fn is_request_breaker_operation(op: &'static str) -> bool {
|
||||
matches!(op, "lock" | "lock_batch")
|
||||
}
|
||||
|
||||
/// Recent history of the shared lock channel to one peer (issue #7363).
|
||||
///
|
||||
@@ -91,44 +64,15 @@ struct LockPeerChannelHealth {
|
||||
last_success: Option<Instant>,
|
||||
last_eviction: Option<Instant>,
|
||||
consecutive_timeouts: u32,
|
||||
request_consecutive_timeouts: u32,
|
||||
request_backoff_until: Option<Instant>,
|
||||
request_backoff: Duration,
|
||||
request_probe_generation: Option<u64>,
|
||||
next_probe_generation: u64,
|
||||
request_in_flight: usize,
|
||||
late_release_in_flight: usize,
|
||||
/// Timed-out RPCs still running in the background for this peer.
|
||||
detached_rpcs: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default)]
|
||||
struct TimeoutLogHealth {
|
||||
last_logged: Option<Instant>,
|
||||
suppressed: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct LockRequestAdmission {
|
||||
probe_generation: Option<u64>,
|
||||
}
|
||||
|
||||
impl LockRequestAdmission {
|
||||
fn is_probe(self) -> bool {
|
||||
self.probe_generation.is_some()
|
||||
}
|
||||
}
|
||||
|
||||
fn lock_peer_channel_health() -> &'static Mutex<HashMap<String, LockPeerChannelHealth>> {
|
||||
static HEALTH: OnceLock<Mutex<HashMap<String, LockPeerChannelHealth>>> = OnceLock::new();
|
||||
HEALTH.get_or_init(Mutex::default)
|
||||
}
|
||||
|
||||
fn timeout_log_health() -> &'static Mutex<HashMap<(String, &'static str), TimeoutLogHealth>> {
|
||||
static HEALTH: OnceLock<Mutex<HashMap<(String, &'static str), TimeoutLogHealth>>> = OnceLock::new();
|
||||
HEALTH.get_or_init(Mutex::default)
|
||||
}
|
||||
|
||||
fn with_lock_peer_health<R>(addr: &str, update: impl FnOnce(&mut LockPeerChannelHealth) -> R) -> R {
|
||||
let mut peers = lock_peer_channel_health()
|
||||
.lock()
|
||||
@@ -399,163 +343,17 @@ impl RemoteClient {
|
||||
)
|
||||
}
|
||||
|
||||
fn request_limit() -> usize {
|
||||
rustfs_utils::get_env_usize(
|
||||
rustfs_config::ENV_OBJECT_LOCK_RPC_REQUEST_LIMIT,
|
||||
rustfs_config::DEFAULT_OBJECT_LOCK_RPC_REQUEST_LIMIT,
|
||||
)
|
||||
.max(1)
|
||||
}
|
||||
|
||||
fn liveness_window(deadline: Duration) -> Duration {
|
||||
deadline.saturating_mul(LOCK_RPC_LIVENESS_WINDOW_DEADLINES)
|
||||
}
|
||||
|
||||
fn record_rpc_success(&self, op: &'static str, admission: Option<LockRequestAdmission>) {
|
||||
fn record_rpc_success(&self) {
|
||||
with_lock_peer_health(&self.addr, |health| {
|
||||
health.last_success = Some(Instant::now());
|
||||
health.consecutive_timeouts = 0;
|
||||
if is_request_breaker_operation(op) {
|
||||
let is_probe = admission.is_some_and(LockRequestAdmission::is_probe);
|
||||
if health.request_backoff_until.is_none() || is_probe {
|
||||
health.request_consecutive_timeouts = 0;
|
||||
health.request_backoff_until = None;
|
||||
health.request_backoff = Duration::ZERO;
|
||||
}
|
||||
if is_probe && admission.is_some_and(|admission| health.request_probe_generation == admission.probe_generation) {
|
||||
health.request_probe_generation = None;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn admit_late_release(&self) -> bool {
|
||||
with_lock_peer_health(&self.addr, |health| {
|
||||
if health.late_release_in_flight >= LATE_RELEASE_IN_FLIGHT_LIMIT {
|
||||
false
|
||||
} else {
|
||||
health.late_release_in_flight += 1;
|
||||
true
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn release_late_release(&self) {
|
||||
with_lock_peer_health(&self.addr, |health| {
|
||||
health.late_release_in_flight = health.late_release_in_flight.saturating_sub(1);
|
||||
});
|
||||
}
|
||||
|
||||
/// Admit a new lock acquisition unless this peer is in the request-level
|
||||
/// backoff window. A single half-open probe is allowed after the window;
|
||||
/// channel liveness and request admission are deliberately separate.
|
||||
fn admit_lock_request(&self, op: &'static str) -> Option<LockRequestAdmission> {
|
||||
if !is_request_breaker_operation(op) {
|
||||
return Some(LockRequestAdmission { probe_generation: None });
|
||||
}
|
||||
|
||||
let now = Instant::now();
|
||||
let (admission, suppression_reason) = with_lock_peer_health(&self.addr, |health| {
|
||||
let degraded = health.request_backoff_until.is_some() || health.request_consecutive_timeouts > 0;
|
||||
if degraded && health.request_in_flight >= Self::request_limit() {
|
||||
return (None, "in_flight_limit");
|
||||
}
|
||||
let Some(until) = health.request_backoff_until else {
|
||||
health.request_in_flight += 1;
|
||||
return (Some(LockRequestAdmission { probe_generation: None }), "none");
|
||||
};
|
||||
if now < until || health.request_probe_generation.is_some() {
|
||||
return (None, "breaker_open");
|
||||
}
|
||||
health.next_probe_generation = health.next_probe_generation.saturating_add(1);
|
||||
let generation = health.next_probe_generation;
|
||||
health.request_probe_generation = Some(generation);
|
||||
health.request_in_flight += 1;
|
||||
(
|
||||
Some(LockRequestAdmission {
|
||||
probe_generation: Some(generation),
|
||||
}),
|
||||
"none",
|
||||
)
|
||||
});
|
||||
if admission.is_none() {
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_request_suppressed(&self.addr, op, suppression_reason);
|
||||
}
|
||||
admission
|
||||
}
|
||||
|
||||
fn release_lock_request(&self, op: &'static str, admission: Option<LockRequestAdmission>) {
|
||||
if is_request_breaker_operation(op) && admission.is_some() {
|
||||
with_lock_peer_health(&self.addr, |health| {
|
||||
health.request_in_flight = health.request_in_flight.saturating_sub(1);
|
||||
if admission.is_some_and(|admission| health.request_probe_generation == admission.probe_generation) {
|
||||
health.request_probe_generation = None;
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
fn record_request_timeout(&self, op: &'static str, deadline: Duration) {
|
||||
if !is_request_breaker_operation(op) {
|
||||
return;
|
||||
}
|
||||
|
||||
let now = Instant::now();
|
||||
with_lock_peer_health(&self.addr, |health| {
|
||||
health.request_consecutive_timeouts = health.request_consecutive_timeouts.saturating_add(1);
|
||||
if health.request_consecutive_timeouts < LOCK_RPC_REQUEST_BACKOFF_THRESHOLD {
|
||||
return;
|
||||
}
|
||||
|
||||
let initial = deadline / 2;
|
||||
let next = if health.request_backoff.is_zero() {
|
||||
initial.max(Duration::from_millis(1))
|
||||
} else {
|
||||
health.request_backoff.saturating_mul(2)
|
||||
};
|
||||
health.request_backoff = next.min(LOCK_RPC_REQUEST_BACKOFF_MAX);
|
||||
health.request_backoff_until = Some(now + health.request_backoff);
|
||||
});
|
||||
}
|
||||
|
||||
fn timeout_log_decision(&self, op: &'static str, interval: Duration) -> (bool, u64) {
|
||||
let now = Instant::now();
|
||||
let mut peers = timeout_log_health().lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if peers.len() >= TIMEOUT_LOG_STATE_MAX {
|
||||
peers.retain(|_, state| {
|
||||
state
|
||||
.last_logged
|
||||
.is_some_and(|at| now.saturating_duration_since(at) < TIMEOUT_LOG_STATE_RETENTION)
|
||||
});
|
||||
}
|
||||
let key = (self.addr.clone(), op);
|
||||
if peers.len() >= TIMEOUT_LOG_STATE_MAX
|
||||
&& !peers.contains_key(&key)
|
||||
&& let Some(eviction_key) = peers.keys().next().cloned()
|
||||
{
|
||||
peers.remove(&eviction_key);
|
||||
}
|
||||
let state = peers.entry(key).or_default();
|
||||
if state
|
||||
.last_logged
|
||||
.is_some_and(|at| now.saturating_duration_since(at) < interval)
|
||||
{
|
||||
state.suppressed = state.suppressed.saturating_add(1);
|
||||
return (false, state.suppressed);
|
||||
}
|
||||
|
||||
let suppressed = state.suppressed;
|
||||
state.last_logged = Some(now);
|
||||
state.suppressed = 0;
|
||||
(true, suppressed)
|
||||
}
|
||||
|
||||
fn bounded_acquire_request(request: &LockRequest, deadline: Duration) -> LockRequest {
|
||||
let mut bounded = request.clone();
|
||||
bounded.acquire_timeout = bounded.acquire_timeout.min(deadline);
|
||||
bounded
|
||||
}
|
||||
|
||||
/// Apply the per-peer eviction policy after a failed RPC.
|
||||
async fn maybe_evict_connection(
|
||||
&self,
|
||||
@@ -608,8 +406,6 @@ impl RemoteClient {
|
||||
resource_summary: &str,
|
||||
handle: JoinHandle<std::result::Result<T, tonic::Status>>,
|
||||
late: LateCompletion<T>,
|
||||
late_failure: LateFailure,
|
||||
admission: Option<LockRequestAdmission>,
|
||||
) {
|
||||
let limit = Self::detached_rpc_limit();
|
||||
let admitted = with_lock_peer_health(&self.addr, |health| {
|
||||
@@ -622,10 +418,6 @@ impl RemoteClient {
|
||||
});
|
||||
if !admitted {
|
||||
handle.abort();
|
||||
self.release_lock_request(op, admission);
|
||||
if let Some(late_failure) = late_failure {
|
||||
drop(tokio::spawn(late_failure(())));
|
||||
}
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_detached(op, "aborted");
|
||||
debug!(
|
||||
addr = %self.addr,
|
||||
@@ -640,19 +432,12 @@ impl RemoteClient {
|
||||
let addr = self.addr.clone();
|
||||
tokio::spawn(async move {
|
||||
let outcome = handle.await;
|
||||
with_lock_peer_health(&addr, |health| {
|
||||
health.detached_rpcs = health.detached_rpcs.saturating_sub(1);
|
||||
if is_request_breaker_operation(op) && admission.is_some() {
|
||||
health.request_in_flight = health.request_in_flight.saturating_sub(1);
|
||||
if admission.is_some_and(|admission| health.request_probe_generation == admission.probe_generation) {
|
||||
health.request_probe_generation = None;
|
||||
}
|
||||
}
|
||||
});
|
||||
with_lock_peer_health(&addr, |health| health.detached_rpcs = health.detached_rpcs.saturating_sub(1));
|
||||
match outcome {
|
||||
Ok(Ok(response)) => {
|
||||
with_lock_peer_health(&addr, |health| {
|
||||
health.last_success = Some(Instant::now());
|
||||
health.consecutive_timeouts = 0;
|
||||
});
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "success");
|
||||
if let Some(late) = late {
|
||||
@@ -660,9 +445,6 @@ impl RemoteClient {
|
||||
}
|
||||
}
|
||||
Ok(Err(status)) => {
|
||||
if let Some(late_failure) = late_failure {
|
||||
late_failure(()).await;
|
||||
}
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "error");
|
||||
debug!(
|
||||
addr = %addr,
|
||||
@@ -673,9 +455,6 @@ impl RemoteClient {
|
||||
);
|
||||
}
|
||||
Err(join_error) => {
|
||||
if let Some(late_failure) = late_failure {
|
||||
late_failure(()).await;
|
||||
}
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "join_error");
|
||||
debug!(addr = %addr, op, error = %join_error, "Detached remote lock RPC task ended abnormally");
|
||||
}
|
||||
@@ -706,79 +485,24 @@ impl RemoteClient {
|
||||
}))
|
||||
}
|
||||
|
||||
fn late_release_failure_hook(&self, lock_ids: Vec<LockId>) -> LateFailure {
|
||||
let client = self.clone();
|
||||
Some(Box::new(move |_| {
|
||||
Box::pin(async move {
|
||||
client.release_late_acquisitions(lock_ids).await;
|
||||
})
|
||||
}))
|
||||
}
|
||||
|
||||
/// A lock granted after its caller stopped waiting is an orphan until its
|
||||
/// lease expires; hand it back right away, best effort.
|
||||
async fn release_late_acquisitions(&self, mut lock_ids: Vec<LockId>) {
|
||||
if !self.admit_late_release() {
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_late_release("failed");
|
||||
let (log_warning, suppressed_timeouts) = self.timeout_log_decision("late_release", Self::rpc_timeout());
|
||||
if log_warning {
|
||||
warn!(
|
||||
addr = %self.addr,
|
||||
count = lock_ids.len(),
|
||||
outcome = "budget_exhausted",
|
||||
suppressed_timeouts,
|
||||
"Skipped late remote lock release because the cleanup budget is exhausted"
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
let original_count = lock_ids.len();
|
||||
let mut outcome = "failed";
|
||||
for (attempt, delay) in std::iter::once(Duration::ZERO).chain(LATE_RELEASE_RETRY_DELAYS).enumerate() {
|
||||
if !delay.is_zero() {
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
|
||||
match self.release_locks_batch(&lock_ids).await {
|
||||
Ok(released) => {
|
||||
lock_ids = lock_ids
|
||||
.into_iter()
|
||||
.zip(released)
|
||||
.filter_map(|(lock_id, released)| (!released).then_some(lock_id))
|
||||
.collect();
|
||||
if lock_ids.is_empty() {
|
||||
outcome = "released";
|
||||
break;
|
||||
}
|
||||
outcome = if attempt == LATE_RELEASE_RETRY_DELAYS.len() {
|
||||
"partial"
|
||||
} else {
|
||||
"failed"
|
||||
};
|
||||
}
|
||||
Err(_) => {
|
||||
outcome = "failed";
|
||||
}
|
||||
}
|
||||
}
|
||||
self.release_late_release();
|
||||
async fn release_late_acquisitions(&self, lock_ids: Vec<LockId>) {
|
||||
let outcome = match self.release_locks_batch(&lock_ids).await {
|
||||
Ok(released) if released.iter().all(|released| *released) => "released",
|
||||
Ok(_) => "partial",
|
||||
Err(_) => "failed",
|
||||
};
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_late_release(outcome);
|
||||
if outcome == "released" {
|
||||
debug!(addr = %self.addr, count = original_count, "Released remote locks granted after their caller timed out");
|
||||
debug!(addr = %self.addr, count = lock_ids.len(), "Released remote locks granted after their caller timed out");
|
||||
} else {
|
||||
let (log_warning, suppressed_timeouts) = self.timeout_log_decision("late_release", Self::rpc_timeout());
|
||||
if log_warning {
|
||||
warn!(
|
||||
addr = %self.addr,
|
||||
count = original_count,
|
||||
outcome,
|
||||
suppressed_timeouts,
|
||||
"Could not release every remote lock granted after its caller timed out; the server lease will expire it"
|
||||
);
|
||||
} else {
|
||||
debug!(addr = %self.addr, count = original_count, outcome, suppressed_timeouts, "Suppressed repeated late remote lock release warning");
|
||||
}
|
||||
warn!(
|
||||
addr = %self.addr,
|
||||
count = lock_ids.len(),
|
||||
outcome,
|
||||
"Could not release every remote lock granted after its caller timed out; the server lease will expire it"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -788,74 +512,26 @@ impl RemoteClient {
|
||||
resource_summary: &str,
|
||||
deadline: Duration,
|
||||
future: Fut,
|
||||
cleanup: RpcCleanup<T>,
|
||||
late: LateCompletion<T>,
|
||||
) -> std::result::Result<T, LockError>
|
||||
where
|
||||
Fut: Future<Output = std::result::Result<T, tonic::Status>> + Send + 'static,
|
||||
T: Send + 'static,
|
||||
{
|
||||
let RpcCleanup {
|
||||
late,
|
||||
late_failure,
|
||||
admission,
|
||||
} = cleanup;
|
||||
let mut handle = tokio::spawn(future);
|
||||
match timeout(deadline, &mut handle).await {
|
||||
Ok(Ok(Ok(response))) => {
|
||||
self.record_rpc_success(op, admission);
|
||||
self.release_lock_request(op, admission);
|
||||
self.record_rpc_success();
|
||||
Ok(response)
|
||||
}
|
||||
Ok(Ok(Err(err))) => {
|
||||
let reason = err.to_string();
|
||||
if err.code() == tonic::Code::DeadlineExceeded {
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_timeout(&self.addr, op);
|
||||
self.record_request_timeout(op, deadline);
|
||||
let (log_timeout, suppressed_timeouts) = self.timeout_log_decision(op, deadline);
|
||||
if Self::is_scanner_leader_lock(resource_summary) {
|
||||
debug!(
|
||||
addr = %self.addr,
|
||||
op,
|
||||
timeout_ms = deadline.as_millis(),
|
||||
resource_summary,
|
||||
"Remote lock RPC returned deadline exceeded for scanner leader lock"
|
||||
);
|
||||
} else if log_timeout {
|
||||
warn!(
|
||||
addr = %self.addr,
|
||||
op,
|
||||
timeout_ms = deadline.as_millis(),
|
||||
resource_summary,
|
||||
suppressed_timeouts,
|
||||
"Remote lock RPC returned deadline exceeded"
|
||||
);
|
||||
} else {
|
||||
debug!(
|
||||
addr = %self.addr,
|
||||
op,
|
||||
timeout_ms = deadline.as_millis(),
|
||||
resource_summary,
|
||||
suppressed_timeouts,
|
||||
"Suppressed repeated remote lock RPC deadline"
|
||||
);
|
||||
}
|
||||
self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Timeout, deadline)
|
||||
.await;
|
||||
self.release_lock_request(op, admission);
|
||||
if let Some(late_failure) = late_failure {
|
||||
drop(tokio::spawn(late_failure(())));
|
||||
}
|
||||
return Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), deadline));
|
||||
}
|
||||
// Only evict (and re-dial) the cached channel when the failure is a genuine
|
||||
// transport problem. A server-produced application status (auth denied, peer
|
||||
// lock service not ready, invalid args, ...) arrives on a perfectly healthy
|
||||
// channel; evicting it just churns the connection and pushes the peer toward
|
||||
// the offline threshold for no benefit. See issue #4567.
|
||||
let transport_failure = Self::is_transport_failure(&err);
|
||||
if transport_failure {
|
||||
self.record_request_timeout(op, deadline);
|
||||
}
|
||||
if Self::is_scanner_leader_lock(resource_summary) {
|
||||
debug!(
|
||||
addr = %self.addr,
|
||||
@@ -867,31 +543,6 @@ impl RemoteClient {
|
||||
transport_failure,
|
||||
"Remote lock RPC returned tonic error for scanner leader lock"
|
||||
);
|
||||
} else if transport_failure {
|
||||
let (log_failure, suppressed_failures) = self.timeout_log_decision(op, deadline);
|
||||
if log_failure {
|
||||
warn!(
|
||||
addr = %self.addr,
|
||||
op,
|
||||
timeout_ms = deadline.as_millis(),
|
||||
resource_summary,
|
||||
tonic_code = ?err.code(),
|
||||
tonic_message = err.message(),
|
||||
transport_failure,
|
||||
suppressed_failures,
|
||||
"Remote lock RPC returned transport error"
|
||||
);
|
||||
} else {
|
||||
debug!(
|
||||
addr = %self.addr,
|
||||
op,
|
||||
timeout_ms = deadline.as_millis(),
|
||||
resource_summary,
|
||||
tonic_code = ?err.code(),
|
||||
suppressed_failures,
|
||||
"Suppressed repeated remote lock transport error"
|
||||
);
|
||||
}
|
||||
} else {
|
||||
warn!(
|
||||
addr = %self.addr,
|
||||
@@ -908,10 +559,6 @@ impl RemoteClient {
|
||||
self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Transport, deadline)
|
||||
.await;
|
||||
}
|
||||
self.release_lock_request(op, admission);
|
||||
if transport_failure && let Some(late_failure) = late_failure {
|
||||
drop(tokio::spawn(late_failure(())));
|
||||
}
|
||||
Err(LockError::internal(format!("{op} RPC failed: {reason}")))
|
||||
}
|
||||
Ok(Err(join_error)) => {
|
||||
@@ -922,17 +569,11 @@ impl RemoteClient {
|
||||
error = %join_error,
|
||||
"Remote lock RPC task ended abnormally"
|
||||
);
|
||||
self.release_lock_request(op, admission);
|
||||
if let Some(late_failure) = late_failure {
|
||||
drop(tokio::spawn(late_failure(())));
|
||||
}
|
||||
Err(LockError::internal(format!("{op} RPC task failed: {join_error}")))
|
||||
}
|
||||
Err(_) => {
|
||||
let reason = format!("RPC timed out after {deadline:?}");
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_timeout(&self.addr, op);
|
||||
self.record_request_timeout(op, deadline);
|
||||
let (log_timeout, suppressed_timeouts) = self.timeout_log_decision(op, deadline);
|
||||
if Self::is_scanner_leader_lock(resource_summary) {
|
||||
debug!(
|
||||
addr = %self.addr,
|
||||
@@ -941,28 +582,18 @@ impl RemoteClient {
|
||||
resource_summary,
|
||||
"Remote lock RPC timed out for scanner leader lock"
|
||||
);
|
||||
} else if log_timeout {
|
||||
} else {
|
||||
warn!(
|
||||
addr = %self.addr,
|
||||
op,
|
||||
timeout_ms = deadline.as_millis(),
|
||||
resource_summary,
|
||||
suppressed_timeouts,
|
||||
"Remote lock RPC timed out"
|
||||
);
|
||||
} else {
|
||||
debug!(
|
||||
addr = %self.addr,
|
||||
op,
|
||||
timeout_ms = deadline.as_millis(),
|
||||
resource_summary,
|
||||
suppressed_timeouts,
|
||||
"Suppressed repeated remote lock RPC timeout"
|
||||
);
|
||||
}
|
||||
self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Timeout, deadline)
|
||||
.await;
|
||||
self.detach_timed_out_rpc(op, resource_summary, handle, late, late_failure, admission);
|
||||
self.detach_timed_out_rpc(op, resource_summary, handle, late);
|
||||
Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), deadline))
|
||||
}
|
||||
}
|
||||
@@ -990,14 +621,6 @@ impl RemoteClient {
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn rpc_backoff_failure_response(request: &LockRequest) -> LockResponse {
|
||||
LockResponse::failure("Remote lock RPC timed out: peer request breaker is open", request.acquire_timeout)
|
||||
}
|
||||
|
||||
fn rpc_backoff_failure_batch(requests: &[LockRequest]) -> Vec<LockResponse> {
|
||||
requests.iter().map(Self::rpc_backoff_failure_response).collect()
|
||||
}
|
||||
|
||||
fn build_lock_info(request: &LockRequest, lock_info_json: Option<String>) -> LockInfo {
|
||||
if let Some(lock_info_json) = lock_info_json {
|
||||
match serde_json::from_str::<LockInfo>(&lock_info_json) {
|
||||
@@ -1057,45 +680,22 @@ impl RemoteClient {
|
||||
impl LockClient for RemoteClient {
|
||||
async fn acquire_lock(&self, request: &LockRequest) -> Result<LockResponse> {
|
||||
info!("remote acquire_exclusive for {}", request.resource);
|
||||
let admission = match self.admit_lock_request("lock") {
|
||||
Some(admission) => admission,
|
||||
None => return Ok(Self::rpc_backoff_failure_response(request)),
|
||||
};
|
||||
|
||||
let rpc_timeout = Self::rpc_timeout();
|
||||
let bounded_request = Self::bounded_acquire_request(request, rpc_timeout);
|
||||
let mut client = match self.get_client().await {
|
||||
Ok(client) => client,
|
||||
Err(err) => {
|
||||
self.record_request_timeout("lock", rpc_timeout);
|
||||
self.release_lock_request("lock", Some(admission));
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = request.resource.to_string();
|
||||
let args = match serde_json::to_string(&bounded_request) {
|
||||
Ok(args) => args,
|
||||
Err(err) => {
|
||||
self.release_lock_request("lock", Some(admission));
|
||||
return Err(LockError::internal(format!("Failed to serialize request: {err}")));
|
||||
}
|
||||
};
|
||||
let mut req = Request::new(GenerallyLockRequest { args });
|
||||
req.set_timeout(rpc_timeout);
|
||||
if let Err(err) = attach_lock_mutation_body_digest(&mut req) {
|
||||
self.release_lock_request("lock", Some(admission));
|
||||
return Err(err.into());
|
||||
}
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
|
||||
});
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
let late = self.late_release_hook(request.lock_id.clone());
|
||||
let late_failure = self.late_release_failure_hook(vec![request.lock_id.clone()]);
|
||||
|
||||
let resp = match self
|
||||
.execute_rpc(
|
||||
"lock",
|
||||
&resource_summary,
|
||||
rpc_timeout,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.lock(req).await },
|
||||
RpcCleanup::new(late, late_failure, Some(admission)),
|
||||
late,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -1125,54 +725,26 @@ impl LockClient for RemoteClient {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let admission = match self.admit_lock_request("lock_batch") {
|
||||
Some(admission) => admission,
|
||||
None => return Ok(Self::rpc_backoff_failure_batch(requests)),
|
||||
};
|
||||
|
||||
let rpc_timeout = Self::rpc_timeout();
|
||||
let bounded_requests = requests
|
||||
.iter()
|
||||
.map(|request| Self::bounded_acquire_request(request, rpc_timeout))
|
||||
.collect::<Vec<_>>();
|
||||
let mut client = match self.get_client().await {
|
||||
Ok(client) => client,
|
||||
Err(err) => {
|
||||
self.record_request_timeout("lock_batch", rpc_timeout);
|
||||
self.release_lock_request("lock_batch", Some(admission));
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = Self::summarize_resources(requests);
|
||||
let args = match bounded_requests
|
||||
.iter()
|
||||
.map(|request| {
|
||||
serde_json::to_string(request).map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()
|
||||
{
|
||||
Ok(args) => args,
|
||||
Err(err) => {
|
||||
self.release_lock_request("lock_batch", Some(admission));
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
let mut req = Request::new(BatchGenerallyLockRequest { args });
|
||||
req.set_timeout(rpc_timeout);
|
||||
if let Err(err) = attach_lock_mutation_body_digest(&mut req) {
|
||||
self.release_lock_request("lock_batch", Some(admission));
|
||||
return Err(err.into());
|
||||
}
|
||||
let mut req = Request::new(BatchGenerallyLockRequest {
|
||||
args: requests
|
||||
.iter()
|
||||
.map(|request| {
|
||||
serde_json::to_string(request).map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?,
|
||||
});
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
let late = self.late_release_batch_hook(requests.iter().map(|request| request.lock_id.clone()).collect());
|
||||
let late_failure = self.late_release_failure_hook(requests.iter().map(|request| request.lock_id.clone()).collect());
|
||||
|
||||
let resp = match self
|
||||
.execute_rpc(
|
||||
"lock_batch",
|
||||
&resource_summary,
|
||||
rpc_timeout,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.lock_batch(req).await },
|
||||
RpcCleanup::new(late, late_failure, Some(admission)),
|
||||
late,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -1219,7 +791,7 @@ impl LockClient for RemoteClient {
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.un_lock(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
None,
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
@@ -1253,7 +825,7 @@ impl LockClient for RemoteClient {
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.un_lock_batch(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
None,
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
@@ -1281,7 +853,7 @@ impl LockClient for RemoteClient {
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.refresh(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
None,
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
@@ -1307,7 +879,7 @@ impl LockClient for RemoteClient {
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.force_un_lock(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
None,
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
@@ -1322,31 +894,26 @@ impl LockClient for RemoteClient {
|
||||
|
||||
// Since there's no direct status query in the gRPC service,
|
||||
// we attempt a non-blocking lock acquisition to check if the resource is available
|
||||
let probe_lock_id = LockId::new_unique(&lock_id.resource);
|
||||
let rpc_timeout = Self::rpc_timeout();
|
||||
let status_request = Self::create_unlock_request(&probe_lock_id).with_ttl(rpc_timeout);
|
||||
let bounded_status_request = Self::bounded_acquire_request(&status_request, rpc_timeout);
|
||||
let status_request = Self::create_unlock_request(lock_id);
|
||||
let resource_summary = status_request.resource.to_string();
|
||||
let mut client = self.get_client().await?;
|
||||
let args = serde_json::to_string(&bounded_status_request)
|
||||
let args = serde_json::to_string(&status_request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?;
|
||||
|
||||
// Try to acquire a very short-lived lock to test availability
|
||||
let mut req = Request::new(GenerallyLockRequest { args: args.clone() });
|
||||
req.set_timeout(rpc_timeout);
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
// A probe lock granted after the deadline must not linger on the peer.
|
||||
let late = self.late_release_hook(probe_lock_id.clone());
|
||||
let late_failure = self.late_release_failure_hook(vec![probe_lock_id]);
|
||||
let late = self.late_release_hook(lock_id.clone());
|
||||
|
||||
// Try exclusive lock first with very short timeout
|
||||
let resp = match self
|
||||
.execute_rpc(
|
||||
"check_status",
|
||||
&resource_summary,
|
||||
rpc_timeout,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.lock(req).await },
|
||||
RpcCleanup::new(late, late_failure, None),
|
||||
late,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -1358,7 +925,6 @@ impl LockClient for RemoteClient {
|
||||
// If we successfully acquired the lock, the resource was free.
|
||||
// Immediately release it on a best-effort basis.
|
||||
let mut release_req = Request::new(GenerallyLockRequest { args });
|
||||
release_req.set_timeout(rpc_timeout);
|
||||
attach_lock_mutation_body_digest(&mut release_req)?;
|
||||
if let Ok(mut client) = self.get_client().await {
|
||||
let _ = self
|
||||
@@ -1367,7 +933,7 @@ impl LockClient for RemoteClient {
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.un_lock(release_req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
@@ -1432,7 +998,7 @@ impl LockClient for RemoteClient {
|
||||
Self::ONLINE_CHECK_RESOURCE,
|
||||
online_timeout,
|
||||
async move { client.ping(ping_req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -1567,94 +1133,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_breaker_opens_after_three_timeouts_and_allows_one_probe() {
|
||||
let addr = "http://breaker-test";
|
||||
let client = RemoteClient::new(addr.to_string());
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
|
||||
for _ in 0..LOCK_RPC_REQUEST_BACKOFF_THRESHOLD {
|
||||
client.record_request_timeout("lock", Duration::from_millis(100));
|
||||
}
|
||||
|
||||
assert!(
|
||||
client.admit_lock_request("lock").is_none(),
|
||||
"open breaker must reject requests during backoff"
|
||||
);
|
||||
with_lock_peer_health(addr, |health| {
|
||||
health.request_backoff_until = Some(Instant::now() - Duration::from_millis(1));
|
||||
health.request_probe_generation = None;
|
||||
health.request_in_flight = 0;
|
||||
});
|
||||
|
||||
let probe = client
|
||||
.admit_lock_request("lock")
|
||||
.expect("half-open breaker must admit one probe");
|
||||
assert!(probe.is_probe());
|
||||
assert!(client.admit_lock_request("lock").is_none(), "only one half-open probe may run");
|
||||
client.release_lock_request("lock", Some(probe));
|
||||
assert!(lock_peer_health_for_test(addr).request_probe_generation.is_none());
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn old_request_completion_cannot_clear_a_new_probe() {
|
||||
let addr = "http://breaker-token-test";
|
||||
let client = RemoteClient::new(addr.to_string());
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
with_lock_peer_health(addr, |health| {
|
||||
health.request_backoff_until = Some(Instant::now() - Duration::from_millis(1));
|
||||
health.request_probe_generation = None;
|
||||
health.request_in_flight = 1;
|
||||
});
|
||||
|
||||
let probe = client.admit_lock_request("lock").expect("probe should be admitted");
|
||||
client.release_lock_request("lock", Some(LockRequestAdmission { probe_generation: None }));
|
||||
assert!(lock_peer_health_for_test(addr).request_probe_generation.is_some());
|
||||
client.record_rpc_success("lock", Some(LockRequestAdmission { probe_generation: None }));
|
||||
assert!(lock_peer_health_for_test(addr).request_probe_generation.is_some());
|
||||
client.release_lock_request("lock", Some(probe));
|
||||
assert!(lock_peer_health_for_test(addr).request_probe_generation.is_none());
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_probe_completion_cannot_clear_a_later_generation() {
|
||||
let addr = "http://breaker-generation-test";
|
||||
let client = RemoteClient::new(addr.to_string());
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
with_lock_peer_health(addr, |health| {
|
||||
health.request_backoff_until = Some(Instant::now() - Duration::from_millis(1));
|
||||
});
|
||||
|
||||
let first = client.admit_lock_request("lock").expect("first probe should be admitted");
|
||||
client.record_request_timeout("lock", Duration::from_millis(10));
|
||||
with_lock_peer_health(addr, |health| {
|
||||
health.request_backoff_until = Some(Instant::now() - Duration::from_millis(1));
|
||||
health.request_in_flight = 0;
|
||||
health.request_probe_generation = None;
|
||||
health.request_consecutive_timeouts = 0;
|
||||
});
|
||||
let second = client.admit_lock_request("lock").expect("second probe should be admitted");
|
||||
assert_ne!(first.probe_generation, second.probe_generation);
|
||||
|
||||
client.release_lock_request("lock", Some(first));
|
||||
assert_eq!(lock_peer_health_for_test(addr).request_probe_generation, second.probe_generation);
|
||||
client.release_lock_request("lock", Some(second));
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bounded_acquire_request_matches_transport_deadline() {
|
||||
let request = test_lock_request(Duration::from_secs(30));
|
||||
let bounded = RemoteClient::bounded_acquire_request(&request, Duration::from_secs(3));
|
||||
assert_eq!(bounded.acquire_timeout, Duration::from_secs(3));
|
||||
|
||||
let request = test_lock_request(Duration::from_secs(1));
|
||||
let bounded = RemoteClient::bounded_acquire_request(&request, Duration::from_secs(3));
|
||||
assert_eq!(bounded.acquire_timeout, Duration::from_secs(1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn acquired_lock_ids_picks_only_granted_batch_entries() {
|
||||
let lock_ids = vec![
|
||||
|
||||
@@ -41,7 +41,7 @@ use crate::config::com::{
|
||||
};
|
||||
use crate::data_movement;
|
||||
use crate::data_movement::backpressure::{self, DataMovementOperation};
|
||||
use crate::data_usage::{DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, load_data_usage_cache};
|
||||
use crate::data_usage::DATA_USAGE_CACHE_NAME;
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::disk::{BUCKET_META_PREFIX, DiskAPI, RUSTFS_META_BUCKET};
|
||||
use crate::error::{Error, Result};
|
||||
@@ -135,7 +135,6 @@ const DECOMMISSION_CAPACITY_RELEASE_FAILED: &str = "failed";
|
||||
const DECOMMISSION_CAPACITY_RELEASE_COMPLETED: &str = "completed";
|
||||
pub(crate) const DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX: &str = "decommission/capacity-target";
|
||||
const DECOMMISSION_CAPACITY_TARGET_LOCK_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(250);
|
||||
const DECOMMISSION_CAPACITY_TARGET_GATE_MAX_ATTEMPTS: usize = 12;
|
||||
const DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_PREFIX: &str = "target pool ";
|
||||
const DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_SUFFIX: &str = " target capacity mutation gate is busy";
|
||||
const METRIC_DECOMMISSION_CAPACITY_CONFLICTS_TOTAL: &str = "rustfs_decommission_capacity_conflicts_total";
|
||||
@@ -148,21 +147,12 @@ const DECOMMISSION_LISTING_MAX_ATTEMPTS: usize = 3;
|
||||
const DECOMMISSION_LISTING_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(5);
|
||||
pub(crate) const DECOMMISSION_ENTRY_MAX_ATTEMPTS: usize = 3;
|
||||
const DECOMMISSION_CAPACITY_INTENT_CONFLICT_MAX_ATTEMPTS: usize = 12;
|
||||
// A scanner pause backlog handoff conflict is a transient membership
|
||||
// transition: the native authority is expected to converge once surviving
|
||||
// replicas advance their durable commit. Bound the wait so a genuinely
|
||||
// divergent authority still reaches a terminal, attributable failure.
|
||||
const DECOMMISSION_SCANNER_BACKLOG_HANDOFF_MAX_ATTEMPTS: usize = 12;
|
||||
const DECOMMISSION_SOURCE_CLEANUP_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(100);
|
||||
pub(crate) const DECOMMISSION_VERSION_COPY_ATTEMPTS: usize = 3;
|
||||
const DECOMMISSION_COPY_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(50);
|
||||
const DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT: usize = 100;
|
||||
const DECOMMISSION_TERMINAL_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(1);
|
||||
const DECOMMISSION_CANCEL_TARGET_LOCK_MAX_ATTEMPTS: usize = 3;
|
||||
|
||||
fn decommission_capacity_target_gate_retry_exhausted(attempt: usize) -> bool {
|
||||
attempt.saturating_add(1) >= DECOMMISSION_CAPACITY_TARGET_GATE_MAX_ATTEMPTS
|
||||
}
|
||||
const DECOMMISSION_DURABLE_ILM_RECEIPT_ROOT: &str = "decommission/ilm-receipts";
|
||||
const DECOMMISSION_DURABLE_ILM_MANIFEST_ROOT: &str = "decommission/ilm-manifests";
|
||||
const DECOMMISSION_DURABLE_ILM_RECEIPT_SCHEMA: &str = "v2";
|
||||
@@ -930,29 +920,6 @@ fn capacity_target_physical_bytes(data_bytes: usize, layout: DecommissionErasure
|
||||
Ok(capacity_mul_div_ceil(data_bytes, layout.width(), layout.data))
|
||||
}
|
||||
|
||||
async fn decommission_owned_physical_usage(sets: &Sets, layout: DecommissionErasureLayout) -> HashMap<i32, usize> {
|
||||
let mut usage = HashMap::with_capacity(sets.disk_set.len());
|
||||
for (set_index, set) in sets.disk_set.iter().enumerate() {
|
||||
let Ok(cache) = load_data_usage_cache(set.as_ref(), DATA_USAGE_CACHE_NAME).await else {
|
||||
continue;
|
||||
};
|
||||
let data_usage = cache.dui(DATA_USAGE_ROOT, &[]);
|
||||
if !data_usage.is_complete_bucket_usage_snapshot() {
|
||||
continue;
|
||||
}
|
||||
let Ok(logical_bytes) = usize::try_from(data_usage.objects_total_size) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(set_index) = i32::try_from(set_index) else {
|
||||
continue;
|
||||
};
|
||||
// The data-usage snapshot excludes unrelated files on a shared mount.
|
||||
// Convert logical object bytes to the source set's physical EC footprint.
|
||||
usage.insert(set_index, capacity_mul_div_ceil(logical_bytes, layout.width(), layout.data));
|
||||
}
|
||||
usage
|
||||
}
|
||||
|
||||
fn worst_decommission_target_layout(targets: &[DecommissionPoolCapacityInfo]) -> Result<DecommissionErasureLayout> {
|
||||
targets
|
||||
.iter()
|
||||
@@ -1029,30 +996,6 @@ fn decommission_capacity_retry_kind(err: &Error, intent_conflict_attempt: usize)
|
||||
.then_some(DecommissionCapacityRetryKind::IntentConflict)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum DecommissionScannerBacklogRetryKind {
|
||||
HandoffConflict,
|
||||
}
|
||||
|
||||
fn is_decommission_scanner_backlog_handoff_conflict(err: &Error) -> bool {
|
||||
data_movement::data_movement_stage_source_as::<data_movement::scanner_backlog::ScannerPauseBacklogRetirementError>(err)
|
||||
.is_some_and(data_movement::scanner_backlog::ScannerPauseBacklogRetirementError::is_retryable_handoff_conflict)
|
||||
}
|
||||
|
||||
/// Classify a retryable scanner pause backlog handoff conflict.
|
||||
///
|
||||
/// The typed planner error is the only signal consulted: string matching would
|
||||
/// turn unrelated retirement failures into retries. `handoff_conflict_attempt`
|
||||
/// is the number of retries already consumed by this entry.
|
||||
fn decommission_scanner_backlog_retry_kind(
|
||||
err: &Error,
|
||||
handoff_conflict_attempt: usize,
|
||||
) -> Option<DecommissionScannerBacklogRetryKind> {
|
||||
(handoff_conflict_attempt < DECOMMISSION_SCANNER_BACKLOG_HANDOFF_MAX_ATTEMPTS
|
||||
&& is_decommission_scanner_backlog_handoff_conflict(err))
|
||||
.then_some(DecommissionScannerBacklogRetryKind::HandoffConflict)
|
||||
}
|
||||
|
||||
fn ensure_decommission_capacity_target_fence(
|
||||
guard: &rustfs_lock::NamespaceLockGuard,
|
||||
target_pool_index: usize,
|
||||
@@ -1889,132 +1832,6 @@ fn ensure_external_decommission_target_admission(
|
||||
Err(Error::SlowDown)
|
||||
}
|
||||
|
||||
fn grow_decommission_target_reservation(
|
||||
meta: &mut PoolMeta,
|
||||
capacity_infos: &[DecommissionPoolCapacityInfo],
|
||||
source_pool_index: usize,
|
||||
target_pool_index: usize,
|
||||
expected_data_bytes: usize,
|
||||
mutation_id: uuid::Uuid,
|
||||
now: OffsetDateTime,
|
||||
) -> Result<bool> {
|
||||
let original = meta.clone();
|
||||
let result = (|| {
|
||||
let target_capacity = capacity_infos
|
||||
.iter()
|
||||
.find(|capacity| capacity.pool_index == target_pool_index)
|
||||
.ok_or_else(|| {
|
||||
decommission_capacity_blocked_error("target capacity snapshot is missing during reservation growth")
|
||||
})?;
|
||||
let pool_count = meta.pools.len();
|
||||
let pool = meta
|
||||
.pools
|
||||
.get_mut(source_pool_index)
|
||||
.ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?;
|
||||
let reservation = pool
|
||||
.decommission
|
||||
.as_mut()
|
||||
.and_then(|info| info.capacity_reservation.as_mut())
|
||||
.filter(|reservation| reservation.active())
|
||||
.ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared during reservation growth"))?;
|
||||
let target_layout = reservation
|
||||
.targets
|
||||
.iter()
|
||||
.find(|target| target.pool_index == target_pool_index)
|
||||
.map(|target| target.layout)
|
||||
.unwrap_or(target_capacity.layout);
|
||||
let data_bytes = expected_data_bytes.max(1);
|
||||
let required_target_bytes = capacity_target_physical_bytes(data_bytes, target_layout)?;
|
||||
let required_peak = required_target_bytes.saturating_mul(1usize.saturating_add(reservation.temporary_copies));
|
||||
let existing_target = reservation
|
||||
.targets
|
||||
.iter()
|
||||
.find(|target| target.pool_index == target_pool_index);
|
||||
if let Some(target) = existing_target
|
||||
&& target.pending_physical_bytes > 0
|
||||
&& target.pending_mutation_id != Some(mutation_id)
|
||||
{
|
||||
return Err(decommission_capacity_blocked_error(format!(
|
||||
"source pool {source_pool_index} target pool {target_pool_index} has an unresolved target capacity intent"
|
||||
)));
|
||||
}
|
||||
let existing_remaining = existing_target
|
||||
.map(|target| target.remaining_reserved_physical_bytes(reservation.temporary_copies))
|
||||
.unwrap_or_default();
|
||||
if required_peak <= existing_remaining {
|
||||
return Ok(false);
|
||||
}
|
||||
if reservation.source_data_equivalent_bytes > 0 {
|
||||
return Err(decommission_capacity_blocked_error(format!(
|
||||
"source pool {source_pool_index} reservation estimate is exhausted"
|
||||
)));
|
||||
}
|
||||
if let Some(target) = existing_target
|
||||
&& target_capacity.physical_free <= target.physical_free_at_reservation
|
||||
{
|
||||
return Err(decommission_capacity_blocked_error(format!(
|
||||
"source pool {source_pool_index} target pool {target_pool_index} reservation is exhausted"
|
||||
)));
|
||||
}
|
||||
let current_peak = reservation.remaining_peak_physical_bytes();
|
||||
let additional_peak = required_peak.saturating_sub(existing_remaining);
|
||||
let additional_predicted = additional_peak.div_ceil(1usize.saturating_add(reservation.temporary_copies));
|
||||
let additional_source_data = capacity_source_data_equivalent(additional_predicted, target_layout)?;
|
||||
reservation.source_data_equivalent_bytes = reservation
|
||||
.source_data_equivalent_bytes
|
||||
.saturating_add(additional_source_data);
|
||||
reservation.source_physical_bytes = reservation
|
||||
.source_physical_bytes
|
||||
.saturating_add(capacity_target_physical_bytes(additional_source_data, reservation.source_layout)?);
|
||||
reservation.predicted_physical_bytes = reservation
|
||||
.predicted_physical_bytes
|
||||
.saturating_add(additional_predicted)
|
||||
.max(capacity_target_physical_bytes(reservation.source_data_equivalent_bytes, target_layout)?);
|
||||
reservation.temporary_physical_bytes = reservation
|
||||
.predicted_physical_bytes
|
||||
.saturating_mul(reservation.temporary_copies);
|
||||
reservation.peak_physical_bytes = reservation
|
||||
.predicted_physical_bytes
|
||||
.saturating_add(reservation.temporary_physical_bytes);
|
||||
let allocation_growth = reservation.remaining_peak_physical_bytes().saturating_sub(current_peak);
|
||||
if let Some(target) = reservation
|
||||
.targets
|
||||
.iter_mut()
|
||||
.find(|target| target.pool_index == target_pool_index)
|
||||
{
|
||||
target.reserved_physical_bytes = target.reserved_physical_bytes.saturating_add(allocation_growth);
|
||||
target.physical_total_at_reservation = target.physical_total_at_reservation.max(target_capacity.physical_total);
|
||||
target.physical_free_at_reservation = target.physical_free_at_reservation.max(target_capacity.physical_free);
|
||||
} else {
|
||||
reservation.targets.push(DecommissionCapacityTarget {
|
||||
pool_index: target_pool_index,
|
||||
layout: target_layout,
|
||||
physical_total_at_reservation: target_capacity.physical_total,
|
||||
physical_free_at_reservation: target_capacity.physical_free,
|
||||
reserved_physical_bytes: allocation_growth,
|
||||
consumed_physical_bytes: 0,
|
||||
observed_physical_bytes: 0,
|
||||
inflight_physical_bytes: 0,
|
||||
pending_physical_bytes: 0,
|
||||
pending_mutation_id: None,
|
||||
temporary_mutations: Vec::new(),
|
||||
});
|
||||
}
|
||||
renew_decommission_capacity_reservation(reservation, now, true);
|
||||
pool.last_update = pool.last_update.max(now);
|
||||
Ok(true)
|
||||
})();
|
||||
if result.is_ok() {
|
||||
if let Err(err) = ensure_decommission_capacity_reservations_available(meta, capacity_infos, "reservation growth") {
|
||||
*meta = original;
|
||||
return Err(err);
|
||||
}
|
||||
} else {
|
||||
*meta = original;
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn ensure_decommission_target_owner_admission(
|
||||
meta: &PoolMeta,
|
||||
owner: DecommissionCapacityOwner,
|
||||
@@ -11556,11 +11373,10 @@ impl ECStore {
|
||||
}
|
||||
})
|
||||
.filter(|reservation| {
|
||||
!temporary_release
|
||||
|| reservation
|
||||
.targets
|
||||
.iter()
|
||||
.any(|target| target.pool_index == target_pool_index)
|
||||
reservation
|
||||
.targets
|
||||
.iter()
|
||||
.any(|target| target.pool_index == target_pool_index)
|
||||
})
|
||||
.map(|reservation| (owner, reservation.model_version))
|
||||
});
|
||||
@@ -11619,11 +11435,10 @@ impl ECStore {
|
||||
} else {
|
||||
reservation.admits_owner(owner, OffsetDateTime::now_utc())
|
||||
}) && reservation.model_version == model_version
|
||||
&& (!temporary_release
|
||||
|| reservation
|
||||
.targets
|
||||
.iter()
|
||||
.any(|target| target.pool_index == target_pool_index))
|
||||
&& reservation
|
||||
.targets
|
||||
.iter()
|
||||
.any(|target| target.pool_index == target_pool_index)
|
||||
})
|
||||
.is_some();
|
||||
if !owner_current {
|
||||
@@ -11632,22 +11447,9 @@ impl ECStore {
|
||||
));
|
||||
}
|
||||
let capacity_infos = self.get_decommission_all_pool_capacity_infos().await?;
|
||||
let reservation_grew = if temporary_release {
|
||||
false
|
||||
} else {
|
||||
let data_bytes = expected_data_bytes.unwrap_or(1).max(1);
|
||||
let grew = grow_decommission_target_reservation(
|
||||
&mut snapshot,
|
||||
&capacity_infos,
|
||||
source_pool_index,
|
||||
target_pool_index,
|
||||
data_bytes,
|
||||
mutation_id,
|
||||
OffsetDateTime::now_utc(),
|
||||
)?;
|
||||
if !temporary_release {
|
||||
ensure_decommission_capacity_reservations_available(&snapshot, &capacity_infos, "mutation")?;
|
||||
grew
|
||||
};
|
||||
}
|
||||
let target_layout = snapshot
|
||||
.pools
|
||||
.get(source_pool_index)
|
||||
@@ -11713,7 +11515,7 @@ impl ECStore {
|
||||
OffsetDateTime::now_utc(),
|
||||
)?
|
||||
};
|
||||
if reservation_grew || pending_added > 0 {
|
||||
if pending_added > 0 {
|
||||
let outcome = snapshot
|
||||
.save_no_lock_armed(self.pools.clone(), &mut save_guard, write_guard.lock_lost_signal(), &[source_pool_index])
|
||||
.await?;
|
||||
@@ -12555,11 +12357,7 @@ impl ECStore {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn get_decommission_pool_capacity_info(
|
||||
&self,
|
||||
idx: usize,
|
||||
use_owned_usage: bool,
|
||||
) -> Result<DecommissionPoolCapacityInfo> {
|
||||
async fn get_decommission_pool_capacity_info(&self, idx: usize) -> Result<DecommissionPoolCapacityInfo> {
|
||||
if let Some(sets) = self.pools.get(idx) {
|
||||
let mut info = sets.storage_info_snapshot().await;
|
||||
info.backend = StorageAdminApi::backend_info(self).await;
|
||||
@@ -12591,13 +12389,8 @@ impl ECStore {
|
||||
layout.data, layout.parity
|
||||
)));
|
||||
}
|
||||
let owned_physical_used = if use_owned_usage {
|
||||
Some(decommission_owned_physical_usage(sets, layout).await)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let (physical_total, physical_free, physical_used) =
|
||||
decommission_physical_pool_capacity(&info.disks, idx, layout, space, owned_physical_used.as_ref());
|
||||
decommission_physical_pool_capacity(&info.disks, idx, layout, space);
|
||||
|
||||
Ok(DecommissionPoolCapacityInfo {
|
||||
pool_index: idx,
|
||||
@@ -12620,20 +12413,7 @@ impl ECStore {
|
||||
|
||||
let mut capacity_infos = Vec::with_capacity(self.pools.len());
|
||||
for idx in 0..self.pools.len() {
|
||||
capacity_infos.push(self.get_decommission_pool_capacity_info(idx, false).await?);
|
||||
}
|
||||
Ok(capacity_infos)
|
||||
}
|
||||
|
||||
async fn get_decommission_all_pool_capacity_infos_with_owned_usage(&self) -> Result<Vec<DecommissionPoolCapacityInfo>> {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
if let Some(capacity_infos) = take_decommission_capacity_info_override_for_test(self.id) {
|
||||
return Ok(capacity_infos);
|
||||
}
|
||||
|
||||
let mut capacity_infos = Vec::with_capacity(self.pools.len());
|
||||
for idx in 0..self.pools.len() {
|
||||
capacity_infos.push(self.get_decommission_pool_capacity_info(idx, true).await?);
|
||||
capacity_infos.push(self.get_decommission_pool_capacity_info(idx).await?);
|
||||
}
|
||||
Ok(capacity_infos)
|
||||
}
|
||||
@@ -12666,14 +12446,10 @@ impl ECStore {
|
||||
.get(idx)
|
||||
.and_then(|pool| pool.decommission.as_ref())
|
||||
.and_then(|info| info.capacity_reservation.as_ref())
|
||||
.filter(|reservation| reservation.lease_active_at(OffsetDateTime::now_utc()))
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
if !reservation.lease_active_at(OffsetDateTime::now_utc()) {
|
||||
return Err(decommission_capacity_blocked_error(
|
||||
"decommission capacity reservation lease is not active",
|
||||
));
|
||||
}
|
||||
Ok(Some(DecommissionCapacityOwner {
|
||||
source_pool_index: idx,
|
||||
operation_id: reservation.operation_id,
|
||||
@@ -12735,37 +12511,15 @@ impl ECStore {
|
||||
// different target so one mutation never holds two target gates.
|
||||
discard_decommission_capacity_target_permit_except(self.id, owner, None);
|
||||
}
|
||||
if let Some((pool_index, _)) = reservation
|
||||
reservation
|
||||
.targets
|
||||
.iter()
|
||||
.filter_map(candidate)
|
||||
.max_by_key(|(_, remaining)| *remaining)
|
||||
{
|
||||
return Ok(pool_index);
|
||||
}
|
||||
if reservation.source_data_equivalent_bytes > 0 {
|
||||
return Err(decommission_capacity_blocked_error(format!(
|
||||
"source pool {} has no target allocation for {expected_data_bytes} data bytes",
|
||||
owner.source_pool_index
|
||||
)));
|
||||
}
|
||||
let capacities = self.get_decommission_all_pool_capacity_infos().await?;
|
||||
let active_sources = active_decommission_source_indices(&pool_meta);
|
||||
capacities
|
||||
.into_iter()
|
||||
.filter(|capacity| {
|
||||
capacity.pool_index != owner.source_pool_index
|
||||
&& !active_sources.contains(&capacity.pool_index)
|
||||
&& pool_meta
|
||||
.pools
|
||||
.get(capacity.pool_index)
|
||||
.is_some_and(is_decommission_start_active_pool)
|
||||
})
|
||||
.max_by_key(|capacity| capacity.physical_free)
|
||||
.map(|capacity| capacity.pool_index)
|
||||
.map(|(pool_index, _)| pool_index)
|
||||
.ok_or_else(|| {
|
||||
decommission_capacity_blocked_error(format!(
|
||||
"source pool {} has no target pool for {expected_data_bytes} data bytes",
|
||||
"source pool {} has no target allocation for {expected_data_bytes} data bytes",
|
||||
owner.source_pool_index
|
||||
))
|
||||
})
|
||||
@@ -13261,7 +13015,7 @@ impl ECStore {
|
||||
let capacity_infos = if self.pools.is_empty() {
|
||||
Vec::new()
|
||||
} else {
|
||||
self.get_decommission_all_pool_capacity_infos_with_owned_usage().await?
|
||||
self.get_decommission_all_pool_capacity_infos().await?
|
||||
};
|
||||
let target_fence_proof = crate::services::notification_sys::acquire_decommission_target_fence_fleet_proof();
|
||||
let mut pool_meta = self.pool_meta.write().await;
|
||||
@@ -13962,20 +13716,6 @@ impl ECStore {
|
||||
match self.acquire_decommission_capacity_target_guard(target_pool_index).await {
|
||||
Ok(guard) => break guard,
|
||||
Err(err) if is_decommission_capacity_target_gate_busy(&err) => {
|
||||
if decommission_capacity_target_gate_retry_exhausted(wait_attempt) {
|
||||
warn!(
|
||||
event = EVENT_DECOMMISSION_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
target_pool_index,
|
||||
attempts = wait_attempt.saturating_add(1),
|
||||
state = "capacity_gate_retry_exhausted",
|
||||
error = %err,
|
||||
"Decommission target capacity gate remained busy; pausing for supervised retry"
|
||||
);
|
||||
return Err(err);
|
||||
}
|
||||
wait_attempt = wait_attempt.saturating_add(1);
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
@@ -14314,7 +14054,6 @@ impl ECStore {
|
||||
for entry_attempt in 1..=DECOMMISSION_ENTRY_MAX_ATTEMPTS {
|
||||
let attempt_result = {
|
||||
let mut conflict_attempt = 0;
|
||||
let mut scanner_backlog_handoff_attempt = 0;
|
||||
loop {
|
||||
let result = self
|
||||
.decommission_entry_attempt(
|
||||
@@ -14329,7 +14068,6 @@ impl ECStore {
|
||||
replication_config.clone(),
|
||||
expected_bucket_incarnation_id,
|
||||
entry_attempt,
|
||||
scanner_backlog_handoff_attempt,
|
||||
source_changed_exhaustions.as_ref(),
|
||||
&mut counted_versions,
|
||||
)
|
||||
@@ -14338,48 +14076,17 @@ impl ECStore {
|
||||
.as_ref()
|
||||
.err()
|
||||
.and_then(|err| decommission_capacity_retry_kind(err, conflict_attempt));
|
||||
if let Some(DecommissionCapacityRetryKind::IntentConflict) = retry {
|
||||
conflict_attempt += 1;
|
||||
let retry_delay =
|
||||
decommission_retry_backoff_delay(DECOMMISSION_SOURCE_CLEANUP_RETRY_DELAY, conflict_attempt);
|
||||
if wait_decommission_retry_backoff(&rx, retry_delay).await {
|
||||
decommission_cancel_signal_result(rx.is_cancelled())?;
|
||||
let retry_attempt = match retry {
|
||||
Some(DecommissionCapacityRetryKind::IntentConflict) => {
|
||||
conflict_attempt += 1;
|
||||
conflict_attempt
|
||||
}
|
||||
continue;
|
||||
None => break result,
|
||||
};
|
||||
let retry_delay = decommission_retry_backoff_delay(DECOMMISSION_SOURCE_CLEANUP_RETRY_DELAY, retry_attempt);
|
||||
if wait_decommission_retry_backoff(&rx, retry_delay).await {
|
||||
decommission_cancel_signal_result(rx.is_cancelled())?;
|
||||
}
|
||||
// Scanner pause backlog handoff conflicts resolve when the
|
||||
// surviving membership converges. Retry on a bounded
|
||||
// backoff that never writes failure state, clears
|
||||
// `start_time`, or releases the capacity reservation.
|
||||
let scanner_retry = result
|
||||
.as_ref()
|
||||
.err()
|
||||
.and_then(|err| decommission_scanner_backlog_retry_kind(err, scanner_backlog_handoff_attempt));
|
||||
if let Some(DecommissionScannerBacklogRetryKind::HandoffConflict) = scanner_retry {
|
||||
scanner_backlog_handoff_attempt += 1;
|
||||
let retry_delay = decommission_retry_backoff_delay(
|
||||
DECOMMISSION_SOURCE_CLEANUP_RETRY_DELAY,
|
||||
scanner_backlog_handoff_attempt,
|
||||
);
|
||||
info!(
|
||||
event = EVENT_DECOMMISSION_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
state = "scanner_backlog_handoff_retry",
|
||||
pool_index = idx,
|
||||
bucket = %bucket,
|
||||
object = %entry.name,
|
||||
attempt = scanner_backlog_handoff_attempt,
|
||||
max_attempts = DECOMMISSION_SCANNER_BACKLOG_HANDOFF_MAX_ATTEMPTS,
|
||||
retry_delay_ms = retry_delay.as_millis(),
|
||||
"Decommission scanner pause backlog handoff conflict; retrying entry"
|
||||
);
|
||||
if wait_decommission_retry_backoff(&rx, retry_delay).await {
|
||||
decommission_cancel_signal_result(rx.is_cancelled())?;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
break result;
|
||||
}
|
||||
};
|
||||
match attempt_result {
|
||||
@@ -14427,7 +14134,6 @@ impl ECStore {
|
||||
replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
|
||||
expected_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
entry_attempt: usize,
|
||||
scanner_backlog_handoff_attempt: usize,
|
||||
source_changed_exhaustions: &AtomicUsize,
|
||||
counted_versions: &mut HashSet<(Option<uuid::Uuid>, bool)>,
|
||||
) -> Result<DecommissionEntryAttemptOutcome> {
|
||||
@@ -14487,56 +14193,27 @@ impl ECStore {
|
||||
if data_movement::scanner_backlog::is_scanner_pause_backlog(&bucket, &entry.name) {
|
||||
let outcome = self
|
||||
.retire_scanner_pause_backlog_entry(rx, idx, generation, Arc::clone(&set), fivs.clone(), capacity_owner)
|
||||
.await;
|
||||
return match outcome {
|
||||
Ok(DecommissionEntryAttemptOutcome::Complete) => {
|
||||
let mut pool_meta = self.pool_meta.write().await;
|
||||
ensure_decommission_generation(&pool_meta, idx, generation)?;
|
||||
if let Some(version) = fivs.versions.first()
|
||||
&& counted_versions.insert((version.version_id, false))
|
||||
{
|
||||
count_decommission_item(&mut pool_meta, idx, decommission_item_size(version.size), false)?;
|
||||
}
|
||||
track_decommission_current_object(&mut pool_meta, idx, &bucket, &entry.name)?;
|
||||
drop(pool_meta);
|
||||
self.track_decommission_entry_progress_stage(
|
||||
idx,
|
||||
generation,
|
||||
&bucket,
|
||||
&entry.name,
|
||||
DECOMMISSION_STAGE_ENTRY_FINISHED,
|
||||
)
|
||||
.await?;
|
||||
Ok(DecommissionEntryAttemptOutcome::Complete)
|
||||
.await?;
|
||||
if matches!(outcome, DecommissionEntryAttemptOutcome::Complete) {
|
||||
let mut pool_meta = self.pool_meta.write().await;
|
||||
ensure_decommission_generation(&pool_meta, idx, generation)?;
|
||||
if let Some(version) = fivs.versions.first()
|
||||
&& counted_versions.insert((version.version_id, false))
|
||||
{
|
||||
count_decommission_item(&mut pool_meta, idx, decommission_item_size(version.size), false)?;
|
||||
}
|
||||
Ok(outcome) => Ok(outcome),
|
||||
Err(err) => {
|
||||
// The retry loop owns the handoff-conflict budget, so a
|
||||
// retryable conflict must not be attributed here. Once that
|
||||
// budget is exhausted the failure is terminal and belongs to
|
||||
// this entry rather than to whichever object was processed
|
||||
// last.
|
||||
let retryable_handoff = data_movement::data_movement_stage_source_as::<
|
||||
data_movement::scanner_backlog::ScannerPauseBacklogRetirementError,
|
||||
>(&err)
|
||||
.is_some_and(
|
||||
data_movement::scanner_backlog::ScannerPauseBacklogRetirementError::is_retryable_handoff_conflict,
|
||||
);
|
||||
let terminal = !retryable_handoff
|
||||
|| scanner_backlog_handoff_attempt >= DECOMMISSION_SCANNER_BACKLOG_HANDOFF_MAX_ATTEMPTS;
|
||||
if terminal {
|
||||
let mut pool_meta = self.pool_meta.write().await;
|
||||
ensure_decommission_generation(&pool_meta, idx, generation)?;
|
||||
if let Some(version) = fivs.versions.first()
|
||||
&& counted_versions.insert((version.version_id, true))
|
||||
{
|
||||
count_decommission_item(&mut pool_meta, idx, decommission_item_size(version.size), true)?;
|
||||
}
|
||||
track_decommission_current_object(&mut pool_meta, idx, &bucket, &entry.name)?;
|
||||
}
|
||||
Err(err)
|
||||
}
|
||||
};
|
||||
track_decommission_current_object(&mut pool_meta, idx, &bucket, &entry.name)?;
|
||||
drop(pool_meta);
|
||||
self.track_decommission_entry_progress_stage(
|
||||
idx,
|
||||
generation,
|
||||
&bucket,
|
||||
&entry.name,
|
||||
DECOMMISSION_STAGE_ENTRY_FINISHED,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
return Ok(outcome);
|
||||
}
|
||||
|
||||
let pending_mutations = if let Some(owner) = capacity_owner {
|
||||
@@ -15582,7 +15259,7 @@ impl ECStore {
|
||||
.is_some_and(|reservation| !reservation.active())
|
||||
};
|
||||
if needs_capacity_reservation {
|
||||
let capacity_infos = self.get_decommission_all_pool_capacity_infos_with_owned_usage().await?;
|
||||
let capacity_infos = self.get_decommission_all_pool_capacity_infos().await?;
|
||||
let mut pool_meta = self.pool_meta.write().await;
|
||||
let version = pool_meta.version;
|
||||
pool_meta.version = POOL_META_VERSION;
|
||||
@@ -16446,7 +16123,7 @@ impl ECStore {
|
||||
self.ensure_decommission_rebalance_idle_after_refresh_under_start_gate()
|
||||
.await?;
|
||||
|
||||
let all_capacity_infos = self.get_decommission_all_pool_capacity_infos_with_owned_usage().await?;
|
||||
let all_capacity_infos = self.get_decommission_all_pool_capacity_infos().await?;
|
||||
let target_fence_proof_available =
|
||||
crate::services::notification_sys::acquire_decommission_target_fence_fleet_proof().is_some();
|
||||
// Signal cancellation before waiting for the movement writer so active
|
||||
@@ -20380,45 +20057,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn expired_decommission_capacity_lease_requires_recovery_before_entry() {
|
||||
let (_temp_dirs, store, _other_store) =
|
||||
crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
|
||||
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
|
||||
set_decommission_capacity_info_overrides_for_test(
|
||||
store.id,
|
||||
vec![vec![
|
||||
DecommissionPoolCapacityInfo::for_test(0, layout, 0, 1, 1),
|
||||
DecommissionPoolCapacityInfo::for_test(1, layout, 2, 2, 0),
|
||||
DecommissionPoolCapacityInfo::for_test(2, layout, 2, 2, 0),
|
||||
]],
|
||||
);
|
||||
store
|
||||
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
|
||||
.await
|
||||
.expect("activate the expired lease reservation");
|
||||
let generation = store
|
||||
.active_decommission_generation(0)
|
||||
.await
|
||||
.expect("read the active decommission generation");
|
||||
{
|
||||
let mut pool_meta = store.pool_meta.write().await;
|
||||
let reservation = pool_meta.pools[0]
|
||||
.decommission
|
||||
.as_mut()
|
||||
.and_then(|info| info.capacity_reservation.as_mut())
|
||||
.expect("the expired lease reservation should exist");
|
||||
reservation.expires_at = OffsetDateTime::now_utc() - Duration::seconds(1);
|
||||
}
|
||||
|
||||
let err = store
|
||||
.decommission_capacity_owner_for_worker(0, generation)
|
||||
.await
|
||||
.expect_err("an expired capacity lease must not fall back to ownerless migration");
|
||||
assert!(is_decommission_capacity_blocked_error(&err));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn expired_capacity_owner_nonce_rejects_stale_put_and_multipart_without_consuming() {
|
||||
@@ -22006,45 +21644,6 @@ mod tests {
|
||||
assert!(!is_decommission_target_capacity_error(&Error::SlowDown));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_scanner_backlog_handoff_retry_is_typed_and_bounded() {
|
||||
use data_movement::scanner_backlog::ScannerPauseBacklogRetirementError;
|
||||
|
||||
let handoff = data_movement::data_movement_context_error(
|
||||
"scanner pause backlog retirement planner failed: competing proofs".to_string(),
|
||||
ScannerPauseBacklogRetirementError::HandoffConflict {
|
||||
reason: "competing maximum-membership commit proofs".to_string(),
|
||||
},
|
||||
);
|
||||
assert!(is_decommission_scanner_backlog_handoff_conflict(&handoff));
|
||||
assert_eq!(
|
||||
decommission_scanner_backlog_retry_kind(&handoff, DECOMMISSION_SCANNER_BACKLOG_HANDOFF_MAX_ATTEMPTS - 1),
|
||||
Some(DecommissionScannerBacklogRetryKind::HandoffConflict)
|
||||
);
|
||||
assert_eq!(
|
||||
decommission_scanner_backlog_retry_kind(&handoff, DECOMMISSION_SCANNER_BACKLOG_HANDOFF_MAX_ATTEMPTS),
|
||||
None,
|
||||
"a divergent handoff must not retry forever"
|
||||
);
|
||||
|
||||
for non_retryable in [
|
||||
ScannerPauseBacklogRetirementError::AuthorityConflict {
|
||||
reason: "larger source authority".to_string(),
|
||||
},
|
||||
ScannerPauseBacklogRetirementError::InvalidRecord {
|
||||
reason: "unread member".to_string(),
|
||||
},
|
||||
] {
|
||||
let err = data_movement::data_movement_context_error("planner failed".to_string(), non_retryable);
|
||||
assert!(!is_decommission_scanner_backlog_handoff_conflict(&err));
|
||||
assert_eq!(
|
||||
decommission_scanner_backlog_retry_kind(&err, 0),
|
||||
None,
|
||||
"only a handoff conflict may be retried"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_skip_decommission_delete_marker_characterizes_empty_marker_without_replication() {
|
||||
let version = rustfs_filemeta::FileInfo {
|
||||
@@ -22880,7 +22479,6 @@ fn decommission_physical_pool_capacity(
|
||||
pool_index: usize,
|
||||
layout: DecommissionErasureLayout,
|
||||
logical: PoolSpaceInfo,
|
||||
owned_physical_used: Option<&HashMap<i32, usize>>,
|
||||
) -> (usize, usize, usize) {
|
||||
let width = layout.width();
|
||||
let mut sets: HashMap<i32, Vec<&rustfs_madmin::Disk>> = HashMap::new();
|
||||
@@ -22920,20 +22518,13 @@ fn decommission_physical_pool_capacity(
|
||||
.map(|disk| disk.available_space as usize)
|
||||
.min()
|
||||
.unwrap_or_default();
|
||||
let physical_used_for_set = owned_physical_used
|
||||
.and_then(|owned| owned.get(&set_disks[0].set_index).copied())
|
||||
.unwrap_or_else(|| {
|
||||
set_disks
|
||||
.iter()
|
||||
.map(|disk| {
|
||||
(disk.used_space as usize).max((disk.total_space as usize).saturating_sub(disk.available_space as usize))
|
||||
})
|
||||
.max()
|
||||
.unwrap_or_default()
|
||||
.saturating_mul(width)
|
||||
});
|
||||
let max_used = set_disks
|
||||
.iter()
|
||||
.map(|disk| (disk.used_space as usize).max((disk.total_space as usize).saturating_sub(disk.available_space as usize)))
|
||||
.max()
|
||||
.unwrap_or_default();
|
||||
physical_total = physical_total.saturating_add(min_total.saturating_mul(width));
|
||||
physical_used = physical_used.saturating_add(physical_used_for_set);
|
||||
physical_used = physical_used.saturating_add(max_used.saturating_mul(width));
|
||||
if set_disks.len() >= width {
|
||||
physical_free = physical_free.saturating_add(min_free.saturating_mul(width));
|
||||
}
|
||||
@@ -23116,8 +22707,6 @@ pub(crate) fn fallback_free_capacity_dedup(disks: &[rustfs_madmin::Disk]) -> usi
|
||||
|
||||
#[cfg(test)]
|
||||
mod pools_tests {
|
||||
use std::collections::HashMap;
|
||||
|
||||
use super::DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF;
|
||||
use super::persist_v3_pool_meta_for_test;
|
||||
use super::record_decommission_entry_error;
|
||||
@@ -23151,11 +22740,10 @@ mod pools_tests {
|
||||
ensure_decommission_start_rebalance_meta_allowed, ensure_decommission_start_target_capacity,
|
||||
ensure_decommission_terminal_operation_supported, ensure_decommission_unresolved_verification_disk_count,
|
||||
ensure_local_decommission_pool_leaders, ensure_pool_meta_write_fence, ensure_valid_decommission_pool_index, get_by_index,
|
||||
grow_decommission_target_reservation, guard_decommission_cancelers, has_active_decommission_canceler,
|
||||
is_decommission_active, is_decommission_cancel_requested, load_decommission_entry_versions,
|
||||
local_decommission_queue_prefix, mark_decommission_bucket_done, merge_decommission_durable_ilm_receipts,
|
||||
merge_pool_meta_updates_for_save, merge_pool_status_refresh, missing_decommission_worker_prefix,
|
||||
next_decommission_capacity_generation, observe_decommission_terminal_reload_result,
|
||||
guard_decommission_cancelers, has_active_decommission_canceler, is_decommission_active, is_decommission_cancel_requested,
|
||||
load_decommission_entry_versions, local_decommission_queue_prefix, mark_decommission_bucket_done,
|
||||
merge_decommission_durable_ilm_receipts, merge_pool_meta_updates_for_save, merge_pool_status_refresh,
|
||||
missing_decommission_worker_prefix, next_decommission_capacity_generation, observe_decommission_terminal_reload_result,
|
||||
parse_decommission_durable_ilm_receipt_path, pool_meta_has_active_decommission, publish_pool_meta_updates,
|
||||
read_pool_meta_replica, reconcile_decommission_meta_buckets, reconcile_decommission_unresolved_entries_for_completion,
|
||||
record_decommission_unresolved_entry, recover_decommission_capacity_reservations,
|
||||
@@ -23181,9 +22769,8 @@ mod pools_tests {
|
||||
with_decommission_entry_context,
|
||||
};
|
||||
use super::{
|
||||
DECOMMISSION_CAPACITY_TARGET_GATE_MAX_ATTEMPTS, DecommissionCapacityAdmission, DecommissionCapacityOwner,
|
||||
DecommissionCapacityReleaseProof, DecommissionCapacityReservation, DecommissionCapacityTemporaryMutation,
|
||||
decommission_capacity_mutation_id, decommission_capacity_target_gate_retry_exhausted,
|
||||
DecommissionCapacityAdmission, DecommissionCapacityOwner, DecommissionCapacityReleaseProof,
|
||||
DecommissionCapacityReservation, DecommissionCapacityTemporaryMutation, decommission_capacity_mutation_id,
|
||||
ensure_decommission_target_owner_admission, ensure_exact_delete_capacity_namespace_fences,
|
||||
ensure_external_decommission_target_admission, is_decommission_capacity_blocked_error,
|
||||
plan_exact_delete_capacity_reconciliations, record_decommission_target_consumption, release_decommission_target_inflight,
|
||||
@@ -23290,17 +22877,6 @@ mod pools_tests {
|
||||
identity: StdMutex<Option<(Vec<u8>, String)>>,
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_target_gate_retry_is_bounded() {
|
||||
assert!(!decommission_capacity_target_gate_retry_exhausted(0));
|
||||
assert!(!decommission_capacity_target_gate_retry_exhausted(
|
||||
DECOMMISSION_CAPACITY_TARGET_GATE_MAX_ATTEMPTS - 2
|
||||
));
|
||||
assert!(decommission_capacity_target_gate_retry_exhausted(
|
||||
DECOMMISSION_CAPACITY_TARGET_GATE_MAX_ATTEMPTS - 1
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pool_meta_recovery_reconciles_prepare_and_partial_commit_without_format_downgrade() {
|
||||
for previous_version in [POOL_META_V1_VERSION, POOL_META_VERSION, POOL_META_GENERATION_VERSION] {
|
||||
@@ -28722,44 +28298,6 @@ mod pools_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_capacity_reservation_grows_after_stale_empty_snapshot() {
|
||||
let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(1);
|
||||
let layout = DecommissionErasureLayout { data: 3, parity: 1 };
|
||||
let capacity_infos = vec![
|
||||
DecommissionPoolCapacityInfo::for_test(0, layout, 0, 0, 0),
|
||||
DecommissionPoolCapacityInfo::for_test(1, layout, 64, 64, 0),
|
||||
];
|
||||
let mut meta = PoolMeta {
|
||||
version: POOL_META_VERSION,
|
||||
pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
|
||||
..Default::default()
|
||||
};
|
||||
meta.decommission(0, capacity_infos[0].space).unwrap();
|
||||
reserve_decommission_start_target_capacity(
|
||||
&mut meta,
|
||||
&[0],
|
||||
&capacity_infos,
|
||||
uuid::Uuid::new_v4(),
|
||||
1,
|
||||
now,
|
||||
DECOMMISSION_CAPACITY_MODEL_VERSION,
|
||||
)
|
||||
.expect("an empty scanner snapshot should still start decommission");
|
||||
let grew = grow_decommission_target_reservation(&mut meta, &capacity_infos, 0, 1, 1, uuid::Uuid::new_v4(), now)
|
||||
.expect("the reservation should grow against current target capacity");
|
||||
assert!(grew);
|
||||
let reservation = meta.pools[0]
|
||||
.decommission
|
||||
.as_ref()
|
||||
.and_then(|info| info.capacity_reservation.as_ref())
|
||||
.unwrap();
|
||||
assert_eq!(reservation.source_data_equivalent_bytes, 2);
|
||||
assert_eq!(reservation.predicted_physical_bytes, 3);
|
||||
assert_eq!(reservation.targets.len(), 1);
|
||||
assert_eq!(reservation.targets[0].reserved_physical_bytes, 6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_capacity_model_reserves_versions_delete_markers_and_temporary_copies_from_physical_usage() {
|
||||
let reservation = build_decommission_capacity_reservation(
|
||||
@@ -29316,46 +28854,11 @@ mod pools_tests {
|
||||
total: 0,
|
||||
used: 0,
|
||||
},
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(capacity, (400, 40, 360));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_physical_capacity_prefers_rustfs_owned_usage_over_filesystem_usage() {
|
||||
let disks = [10_u64, 20, 30, 40]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(disk_index, available_space)| rustfs_madmin::Disk {
|
||||
endpoint: format!("http://node-{disk_index}"),
|
||||
drive_path: format!("/disk-{disk_index}"),
|
||||
state: "ok".to_string(),
|
||||
total_space: 100,
|
||||
used_space: 100 - available_space,
|
||||
available_space,
|
||||
pool_index: 0,
|
||||
set_index: 0,
|
||||
disk_index: disk_index as i32,
|
||||
..Default::default()
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let owned = HashMap::from([(0, 12)]);
|
||||
let capacity = decommission_physical_pool_capacity(
|
||||
&disks,
|
||||
0,
|
||||
DecommissionErasureLayout { data: 2, parity: 2 },
|
||||
PoolSpaceInfo {
|
||||
free: 0,
|
||||
total: 0,
|
||||
used: 0,
|
||||
},
|
||||
Some(&owned),
|
||||
);
|
||||
|
||||
assert_eq!(capacity, (400, 40, 12));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_terminal_transitions_release_capacity_reservations() {
|
||||
let now = OffsetDateTime::UNIX_EPOCH + Duration::hours(1);
|
||||
|
||||
@@ -481,9 +481,6 @@ mod decommission_lock_order_tests {
|
||||
if let Some(deployment_id) = other_store.ctx.deployment_id() {
|
||||
ctx.set_deployment_id(deployment_id);
|
||||
}
|
||||
for pool in &mut pools {
|
||||
Arc::make_mut(pool).set_instance_ctx_for_test(Arc::clone(&ctx));
|
||||
}
|
||||
let store = Arc::new(crate::store::ECStore {
|
||||
id: uuid::Uuid::new_v4(),
|
||||
disk_map: other_store.disk_map.clone(),
|
||||
@@ -2886,89 +2883,6 @@ mod decommission_lock_order_tests {
|
||||
assert!(upload_info.parts.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn unversioned_decommission_multipart_uses_reserved_target() {
|
||||
run_large_stack_current_thread_async_test(
|
||||
"unversioned-decommission-multipart-target",
|
||||
unversioned_decommission_multipart_uses_reserved_target_case,
|
||||
);
|
||||
}
|
||||
|
||||
async fn unversioned_decommission_multipart_uses_reserved_target_case() {
|
||||
let (_temp_dirs, store, _other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
|
||||
let bucket = test_bucket("unversioned-decommission-multipart-target");
|
||||
let object = "unversioned-existing-unreserved-target.bin";
|
||||
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
|
||||
let capacity_snapshot = || {
|
||||
vec![
|
||||
DecommissionPoolCapacityInfo::for_test(0, layout, 0, 1, 1),
|
||||
DecommissionPoolCapacityInfo::for_test(1, layout, 2, 2, 0),
|
||||
DecommissionPoolCapacityInfo::for_test(2, layout, 100, 100, 0),
|
||||
]
|
||||
};
|
||||
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create the unversioned multipart target bucket");
|
||||
let incarnation = store
|
||||
.bucket_incarnation_id(&bucket)
|
||||
.await
|
||||
.expect("load the unversioned multipart target incarnation");
|
||||
|
||||
// Force the ordinary pool selector to choose pool 2. A capacity-owned
|
||||
// upload must ignore this unreserved existing object and use pool 1.
|
||||
let mut existing = PutObjReader::from_vec(b"existing unreserved target".to_vec());
|
||||
store.pools[2]
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut existing,
|
||||
&ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed the unreserved target object");
|
||||
|
||||
set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot()]);
|
||||
store
|
||||
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
|
||||
.await
|
||||
.expect("activate the unversioned multipart reservation");
|
||||
{
|
||||
let pool_meta = store.pool_meta.read().await;
|
||||
let targets = &pool_meta.pools[0]
|
||||
.decommission
|
||||
.as_ref()
|
||||
.and_then(|info| info.capacity_reservation.as_ref())
|
||||
.expect("the unversioned multipart reservation should exist")
|
||||
.targets;
|
||||
assert_eq!(targets.len(), 1, "only pool 1 should be reserved");
|
||||
assert_eq!(targets[0].pool_index, 1);
|
||||
}
|
||||
|
||||
let owner = decommission_capacity_owner(&*store.pool_meta.read().await).with_mutation_id(uuid::Uuid::new_v4());
|
||||
let mut upload_opts = ObjectOptions {
|
||||
data_movement: true,
|
||||
src_pool_idx: 0,
|
||||
versioned: false,
|
||||
version_id: None,
|
||||
mod_time: Some(time::OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(97)),
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
..Default::default()
|
||||
};
|
||||
owner.apply_to(&mut upload_opts);
|
||||
|
||||
let (_, target_pool_idx, _) = store
|
||||
.handle_new_multipart_upload_with_pool_idx(&bucket, object, &upload_opts, None)
|
||||
.await
|
||||
.expect("unversioned decommission multipart must use the reserved target");
|
||||
assert_eq!(target_pool_idx, 1, "the reservation target must override the existing unreserved object");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn data_movement_multipart_abort_restart_reconciles_inflight_after_release_save_loss() {
|
||||
@@ -6648,19 +6562,7 @@ mod decommission_lock_order_tests {
|
||||
|
||||
let restore_get_barrier = if matches!(mutation, ExternalObjectMutation::Restore) {
|
||||
let tier_name = format!("ORDERRESTORE{}", &uuid::Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let source_tiers = store.tier_config_mgr();
|
||||
let restore_tiers = other_store.tier_config_mgr();
|
||||
let backend = register_mock_tier(&source_tiers, &tier_name).await;
|
||||
// Both peers must resolve the same persisted backend identity, including its prefix.
|
||||
let tier_config = source_tiers.read().await.tiers[&tier_name].clone();
|
||||
restore_tiers.write().await.tiers.insert(tier_name.clone(), tier_config);
|
||||
crate::services::tier::tier::TierConfigMgr::install_test_driver_in(
|
||||
&restore_tiers,
|
||||
&tier_name,
|
||||
Box::new(backend.clone()),
|
||||
)
|
||||
.await
|
||||
.expect("install the same mock tier on the restoring peer");
|
||||
let backend = register_mock_tier(&store.pools[2].instance_ctx().tier_config_mgr(), &tier_name).await;
|
||||
store.pools[2]
|
||||
.transition_object(
|
||||
&bucket,
|
||||
@@ -6934,10 +6836,7 @@ mod decommission_lock_order_tests {
|
||||
}
|
||||
});
|
||||
if let Some(get_barrier) = restore_get_barrier.as_ref() {
|
||||
tokio::select! {
|
||||
() = get_barrier.wait_until_paused() => {}
|
||||
result = &mut ordinary_mutation => panic!("restore finished before the tier GET barrier: {result:?}"),
|
||||
}
|
||||
get_barrier.wait_until_paused().await;
|
||||
let read_opts = ObjectOptions {
|
||||
skip_decommissioned: true,
|
||||
..Default::default()
|
||||
|
||||
@@ -324,15 +324,6 @@ impl Sets {
|
||||
&self.ctx
|
||||
}
|
||||
|
||||
/// Keep simulated peers' metadata ownership separate while sharing disks and lock clients.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn set_instance_ctx_for_test(&mut self, ctx: Arc<InstanceContext>) {
|
||||
for set in &mut self.disk_set {
|
||||
Arc::make_mut(set).set_instance_ctx_for_test(Arc::clone(&ctx));
|
||||
}
|
||||
self.ctx = ctx;
|
||||
}
|
||||
|
||||
async fn monitor_and_connect_endpoints_task(sets: Weak<Sets>, mut rx: Receiver<()>) {
|
||||
let startup_delay = tokio::time::sleep(Duration::from_secs(5));
|
||||
tokio::pin!(startup_delay);
|
||||
@@ -799,7 +790,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for Sets {
|
||||
version_id: dst_opts.version_id.clone(),
|
||||
mod_time: dst_opts.mod_time,
|
||||
http_preconditions: dst_opts.http_preconditions.clone(),
|
||||
quota_admission: dst_opts.quota_admission,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -1493,12 +1483,12 @@ impl Sets {
|
||||
object: &str,
|
||||
version_id: &str,
|
||||
opts: &HealOpts,
|
||||
proof: crate::set_disk::AbsenceProofRequest<'_>,
|
||||
retirement: Option<&crate::bucket::retirement::MarkerRetirementContext<'_>>,
|
||||
) -> Result<(HealResultItem, Option<Error>, Option<crate::set_disk::HealedObjectAbsence>)> {
|
||||
let mut absence = None;
|
||||
let (item, error) = self
|
||||
.get_disks_for_heal_object(object, opts)?
|
||||
.heal_object_with_retirement(bucket, object, version_id, opts, &mut absence, proof)
|
||||
.heal_object_with_retirement(bucket, object, version_id, opts, &mut absence, retirement)
|
||||
.await?;
|
||||
// A caller-owned lock does not expose its lease to this boundary.
|
||||
// Keep cleanup unverified when that lease cannot be checked here.
|
||||
|
||||
@@ -642,21 +642,6 @@ pub(crate) fn data_movement_stage_source(err: &Error) -> Option<&Error> {
|
||||
.downcast_ref::<Error>()
|
||||
}
|
||||
|
||||
/// Recover a concrete typed error wrapped by [`data_movement_stage_error`].
|
||||
pub(crate) fn data_movement_stage_source_as<T>(err: &Error) -> Option<&T>
|
||||
where
|
||||
T: std::error::Error + 'static,
|
||||
{
|
||||
let Error::Io(io_err) = err else {
|
||||
return None;
|
||||
};
|
||||
io_err
|
||||
.get_ref()?
|
||||
.downcast_ref::<DataMovementStageError>()?
|
||||
.source
|
||||
.downcast_ref::<T>()
|
||||
}
|
||||
|
||||
fn schedule_data_movement_multipart_abort_cleanup(
|
||||
store: Arc<ECStore>,
|
||||
target_pool_idx: usize,
|
||||
|
||||
@@ -12,7 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::data_movement::data_movement_context_error;
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::{Error, Result, is_err_object_not_found, is_err_version_not_found};
|
||||
use crate::object_api::ObjectOptions;
|
||||
@@ -28,28 +27,6 @@ use tokio::io::AsyncReadExt;
|
||||
pub const MAX_SCANNER_PAUSE_BACKLOG_BYTES: u64 = 64 * 1024;
|
||||
pub(crate) const SCANNER_PAUSE_BACKLOG_PATH: &str = "buckets/.scanner-pause-backlog.json";
|
||||
|
||||
/// Typed failure from the scanner-owned native retirement planner.
|
||||
///
|
||||
/// Decommission must preserve the source when the native authority cannot be
|
||||
/// proven. The handoff variant is deliberately narrower than other authority
|
||||
/// failures so the worker can retry without converting a transient membership
|
||||
/// transition into a terminal pool failure.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ScannerPauseBacklogRetirementError {
|
||||
#[error("scanner pause backlog retirement has a retryable handoff conflict: {reason}")]
|
||||
HandoffConflict { reason: String },
|
||||
#[error("scanner pause backlog retirement has an unsafe authority conflict: {reason}")]
|
||||
AuthorityConflict { reason: String },
|
||||
#[error("scanner pause backlog retirement found an invalid native record: {reason}")]
|
||||
InvalidRecord { reason: String },
|
||||
}
|
||||
|
||||
impl ScannerPauseBacklogRetirementError {
|
||||
pub fn is_retryable_handoff_conflict(&self) -> bool {
|
||||
matches!(self, Self::HandoffConflict { .. })
|
||||
}
|
||||
}
|
||||
|
||||
/// A bounded, storage-fenced native replica. Only a confirmed missing object
|
||||
/// has no payload; read failures never enter the Scanner verifier.
|
||||
pub struct ScannerPauseBacklogRetirementReplica {
|
||||
@@ -67,10 +44,7 @@ pub struct ScannerPauseBacklogRetirementPlan {
|
||||
}
|
||||
|
||||
pub type ScannerPauseBacklogRetirementPlanner =
|
||||
fn(
|
||||
usize,
|
||||
&[ScannerPauseBacklogRetirementReplica],
|
||||
) -> std::result::Result<Option<ScannerPauseBacklogRetirementPlan>, ScannerPauseBacklogRetirementError>;
|
||||
fn(usize, &[ScannerPauseBacklogRetirementReplica]) -> std::result::Result<Option<ScannerPauseBacklogRetirementPlan>, String>;
|
||||
|
||||
static RETIREMENT_PLANNER: OnceLock<ScannerPauseBacklogRetirementPlanner> = OnceLock::new();
|
||||
|
||||
@@ -187,8 +161,7 @@ pub(crate) fn plan_scanner_pause_backlog_retirement(
|
||||
data: read.replica.data.clone(),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
planner(source_pool_index, &snapshots)
|
||||
.map_err(|err| data_movement_context_error(format!("scanner pause backlog retirement planner failed: {err}"), err))
|
||||
planner(source_pool_index, &snapshots).map_err(Error::other)
|
||||
}
|
||||
|
||||
/// The native writer and retirement handoff use the same conditional, full-tail
|
||||
|
||||
@@ -30,8 +30,6 @@ use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
|
||||
use rustfs_madmin::{info_commands::DiskMetrics, metrics::TimedAction};
|
||||
#[cfg(not(test))]
|
||||
use std::sync::OnceLock;
|
||||
#[cfg(test)]
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
path::PathBuf,
|
||||
@@ -630,8 +628,6 @@ pub struct DiskHealthTracker {
|
||||
/// Authoritative atomically published runtime/status pair.
|
||||
state_snapshot: AtomicU64,
|
||||
transition_lock: std::sync::Mutex<()>,
|
||||
#[cfg(test)]
|
||||
test_forced_offline: AtomicBool,
|
||||
}
|
||||
|
||||
fn pack_health_state(runtime_state: RuntimeDriveHealthState, status: u32) -> u64 {
|
||||
@@ -979,8 +975,6 @@ impl DiskHealthTracker {
|
||||
last_capacity_probe_unix_secs: AtomicI64::new(0),
|
||||
state_snapshot: AtomicU64::new(pack_health_state(RuntimeDriveHealthState::Online, DISK_HEALTH_OK)),
|
||||
transition_lock: std::sync::Mutex::new(()),
|
||||
#[cfg(test)]
|
||||
test_forced_offline: AtomicBool::new(false),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1051,13 +1045,6 @@ impl DiskHealthTracker {
|
||||
self.publish_state(state, status);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn force_offline_for_test(&self) {
|
||||
self.test_forced_offline.store(true, Ordering::Release);
|
||||
let _guard = self.transition_lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
self.publish_state(RuntimeDriveHealthState::Offline, DISK_HEALTH_FAULTY);
|
||||
}
|
||||
|
||||
pub fn swap_ok_to_faulty(&self) -> bool {
|
||||
let _guard = self.transition_lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
let (_, status) = unpack_health_state(self.state_snapshot.load(Ordering::Acquire));
|
||||
@@ -1136,8 +1123,6 @@ impl DiskHealthTracker {
|
||||
/// Remote disks are marked faulty on timeout/network errors; the init loop retries with the
|
||||
/// same [`DiskStore`] handles, which would otherwise fail immediately at `is_faulty()`.
|
||||
pub fn reset_for_store_init_retry(&self, endpoint: &Endpoint) {
|
||||
#[cfg(test)]
|
||||
self.test_forced_offline.store(false, Ordering::Release);
|
||||
self.reset_for_store_init_retry_at(endpoint, current_unix_time());
|
||||
}
|
||||
|
||||
@@ -1157,10 +1142,6 @@ impl DiskHealthTracker {
|
||||
}
|
||||
|
||||
pub fn mark_recovery_success(&self, endpoint: &Endpoint, reason: &'static str) -> bool {
|
||||
#[cfg(test)]
|
||||
if self.test_forced_offline.load(Ordering::Acquire) {
|
||||
return false;
|
||||
}
|
||||
let _guard = self.transition_lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
let current = self.runtime_state();
|
||||
let next = match current {
|
||||
@@ -1459,11 +1440,6 @@ impl LocalDiskWrapper {
|
||||
self.health.force_runtime_state_for_test(state);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn force_offline_for_test(&self) {
|
||||
self.health.force_offline_for_test();
|
||||
}
|
||||
|
||||
/// Same as [`DiskHealthTracker::reset_for_store_init_retry`]: undo a transient faulty mark before another format load attempt.
|
||||
pub fn reset_health_for_store_init_retry(&self) {
|
||||
self.health.reset_for_store_init_retry(&self.disk.endpoint());
|
||||
|
||||
@@ -244,16 +244,13 @@ impl StdError for ConditionalFileNotCommittedError {
|
||||
}
|
||||
}
|
||||
|
||||
fn classify_internode_disk_error(error: &InternodeHttpError) -> Option<DiskError> {
|
||||
fn classify_internode_missing_error(error: &InternodeHttpError) -> Option<DiskError> {
|
||||
if error.is_remote_file_not_found() {
|
||||
return Some(DiskError::FileNotFound);
|
||||
}
|
||||
if error.is_remote_volume_not_found() {
|
||||
return Some(DiskError::VolumeNotFound);
|
||||
}
|
||||
if error.is_remote_file_corrupt() {
|
||||
return Some(DiskError::FileCorrupt);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
@@ -529,10 +526,13 @@ fn io_error_chain_contains_kind(io_error: &std::io::Error, kind: std::io::ErrorK
|
||||
|
||||
impl From<std::io::Error> for DiskError {
|
||||
fn from(e: std::io::Error) -> Self {
|
||||
if let Some(error) = e.get_ref().and_then(|source| source.downcast_ref::<InternodeHttpError>())
|
||||
&& let Some(classified) = classify_internode_disk_error(error)
|
||||
{
|
||||
return classified;
|
||||
if let Some(error) = e.get_ref().and_then(|source| source.downcast_ref::<InternodeHttpError>()) {
|
||||
if error.is_remote_file_not_found() {
|
||||
return DiskError::FileNotFound;
|
||||
}
|
||||
if error.is_remote_volume_not_found() {
|
||||
return DiskError::VolumeNotFound;
|
||||
}
|
||||
}
|
||||
let e = match e.downcast::<TerminalReadError>() {
|
||||
Ok(terminal_error) => {
|
||||
@@ -541,7 +541,7 @@ impl From<std::io::Error> for DiskError {
|
||||
&& let Some(internode_error) = io_error
|
||||
.get_ref()
|
||||
.and_then(|source| source.downcast_ref::<InternodeHttpError>())
|
||||
&& let Some(classified) = classify_internode_disk_error(internode_error)
|
||||
&& let Some(classified) = classify_internode_missing_error(internode_error)
|
||||
{
|
||||
return classified;
|
||||
}
|
||||
@@ -963,7 +963,6 @@ mod tests {
|
||||
for (remote_error, expected) in [
|
||||
(rustfs_rio::new_test_remote_file_not_found_http_io_error(), DiskError::FileNotFound),
|
||||
(rustfs_rio::new_test_remote_volume_not_found_http_io_error(), DiskError::VolumeNotFound),
|
||||
(rustfs_rio::new_test_remote_file_corrupt_http_io_error(), DiskError::FileCorrupt),
|
||||
] {
|
||||
let wrapped = terminal_read_error_to_io(DiskError::Io(remote_error));
|
||||
assert_eq!(DiskError::from(wrapped), expected);
|
||||
@@ -1534,23 +1533,25 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_internode_disk_errors_preserve_disk_error_types() {
|
||||
fn test_internode_missing_errors_preserve_disk_error_types() {
|
||||
let file_missing = DiskError::from(rustfs_rio::new_test_remote_file_not_found_http_io_error());
|
||||
let volume_missing = DiskError::from(rustfs_rio::new_test_remote_volume_not_found_http_io_error());
|
||||
let file_corrupt = DiskError::from(rustfs_rio::new_test_remote_file_corrupt_http_io_error());
|
||||
let unmarked_server_error = DiskError::from(rustfs_rio::new_test_internode_http_io_error(
|
||||
rustfs_rio::InternodeHttpErrorKind::HttpStatus(http::StatusCode::INTERNAL_SERVER_ERROR),
|
||||
));
|
||||
|
||||
assert_eq!(file_missing, DiskError::FileNotFound);
|
||||
assert_eq!(volume_missing, DiskError::VolumeNotFound);
|
||||
assert_eq!(file_corrupt, DiskError::FileCorrupt);
|
||||
assert!(matches!(unmarked_server_error, DiskError::Io(_)));
|
||||
for error in [file_missing, volume_missing, file_corrupt] {
|
||||
assert_eq!(error.clone(), error);
|
||||
for missing in [file_missing, volume_missing] {
|
||||
assert_eq!(missing.clone(), missing);
|
||||
assert_eq!(
|
||||
crate::disk::error_reduce::reduce_write_quorum_errs(&[Some(error.clone()), Some(error.clone()), None], &[], 2),
|
||||
Some(error)
|
||||
crate::disk::error_reduce::reduce_write_quorum_errs(
|
||||
&[Some(missing.clone()), Some(missing.clone()), None],
|
||||
&[],
|
||||
2
|
||||
),
|
||||
Some(missing)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
+121
-495
@@ -94,8 +94,6 @@ const STALE_TMP_OBJECT_EXPIRY: Duration = Duration::from_secs(24 * 60 * 60);
|
||||
#[cfg(test)]
|
||||
tokio::task_local! {
|
||||
static DIRECTORY_LISTING_ENTRY_PROBE_COUNT: Arc<AtomicUsize>;
|
||||
/// `(sampled, complete)` directory reads issued by the delete-residue probe.
|
||||
static DELETE_RESIDUE_PROBE_READS: Arc<(AtomicUsize, AtomicUsize)>;
|
||||
}
|
||||
const RUSTFS_META_TMP_OLD_BUCKET: &str = ".rustfs.sys/tmp-old";
|
||||
const INLINE_METADATA_ROLLBACK_DIR_XOR: u128 = 0x7275737466735f696e6c696e655f7262;
|
||||
@@ -107,21 +105,6 @@ pub(crate) const RESERVED_DELETE_DATA_DIR_MARKER_PREFIX: &str = "reserve-delete-
|
||||
/// under-filled batch settles the common case without materializing large
|
||||
/// child sets; a full batch cannot prove no listable child hides behind it.
|
||||
const DELETE_RESIDUE_PROBE_LIMIT: i32 = 8;
|
||||
/// Most directory reads one delete-residue probe spends walking down through
|
||||
/// the ancestors of a deleted key before it gives up and lets the prefix
|
||||
/// surface. A genuine prefix settles within its depth (the first object's
|
||||
/// `xl.meta` ends the walk), so the budget only caps the cost of hiding a
|
||||
/// large residue tree, which then hides one level at a time instead.
|
||||
const DELETE_RESIDUE_PROBE_READ_BUDGET: usize = 32;
|
||||
|
||||
/// One directory read owed by the delete-residue probe.
|
||||
enum ResidueProbeStep {
|
||||
/// Read a bounded batch of `dir` and probe what it holds.
|
||||
Sample(String),
|
||||
/// Read all of `dir` once every child in `sampled` proved unlistable, and
|
||||
/// probe the rest.
|
||||
Remainder { dir: String, sampled: Vec<String> },
|
||||
}
|
||||
|
||||
/// A `part.N` file with a positive part number, the shape erasure data takes
|
||||
/// inside a version data dir.
|
||||
@@ -1899,61 +1882,6 @@ impl AsyncWrite for DirectWriter {
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_write_vectored(
|
||||
self: std::pin::Pin<&mut Self>,
|
||||
cx: &mut std::task::Context<'_>,
|
||||
bufs: &[std::io::IoSlice<'_>],
|
||||
) -> std::task::Poll<std::io::Result<usize>> {
|
||||
// Bitrot writes arrive as [hash, data]. Coalesce both slices into the
|
||||
// aligned bounce buffer so the O_DIRECT path has the same byte-stream
|
||||
// contract as the buffered writer instead of relying on the trait's
|
||||
// first-slice fallback.
|
||||
let this = self.get_mut();
|
||||
loop {
|
||||
match &mut this.state {
|
||||
DirectWriteState::Busy(_) => {
|
||||
std::task::ready!(this.poll_drive_busy(cx))?;
|
||||
}
|
||||
DirectWriteState::Idle(inner_opt) => {
|
||||
let inner = inner_opt.as_mut().expect("idle direct writer must hold inner state");
|
||||
let capacity = inner.buf.len;
|
||||
let space = capacity - inner.filled;
|
||||
let mut remaining = space;
|
||||
let mut written = 0;
|
||||
for src in bufs {
|
||||
if remaining == 0 {
|
||||
break;
|
||||
}
|
||||
let take = src.len().min(remaining);
|
||||
let start = inner.filled + written;
|
||||
inner.buf.as_mut_slice()[start..start + take].copy_from_slice(&src[..take]);
|
||||
written += take;
|
||||
remaining -= take;
|
||||
}
|
||||
|
||||
if written == 0 {
|
||||
return std::task::Poll::Ready(Ok(0));
|
||||
}
|
||||
inner.filled += written;
|
||||
|
||||
if inner.filled == capacity {
|
||||
let mut inner = inner_opt.take().expect("idle direct writer must hold inner state");
|
||||
let handle = tokio::task::spawn_blocking(move || {
|
||||
let res = inner.flush_batch();
|
||||
(inner, res)
|
||||
});
|
||||
this.state = DirectWriteState::Busy(handle);
|
||||
}
|
||||
return std::task::Poll::Ready(Ok(written));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn is_write_vectored(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn poll_flush(self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> std::task::Poll<std::io::Result<()>> {
|
||||
// Only drive an in-flight batch to completion. Sub-alignment staged
|
||||
// bytes cannot be flushed mid-stream (they would misalign the next
|
||||
@@ -7611,7 +7539,7 @@ impl LocalDisk {
|
||||
objs_returned: &mut i32,
|
||||
skip_current_dir_object: bool,
|
||||
multipart_dir_to_skip: Option<HashSet<String>>,
|
||||
) -> Result<bool>
|
||||
) -> Result<()>
|
||||
where
|
||||
W: AsyncWrite + Unpin + Send,
|
||||
{
|
||||
@@ -7629,7 +7557,7 @@ impl LocalDisk {
|
||||
};
|
||||
|
||||
if opts.limit > 0 && *objs_returned >= opts.limit {
|
||||
return Ok(true);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// TODO(backlog): add directory listing lock to prevent concurrent enumeration
|
||||
@@ -7690,12 +7618,12 @@ impl LocalDisk {
|
||||
return Err(DiskError::FileNotFound);
|
||||
}
|
||||
|
||||
return Ok(false);
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
|
||||
if entries.is_empty() {
|
||||
return Ok(false);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
current = current.trim_matches('/').to_owned();
|
||||
@@ -7709,7 +7637,7 @@ impl LocalDisk {
|
||||
let entry = item.clone();
|
||||
// check limit
|
||||
if opts.limit > 0 && *objs_returned >= opts.limit {
|
||||
return Ok(true);
|
||||
return Ok(());
|
||||
}
|
||||
// check multipart dir
|
||||
if skip_current_dir_object
|
||||
@@ -7814,12 +7742,12 @@ impl LocalDisk {
|
||||
prefix = "".to_owned();
|
||||
|
||||
for entry in entries.iter() {
|
||||
if entry.is_empty() {
|
||||
continue;
|
||||
if opts.limit > 0 && *objs_returned >= opts.limit {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if opts.limit > 0 && *objs_returned >= opts.limit {
|
||||
return Ok(true);
|
||||
if entry.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let name = path_join_buf(&[current.as_str(), entry.as_str()]);
|
||||
@@ -7835,7 +7763,7 @@ impl LocalDisk {
|
||||
// moment the limit is reached, same as the check below this
|
||||
// loop guards against for the current entry itself.
|
||||
if opts.limit > 0 && *objs_returned >= opts.limit {
|
||||
return Ok(true);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let (pop, skip_object, dir_to_skip, scan_required) = dir_stack.pop().expect("operation should succeed");
|
||||
@@ -7849,25 +7777,23 @@ impl LocalDisk {
|
||||
.await?;
|
||||
|
||||
let scan_path = pop.clone();
|
||||
if opts.recursive && scan_required {
|
||||
match Box::pin(self.scan_dir(pop, prefix.clone(), opts, out, objs_returned, skip_object, dir_to_skip)).await {
|
||||
Ok(true) => return Ok(true),
|
||||
Ok(false) => {}
|
||||
Err(er) => {
|
||||
if !er.is_metacache_output_stream_closed() {
|
||||
error!(
|
||||
event = EVENT_DISK_LOCAL_SCAN_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
path = %scan_path,
|
||||
operation = "scan_dir",
|
||||
error = ?er,
|
||||
"Disk local scan failed"
|
||||
);
|
||||
}
|
||||
return Err(er);
|
||||
}
|
||||
if opts.recursive
|
||||
&& scan_required
|
||||
&& let Err(er) =
|
||||
Box::pin(self.scan_dir(pop, prefix.clone(), opts, out, objs_returned, skip_object, dir_to_skip)).await
|
||||
{
|
||||
if !er.is_metacache_output_stream_closed() {
|
||||
error!(
|
||||
event = EVENT_DISK_LOCAL_SCAN_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
path = %scan_path,
|
||||
operation = "scan_dir",
|
||||
error = ?er,
|
||||
"Disk local scan failed"
|
||||
);
|
||||
}
|
||||
return Err(er);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7878,7 +7804,7 @@ impl LocalDisk {
|
||||
// tail, permanently skipping those keys on the next page instead
|
||||
// of just deferring them to it.
|
||||
if opts.limit > 0 && *objs_returned >= opts.limit {
|
||||
return Ok(true);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut meta = MetaCacheEntry {
|
||||
@@ -8021,7 +7947,7 @@ impl LocalDisk {
|
||||
|
||||
while let Some((dir, skip_object, dir_to_skip, scan_required)) = dir_stack.pop() {
|
||||
if opts.limit > 0 && *objs_returned >= opts.limit {
|
||||
return Ok(true);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
write_metacache_obj(
|
||||
@@ -8034,29 +7960,27 @@ impl LocalDisk {
|
||||
.await?;
|
||||
|
||||
let scan_path = dir.clone();
|
||||
if opts.recursive && scan_required {
|
||||
match Box::pin(self.scan_dir(dir, prefix.clone(), opts, out, objs_returned, skip_object, dir_to_skip)).await {
|
||||
Ok(true) => return Ok(true),
|
||||
Ok(false) => {}
|
||||
Err(er) => {
|
||||
if !er.is_metacache_output_stream_closed() {
|
||||
error!(
|
||||
event = EVENT_DISK_LOCAL_SCAN_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
path = %scan_path,
|
||||
operation = "scan_dir",
|
||||
error = ?er,
|
||||
"Disk local recursive scan failed"
|
||||
);
|
||||
}
|
||||
return Err(er);
|
||||
}
|
||||
if opts.recursive
|
||||
&& scan_required
|
||||
&& let Err(er) =
|
||||
Box::pin(self.scan_dir(dir, prefix.clone(), opts, out, objs_returned, skip_object, dir_to_skip)).await
|
||||
{
|
||||
if !er.is_metacache_output_stream_closed() {
|
||||
error!(
|
||||
event = EVENT_DISK_LOCAL_SCAN_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
path = %scan_path,
|
||||
operation = "scan_dir",
|
||||
error = ?er,
|
||||
"Disk local recursive scan failed"
|
||||
);
|
||||
}
|
||||
return Err(er);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(false)
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Whether the backing directory of plain object `object_name` also holds
|
||||
@@ -8123,116 +8047,72 @@ impl LocalDisk {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Whether the metadata-less directory `dir_name` holds nothing that a
|
||||
/// listing could show: every leaf under it is the data dir of a deleted
|
||||
/// version (a non-nil UUID directory of `part.N` files and
|
||||
/// delete-transaction markers) or an empty directory. That is what an
|
||||
/// interrupted or deferred version delete leaves behind once the `xl.meta`
|
||||
/// is gone, and neither the object directory nor the date-style ancestors
|
||||
/// above it may surface as prefixes (#6898). Real objects are directories
|
||||
/// carrying their own `xl.meta`, so the first file met outside a UUID data
|
||||
/// dir, or any stray entry inside one, proves a genuine prefix and ends the
|
||||
/// walk at the depth of the first object.
|
||||
///
|
||||
/// Reads stay small on genuine prefixes: each directory is sampled with one
|
||||
/// bounded batch, the sampled children are probed depth-first, and the
|
||||
/// directory is only read in full once every sampled child proved to hold
|
||||
/// nothing listable. A total read budget makes a residue tree larger than
|
||||
/// the budget surface and be hidden one level down instead, so the cost on
|
||||
/// a genuine prefix never exceeds its depth in small directory reads. A
|
||||
/// Whether the metadata-less directory `dir_name` holds nothing but the
|
||||
/// data dirs of deleted versions: it is itself a non-nil UUID directory of
|
||||
/// `part.N` files and delete-transaction markers, or every child is one.
|
||||
/// That is what an interrupted or deferred version delete leaves behind
|
||||
/// once the `xl.meta` is gone, and it must not surface as a prefix. Real
|
||||
/// object children are directories carrying their own `xl.meta`, so the
|
||||
/// first non-UUID child, stray file, or subdirectory inside a UUID child
|
||||
/// proves the directory is a genuine prefix. Reads are bounded: a
|
||||
/// directory that vanishes mid-probe holds nothing listable.
|
||||
async fn directory_is_delete_residue(&self, bucket: &str, dir_name: &str, stall: Option<Duration>) -> Result<bool> {
|
||||
let dir_name = dir_name.trim_end_matches(SLASH_SEPARATOR);
|
||||
let mut reads = 0usize;
|
||||
let mut pending = vec![ResidueProbeStep::Sample(dir_name.to_owned())];
|
||||
while let Some(step) = pending.pop() {
|
||||
if reads >= DELETE_RESIDUE_PROBE_READ_BUDGET {
|
||||
let Some(entries) = self.read_dir_for_residue_probe(bucket, dir_name, stall).await? else {
|
||||
return Ok(false);
|
||||
};
|
||||
if entries.is_empty() {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let is_data_dir = dir_name
|
||||
.rsplit(SLASH_SEPARATOR)
|
||||
.next()
|
||||
.is_some_and(|name| Uuid::parse_str(name).is_ok_and(|uuid| !uuid.is_nil()));
|
||||
if is_data_dir && entries.iter().all(|entry| is_metadata_less_data_dir_entry(entry)) {
|
||||
return Ok(true);
|
||||
}
|
||||
|
||||
for entry in entries {
|
||||
let Some(child) = entry.strip_suffix(SLASH_SEPARATOR) else {
|
||||
return Ok(false);
|
||||
};
|
||||
if !Uuid::parse_str(child).is_ok_and(|uuid| !uuid.is_nil()) {
|
||||
return Ok(false);
|
||||
}
|
||||
reads += 1;
|
||||
|
||||
let (dir, entries, complete) = match step {
|
||||
ResidueProbeStep::Sample(dir) => {
|
||||
let Some((entries, complete)) = self.read_dir_for_residue_probe(bucket, &dir, stall, false).await? else {
|
||||
continue;
|
||||
};
|
||||
(dir, entries, complete)
|
||||
}
|
||||
ResidueProbeStep::Remainder { dir, sampled } => {
|
||||
let Some((entries, _)) = self.read_dir_for_residue_probe(bucket, &dir, stall, true).await? else {
|
||||
continue;
|
||||
};
|
||||
let entries = entries
|
||||
.into_iter()
|
||||
.filter(|entry| !sampled.contains(entry))
|
||||
.collect::<Vec<_>>();
|
||||
(dir, entries, true)
|
||||
}
|
||||
};
|
||||
|
||||
let is_data_dir = dir
|
||||
.rsplit(SLASH_SEPARATOR)
|
||||
.next()
|
||||
.is_some_and(|name| Uuid::parse_str(name).is_ok_and(|uuid| !uuid.is_nil()));
|
||||
if is_data_dir {
|
||||
if !entries.iter().all(|entry| is_metadata_less_data_dir_entry(entry)) {
|
||||
// A subdirectory, an `xl.meta`, or an unknown file inside a
|
||||
// UUID directory: not plain delete residue.
|
||||
return Ok(false);
|
||||
}
|
||||
if !complete {
|
||||
// Only the sampled part files were seen; the rest must be
|
||||
// read before the data dir counts as plain residue.
|
||||
pending.push(ResidueProbeStep::Remainder { dir, sampled: entries });
|
||||
}
|
||||
let child_path = path_join_buf(&[dir_name, child]);
|
||||
let Some(child_entries) = self.read_dir_for_residue_probe(bucket, &child_path, stall).await? else {
|
||||
continue;
|
||||
};
|
||||
if !child_entries.iter().all(|entry| is_metadata_less_data_dir_entry(entry)) {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let mut children = Vec::with_capacity(entries.len());
|
||||
for entry in &entries {
|
||||
let Some(child) = entry.strip_suffix(SLASH_SEPARATOR) else {
|
||||
// A file outside a plain data dir: `xl.meta` or something
|
||||
// this probe does not understand. Either way, not residue.
|
||||
return Ok(false);
|
||||
};
|
||||
children.push(path_join_buf(&[&dir, child]));
|
||||
}
|
||||
if !complete {
|
||||
// Revisit the unsampled siblings only after every sampled
|
||||
// child, probed first, turned out to hold nothing listable.
|
||||
pending.push(ResidueProbeStep::Remainder { dir, sampled: entries });
|
||||
}
|
||||
pending.extend(children.into_iter().map(ResidueProbeStep::Sample));
|
||||
}
|
||||
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Read `dir` for the residue probe: one bounded batch, or the complete
|
||||
/// directory when `complete` is set. The flag in the result says whether
|
||||
/// the batch held the whole directory. `None` when the directory does not
|
||||
/// exist any more.
|
||||
async fn read_dir_for_residue_probe(
|
||||
&self,
|
||||
bucket: &str,
|
||||
dir: &str,
|
||||
stall: Option<Duration>,
|
||||
complete: bool,
|
||||
) -> Result<Option<(Vec<String>, bool)>> {
|
||||
let count = if complete { -1 } else { DELETE_RESIDUE_PROBE_LIMIT };
|
||||
#[cfg(test)]
|
||||
let _ = DELETE_RESIDUE_PROBE_READS.try_with(|reads| {
|
||||
let counter = if complete { &reads.1 } else { &reads.0 };
|
||||
counter.fetch_add(1, Ordering::Relaxed)
|
||||
});
|
||||
match with_walk_stall_timeout(stall, self.list_dir("", bucket, dir, count)).await {
|
||||
Ok(entries) => {
|
||||
let complete = complete || entries.len() < DELETE_RESIDUE_PROBE_LIMIT as usize;
|
||||
Ok(Some((entries, complete)))
|
||||
/// Read `dir` with a bounded batch first and a complete read only when the
|
||||
/// batch was full. `None` when the directory does not exist any more.
|
||||
async fn read_dir_for_residue_probe(&self, bucket: &str, dir: &str, stall: Option<Duration>) -> Result<Option<Vec<String>>> {
|
||||
for count in [DELETE_RESIDUE_PROBE_LIMIT, -1] {
|
||||
let entries = match with_walk_stall_timeout(stall, self.list_dir("", bucket, dir, count)).await {
|
||||
Ok(entries) => entries,
|
||||
Err(err) => {
|
||||
if err == DiskError::VolumeNotFound || err == Error::FileNotFound {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
if count < 0 || entries.len() < count as usize {
|
||||
return Ok(Some(entries));
|
||||
}
|
||||
Err(err) if err == DiskError::VolumeNotFound || err == Error::FileNotFound => Ok(None),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Whether anything under `dir_name` would appear in a listing. With
|
||||
@@ -10180,25 +10060,21 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
}
|
||||
|
||||
let limit_reached = self
|
||||
.scan_dir(
|
||||
opts.base_dir.clone(),
|
||||
opts.filter_prefix.clone().unwrap_or_default(),
|
||||
&opts,
|
||||
&mut out,
|
||||
&mut objs_returned,
|
||||
skip_current_dir_object,
|
||||
if multipart_dir_to_skip.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(multipart_dir_to_skip)
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
self.scan_dir(
|
||||
opts.base_dir.clone(),
|
||||
opts.filter_prefix.clone().unwrap_or_default(),
|
||||
&opts,
|
||||
&mut out,
|
||||
&mut objs_returned,
|
||||
skip_current_dir_object,
|
||||
if multipart_dir_to_skip.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(multipart_dir_to_skip)
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
if let Some(flag) = opts.producer_limit_reached.as_ref() {
|
||||
flag.store(limit_reached, std::sync::atomic::Ordering::Release);
|
||||
}
|
||||
out.close().await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -18843,9 +18719,9 @@ mod test {
|
||||
let (fast_path_names, fast_path_probes) = scan_prefixes(&disk, bucket, true).await;
|
||||
|
||||
assert_eq!(conservative_names, expected_names);
|
||||
// The fast path hides the empty `stale/` chain too, at the cost of a
|
||||
// few bounded directory reads rather than the metadata probes.
|
||||
assert_eq!(fast_path_names, expected_names);
|
||||
let mut expected_fast_path_names = expected_names.clone();
|
||||
expected_fast_path_names.push("stale/".to_owned());
|
||||
assert_eq!(fast_path_names, expected_fast_path_names);
|
||||
let expected_probes = PREFIX_COUNT * 3 + 3;
|
||||
assert_eq!(conservative_probes, expected_probes);
|
||||
assert_eq!(fast_path_probes, 0);
|
||||
@@ -18956,9 +18832,9 @@ mod test {
|
||||
}
|
||||
|
||||
// Directories whose only content is a deleted version's data dir are
|
||||
// not prefixes, and neither are their ancestors.
|
||||
// not prefixes; their ancestors stay ordinary directories until an
|
||||
// empty listing reclaims them.
|
||||
assert_eq!(scan_names(&disk, bucket, "residue/2026/").await, Vec::<String>::new());
|
||||
assert_eq!(scan_names(&disk, bucket, "residue/").await, Vec::<String>::new());
|
||||
assert_eq!(scan_names(&disk, bucket, "committed/").await, Vec::<String>::new());
|
||||
|
||||
// UUID-named directories holding real objects, an object keyed by a
|
||||
@@ -18966,258 +18842,18 @@ mod test {
|
||||
assert_eq!(scan_names(&disk, bucket, "uploads/").await, vec![format!("uploads/{upload}/")]);
|
||||
assert_eq!(scan_names(&disk, bucket, "named/").await, vec![format!("named/{named}")]);
|
||||
assert_eq!(scan_names(&disk, bucket, "mixed/").await, vec!["mixed/child".to_owned()]);
|
||||
assert_eq!(
|
||||
scan_names(&disk, bucket, "").await,
|
||||
vec!["mixed/".to_owned(), "named/".to_owned(), "uploads/".to_owned()]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_scan_dir_nonrecursive_fast_path_hides_delete_residue_ancestors() {
|
||||
use rustfs_filemeta::MetacacheReader;
|
||||
use tempfile::tempdir;
|
||||
|
||||
let dir = tempdir().expect("tempdir should be created");
|
||||
let bucket = "test-bucket";
|
||||
let bucket_dir = dir.path().join(bucket);
|
||||
|
||||
async fn write_object(object_dir: &Path, object_name: &str) {
|
||||
fs::create_dir_all(object_dir)
|
||||
.await
|
||||
.expect("object directory should be created");
|
||||
let mut metadata = FileMeta::default();
|
||||
let mut file_info = FileInfo::new(object_name, 1, 1);
|
||||
file_info.mod_time = Some(OffsetDateTime::now_utc());
|
||||
metadata.add_version(file_info).expect("metadata should be valid");
|
||||
fs::write(
|
||||
object_dir.join(STORAGE_FORMAT_FILE),
|
||||
metadata.marshal_msg().expect("metadata should encode"),
|
||||
)
|
||||
.await
|
||||
.expect("object metadata should be written");
|
||||
}
|
||||
|
||||
async fn write_residue(object_dir: &Path, committed: bool) {
|
||||
let residue = object_dir.join(Uuid::new_v4().to_string());
|
||||
fs::create_dir_all(&residue).await.expect("residue should be created");
|
||||
fs::write(residue.join("part.1"), b"stale")
|
||||
.await
|
||||
.expect("stale part should be written");
|
||||
if committed {
|
||||
fs::write(residue.join(format!("{DELETE_DATA_DIR_MARKER_PREFIX}{}", Uuid::new_v4())), [])
|
||||
.await
|
||||
.expect("delete marker should be written");
|
||||
}
|
||||
}
|
||||
|
||||
// The reported shape: a date-partitioned key deleted on an older build,
|
||||
// whose data dir survived without a delete-transaction marker. Every
|
||||
// ancestor up to `metrics/` holds nothing else.
|
||||
write_residue(&bucket_dir.join("metrics/kubelet/2026/08/28/23/74992556388248657933757.parquet"), false).await;
|
||||
// Several committed residues under one hour plus an empty sibling hour.
|
||||
write_residue(&bucket_dir.join("metrics/cpu/2026/08/28/22/a.parquet"), true).await;
|
||||
write_residue(&bucket_dir.join("metrics/cpu/2026/08/28/22/b.parquet"), true).await;
|
||||
fs::create_dir_all(bucket_dir.join("metrics/cpu/2026/08/28/21"))
|
||||
.await
|
||||
.expect("empty hour directory should be created");
|
||||
|
||||
// A live object deep under an otherwise identical tree keeps every
|
||||
// ancestor visible, even beside residue.
|
||||
write_residue(&bucket_dir.join("logs/default/2026/08/28/23/old.parquet"), false).await;
|
||||
write_object(
|
||||
&bucket_dir.join("logs/default/2026/08/28/23/live.parquet"),
|
||||
"logs/default/2026/08/28/23/live.parquet",
|
||||
)
|
||||
.await;
|
||||
|
||||
// A prefix with more residue directories than the probe budget stays
|
||||
// visible rather than costing an unbounded walk.
|
||||
for hour in 0..(DELETE_RESIDUE_PROBE_READ_BUDGET + 1) {
|
||||
write_residue(&bucket_dir.join(format!("bulk/2026/08/28/{hour:02}/a.parquet")), false).await;
|
||||
}
|
||||
// Exactly at the budget: `edge/` with N object dirs costs one sampled
|
||||
// read of `edge`, two reads per object dir (its own and its data dir),
|
||||
// and one remainder read of `edge` once its 8-entry sample was full.
|
||||
for object in 0..15 {
|
||||
write_residue(&bucket_dir.join(format!("edge-hide/{object:02}.parquet")), false).await;
|
||||
}
|
||||
for object in 0..16 {
|
||||
write_residue(&bucket_dir.join(format!("edge-show/{object:02}.parquet")), false).await;
|
||||
}
|
||||
|
||||
// A file that is not `xl.meta` outside a UUID data dir is not residue
|
||||
// either: the probe does not guess, the prefix surfaces.
|
||||
fs::create_dir_all(bucket_dir.join("stray/2026/08"))
|
||||
.await
|
||||
.expect("stray directory should be created");
|
||||
fs::write(bucket_dir.join("stray/2026/08/notes.txt"), b"?")
|
||||
.await
|
||||
.expect("stray file should be written");
|
||||
|
||||
// A UUID directory holding a subdirectory is not a data dir shape the
|
||||
// probe understands, so it surfaces even when the subdirectory is empty.
|
||||
let odd = Uuid::new_v4().to_string();
|
||||
fs::create_dir_all(bucket_dir.join("odd").join(&odd).join("sub"))
|
||||
.await
|
||||
.expect("odd directory should be created");
|
||||
|
||||
let endpoint =
|
||||
Endpoint::try_from(dir.path().to_str().expect("tempdir path should be UTF-8")).expect("endpoint should parse");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should initialize");
|
||||
|
||||
async fn scan_names(disk: &LocalDisk, bucket: &str, current: &str) -> Vec<String> {
|
||||
let (reader, mut writer) = tokio::io::duplex(64 * 1024);
|
||||
let mut output = MetacacheWriter::new(&mut writer);
|
||||
let opts = WalkDirOptions {
|
||||
bucket: bucket.to_string(),
|
||||
base_dir: current.to_string(),
|
||||
skip_hidden_prefix_check: true,
|
||||
..Default::default()
|
||||
};
|
||||
let mut objects_returned = 0;
|
||||
disk.scan_dir(
|
||||
current.to_string(),
|
||||
"".to_string(),
|
||||
&opts,
|
||||
&mut output,
|
||||
&mut objects_returned,
|
||||
false,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("scan_dir should succeed");
|
||||
output.close().await.expect("metacache writer should close");
|
||||
drop(output);
|
||||
drop(writer);
|
||||
|
||||
let mut names = MetacacheReader::new(reader)
|
||||
.read_all()
|
||||
.await
|
||||
.expect("scan output should decode")
|
||||
.into_iter()
|
||||
.map(|entry| entry.name)
|
||||
.collect::<Vec<_>>();
|
||||
names.sort();
|
||||
names
|
||||
}
|
||||
|
||||
// Every level of a tree whose only leaves are deleted data dirs is
|
||||
// hidden, not just the object directory itself.
|
||||
assert_eq!(scan_names(&disk, bucket, "metrics/kubelet/2026/08/28/").await, Vec::<String>::new());
|
||||
assert_eq!(scan_names(&disk, bucket, "metrics/kubelet/").await, Vec::<String>::new());
|
||||
assert_eq!(scan_names(&disk, bucket, "metrics/cpu/2026/08/28/").await, Vec::<String>::new());
|
||||
assert_eq!(scan_names(&disk, bucket, "metrics/").await, Vec::<String>::new());
|
||||
assert_eq!(
|
||||
scan_names(&disk, bucket, "logs/default/2026/08/28/").await,
|
||||
vec!["logs/default/2026/08/28/23/".to_owned()]
|
||||
);
|
||||
assert_eq!(scan_names(&disk, bucket, "logs/").await, vec!["logs/default/".to_owned()]);
|
||||
assert_eq!(scan_names(&disk, bucket, "bulk/2026/08/").await, vec!["bulk/2026/08/28/".to_owned()]);
|
||||
// Listed directly, every object dir is probed on its own and hidden;
|
||||
// the budget only bites where the whole tree hangs off one entry, so
|
||||
// the root listing below shows `edge-show/` but not `edge-hide/`.
|
||||
assert_eq!(scan_names(&disk, bucket, "edge-hide/").await, Vec::<String>::new());
|
||||
assert_eq!(scan_names(&disk, bucket, "edge-show/").await, Vec::<String>::new());
|
||||
assert_eq!(scan_names(&disk, bucket, "stray/").await, vec!["stray/2026/".to_owned()]);
|
||||
assert_eq!(scan_names(&disk, bucket, "odd/").await, vec![format!("odd/{odd}/")]);
|
||||
assert_eq!(
|
||||
scan_names(&disk, bucket, "").await,
|
||||
vec![
|
||||
"bulk/".to_owned(),
|
||||
"edge-show/".to_owned(),
|
||||
"logs/".to_owned(),
|
||||
"odd/".to_owned(),
|
||||
"stray/".to_owned(),
|
||||
"committed/".to_owned(),
|
||||
"mixed/".to_owned(),
|
||||
"named/".to_owned(),
|
||||
"residue/".to_owned(),
|
||||
"uploads/".to_owned(),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_scan_dir_nonrecursive_fast_path_residue_probe_reads_stay_within_prefix_depth() {
|
||||
use rustfs_filemeta::MetacacheReader;
|
||||
use tempfile::tempdir;
|
||||
|
||||
let dir = tempdir().expect("tempdir should be created");
|
||||
let bucket = "test-bucket";
|
||||
let bucket_dir = dir.path().join(bucket);
|
||||
|
||||
async fn write_object(object_dir: &Path, object_name: &str) {
|
||||
fs::create_dir_all(object_dir)
|
||||
.await
|
||||
.expect("object directory should be created");
|
||||
let mut metadata = FileMeta::default();
|
||||
let mut file_info = FileInfo::new(object_name, 1, 1);
|
||||
file_info.mod_time = Some(OffsetDateTime::now_utc());
|
||||
metadata.add_version(file_info).expect("metadata should be valid");
|
||||
fs::write(
|
||||
object_dir.join(STORAGE_FORMAT_FILE),
|
||||
metadata.marshal_msg().expect("metadata should encode"),
|
||||
)
|
||||
.await
|
||||
.expect("object metadata should be written");
|
||||
}
|
||||
|
||||
// A genuine date-partitioned prefix wider than the probe batch at two
|
||||
// levels: 20 days, each hour holding 100 objects.
|
||||
const DAYS: usize = 20;
|
||||
const OBJECTS: usize = 100;
|
||||
for day in 0..DAYS {
|
||||
for object in 0..OBJECTS {
|
||||
let key = format!("data/stream/2026/08/{day:02}/23/{object:03}.parquet");
|
||||
write_object(&bucket_dir.join(&key), &key).await;
|
||||
}
|
||||
}
|
||||
|
||||
let endpoint =
|
||||
Endpoint::try_from(dir.path().to_str().expect("tempdir path should be UTF-8")).expect("endpoint should parse");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should initialize");
|
||||
|
||||
let reads = Arc::new((AtomicUsize::new(0), AtomicUsize::new(0)));
|
||||
let (reader, mut writer) = tokio::io::duplex(64 * 1024);
|
||||
let mut output = MetacacheWriter::new(&mut writer);
|
||||
let opts = WalkDirOptions {
|
||||
bucket: bucket.to_string(),
|
||||
base_dir: "data/".to_string(),
|
||||
skip_hidden_prefix_check: true,
|
||||
..Default::default()
|
||||
};
|
||||
let mut objects_returned = 0;
|
||||
DELETE_RESIDUE_PROBE_READS
|
||||
.scope(
|
||||
Arc::clone(&reads),
|
||||
disk.scan_dir(
|
||||
"data/".to_string(),
|
||||
"".to_string(),
|
||||
&opts,
|
||||
&mut output,
|
||||
&mut objects_returned,
|
||||
false,
|
||||
None,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("scan_dir should succeed");
|
||||
output.close().await.expect("metacache writer should close");
|
||||
drop(output);
|
||||
drop(writer);
|
||||
|
||||
let names = MetacacheReader::new(reader)
|
||||
.read_all()
|
||||
.await
|
||||
.expect("scan output should decode")
|
||||
.into_iter()
|
||||
.map(|entry| entry.name)
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(names, vec!["data/stream/".to_owned()]);
|
||||
|
||||
// stream, 2026, 08, one sampled day, its hour, one sampled object:
|
||||
// the walk ends at the first `xl.meta`, one bounded read per level,
|
||||
// without ever reading a wide directory in full.
|
||||
let (sampled, complete) = (reads.0.load(Ordering::Relaxed), reads.1.load(Ordering::Relaxed));
|
||||
assert_eq!(sampled, 6, "one sampled read per level down to the first object");
|
||||
assert_eq!(complete, 0, "a genuine prefix must never cost a complete directory read");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_scan_dir_nonrecursive_skips_dirs_with_only_hidden_delete_markers() {
|
||||
use rustfs_filemeta::MetacacheReader;
|
||||
@@ -22197,7 +21833,7 @@ mod test {
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test]
|
||||
async fn direct_writer_state_machine_round_trips_over_plain_file() {
|
||||
use std::io::{IoSlice, Read};
|
||||
use std::io::Read;
|
||||
use tempfile::tempdir;
|
||||
|
||||
let dir = tempdir().expect("tempdir");
|
||||
@@ -22216,16 +21852,6 @@ mod test {
|
||||
|
||||
let mut writer = DirectWriter::from_std_file_for_test(file, align, capacity);
|
||||
let mut off = 0;
|
||||
if content.len() >= 2 {
|
||||
let split = content.len().min(300);
|
||||
let first = split / 2;
|
||||
let written = writer
|
||||
.write_vectored(&[IoSlice::new(&content[..first]), IoSlice::new(&content[first..split])])
|
||||
.await
|
||||
.expect("vectored write");
|
||||
assert_eq!(written, split, "vectored write must consume both hash/data-like slices");
|
||||
off = split;
|
||||
}
|
||||
while off < content.len() {
|
||||
let end = (off + 300).min(content.len());
|
||||
writer.write_all(&content[off..end]).await.expect("write_all");
|
||||
|
||||
@@ -70,12 +70,7 @@ use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
|
||||
use rustfs_madmin::info_commands::DiskMetrics;
|
||||
use rustfs_rio::ChunkReaderBox;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
fmt::Debug,
|
||||
path::PathBuf,
|
||||
sync::{Arc, atomic::AtomicBool},
|
||||
time::Duration,
|
||||
};
|
||||
use std::{fmt::Debug, path::PathBuf, sync::Arc, time::Duration};
|
||||
use time::OffsetDateTime;
|
||||
use tokio::io::{AsyncRead, AsyncWrite};
|
||||
use uuid::Uuid;
|
||||
@@ -825,7 +820,7 @@ impl Disk {
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
) -> Result<RenameDataResp> {
|
||||
let Some(scope) = crate::store::bucket_heal_scope_for_object(dst_volume, dst_path) else {
|
||||
let Some(scope) = crate::store::bucket_heal_scope(dst_volume) else {
|
||||
return self
|
||||
.rename_data_borrowed_with_fence(src_volume, src_path, fi, dst_volume, dst_path, None)
|
||||
.await;
|
||||
@@ -984,14 +979,6 @@ impl Disk {
|
||||
Disk::Remote(remote_disk) => remote_disk.force_runtime_state_for_test(state),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn force_offline_for_test(&self) {
|
||||
match self {
|
||||
Disk::Local(local_disk) => local_disk.force_offline_for_test(),
|
||||
Disk::Remote(remote_disk) => remote_disk.force_offline_for_test(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -1484,12 +1471,6 @@ pub struct WalkDirOptions {
|
||||
// Override the remote stream stall timeout for long background walks.
|
||||
#[serde(default)]
|
||||
pub stall_timeout_ms: Option<u64>,
|
||||
|
||||
/// In-process completion state for bounded local walks. This is skipped
|
||||
/// from RPC serialization; remote peers retain the legacy natural-EOF
|
||||
/// behavior until they support an explicit capability.
|
||||
#[serde(skip)]
|
||||
pub producer_limit_reached: Option<Arc<AtomicBool>>,
|
||||
}
|
||||
|
||||
impl WalkDirOptions {
|
||||
@@ -1813,7 +1794,6 @@ mod tests {
|
||||
skip_total_timeout: false,
|
||||
timeout_ms: Some(10_000),
|
||||
stall_timeout_ms: Some(20_000),
|
||||
producer_limit_reached: None,
|
||||
};
|
||||
|
||||
assert_eq!(opts.bucket, "test-bucket");
|
||||
|
||||
@@ -2573,11 +2573,6 @@ impl Erasure {
|
||||
.with_deferred_parity_reopeners(deferred_reopeners);
|
||||
|
||||
let integrity = reader.integrity.clone();
|
||||
// Copy sources use demand-bound scheduling for backpressure, but retain
|
||||
// the ordinary GET read quorum. Only the opt-in GET rollout requires a
|
||||
// surplus source; available surplus shards and integrity proofs are
|
||||
// still verified before any reconstructed copy bytes are emitted.
|
||||
let require_surplus_source = reader.demand_bound_lockstep && matches!(decode_read_policy(), DecodeReadPolicy::Default);
|
||||
|
||||
let start = offset / self.block_size;
|
||||
let end = end_offset.saturating_sub(1) / self.block_size;
|
||||
@@ -2672,6 +2667,7 @@ impl Erasure {
|
||||
// `shards` are borrowed again below. In the `Stop` case that
|
||||
// drop is what cancels the still-in-flight prefetch read.
|
||||
let (flow, next): (Option<StripeFlow>, Option<StripeReadOutput>) = {
|
||||
let require_surplus_source = reader.demand_bound_lockstep;
|
||||
let read_fut = read_stripe_timed(&mut reader, stage_metrics_enabled);
|
||||
let emit_fut = self.emit_decoded_stripe(
|
||||
writer,
|
||||
@@ -2732,7 +2728,7 @@ impl Erasure {
|
||||
&mut written,
|
||||
&mut ret_err,
|
||||
stage_metrics_enabled,
|
||||
require_surplus_source,
|
||||
reader.demand_bound_lockstep,
|
||||
integrity.as_ref(),
|
||||
offset,
|
||||
)
|
||||
@@ -2775,7 +2771,7 @@ impl Erasure {
|
||||
&mut written,
|
||||
&mut ret_err,
|
||||
stage_metrics_enabled,
|
||||
require_surplus_source,
|
||||
reader.demand_bound_lockstep,
|
||||
integrity.as_ref(),
|
||||
offset,
|
||||
)
|
||||
@@ -3453,150 +3449,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
async fn copy_source_bitrot_shards(erasure: &Erasure, payload: &[u8], corrupt_parity: bool) -> Vec<Vec<u8>> {
|
||||
let mut writers: Vec<_> = (0..erasure.data_shards + erasure.parity_shards)
|
||||
.map(|_| BitrotWriter::new(Cursor::new(Vec::new()), erasure.shard_size(), HashAlgorithm::HighwayHash256))
|
||||
.collect();
|
||||
for block in payload.chunks(erasure.block_size) {
|
||||
let shards = erasure.encode_data(block).expect("copy source should encode");
|
||||
for (index, shard) in shards.iter().enumerate() {
|
||||
let mut bytes = shard.to_vec();
|
||||
if corrupt_parity && index == erasure.data_shards {
|
||||
// A stale source can have a valid per-shard bitrot checksum.
|
||||
// Reframe the changed parity to exercise reconstruction verification.
|
||||
bytes[0] ^= 0x80;
|
||||
}
|
||||
writers[index].write(&bytes).await.expect("copy source should frame");
|
||||
}
|
||||
}
|
||||
writers.into_iter().map(|writer| writer.into_inner().into_inner()).collect()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn copy_source_reads_exact_quorum_across_stripes_and_ranges() {
|
||||
const BLOCK_SIZE: usize = 768;
|
||||
let payload: Vec<_> = (0..BLOCK_SIZE * 3 + 37)
|
||||
.map(|index| (index.wrapping_mul(31) % 251) as u8)
|
||||
.collect();
|
||||
|
||||
for uses_legacy in [false, true] {
|
||||
let erasure = Erasure::new_with_options(12, 4, BLOCK_SIZE, uses_legacy);
|
||||
let buffers = copy_source_bitrot_shards(&erasure, &payload, false).await;
|
||||
for offline in [[0, 1, 2, 3], [0, 5, 12, 15]] {
|
||||
for (offset, length) in [(0, payload.len()), (BLOCK_SIZE + 17, payload.len() - BLOCK_SIZE - 17)] {
|
||||
let frame_offset = offset / BLOCK_SIZE * (erasure.shard_size() + HashAlgorithm::HighwayHash256.size());
|
||||
let positioned: Vec<_> = buffers.iter().map(|bytes| bytes[frame_offset..].to_vec()).collect();
|
||||
let (mut readers, handles) =
|
||||
readers_with_deferred_parity(&positioned, 12, erasure.shard_size(), &HashAlgorithm::HighwayHash256, &[]);
|
||||
for index in offline {
|
||||
readers[index] = None;
|
||||
}
|
||||
let mut output = Vec::new();
|
||||
let (written, error) = crate::set_disk::with_get_object_read_policy(
|
||||
crate::set_disk::GetObjectReadPolicy::CopySource,
|
||||
erasure.decode_with_stripe_handles(&mut output, readers, offset, length, payload.len(), None, handles),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(error.is_none(), "legacy={uses_legacy}, offline={offline:?}, offset={offset}: {error:?}");
|
||||
assert_eq!(written, length);
|
||||
assert_eq!(output, payload[offset..offset + length]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn copy_source_recovers_exact_quorum_after_first_stripe() {
|
||||
const BLOCK_SIZE: usize = 768;
|
||||
let payload: Vec<_> = (0..BLOCK_SIZE * 3 + 37)
|
||||
.map(|index| (index.wrapping_mul(31) % 251) as u8)
|
||||
.collect();
|
||||
for uses_legacy in [false, true] {
|
||||
let erasure = Erasure::new_with_options(12, 4, BLOCK_SIZE, uses_legacy);
|
||||
let buffers = copy_source_bitrot_shards(&erasure, &payload, false).await;
|
||||
let first_frame = erasure.shard_size() + HashAlgorithm::HighwayHash256.size();
|
||||
let (readers, handles) = readers_with_deferred_parity(
|
||||
&buffers,
|
||||
12,
|
||||
erasure.shard_size(),
|
||||
&HashAlgorithm::HighwayHash256,
|
||||
&[(0, first_frame), (1, first_frame), (2, first_frame), (3, first_frame)],
|
||||
);
|
||||
let mut output = Vec::new();
|
||||
let (written, error) = crate::set_disk::with_get_object_read_policy(
|
||||
crate::set_disk::GetObjectReadPolicy::CopySource,
|
||||
erasure.decode_with_stripe_handles(&mut output, readers, 0, payload.len(), payload.len(), None, handles),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(error.is_none(), "legacy={uses_legacy}: late shard loss should recover: {error:?}");
|
||||
assert_eq!(written, payload.len());
|
||||
assert_eq!(output, payload);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn copy_source_below_read_quorum_fails_before_output() {
|
||||
const BLOCK_SIZE: usize = 768;
|
||||
let payload = vec![0x5a; BLOCK_SIZE * 2 + 37];
|
||||
for uses_legacy in [false, true] {
|
||||
let erasure = Erasure::new_with_options(12, 4, BLOCK_SIZE, uses_legacy);
|
||||
let buffers = copy_source_bitrot_shards(&erasure, &payload, false).await;
|
||||
let (mut readers, handles) =
|
||||
readers_with_deferred_parity(&buffers, 12, erasure.shard_size(), &HashAlgorithm::HighwayHash256, &[]);
|
||||
for reader in readers.iter_mut().take(5) {
|
||||
*reader = None;
|
||||
}
|
||||
let mut output = Vec::new();
|
||||
let (written, error) = crate::set_disk::with_get_object_read_policy(
|
||||
crate::set_disk::GetObjectReadPolicy::CopySource,
|
||||
erasure.decode_with_stripe_handles(&mut output, readers, 0, payload.len(), payload.len(), None, handles),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(written, 0);
|
||||
assert!(output.is_empty());
|
||||
assert_eq!(
|
||||
error
|
||||
.expect("eleven shards cannot reconstruct twelve data shards")
|
||||
.to_string(),
|
||||
Error::ErasureReadQuorum.to_string()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn copy_source_rejects_corrupt_surplus_parity_before_output() {
|
||||
const BLOCK_SIZE: usize = 768;
|
||||
let payload = vec![0x5a; BLOCK_SIZE * 2 + 37];
|
||||
for uses_legacy in [false, true] {
|
||||
let erasure = Erasure::new_with_options(12, 4, BLOCK_SIZE, uses_legacy);
|
||||
let buffers = copy_source_bitrot_shards(&erasure, &payload, true).await;
|
||||
let (mut readers, handles) =
|
||||
readers_with_deferred_parity(&buffers, 12, erasure.shard_size(), &HashAlgorithm::HighwayHash256, &[]);
|
||||
for reader in readers.iter_mut().take(3) {
|
||||
*reader = None;
|
||||
}
|
||||
let mut output = Vec::new();
|
||||
let (written, error) = crate::set_disk::with_get_object_read_policy(
|
||||
crate::set_disk::GetObjectReadPolicy::CopySource,
|
||||
erasure.decode_with_stripe_handles(&mut output, readers, 0, payload.len(), payload.len(), None, handles),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(written, 0);
|
||||
assert!(output.is_empty());
|
||||
let error = error.expect("copy must reject inconsistent surplus parity despite valid bitrot framing");
|
||||
assert_eq!(error.kind(), ErrorKind::InvalidData);
|
||||
assert!(error.to_string().contains("inconsistent read source shards"));
|
||||
}
|
||||
}
|
||||
|
||||
/// Guards the shard-buffer reuse in `ParallelReader`: a multi-stripe
|
||||
/// `Erasure::decode` that reconstructs missing data shards on every stripe
|
||||
/// must still return byte-exact output. Reconstructed and parity buffers are
|
||||
|
||||
@@ -587,9 +587,7 @@ pub(crate) async fn verify_deep_parts(
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
let part_status = statuses.get_mut(&part_index).ok_or(DiskError::FileCorrupt)?;
|
||||
// Deep verification must consume every encoded stripe, including a
|
||||
// partial final stripe, before certifying the shard's integrity.
|
||||
let length = usize::try_from(erasure.shard_file_size(part.size as i64)).map_err(|_| DiskError::FileCorrupt)?;
|
||||
let length = erasure.shard_file_offset(0, part.size, part.size);
|
||||
let mut readers = Vec::with_capacity(disks.len());
|
||||
for (index, disk) in disks.iter().enumerate() {
|
||||
if part_status[index] != CHECK_PART_UNKNOWN {
|
||||
@@ -633,9 +631,6 @@ pub(crate) async fn verify_deep_parts(
|
||||
part_status[index] = CHECK_PART_FILE_CORRUPT;
|
||||
}
|
||||
Err(error) => {
|
||||
if !matches!(error, DiskError::FileNotFound | DiskError::FileVersionNotFound | DiskError::FileCorrupt) {
|
||||
return Err(error);
|
||||
}
|
||||
readers.push(None);
|
||||
part_status[index] = conv_part_err_to_int(&Some(error));
|
||||
}
|
||||
|
||||
@@ -41,7 +41,7 @@ impl Debug for NamespaceLockFence {
|
||||
}
|
||||
|
||||
impl NamespaceLockFence {
|
||||
pub(crate) fn new() -> Self {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
signals: Arc::default(),
|
||||
#[cfg(test)]
|
||||
@@ -153,7 +153,7 @@ pub struct ObjectLockConfigSnapshot {
|
||||
state: crate::bucket::metadata_sys::ObjectLockConfigState,
|
||||
lifecycle_fence: NamespaceLockFence,
|
||||
_lifecycle_guard: Option<rustfs_lock::NamespaceLockGuard>,
|
||||
metadata_transaction_guard: Option<Arc<rustfs_lock::NamespaceLockGuard>>,
|
||||
metadata_transaction_guard: Option<rustfs_lock::NamespaceLockGuard>,
|
||||
}
|
||||
|
||||
impl ObjectLockConfigSnapshot {
|
||||
@@ -170,7 +170,6 @@ impl ObjectLockConfigSnapshot {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn for_store_bucket(
|
||||
store_id: Uuid,
|
||||
bucket: &str,
|
||||
@@ -211,7 +210,7 @@ impl ObjectLockConfigSnapshot {
|
||||
state,
|
||||
lifecycle_fence,
|
||||
_lifecycle_guard: Some(lifecycle_guard),
|
||||
metadata_transaction_guard: Some(Arc::new(metadata_transaction_guard)),
|
||||
metadata_transaction_guard: Some(metadata_transaction_guard),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -222,7 +221,7 @@ impl ObjectLockConfigSnapshot {
|
||||
config_revision: OffsetDateTime,
|
||||
state: crate::bucket::metadata_sys::ObjectLockConfigState,
|
||||
lifecycle_fence: NamespaceLockFence,
|
||||
metadata_transaction_guard: Arc<rustfs_lock::NamespaceLockGuard>,
|
||||
metadata_transaction_guard: rustfs_lock::NamespaceLockGuard,
|
||||
) -> Self {
|
||||
Self {
|
||||
store_id: Some(store_id),
|
||||
@@ -266,23 +265,6 @@ impl ObjectLockConfigSnapshot {
|
||||
.is_some_and(|guard| !guard.is_lock_lost())
|
||||
}
|
||||
|
||||
/// Share the held transaction lock without queuing another reader behind
|
||||
/// a metadata writer that is itself waiting for this snapshot to drop.
|
||||
pub(crate) fn metadata_transaction_guard_for(
|
||||
&self,
|
||||
store_id: Uuid,
|
||||
bucket: &str,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
) -> Option<Arc<rustfs_lock::NamespaceLockGuard>> {
|
||||
let bucket_incarnation_id = self.bucket_incarnation_id?;
|
||||
if expected_incarnation_id.is_some_and(|expected| expected != bucket_incarnation_id)
|
||||
|| !self.is_valid_for_destructive_put(store_id, bucket, bucket_incarnation_id)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
self.metadata_transaction_guard.clone()
|
||||
}
|
||||
|
||||
pub(crate) fn add_lock_fences(&self, opts: &mut ObjectOptions) {
|
||||
opts.bucket_lifecycle_lock_fence
|
||||
.get_or_insert_with(NamespaceLockFence::new)
|
||||
|
||||
@@ -42,7 +42,6 @@ use std::sync::{
|
||||
atomic::{AtomicBool, AtomicUsize, Ordering},
|
||||
};
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
use tokio::sync::Notify;
|
||||
use tokio::time::{sleep, timeout};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, info, warn};
|
||||
@@ -62,14 +61,6 @@ const TIER_CONFIG_RELOAD_RETRY_CAP: Duration = Duration::from_secs(5);
|
||||
const REMOTE_VERSION_STATE_PROBE_INTERVAL: Duration = Duration::from_secs(10);
|
||||
const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30);
|
||||
/// Fallback poll cadence while startup has not yet published the notification
|
||||
/// system. IAM finalization starts the probe before `init_notification_runtime`
|
||||
/// runs, so the first pass always fails closed; waiting a full probe interval
|
||||
/// there left a single node without its durable quota capability for ten
|
||||
/// seconds after `/health` reported ok (rustfs/rustfs#8014). Publication wakes
|
||||
/// the probe immediately through `NOTIFICATION_SYS_PUBLISHED`; this bound only
|
||||
/// covers a wakeup that races the availability check.
|
||||
const REMOTE_VERSION_STATE_PROBE_BOOTSTRAP_INTERVAL: Duration = Duration::from_millis(100);
|
||||
const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 2;
|
||||
const TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION: u32 = 3;
|
||||
const DECOMMISSION_TARGET_FENCE_POLICY_SUPPORTED_VERSION: u32 = 4;
|
||||
@@ -347,9 +338,6 @@ static LEGACY_TRANSITION_STATE_RECONCILE_FLEET_PROOF: OnceLock<std::sync::RwLock
|
||||
static ILM_RECOVERY_EXPORT_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static TRANSITION_TRANSACTION_COMPACTION_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock<String> = OnceLock::new();
|
||||
/// Signalled once `GLOBAL_NOTIFICATION_SYS` is published so the fleet probe can
|
||||
/// run its first real pass without waiting for the bootstrap poll.
|
||||
static NOTIFICATION_SYS_PUBLISHED: Notify = Notify::const_new();
|
||||
static ILM_RECOVERY_EXPORT_LOCAL_PROCESS_EPOCH: LazyLock<Uuid> = LazyLock::new(Uuid::new_v4);
|
||||
|
||||
fn cross_pool_fence_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
|
||||
@@ -380,24 +368,6 @@ fn transition_transaction_compaction_fleet_proof_slot() -> &'static std::sync::R
|
||||
TRANSITION_TRANSACTION_COMPACTION_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
|
||||
}
|
||||
|
||||
fn all_fleet_capability_proof_slots() -> [&'static std::sync::RwLock<FleetCapabilityProofState>; 7] {
|
||||
[
|
||||
remote_version_state_fleet_proof_slot(),
|
||||
cross_pool_fence_fleet_proof_slot(),
|
||||
tier_delete_journal_fleet_proof_slot(),
|
||||
decommission_target_fence_fleet_proof_slot(),
|
||||
legacy_transition_state_reconcile_fleet_proof_slot(),
|
||||
ilm_recovery_export_fleet_proof_slot(),
|
||||
transition_transaction_compaction_fleet_proof_slot(),
|
||||
]
|
||||
}
|
||||
|
||||
fn revoke_all_fleet_capability_proofs() {
|
||||
for slot in all_fleet_capability_proof_slots() {
|
||||
revoke_fleet_capability_proof(slot);
|
||||
}
|
||||
}
|
||||
|
||||
fn revoke_fleet_capability_proof_state(state: &mut FleetCapabilityProofState) {
|
||||
if let Some(proof) = state.proof.take() {
|
||||
proof.generation.revoke();
|
||||
@@ -483,20 +453,6 @@ pub(crate) fn acquire_remote_version_state_fleet_proof() -> Option<RemoteVersion
|
||||
acquire_fleet_capability_proof_from(&state, expected_topology, Instant::now()).map(RemoteVersionStateFleetProofToken)
|
||||
}
|
||||
|
||||
pub(crate) fn acquire_remote_version_state_writer_fleet_proof() -> Option<RemoteVersionStateFleetProofToken> {
|
||||
let requested = rustfs_utils::get_env_bool(
|
||||
rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_WRITE,
|
||||
rustfs_config::DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE,
|
||||
);
|
||||
let fleet_confirmed = rustfs_utils::get_env_bool(
|
||||
rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
|
||||
rustfs_config::DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
|
||||
);
|
||||
(requested && fleet_confirmed)
|
||||
.then(acquire_remote_version_state_fleet_proof)
|
||||
.flatten()
|
||||
}
|
||||
|
||||
fn acquire_fleet_capability_proof_from(
|
||||
state: &FleetCapabilityProofState,
|
||||
expected_topology: &str,
|
||||
@@ -1256,7 +1212,15 @@ fn insert_remote_version_state_peer(peer_epochs: &mut BTreeMap<String, Uuid>, pe
|
||||
pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.set(topology_fingerprint.clone()).is_err() {
|
||||
if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() != Some(&topology_fingerprint) {
|
||||
for slot in all_fleet_capability_proof_slots() {
|
||||
for slot in [
|
||||
remote_version_state_fleet_proof_slot(),
|
||||
cross_pool_fence_fleet_proof_slot(),
|
||||
tier_delete_journal_fleet_proof_slot(),
|
||||
decommission_target_fence_fleet_proof_slot(),
|
||||
legacy_transition_state_reconcile_fleet_proof_slot(),
|
||||
ilm_recovery_export_fleet_proof_slot(),
|
||||
transition_transaction_compaction_fleet_proof_slot(),
|
||||
] {
|
||||
mark_fleet_capability_topology_conflict(slot);
|
||||
}
|
||||
}
|
||||
@@ -1264,59 +1228,53 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
}
|
||||
|
||||
tokio::spawn(async move {
|
||||
let mut notification_sys_unavailable_logged = false;
|
||||
loop {
|
||||
let Some(notification_sys) = get_global_notification_sys() else {
|
||||
// Startup publishes the notification system after this probe
|
||||
// is started. Stay failed closed, but retry on the bootstrap
|
||||
// cadence so a single node gains its capabilities as soon as
|
||||
// the system appears instead of one full probe interval later.
|
||||
revoke_all_fleet_capability_proofs();
|
||||
if !notification_sys_unavailable_logged {
|
||||
notification_sys_unavailable_logged = true;
|
||||
debug!(
|
||||
event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||
state = "waiting_for_notification_system",
|
||||
"notification capability probe"
|
||||
);
|
||||
}
|
||||
let _ = timeout(REMOTE_VERSION_STATE_PROBE_BOOTSTRAP_INTERVAL, NOTIFICATION_SYS_PUBLISHED.notified()).await;
|
||||
continue;
|
||||
};
|
||||
notification_sys_unavailable_logged = false;
|
||||
let notification_sys = get_global_notification_sys();
|
||||
let remote_version_state_probe = async {
|
||||
timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_remote_version_state_fleet(&topology_fingerprint),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other("remote version state fleet capability probe timed out")))
|
||||
match notification_sys.as_ref() {
|
||||
Some(notification_sys) => timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_remote_version_state_fleet(&topology_fingerprint),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other("remote version state fleet capability probe timed out"))),
|
||||
None => Err(Error::other("remote version state fleet capability notification system is unavailable")),
|
||||
}
|
||||
};
|
||||
let cross_pool_fence_probe = async {
|
||||
timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_cross_pool_fence_fleet(&topology_fingerprint),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other("cross-pool fence fleet capability probe timed out")))
|
||||
match notification_sys.as_ref() {
|
||||
Some(notification_sys) => timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_cross_pool_fence_fleet(&topology_fingerprint),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other("cross-pool fence fleet capability probe timed out"))),
|
||||
None => Err(Error::other("cross-pool fence fleet capability notification system is unavailable")),
|
||||
}
|
||||
};
|
||||
let recovery_export_probe = async {
|
||||
timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_ilm_recovery_export_fleet(&topology_fingerprint),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other("ILM recovery export fleet capability probe timed out")))
|
||||
match notification_sys.as_ref() {
|
||||
Some(notification_sys) => timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_ilm_recovery_export_fleet(&topology_fingerprint),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other("ILM recovery export fleet capability probe timed out"))),
|
||||
None => Err(Error::other("ILM recovery export fleet capability notification system is unavailable")),
|
||||
}
|
||||
};
|
||||
let transition_transaction_compaction_probe = async {
|
||||
timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_transition_transaction_compaction_fleet(&topology_fingerprint),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other("transition transaction compaction fleet capability probe timed out")))
|
||||
match notification_sys.as_ref() {
|
||||
Some(notification_sys) => timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_transition_transaction_compaction_fleet(&topology_fingerprint),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other("transition transaction compaction fleet capability probe timed out"))),
|
||||
None => Err(Error::other(
|
||||
"transition transaction compaction fleet capability notification system is unavailable",
|
||||
)),
|
||||
}
|
||||
};
|
||||
let (result, fence_probe, recovery_export_result, transition_transaction_compaction_result) = tokio::join!(
|
||||
remote_version_state_probe,
|
||||
@@ -1341,7 +1299,13 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.topology_conflict;
|
||||
if topology_conflict {
|
||||
revoke_all_fleet_capability_proofs();
|
||||
revoke_fleet_capability_proof(remote_version_state_fleet_proof_slot());
|
||||
revoke_fleet_capability_proof(cross_pool_fence_fleet_proof_slot());
|
||||
revoke_fleet_capability_proof(tier_delete_journal_fleet_proof_slot());
|
||||
revoke_fleet_capability_proof(decommission_target_fence_fleet_proof_slot());
|
||||
revoke_fleet_capability_proof(legacy_transition_state_reconcile_fleet_proof_slot());
|
||||
revoke_fleet_capability_proof(ilm_recovery_export_fleet_proof_slot());
|
||||
revoke_fleet_capability_proof(transition_transaction_compaction_fleet_proof_slot());
|
||||
} else if let Some(err) = publish_fleet_capability_probe_result(
|
||||
remote_version_state_fleet_proof_slot(),
|
||||
&topology_fingerprint,
|
||||
@@ -1464,11 +1428,9 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
}
|
||||
|
||||
pub async fn new_global_notification_sys(eps: EndpointServerPools) -> Result<()> {
|
||||
if GLOBAL_NOTIFICATION_SYS.set(Arc::new(NotificationSys::new(eps).await)).is_ok() {
|
||||
// `notify_one` stores a permit, so a probe that checks availability
|
||||
// just before this publication still wakes without losing the signal.
|
||||
NOTIFICATION_SYS_PUBLISHED.notify_one();
|
||||
}
|
||||
let _ = GLOBAL_NOTIFICATION_SYS
|
||||
.set(Arc::new(NotificationSys::new(eps).await))
|
||||
.map_err(|_| Error::other("init notification_sys fail"));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -3943,49 +3905,6 @@ mod tests {
|
||||
stats
|
||||
}
|
||||
|
||||
/// rustfs/rustfs#8014: IAM finalization starts the fleet probe before the
|
||||
/// notification system is published. The first probe therefore fails
|
||||
/// closed, and a single node used to wait a full probe interval before the
|
||||
/// durable quota capability appeared, even though `/health` was already
|
||||
/// reporting ok. The probe must retry promptly while it waits for startup
|
||||
/// to publish the notification system.
|
||||
#[tokio::test]
|
||||
async fn fleet_probe_publishes_promptly_once_the_notification_system_appears() {
|
||||
if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get().is_some() || GLOBAL_NOTIFICATION_SYS.get().is_some() {
|
||||
// Process-wide state was bound by another test in this binary. The
|
||||
// startup ordering under test needs a fresh process; nextest
|
||||
// (the authoritative runner) always provides one.
|
||||
eprintln!("skipping: fleet probe globals already bound in this process");
|
||||
return;
|
||||
}
|
||||
|
||||
// Share the fingerprint every other proof-installing test in this crate
|
||||
// binds, so the plain `cargo test` fallback cannot see two topologies.
|
||||
start_remote_version_state_fleet_probe("object-transaction-fencing-test".to_string());
|
||||
sleep(Duration::from_millis(300)).await;
|
||||
assert!(
|
||||
acquire_cross_pool_fence_fleet_proof().is_none(),
|
||||
"no capability proof may exist before the notification system is published"
|
||||
);
|
||||
|
||||
let published_at = Instant::now();
|
||||
new_global_notification_sys(EndpointServerPools::default())
|
||||
.await
|
||||
.expect("single-node notification system initializes");
|
||||
|
||||
let observed = timeout(Duration::from_secs(2), async {
|
||||
while acquire_cross_pool_fence_fleet_proof().is_none() {
|
||||
sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
observed.is_ok(),
|
||||
"single-node cross-pool fence proof should publish within 2s of the notification system, waited {:?}",
|
||||
published_at.elapsed()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merging_peer_rings_sums_each_tier_without_double_counting() {
|
||||
let mut cluster = DailyAllTierStats::from([("WARM".to_string(), ring(10))]);
|
||||
|
||||
@@ -9,12 +9,11 @@ use super::meta::{
|
||||
};
|
||||
use super::worker::{
|
||||
rebalance_max_attempts, rebalance_meta_lock_error, resolve_load_rebalance_stats_update_result,
|
||||
resolve_rebalance_deferred_last_error, resolve_rebalance_meta_load_result, resolve_rebalance_meta_save_result,
|
||||
retry_rebalance_metadata_access,
|
||||
resolve_rebalance_meta_load_result, resolve_rebalance_meta_save_result, retry_rebalance_metadata_access,
|
||||
};
|
||||
use super::{
|
||||
DiskStat, EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE, REBAL_META_NAME,
|
||||
RebalStatus, RebalanceDeferKind, RebalanceInfo, RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord,
|
||||
RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord,
|
||||
encode_rebalance_stop_propagation_record,
|
||||
};
|
||||
use crate::core::pools::{
|
||||
@@ -1104,7 +1103,6 @@ impl ECStore {
|
||||
bucket: String,
|
||||
last_error: String,
|
||||
expected_id: &str,
|
||||
kind: RebalanceDeferKind,
|
||||
) -> Result<()> {
|
||||
let mut rebalance_meta = self.rebalance_meta.write().await;
|
||||
ensure_rebalance_worker_active(rebalance_meta.as_ref(), expected_id, "defer rebalance bucket")?;
|
||||
@@ -1118,8 +1116,7 @@ impl ECStore {
|
||||
};
|
||||
|
||||
defer_bucket_in_rebalance_queue(pool_stat, &bucket)?;
|
||||
let pending_entry_defer = pool_stat.info.last_error.clone();
|
||||
pool_stat.info.last_error = resolve_rebalance_deferred_last_error(kind, pending_entry_defer.as_deref(), &last_error);
|
||||
pool_stat.info.last_error = Some(last_error);
|
||||
meta.last_refreshed_at = Some(OffsetDateTime::now_utc());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1209,7 +1209,7 @@ mod tests {
|
||||
cancel: &warning_cancel,
|
||||
};
|
||||
let warning_result = store
|
||||
.finish_rebalance_entry_after_cleanup(&warning_cleanup_context, async { Err(Error::FileAccessDenied.into()) })
|
||||
.finish_rebalance_entry_after_cleanup(&warning_cleanup_context, async { Err(Error::SlowDown.into()) })
|
||||
.await
|
||||
.expect("cleanup warnings should not fail the completed migration");
|
||||
assert!(matches!(warning_result, RebalanceEntryCleanupResult::Completed { warning: Some(_) }));
|
||||
@@ -1250,35 +1250,6 @@ mod tests {
|
||||
let pool_stats = &meta.as_ref().expect("rebalance metadata should exist").pool_stats[0];
|
||||
assert_eq!(pool_stats.bytes, 0, "deferred cleanup must not commit completion stats");
|
||||
assert_eq!(pool_stats.cleanup_warnings.count, 1, "deferred cleanup must not add a permanent warning");
|
||||
drop(meta);
|
||||
|
||||
let transient_guard = store
|
||||
.rebalance_run_guard(rebalance_id, "rebalance transient cleanup deferral test")
|
||||
.await
|
||||
.expect("rebalance transient cleanup deferral test guard should be acquired");
|
||||
let transient_cancel = CancellationToken::new();
|
||||
let transient_stats_updates = [&warning_version];
|
||||
let transient_cleanup_context = RebalanceEntryCleanupContext {
|
||||
run_guard: &transient_guard,
|
||||
pool_index: 0,
|
||||
bucket: "bucket",
|
||||
object: "object.bin",
|
||||
stats_updates: &transient_stats_updates,
|
||||
expected_id: rebalance_id,
|
||||
cancel: &transient_cancel,
|
||||
};
|
||||
let transient = store
|
||||
.finish_rebalance_entry_after_cleanup(&transient_cleanup_context, async { Err(Error::SlowDown.into()) })
|
||||
.await
|
||||
.expect("retryable cleanup failures should defer without failing the worker");
|
||||
assert!(matches!(transient, RebalanceEntryCleanupResult::Deferred { .. }));
|
||||
let meta = store.rebalance_meta.read().await;
|
||||
let pool_stats = &meta.as_ref().expect("rebalance metadata should exist").pool_stats[0];
|
||||
assert_eq!(pool_stats.bytes, 0, "retryable cleanup failures must not commit completion stats");
|
||||
assert_eq!(
|
||||
pool_stats.cleanup_warnings.count, 1,
|
||||
"retryable cleanup failures must not add a permanent warning"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -29,7 +29,7 @@ const REBAL_META_FMT: u16 = 1; // Replace with actual format value
|
||||
const REBAL_META_VER: u16 = 1; // Replace with actual version value
|
||||
pub(crate) const REBAL_META_NAME: &str = "rebalance.bin";
|
||||
const DEFAULT_REBALANCE_MAX_ATTEMPTS: usize = 3;
|
||||
pub(crate) const REBALANCE_SOURCE_CLEANUP_MAX_DEFERS: usize = 8;
|
||||
pub(crate) const REBALANCE_SOURCE_CLEANUP_MAX_DEFERS: usize = 3;
|
||||
const REBALANCE_MAX_ATTEMPTS_ENV: &str = "RUSTFS_REBALANCE_MAX_ATTEMPTS";
|
||||
const REBALANCE_STOP_PROPAGATION_ERROR_PREFIX: &str = "rebalance stop propagation incomplete: ";
|
||||
const REBALANCE_LISTING_RETRY_BASE_DELAY: Duration = Duration::from_millis(250);
|
||||
@@ -55,7 +55,7 @@ pub use types::{
|
||||
DiskStat, RebalSaveOpt, RebalStatus, RebalanceCleanupWarningEntry, RebalanceCleanupWarnings, RebalanceInfo, RebalanceMeta,
|
||||
RebalanceStats, RebalanceStopPropagationRecord,
|
||||
};
|
||||
use types::{RebalanceBucketConfigs, RebalanceBucketOutcome, RebalanceDeferKind, RebalanceEntryOutcome};
|
||||
use types::{RebalanceBucketConfigs, RebalanceBucketOutcome, RebalanceEntryOutcome};
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub async fn test_store_with_persisted_rebalance_meta(
|
||||
@@ -222,14 +222,10 @@ async fn test_pool_stores_with_contexts(
|
||||
std::sync::Arc::clone(&ctx)
|
||||
};
|
||||
let make_store = |store_ctx: std::sync::Arc<crate::runtime::instance::InstanceContext>| {
|
||||
let mut store_pools = pools.clone();
|
||||
for pool in &mut store_pools {
|
||||
std::sync::Arc::make_mut(pool).set_instance_ctx_for_test(std::sync::Arc::clone(&store_ctx));
|
||||
}
|
||||
std::sync::Arc::new(crate::store::ECStore {
|
||||
id: uuid::Uuid::new_v4(),
|
||||
disk_map: std::collections::HashMap::new(),
|
||||
pools: store_pools,
|
||||
pools: pools.clone(),
|
||||
peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoint_pools, std::sync::Arc::clone(&store_ctx)),
|
||||
pool_meta: tokio::sync::RwLock::new(pool_meta.clone()),
|
||||
rebalance_meta: tokio::sync::RwLock::new(rebalance_meta.clone()),
|
||||
|
||||
@@ -34,29 +34,27 @@ use super::migration::{
|
||||
};
|
||||
use super::runtime::{
|
||||
RebalanceLocalActivationOutcome, commit_local_rebalance_worker_activation,
|
||||
commit_local_rebalance_worker_activation_candidate, reached_rebalance_source_cleanup_defer_limit,
|
||||
should_fail_repeated_rebalance_bucket_defer, source_cleanup_defer_attempt, stage_local_rebalance_worker_activation,
|
||||
commit_local_rebalance_worker_activation_candidate, should_fail_repeated_rebalance_bucket_defer,
|
||||
source_cleanup_defer_attempt, stage_local_rebalance_worker_activation,
|
||||
};
|
||||
use super::worker::{
|
||||
RebalanceEntryCleanupResult, ensure_rebalance_listing_disks_available, is_transient_rebalance_error,
|
||||
parse_rebalance_max_attempts, rebalance_listing_retry_delay, rebalance_migration_retry_delay,
|
||||
resolve_load_rebalance_stats_update_result, resolve_rebalance_bucket_error, resolve_rebalance_bucket_result,
|
||||
resolve_rebalance_deferred_last_error, resolve_rebalance_entry_cleanup_delete_result,
|
||||
resolve_rebalance_file_info_versions_result, resolve_rebalance_meta_load_result, resolve_rebalance_meta_save_result,
|
||||
resolve_rebalance_migrate_result_error, resolve_rebalance_optional_bucket_config_result, resolve_rebalance_save_task_result,
|
||||
resolve_rebalance_stats_update_result, resolve_rebalance_terminal_error, resolve_rebalance_worker_result,
|
||||
run_rebalance_listing_with_retry, send_rebalance_done_signal, should_cleanup_rebalance_source_entry,
|
||||
should_count_rebalance_version_complete, should_defer_rebalance_entry_failure, should_retry_rebalance_listing,
|
||||
should_skip_rebalance_delete_marker, wait_rebalance_entry_tasks, wait_rebalance_listing_retry, with_rebalance_entry_context,
|
||||
resolve_rebalance_entry_cleanup_delete_result, resolve_rebalance_file_info_versions_result,
|
||||
resolve_rebalance_meta_load_result, resolve_rebalance_meta_save_result, resolve_rebalance_migrate_result_error,
|
||||
resolve_rebalance_optional_bucket_config_result, resolve_rebalance_save_task_result, resolve_rebalance_stats_update_result,
|
||||
resolve_rebalance_terminal_error, resolve_rebalance_worker_result, run_rebalance_listing_with_retry,
|
||||
send_rebalance_done_signal, should_cleanup_rebalance_source_entry, should_count_rebalance_version_complete,
|
||||
should_defer_rebalance_entry_failure, should_retry_rebalance_listing, should_skip_rebalance_delete_marker,
|
||||
wait_rebalance_entry_tasks, wait_rebalance_listing_retry, with_rebalance_entry_context,
|
||||
};
|
||||
use super::{
|
||||
DiskStat, GetObjectReader, ObjectInfo, ObjectOptions, RebalSaveOpt, RebalStatus, RebalanceBucketConfigs,
|
||||
RebalanceBucketOutcome, RebalanceCleanupWarnings, RebalanceDeferKind, RebalanceEntryOutcome, RebalanceInfo, RebalanceMeta,
|
||||
RebalanceStats, RebalanceStopPropagationRecord,
|
||||
};
|
||||
use super::{
|
||||
REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX, REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX, REBALANCE_SOURCE_CLEANUP_MAX_DEFERS,
|
||||
RebalanceBucketOutcome, RebalanceCleanupWarnings, RebalanceEntryOutcome, RebalanceInfo, RebalanceMeta, RebalanceStats,
|
||||
RebalanceStopPropagationRecord,
|
||||
};
|
||||
use super::{REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX, REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX};
|
||||
use crate::bucket::replication::{ReplicationState, ReplicationStatusType, replication_state_to_filemeta};
|
||||
use crate::data_movement;
|
||||
use crate::data_movement::SourceCleanupError;
|
||||
@@ -1777,7 +1775,7 @@ fn test_resolve_rebalance_entry_cleanup_delete_result_ignores_not_found() {
|
||||
|
||||
#[test]
|
||||
fn test_resolve_rebalance_entry_cleanup_delete_result_returns_warning_for_failures() {
|
||||
let result = resolve_rebalance_entry_cleanup_delete_result(Err(Error::FileAccessDenied.into()), "bucket-a", "obj.txt");
|
||||
let result = resolve_rebalance_entry_cleanup_delete_result(Err(Error::SlowDown.into()), "bucket-a", "obj.txt");
|
||||
assert!(matches!(
|
||||
result,
|
||||
RebalanceEntryCleanupResult::Completed { warning: Some(ref message) }
|
||||
@@ -1785,122 +1783,6 @@ fn test_resolve_rebalance_entry_cleanup_delete_result_returns_warning_for_failur
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_rebalance_entry_cleanup_delete_result_defers_transient_failures() {
|
||||
let cases = [
|
||||
(Error::SlowDown, "slow down"),
|
||||
(
|
||||
Error::Lock(rustfs_lock::LockError::timeout("bucket-a/obj.txt@latest", Duration::from_secs(5))),
|
||||
"object lock timeout",
|
||||
),
|
||||
(
|
||||
Error::Lock(rustfs_lock::LockError::network(
|
||||
"peer unavailable",
|
||||
std::io::Error::from(std::io::ErrorKind::ConnectionReset),
|
||||
)),
|
||||
"object lock network failure",
|
||||
),
|
||||
(Error::ErasureWriteQuorum, "write quorum"),
|
||||
(Error::Io(std::io::Error::from(std::io::ErrorKind::TimedOut)), "io timeout"),
|
||||
(
|
||||
Error::other("Lock error: Lock acquisition timeout for resource 'bucket-a/obj.txt@latest' after 5s"),
|
||||
"rendered lock timeout text",
|
||||
),
|
||||
];
|
||||
|
||||
for (err, label) in cases {
|
||||
match resolve_rebalance_entry_cleanup_delete_result(Err(err.into()), "bucket-a", "obj.txt") {
|
||||
RebalanceEntryCleanupResult::Deferred { last_error } => {
|
||||
assert!(
|
||||
last_error.starts_with(REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX),
|
||||
"{label}: {last_error}"
|
||||
);
|
||||
assert!(last_error.contains("bucket-a/obj.txt"), "{label}: {last_error}");
|
||||
}
|
||||
RebalanceEntryCleanupResult::Completed { warning } => {
|
||||
panic!("{label} must defer source cleanup instead of completing the entry with warning {warning:?}")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_rebalance_entry_cleanup_delete_result_defers_stage_wrapped_lock_timeout() {
|
||||
let err = data_movement::data_movement_stage_error_for_test(
|
||||
"rebalance",
|
||||
"delete_object",
|
||||
"bucket-a",
|
||||
"obj.txt",
|
||||
Error::Lock(rustfs_lock::LockError::timeout("bucket-a/obj.txt@latest", Duration::from_secs(5))),
|
||||
);
|
||||
|
||||
assert!(matches!(
|
||||
resolve_rebalance_entry_cleanup_delete_result(Err(err.into()), "bucket-a", "obj.txt"),
|
||||
RebalanceEntryCleanupResult::Deferred { .. }
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_rebalance_entry_cleanup_delete_result_ignores_stage_wrapped_not_found() {
|
||||
let err = data_movement::data_movement_stage_error_for_test(
|
||||
"rebalance",
|
||||
"delete_object",
|
||||
"bucket-a",
|
||||
"obj.txt",
|
||||
Error::ObjectNotFound("bucket-a".to_string(), "obj.txt".to_string()),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
resolve_rebalance_entry_cleanup_delete_result(Err(err.into()), "bucket-a", "obj.txt"),
|
||||
RebalanceEntryCleanupResult::Completed { warning: None }
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_rebalance_deferred_last_error_hides_retryable_cleanup_conflicts() {
|
||||
let entry_error = format!("{REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX} timeout");
|
||||
assert_eq!(
|
||||
resolve_rebalance_deferred_last_error(RebalanceDeferKind::Entry, None, entry_error.as_str()),
|
||||
Some(entry_error.clone()),
|
||||
"transient migration deferrals must stay visible to the completion guards"
|
||||
);
|
||||
|
||||
let cleanup_error = format!("{REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX} lock acquisition timeout");
|
||||
assert_eq!(
|
||||
resolve_rebalance_deferred_last_error(RebalanceDeferKind::SourceCleanup, None, cleanup_error.as_str()),
|
||||
None,
|
||||
"a retryable source cleanup conflict is progress, not a pool failure"
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
resolve_rebalance_deferred_last_error(
|
||||
RebalanceDeferKind::SourceCleanup,
|
||||
Some(entry_error.as_str()),
|
||||
cleanup_error.as_str()
|
||||
),
|
||||
Some(entry_error.clone()),
|
||||
"a cleanup deferral for one bucket must not erase an unresolved migration deferral of the same pool"
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_rebalance_deferred_last_error(
|
||||
RebalanceDeferKind::SourceCleanup,
|
||||
Some(cleanup_error.as_str()),
|
||||
cleanup_error.as_str()
|
||||
),
|
||||
None,
|
||||
"a stale retryable cleanup message must not survive as a pool failure"
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_rebalance_deferred_last_error(
|
||||
RebalanceDeferKind::SourceCleanup,
|
||||
Some(entry_error.as_str()),
|
||||
entry_error.as_str()
|
||||
),
|
||||
Some(entry_error),
|
||||
"repeated cleanup deferrals must keep the pending migration deferral visible"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_rebalance_entry_cleanup_delete_result_defers_source_change() {
|
||||
let result = resolve_rebalance_entry_cleanup_delete_result(Err(SourceCleanupError::SourceChanged), "bucket-a", "obj.txt");
|
||||
@@ -1935,89 +1817,6 @@ fn test_source_cleanup_defer_does_not_fail_repeated_bucket_retry() {
|
||||
assert_eq!(source_cleanup_defer_attempt(&mut source_attempts, "bucket-c"), 1);
|
||||
assert_eq!(source_cleanup_defer_attempt(&mut source_attempts, "bucket-c"), 2);
|
||||
assert_eq!(source_cleanup_defer_attempt(&mut source_attempts, "bucket-c"), 3);
|
||||
|
||||
let mut bounded_attempts = std::collections::HashMap::new();
|
||||
for expected in 1..REBALANCE_SOURCE_CLEANUP_MAX_DEFERS {
|
||||
assert_eq!(source_cleanup_defer_attempt(&mut bounded_attempts, "bucket-d"), expected);
|
||||
assert!(
|
||||
!reached_rebalance_source_cleanup_defer_limit(expected),
|
||||
"a retryable cleanup conflict must stay retryable at deferral {expected}"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
source_cleanup_defer_attempt(&mut bounded_attempts, "bucket-d"),
|
||||
REBALANCE_SOURCE_CLEANUP_MAX_DEFERS
|
||||
);
|
||||
assert!(
|
||||
reached_rebalance_source_cleanup_defer_limit(REBALANCE_SOURCE_CLEANUP_MAX_DEFERS),
|
||||
"an unreclaimable source replica must fail the bucket after the bounded deferral budget"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_defer_rebalance_bucket_keeps_pending_entry_defer_without_surfacing_cleanup_conflicts() {
|
||||
let id = "rebalance-defer-last-error";
|
||||
let meta = RebalanceMeta {
|
||||
id: id.to_string(),
|
||||
pool_stats: vec![RebalanceStats {
|
||||
participating: true,
|
||||
buckets: vec!["bucket-a".to_string(), "bucket-b".to_string()],
|
||||
info: RebalanceInfo {
|
||||
status: RebalStatus::Started,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
let store = test_store_with_rebalance_meta(meta);
|
||||
let cleanup_error = format!("{REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX} lock acquisition timeout");
|
||||
|
||||
store
|
||||
.defer_rebalance_bucket(0, "bucket-a".to_string(), cleanup_error.clone(), id, RebalanceDeferKind::SourceCleanup)
|
||||
.await
|
||||
.expect("a retryable cleanup conflict must be deferrable");
|
||||
{
|
||||
let meta = store.rebalance_meta.read().await;
|
||||
let pool_stat = &meta.as_ref().expect("rebalance metadata should exist").pool_stats[0];
|
||||
assert_eq!(
|
||||
pool_stat.info.last_error, None,
|
||||
"a retryable cleanup conflict is progress and must not surface as a pool failure"
|
||||
);
|
||||
assert_eq!(pool_stat.buckets, vec!["bucket-b".to_string(), "bucket-a".to_string()]);
|
||||
}
|
||||
|
||||
let entry_error = format!("{REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX} slow down");
|
||||
{
|
||||
let mut meta = store.rebalance_meta.write().await;
|
||||
meta.as_mut().expect("rebalance metadata should exist").pool_stats[0]
|
||||
.info
|
||||
.last_error = Some(entry_error.clone());
|
||||
}
|
||||
store
|
||||
.defer_rebalance_bucket(0, "bucket-b".to_string(), cleanup_error, id, RebalanceDeferKind::SourceCleanup)
|
||||
.await
|
||||
.expect("a cleanup deferral must be accepted while another bucket still defers an entry");
|
||||
{
|
||||
let meta = store.rebalance_meta.read().await;
|
||||
let pool_stat = &meta.as_ref().expect("rebalance metadata should exist").pool_stats[0];
|
||||
assert_eq!(
|
||||
pool_stat.info.last_error,
|
||||
Some(entry_error),
|
||||
"a cleanup deferral must not erase the pending migration deferral that blocks goal completion"
|
||||
);
|
||||
assert!(has_deferred_rebalance_error(pool_stat));
|
||||
assert_eq!(pool_stat.buckets, vec!["bucket-a".to_string(), "bucket-b".to_string()]);
|
||||
}
|
||||
|
||||
let migration_error = format!("{REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX} i/o timeout");
|
||||
store
|
||||
.defer_rebalance_bucket(0, "bucket-a".to_string(), migration_error.clone(), id, RebalanceDeferKind::Entry)
|
||||
.await
|
||||
.expect("migration deferrals must keep their last error");
|
||||
let meta = store.rebalance_meta.read().await;
|
||||
let pool_stat = &meta.as_ref().expect("rebalance metadata should exist").pool_stats[0];
|
||||
assert_eq!(pool_stat.info.last_error, Some(migration_error));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -7,14 +7,13 @@ use super::meta::{
|
||||
validate_start_rebalance_state,
|
||||
};
|
||||
use super::worker::{
|
||||
rebalance_max_attempts, resolve_rebalance_bucket_result, resolve_rebalance_meta_save_result,
|
||||
resolve_rebalance_save_task_result, resolve_rebalance_terminal_error, retry_rebalance_metadata_access,
|
||||
send_rebalance_done_signal,
|
||||
resolve_rebalance_bucket_result, resolve_rebalance_meta_save_result, resolve_rebalance_save_task_result,
|
||||
resolve_rebalance_terminal_error, send_rebalance_done_signal,
|
||||
};
|
||||
use super::{
|
||||
EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE,
|
||||
REBALANCE_LISTING_RETRY_BASE_DELAY, REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX, REBALANCE_SOURCE_CLEANUP_MAX_DEFERS,
|
||||
RebalSaveOpt, RebalStatus, RebalanceBucketOutcome, RebalanceDeferKind,
|
||||
REBALANCE_LISTING_RETRY_BASE_DELAY, REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX, RebalSaveOpt, RebalStatus,
|
||||
RebalanceBucketOutcome,
|
||||
};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
@@ -41,12 +40,6 @@ pub(super) fn source_cleanup_defer_attempt(deferred_attempts: &mut HashMap<Strin
|
||||
*attempts
|
||||
}
|
||||
|
||||
/// Retryable source cleanup conflicts are bounded per run, so a source replica that stays
|
||||
/// unreclaimable fails the bucket explicitly instead of deferring forever.
|
||||
pub(super) fn reached_rebalance_source_cleanup_defer_limit(attempt: usize) -> bool {
|
||||
attempt >= REBALANCE_SOURCE_CLEANUP_MAX_DEFERS
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(super) enum RebalanceLocalActivationOutcome {
|
||||
Started,
|
||||
@@ -450,14 +443,13 @@ impl ECStore {
|
||||
};
|
||||
|
||||
if terminal_state_present {
|
||||
if let Err(err) = retry_rebalance_metadata_access(None, rebalance_max_attempts(), || {
|
||||
store.save_rebalance_stats_inner(
|
||||
if let Err(err) = store
|
||||
.save_rebalance_stats_inner(
|
||||
pool_index,
|
||||
RebalSaveOpt::Stats,
|
||||
Some(save_rebalance_id.as_ref()),
|
||||
)
|
||||
})
|
||||
.await
|
||||
.await
|
||||
{
|
||||
let mut rebalance_meta = store.rebalance_meta.write().await;
|
||||
*rebalance_meta = previous_meta;
|
||||
@@ -477,10 +469,9 @@ impl ECStore {
|
||||
}
|
||||
|
||||
if !terminal_state_saved
|
||||
&& let Err(err) = retry_rebalance_metadata_access(None, rebalance_max_attempts(), || {
|
||||
store.save_rebalance_stats_for_id(pool_index, RebalSaveOpt::Stats, save_rebalance_id.as_ref())
|
||||
})
|
||||
.await
|
||||
&& let Err(err) = store
|
||||
.save_rebalance_stats_for_id(pool_index, RebalSaveOpt::Stats, save_rebalance_id.as_ref())
|
||||
.await
|
||||
{
|
||||
let wrapped = Error::other(format!("rebalance save_task stats save failed for pool {pool_index}: {err}"));
|
||||
error!("{} err: {:?}", msg, wrapped);
|
||||
@@ -632,11 +623,6 @@ impl ECStore {
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let defer_kind = if source_cleanup_deferred {
|
||||
RebalanceDeferKind::SourceCleanup
|
||||
} else {
|
||||
RebalanceDeferKind::Entry
|
||||
};
|
||||
warn!(
|
||||
event = EVENT_REBALANCE_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -648,7 +634,7 @@ impl ECStore {
|
||||
"Deferred rebalance bucket after transient object failures"
|
||||
);
|
||||
if let Err(err) = self
|
||||
.defer_rebalance_bucket(pool_index, bucket.clone(), last_error.clone(), rebalance_id.as_ref(), defer_kind)
|
||||
.defer_rebalance_bucket(pool_index, bucket.clone(), last_error.clone(), rebalance_id.as_ref())
|
||||
.await
|
||||
{
|
||||
error!(
|
||||
@@ -668,10 +654,10 @@ impl ECStore {
|
||||
break;
|
||||
}
|
||||
if source_cleanup_deferred {
|
||||
if reached_rebalance_source_cleanup_defer_limit(source_cleanup_attempt) {
|
||||
if source_cleanup_attempt >= super::REBALANCE_SOURCE_CLEANUP_MAX_DEFERS {
|
||||
let err = Error::other(format!(
|
||||
"rebalance bucket {bucket} source cleanup remained unstable after {} deferrals: {last_error}",
|
||||
REBALANCE_SOURCE_CLEANUP_MAX_DEFERS
|
||||
super::REBALANCE_SOURCE_CLEANUP_MAX_DEFERS
|
||||
));
|
||||
warn!(
|
||||
event = EVENT_REBALANCE_BUCKET,
|
||||
|
||||
@@ -51,18 +51,6 @@ pub(super) enum RebalanceEntryOutcome {
|
||||
Deferred { last_error: String },
|
||||
}
|
||||
|
||||
/// Why a rebalance bucket was put back at the end of the queue.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum RebalanceDeferKind {
|
||||
/// A transient object-migration failure. Persisting it as `lastError` keeps the pool
|
||||
/// from being completed at the free-space goal while the entry is still retried.
|
||||
Entry,
|
||||
/// A retryable source-cleanup conflict. The bucket stays queued and is retried, so a
|
||||
/// transient lock conflict must not be recorded as a permanent warning that would block
|
||||
/// pool completion for the rest of the run.
|
||||
SourceCleanup,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
|
||||
pub enum RebalStatus {
|
||||
#[default]
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
use super::migration::MigrationVersionResult;
|
||||
use super::{
|
||||
DEFAULT_REBALANCE_MAX_ATTEMPTS, EVENT_REBALANCE_LISTING, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE, REBAL_META_NAME,
|
||||
REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX, REBALANCE_LISTING_RETRY_BASE_DELAY, REBALANCE_MAX_ATTEMPTS_ENV,
|
||||
REBALANCE_MIGRATION_LOCK_RETRY_CAP, REBALANCE_MIGRATION_RETRY_BASE_DELAY, REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX,
|
||||
RebalanceBucketConfigs, RebalanceBucketOutcome, RebalanceDeferKind, RebalanceEntryOutcome, Result,
|
||||
REBALANCE_LISTING_RETRY_BASE_DELAY, REBALANCE_MAX_ATTEMPTS_ENV, REBALANCE_MIGRATION_LOCK_RETRY_CAP,
|
||||
REBALANCE_MIGRATION_RETRY_BASE_DELAY, REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX, RebalanceBucketConfigs,
|
||||
RebalanceBucketOutcome, RebalanceEntryOutcome, Result,
|
||||
};
|
||||
use crate::cache_value::metacache_set::{ListPathRawOptions, list_path_raw};
|
||||
use crate::core::pools::ListCallback;
|
||||
@@ -109,13 +109,7 @@ pub(super) fn resolve_rebalance_save_task_result(
|
||||
}
|
||||
|
||||
pub(super) fn resolve_rebalance_meta_save_result(result: Result<()>, stage: &str) -> Result<()> {
|
||||
// Keep the source error reachable: the metadata retry policy classifies
|
||||
// transient lock timeouts by inspecting the source chain, so collapsing the
|
||||
// failure into a plain string here would make that retry a no-op.
|
||||
result.map_err(|err| {
|
||||
let rendered = format!("rebalance meta save failed during {stage}: {err}");
|
||||
crate::data_movement::data_movement_context_error(rendered, err)
|
||||
})
|
||||
result.map_err(|err| Error::other(format!("rebalance meta save failed during {stage}: {err}")))
|
||||
}
|
||||
|
||||
pub(super) fn rebalance_meta_lock_error(err: rustfs_lock::LockError, mode: &'static str) -> Error {
|
||||
@@ -183,7 +177,7 @@ pub(super) fn resolve_rebalance_entry_cleanup_delete_result(
|
||||
) -> RebalanceEntryCleanupResult {
|
||||
match result {
|
||||
Ok(_) => RebalanceEntryCleanupResult::Completed { warning: None },
|
||||
Err(SourceCleanupError::Storage(err)) if is_source_cleanup_not_found(&err) => {
|
||||
Err(SourceCleanupError::Storage(err)) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {
|
||||
RebalanceEntryCleanupResult::Completed { warning: None }
|
||||
}
|
||||
Err(SourceCleanupError::SourceChanged) => RebalanceEntryCleanupResult::Deferred {
|
||||
@@ -191,25 +185,12 @@ pub(super) fn resolve_rebalance_entry_cleanup_delete_result(
|
||||
"{REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX} source changed during cleanup preflight for {bucket}/{object_name}"
|
||||
),
|
||||
},
|
||||
// A transient cleanup failure is not evidence that the source replica is gone, so the
|
||||
// entry stays incomplete and the bucket is retried instead of recording a permanent
|
||||
// cleanup warning that would block pool completion.
|
||||
Err(SourceCleanupError::Storage(err)) if is_transient_rebalance_error(&err) => RebalanceEntryCleanupResult::Deferred {
|
||||
last_error: format!(
|
||||
"{REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX} transient source cleanup failure for {bucket}/{object_name} will be retried: {err}"
|
||||
),
|
||||
},
|
||||
Err(SourceCleanupError::Storage(err)) => RebalanceEntryCleanupResult::Completed {
|
||||
warning: Some(format!("rebalance cleanup delete failed for {bucket}/{object_name}: {err}")),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn is_source_cleanup_not_found(err: &Error) -> bool {
|
||||
let err = rebalance_error_source(err);
|
||||
is_err_object_not_found(err) || is_err_version_not_found(err)
|
||||
}
|
||||
|
||||
pub(super) fn resolve_rebalance_migrate_result_error(
|
||||
err: Option<Error>,
|
||||
pool_idx: usize,
|
||||
@@ -229,23 +210,6 @@ pub(super) fn should_defer_rebalance_entry_failure(err: &Error) -> bool {
|
||||
is_transient_rebalance_error(err)
|
||||
}
|
||||
|
||||
pub(super) fn resolve_rebalance_deferred_last_error(
|
||||
kind: RebalanceDeferKind,
|
||||
pending_entry_defer: Option<&str>,
|
||||
last_error: &str,
|
||||
) -> Option<String> {
|
||||
match kind {
|
||||
RebalanceDeferKind::Entry => Some(last_error.to_string()),
|
||||
// A retryable cleanup conflict is progress, not a pool failure, so it must not surface as
|
||||
// `lastError`. It also must not erase an unresolved migration deferral: that marker is the
|
||||
// only signal keeping the pool from completing at the free-space goal while an entry is
|
||||
// still retried, and the two deferrals can be reported by different buckets of one pool.
|
||||
RebalanceDeferKind::SourceCleanup => pending_entry_defer
|
||||
.filter(|pending| pending.starts_with(REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX))
|
||||
.map(str::to_string),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn resolve_load_rebalance_stats_update_result(result: Result<()>) -> Result<()> {
|
||||
result.map_err(|err| Error::other(format!("rebalance metadata stats refresh failed after load: {err}")))
|
||||
}
|
||||
@@ -335,10 +299,7 @@ fn is_rebalance_transient_io_error(err: &std::io::Error) -> bool {
|
||||
|
||||
fn is_rebalance_transient_message(message: &str) -> bool {
|
||||
let message = message.to_ascii_lowercase();
|
||||
// `LockError::Timeout` renders "Lock acquisition timeout for resource ...", while the
|
||||
// namespace-lock layer renders "lock acquisition timed out on ..."; both are retryable.
|
||||
message.contains("lock acquisition timeout")
|
||||
|| message.contains("lock acquisition timed out")
|
||||
message.contains("lock acquisition timed out")
|
||||
|| message.contains("remote lock rpc timed out")
|
||||
|| message.contains("keepalivetimedout")
|
||||
|| message.contains("i/o timeout")
|
||||
@@ -419,8 +380,7 @@ fn is_rebalance_lock_or_rpc_timeout(err: &Error) -> bool {
|
||||
|
||||
fn is_rebalance_lock_or_rpc_timeout_message(message: &str) -> bool {
|
||||
let message = message.to_ascii_lowercase();
|
||||
message.contains("lock acquisition timeout")
|
||||
|| message.contains("lock acquisition timed out")
|
||||
message.contains("lock acquisition timed out")
|
||||
|| message.contains("remote lock rpc timed out")
|
||||
|| message.contains("keepalivetimedout")
|
||||
}
|
||||
@@ -745,29 +705,6 @@ mod error_source_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rebalance_metadata_retry_engages_for_wrapped_meta_save_lock_timeout() {
|
||||
let mut attempts = 0;
|
||||
let result = retry_rebalance_metadata_access(None, 3, || {
|
||||
attempts += 1;
|
||||
std::future::ready(resolve_rebalance_meta_save_result(
|
||||
Err(rebalance_meta_lock_error(
|
||||
rustfs_lock::LockError::timeout(".rustfs.sys/rebalance.bin@latest", Duration::from_secs(5)),
|
||||
"write",
|
||||
)),
|
||||
"save_rebalance_stats for pool 0 opt Stats",
|
||||
))
|
||||
})
|
||||
.await;
|
||||
|
||||
assert_eq!(attempts, 3, "a wrapped meta save lock timeout must stay retryable");
|
||||
let err = result.expect_err("persistent lock contention must not become success");
|
||||
assert!(matches!(
|
||||
rebalance_error_source(&err),
|
||||
Error::Lock(rustfs_lock::LockError::Timeout { .. })
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rebalance_metadata_retry_cancels_a_pending_attempt() {
|
||||
struct DropProbe(Arc<std::sync::atomic::AtomicUsize>);
|
||||
@@ -870,44 +807,4 @@ mod error_source_tests {
|
||||
}
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn rendered_lock_timeout_text_selects_the_lock_backoff() {
|
||||
// The lock backend renders a timeout as "Lock acquisition timeout for resource ...",
|
||||
// so the message matcher must recognize that text when the error arrives re-rendered
|
||||
// instead of as a typed `Error::Lock`.
|
||||
let rendered = rustfs_lock::LockError::timeout("bucket/object@latest", Duration::from_secs(5)).to_string();
|
||||
assert!(
|
||||
rendered.contains("Lock acquisition timeout for resource"),
|
||||
"unexpected lock timeout text: {rendered}"
|
||||
);
|
||||
|
||||
// The lock policy jitters the delay inside its own cap, so a far-out attempt identifies
|
||||
// the selected policy: the linear fallback would return `base * (attempt + 1)`.
|
||||
let far_attempt = 100;
|
||||
assert!(REBALANCE_MIGRATION_RETRY_BASE_DELAY * 101 > REBALANCE_MIGRATION_LOCK_RETRY_CAP);
|
||||
|
||||
let mut error = Error::other(format!("Lock error: {rendered}"));
|
||||
for depth in 0..=3 {
|
||||
assert!(
|
||||
is_transient_rebalance_error(&error),
|
||||
"rendered lock timeout lost retryability at depth {depth}: {error:?}"
|
||||
);
|
||||
assert!(
|
||||
is_rebalance_lock_or_rpc_timeout(&error),
|
||||
"rendered lock timeout lost the lock backoff at depth {depth}: {error:?}"
|
||||
);
|
||||
let delay = rebalance_migration_retry_delay(far_attempt, &error);
|
||||
assert!(
|
||||
delay <= REBALANCE_MIGRATION_LOCK_RETRY_CAP && delay >= Duration::from_millis(1),
|
||||
"rendered lock timeout must stay inside the lock backoff cap at depth {depth}: {delay:?}"
|
||||
);
|
||||
error = crate::data_movement::data_movement_stage_error_for_test(
|
||||
"rebalance_object",
|
||||
"put_object",
|
||||
"bucket",
|
||||
"baseline/00042.bin",
|
||||
error,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -37,7 +37,7 @@ use rustfs_s3_client::{
|
||||
api_s3_datatypes::ListVersionsResult,
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
provider_versions::validate_remote_version_id,
|
||||
transition_api::{BucketLookupType, ObjectInfo, Options, TransitionClient, TransitionCore},
|
||||
transition_api::{BucketLookupType, Options, TransitionClient, TransitionCore},
|
||||
transition_api::{ReadCloser, ReaderImpl},
|
||||
};
|
||||
use rustfs_utils::egress::validate_outbound_url;
|
||||
@@ -278,7 +278,19 @@ impl WarmBackendS3 {
|
||||
let mut stat_opts = GetObjectOptions::default();
|
||||
stat_opts.version_id.clone_from(&version.version_id);
|
||||
let info = self.client.stat_object(&self.bucket, &remote_object, &stat_opts).await?;
|
||||
let metadata = transition_candidate_metadata(info);
|
||||
let mut metadata = info.user_metadata;
|
||||
for (name, value) in &info.metadata {
|
||||
if (name
|
||||
.as_str()
|
||||
.starts_with(rustfs_utils::http::metadata_compat::RUSTFS_INTERNAL_PREFIX)
|
||||
|| name
|
||||
.as_str()
|
||||
.starts_with(rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX))
|
||||
&& let Ok(value) = value.to_str()
|
||||
{
|
||||
metadata.insert(name.as_str().to_string(), value.to_string());
|
||||
}
|
||||
}
|
||||
if transition_candidate_metadata_matches(&metadata, identity)? {
|
||||
if matched_version.is_some() {
|
||||
return Ok(TransitionCandidateProbe::Ambiguous);
|
||||
@@ -301,53 +313,6 @@ impl WarmBackendS3 {
|
||||
advance_version_markers(&mut key_marker, &mut version_id_marker, &versions)?;
|
||||
}
|
||||
}
|
||||
|
||||
async fn probe_unversioned_transition_candidate_identity(
|
||||
&self,
|
||||
object: &str,
|
||||
identity: TransitionCandidateIdentity,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
let remote_object = self.get_dest(object);
|
||||
match self
|
||||
.client
|
||||
.stat_object(&self.bucket, &remote_object, &GetObjectOptions::default())
|
||||
.await
|
||||
{
|
||||
Ok(info) => {
|
||||
let metadata = transition_candidate_metadata(info);
|
||||
if transition_candidate_metadata_matches(&metadata, identity)? {
|
||||
Ok(TransitionCandidateProbe::UnversionedPresent)
|
||||
} else {
|
||||
Ok(TransitionCandidateProbe::Unsupported)
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
let response = to_error_response(&err);
|
||||
if response.code == S3ErrorCode::NoSuchKey {
|
||||
Ok(TransitionCandidateProbe::Missing)
|
||||
} else {
|
||||
Err(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn transition_candidate_metadata(info: ObjectInfo) -> HashMap<String, String> {
|
||||
let mut metadata = info.user_metadata;
|
||||
for (name, value) in &info.metadata {
|
||||
if (name
|
||||
.as_str()
|
||||
.starts_with(rustfs_utils::http::metadata_compat::RUSTFS_INTERNAL_PREFIX)
|
||||
|| name
|
||||
.as_str()
|
||||
.starts_with(rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX))
|
||||
&& let Ok(value) = value.to_str()
|
||||
{
|
||||
metadata.insert(name.as_str().to_string(), value.to_string());
|
||||
}
|
||||
}
|
||||
metadata
|
||||
}
|
||||
|
||||
fn transition_candidate_metadata_matches(
|
||||
@@ -560,13 +525,6 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn scripted_probe_fixture(responses: Vec<String>) -> Option<(WarmBackendS3, tokio::task::JoinHandle<Vec<String>>)> {
|
||||
scripted_probe_fixture_for_tier(responses, "s3").await
|
||||
}
|
||||
|
||||
async fn scripted_probe_fixture_for_tier(
|
||||
responses: Vec<String>,
|
||||
tier_type: &str,
|
||||
) -> Option<(WarmBackendS3, tokio::task::JoinHandle<Vec<String>>)> {
|
||||
let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await {
|
||||
Ok(listener) => listener,
|
||||
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None,
|
||||
@@ -613,7 +571,7 @@ mod tests {
|
||||
max_retries: 1,
|
||||
..Default::default()
|
||||
},
|
||||
tier_type,
|
||||
"s3",
|
||||
)
|
||||
.await
|
||||
.expect("fixture client should build"),
|
||||
@@ -717,37 +675,6 @@ mod tests {
|
||||
assert!(requests[8].to_ascii_lowercase().contains("?versionid=historical-version"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unversioned_candidate_reconciler_uses_head_without_version_apis() {
|
||||
let identity = candidate_identity();
|
||||
let transaction_id = identity.transaction_id;
|
||||
let destination_id = rustfs_utils::crypto::hex(identity.destination_id);
|
||||
let response = format!(
|
||||
"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nx-amz-meta-x-rustfs-internal-transition-transaction-id: {transaction_id}\r\nx-amz-meta-x-rustfs-internal-transition-tier-destination-id: {destination_id}\r\nConnection: close\r\n\r\n"
|
||||
);
|
||||
let Some((backend, fixture)) = scripted_probe_fixture_for_tier(vec![response], "r2").await else {
|
||||
return;
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_transition_candidate_for(
|
||||
&backend,
|
||||
"archive/object",
|
||||
identity,
|
||||
)
|
||||
.await
|
||||
.expect("R2 candidate probe should use the current unversioned object"),
|
||||
TransitionCandidateProbe::UnversionedPresent
|
||||
);
|
||||
let requests = fixture.await.expect("R2 candidate fixture should finish");
|
||||
assert_eq!(requests.len(), 1);
|
||||
let request = requests[0].to_ascii_lowercase();
|
||||
assert!(request.starts_with("head /bucket/archive/object"));
|
||||
assert!(!request.contains("versioning"));
|
||||
assert!(!request.contains("versions"));
|
||||
assert!(!request.contains("versionid="));
|
||||
}
|
||||
|
||||
fn legacy_probe_xml_response(body: &str) -> String {
|
||||
format!(
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/xml\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
|
||||
@@ -1162,10 +1089,6 @@ impl TransitionCandidateReconciler for WarmBackendS3 {
|
||||
object: &str,
|
||||
identity: TransitionCandidateIdentity,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
let capabilities = self.client.provider_version_capabilities();
|
||||
if !capabilities.bucket_versioning_state && !capabilities.list_object_versions {
|
||||
return self.probe_unversioned_transition_candidate_identity(object, identity).await;
|
||||
}
|
||||
let bucket_versioning = self.remote_bucket_versioning().await?;
|
||||
self.probe_transition_candidate_identity(object, identity, bucket_versioning)
|
||||
.await
|
||||
|
||||
@@ -3494,44 +3494,19 @@ fn dangling_delete_grace() -> time::Duration {
|
||||
/// Result of scanning one disk's copy of a directory prefix while deciding
|
||||
/// whether an orphan (metadata-less) directory tree can be safely purged.
|
||||
enum OrphanDirScan {
|
||||
/// The prefix exists on this disk. `blocked` holds every directory that is
|
||||
/// itself, or an ancestor of, object metadata or uncommitted data (closed
|
||||
/// under taking parents, root included when anything under it is blocked);
|
||||
/// `dirs` is the pre-order list of every directory reached that is not
|
||||
/// blocked, and `committed_files` the erasure data and committed delete
|
||||
/// markers found in the unblocked UUID data dirs among them.
|
||||
Scanned {
|
||||
blocked: HashSet<String>,
|
||||
dirs: Vec<String>,
|
||||
/// The subtree holds object metadata or uncommitted data, so it must not be
|
||||
/// purged.
|
||||
HasData,
|
||||
/// The prefix contains only empty directories and/or UUID data directories
|
||||
/// carrying a committed delete marker.
|
||||
Purgeable {
|
||||
empty_dirs: Vec<String>,
|
||||
committed_files: Vec<String>,
|
||||
},
|
||||
/// A directory read failed for a reason other than absence, so nothing
|
||||
/// under the prefix can be classified on this disk.
|
||||
Unreadable,
|
||||
/// The prefix does not exist on this disk.
|
||||
Missing,
|
||||
}
|
||||
|
||||
/// How long an orphan prefix whose purge scan met unpurgeable data is left
|
||||
/// alone before an empty listing scans it again.
|
||||
const ORPHAN_PURGE_BACKOFF: Duration = Duration::from_secs(60);
|
||||
/// Upper bound on remembered backoff entries; the oldest is dropped first.
|
||||
const ORPHAN_PURGE_BACKOFF_MAX_ENTRIES: usize = 4096;
|
||||
|
||||
/// Mark `dir` and every ancestor up to and including `root` as blocked.
|
||||
fn block_orphan_dir_chain(blocked: &mut HashSet<String>, root: &str, dir: &str) {
|
||||
let mut current = dir;
|
||||
loop {
|
||||
if !blocked.insert(current.to_owned()) || current == root {
|
||||
return;
|
||||
}
|
||||
let Some((parent, _)) = current.rsplit_once(SLASH_SEPARATOR) else {
|
||||
return;
|
||||
};
|
||||
current = parent;
|
||||
}
|
||||
}
|
||||
|
||||
fn is_safe_orphan_dir_entry(entry: &str) -> bool {
|
||||
let component = entry.strip_suffix(SLASH_SEPARATOR).unwrap_or(entry);
|
||||
!component.is_empty()
|
||||
@@ -6336,12 +6311,9 @@ impl SetDisks {
|
||||
.await
|
||||
}
|
||||
|
||||
/// Scan a single disk's copy of `prefix` and classify every directory
|
||||
/// under it. Only empty directories and UUID data directories with valid
|
||||
/// committed delete markers are purgeable; anything else blocks its whole
|
||||
/// ancestor chain while sibling subtrees stay purgeable, so committed
|
||||
/// residue is still reclaimed when it shares an ancestor with residue from
|
||||
/// an older build that never wrote markers (#6898).
|
||||
/// Scan a single disk's copy of `prefix` and decide whether it is an orphan
|
||||
/// directory subtree. Only empty directories and UUID data directories with
|
||||
/// valid committed delete markers are purgeable; every child is still scanned.
|
||||
async fn scan_orphan_dir(disk: &DiskStore, bucket: &str, prefix: &str) -> OrphanDirScan {
|
||||
let root = prefix.trim_end_matches(SLASH_SEPARATOR).to_string();
|
||||
let mut stack = vec![root.clone()];
|
||||
@@ -6349,7 +6321,6 @@ impl SetDisks {
|
||||
// so reversing it yields a safe children-first removal order.
|
||||
let mut dirs: Vec<String> = Vec::new();
|
||||
let mut committed_files: Vec<String> = Vec::new();
|
||||
let mut blocked: HashSet<String> = HashSet::new();
|
||||
let mut existed = false;
|
||||
|
||||
while let Some(dir) = stack.pop() {
|
||||
@@ -6364,18 +6335,16 @@ impl SetDisks {
|
||||
}
|
||||
// Classification must fail closed: committed residue is safe to
|
||||
// remove only after every reachable child was inspected.
|
||||
Err(_) => return OrphanDirScan::Unreadable,
|
||||
Err(_) => return OrphanDirScan::HasData,
|
||||
};
|
||||
|
||||
existed = true;
|
||||
let mut child_dirs = Vec::new();
|
||||
let mut files = Vec::new();
|
||||
|
||||
let mut has_data = false;
|
||||
for entry in entries {
|
||||
if !is_safe_orphan_dir_entry(&entry) {
|
||||
has_data = true;
|
||||
break;
|
||||
return OrphanDirScan::HasData;
|
||||
}
|
||||
match entry.strip_suffix(SLASH_SEPARATOR) {
|
||||
Some(child) => child_dirs.push(format!("{dir}{SLASH_SEPARATOR}{child}")),
|
||||
@@ -6383,29 +6352,28 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
if !has_data && !files.is_empty() {
|
||||
if !files.is_empty() {
|
||||
let data_dir_name = dir.rsplit(SLASH_SEPARATOR).next().unwrap_or_default();
|
||||
let is_uuid_data_dir = Uuid::parse_str(data_dir_name).is_ok_and(|uuid| !uuid.is_nil());
|
||||
let has_committed_delete = files.iter().any(|entry| is_committed_delete_marker(entry));
|
||||
has_data = !is_uuid_data_dir || !has_committed_delete || files.iter().any(|entry| entry == STORAGE_FORMAT_FILE);
|
||||
}
|
||||
|
||||
if has_data {
|
||||
// Nothing below a blocked directory is ever removed, so its
|
||||
// children need no classification.
|
||||
block_orphan_dir_chain(&mut blocked, &root, &dir);
|
||||
if !is_uuid_data_dir || !has_committed_delete || files.iter().any(|entry| entry == STORAGE_FORMAT_FILE) {
|
||||
return OrphanDirScan::HasData;
|
||||
}
|
||||
|
||||
committed_files.extend(files.into_iter().map(|entry| path_join_buf(&[&dir, &entry])));
|
||||
dirs.push(dir);
|
||||
stack.extend(child_dirs);
|
||||
continue;
|
||||
}
|
||||
|
||||
committed_files.extend(files.into_iter().map(|entry| format!("{dir}{SLASH_SEPARATOR}{entry}")));
|
||||
dirs.push(dir);
|
||||
stack.extend(child_dirs);
|
||||
}
|
||||
|
||||
if existed {
|
||||
OrphanDirScan::Scanned {
|
||||
blocked,
|
||||
dirs,
|
||||
OrphanDirScan::Purgeable {
|
||||
empty_dirs: dirs,
|
||||
committed_files,
|
||||
}
|
||||
} else {
|
||||
@@ -6500,101 +6468,43 @@ impl SetDisks {
|
||||
/// of this set (the caller should surface the original NotFound), and `Err` on
|
||||
/// a hard disk failure.
|
||||
pub(crate) async fn purge_orphan_dir_object(&self, bucket: &str, object: &str) -> disk::error::Result<bool> {
|
||||
let backoff_key = format!("{bucket}{SLASH_SEPARATOR}{object}");
|
||||
if self.orphan_purge_in_backoff(&backoff_key) {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let disks = self.get_disks_internal().await;
|
||||
|
||||
// Phase 1: classify every online disk. A directory that holds object
|
||||
// data or uncommitted residue on ANY disk blocks itself and its
|
||||
// ancestors on every disk, so a degraded/healable object is never
|
||||
// destroyed; purgeable subtrees beside it are still reclaimed.
|
||||
let mut per_disk: Vec<(usize, Vec<String>, Vec<String>)> = Vec::new();
|
||||
let mut blocked: HashSet<String> = HashSet::new();
|
||||
// Phase 1: classify every online disk. Refuse to purge if ANY disk holds
|
||||
// object data under the prefix, so a degraded/healable object is never
|
||||
// destroyed.
|
||||
let mut per_disk_dirs: Vec<(usize, Vec<String>, Vec<String>)> = Vec::new();
|
||||
let mut existed = false;
|
||||
for (i, disk) in disks.iter().enumerate() {
|
||||
let Some(disk) = disk else { continue };
|
||||
match Self::scan_orphan_dir(disk, bucket, object).await {
|
||||
OrphanDirScan::Unreadable => return Ok(false),
|
||||
OrphanDirScan::Scanned {
|
||||
blocked: disk_blocked,
|
||||
dirs,
|
||||
OrphanDirScan::HasData => return Ok(false),
|
||||
OrphanDirScan::Purgeable {
|
||||
empty_dirs,
|
||||
committed_files,
|
||||
} => {
|
||||
blocked.extend(disk_blocked);
|
||||
per_disk.push((i, dirs, committed_files));
|
||||
existed = true;
|
||||
per_disk_dirs.push((i, empty_dirs, committed_files));
|
||||
}
|
||||
OrphanDirScan::Missing => {}
|
||||
}
|
||||
}
|
||||
if !blocked.is_empty() {
|
||||
// Whatever is purgeable goes now; what blocks the rest will still
|
||||
// block it on the next empty listing, so do not rescan for a while.
|
||||
self.record_orphan_purge_backoff(backoff_key);
|
||||
|
||||
if !existed {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
// Phase 2: remove only the files classified as committed residue in
|
||||
// unblocked data dirs, then remove unblocked directories
|
||||
// children-first. Every directory delete is non-recursive, so a
|
||||
// directory that concurrently gained an object fails with
|
||||
// DirectoryNotEmpty and is skipped — a racing PutObject is never
|
||||
// Phase 2: remove only the files classified as committed residue, then
|
||||
// remove directories children-first. Every directory delete is
|
||||
// non-recursive, so a directory that concurrently gained an object fails
|
||||
// with DirectoryNotEmpty and is skipped — a racing PutObject is never
|
||||
// clobbered.
|
||||
let mut purged = false;
|
||||
for (i, dirs, committed_files) in per_disk {
|
||||
for (i, empty_dirs, committed_files) in per_disk_dirs {
|
||||
let Some(disk) = disks[i].as_ref() else { continue };
|
||||
let dirs = dirs.into_iter().filter(|dir| !blocked.contains(dir)).collect::<Vec<_>>();
|
||||
let committed_files = committed_files
|
||||
.into_iter()
|
||||
.filter(|file| {
|
||||
file.rsplit_once(SLASH_SEPARATOR)
|
||||
.is_some_and(|(data_dir, _)| !blocked.contains(data_dir))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
if dirs.is_empty() && committed_files.is_empty() {
|
||||
continue;
|
||||
}
|
||||
purged = true;
|
||||
Self::delete_purgeable_orphan_entries(disk, bucket, object, dirs, committed_files).await;
|
||||
Self::delete_purgeable_orphan_entries(disk, bucket, object, empty_dirs, committed_files).await;
|
||||
}
|
||||
|
||||
Ok(purged)
|
||||
}
|
||||
|
||||
fn orphan_purge_in_backoff(&self, key: &str) -> bool {
|
||||
let backoff = self
|
||||
.orphan_purge_backoff
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
backoff
|
||||
.get(key)
|
||||
.is_some_and(|scanned_at| scanned_at.elapsed() < ORPHAN_PURGE_BACKOFF)
|
||||
}
|
||||
|
||||
fn record_orphan_purge_backoff(&self, key: String) {
|
||||
let mut backoff = self
|
||||
.orphan_purge_backoff
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
let now = Instant::now();
|
||||
backoff.retain(|_, scanned_at| now.duration_since(*scanned_at) < ORPHAN_PURGE_BACKOFF);
|
||||
if backoff.len() >= ORPHAN_PURGE_BACKOFF_MAX_ENTRIES
|
||||
&& let Some(oldest) = backoff
|
||||
.iter()
|
||||
.min_by_key(|(_, scanned_at)| **scanned_at)
|
||||
.map(|(key, _)| key.clone())
|
||||
{
|
||||
backoff.remove(&oldest);
|
||||
}
|
||||
backoff.insert(key, now);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn clear_orphan_purge_backoff(&self) {
|
||||
self.orphan_purge_backoff
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.clear();
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Reclaim orphaned physical data directories under `bucket/object` that no
|
||||
@@ -7711,15 +7621,13 @@ mod tests {
|
||||
.await
|
||||
.expect("committed delete marker should be written");
|
||||
|
||||
let OrphanDirScan::Scanned {
|
||||
blocked,
|
||||
dirs: empty_dirs,
|
||||
let OrphanDirScan::Purgeable {
|
||||
empty_dirs,
|
||||
committed_files,
|
||||
} = SetDisks::scan_orphan_dir(&disk, "bucket", "pfx/").await
|
||||
else {
|
||||
panic!("committed residue should be classified as purgeable");
|
||||
};
|
||||
assert!(blocked.is_empty(), "committed residue alone blocks nothing");
|
||||
|
||||
let nested_object = residue.join("nested");
|
||||
tokio::fs::create_dir_all(&nested_object)
|
||||
@@ -7761,15 +7669,13 @@ mod tests {
|
||||
.await
|
||||
.expect("committed delete marker should be written");
|
||||
|
||||
let OrphanDirScan::Scanned {
|
||||
blocked,
|
||||
dirs: empty_dirs,
|
||||
let OrphanDirScan::Purgeable {
|
||||
empty_dirs,
|
||||
committed_files,
|
||||
} = SetDisks::scan_orphan_dir(&disk, "bucket", "pfx/").await
|
||||
else {
|
||||
panic!("committed residue should be classified as purgeable");
|
||||
};
|
||||
assert!(blocked.is_empty(), "committed residue alone blocks nothing");
|
||||
tokio::fs::set_permissions(&residue, std::fs::Permissions::from_mode(0o555))
|
||||
.await
|
||||
.expect("residue directory should become read-only");
|
||||
|
||||
@@ -874,7 +874,6 @@ mod ctx;
|
||||
mod metadata;
|
||||
mod ops;
|
||||
pub(crate) use ops::bucket::BucketInfoQuorum;
|
||||
pub(crate) use ops::heal::AbsenceProofRequest;
|
||||
#[cfg(test)]
|
||||
pub(crate) use ops::heal::DanglingDeleteFailure;
|
||||
pub(crate) use ops::heal::HealedObjectAbsence;
|
||||
@@ -3901,12 +3900,6 @@ pub struct SetDisks {
|
||||
/// writes skip the global registry mutex (backlog#1315). `Arc` so clones of
|
||||
/// a set share one generation marker.
|
||||
capacity_dirty_generation: Arc<AtomicU64>,
|
||||
/// Orphan prefixes whose last purge scan met data that can never be
|
||||
/// purged by listing (residue without a committed marker, an in-flight
|
||||
/// write), keyed by `bucket/prefix` with the time of that scan. Empty
|
||||
/// listings of such a prefix are frequent and the scan reads the whole
|
||||
/// subtree, so it is not repeated within `ORPHAN_PURGE_BACKOFF` (#6898).
|
||||
orphan_purge_backoff: Arc<std::sync::Mutex<HashMap<String, std::time::Instant>>>,
|
||||
#[cfg(test)]
|
||||
storage_class_config_override: Arc<std::sync::RwLock<Option<Arc<storageclass::Config>>>>,
|
||||
#[cfg(test)]
|
||||
@@ -3915,72 +3908,6 @@ pub struct SetDisks {
|
||||
>,
|
||||
}
|
||||
|
||||
fn marker_purge_receipt_path(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version: Uuid,
|
||||
purge: &rustfs_common::mrf_channel::MrfDeleteMarkerPurge,
|
||||
) -> String {
|
||||
let object_hash = rustfs_utils::crypto::hex(Sha256::digest(object.as_bytes()));
|
||||
let marker_hash = rustfs_utils::crypto::hex(purge.marker_identity);
|
||||
format!(
|
||||
"marker-purge-receipts/{}/{}/{}-{}-{}.json",
|
||||
bucket, purge.bucket_incarnation_id, version, object_hash, marker_hash
|
||||
)
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
/// Persist a quorum DELETE receipt outside the object marker itself. The
|
||||
/// receipt is written to the metadata bucket so it survives a node restart
|
||||
/// while an inaccessible member still holds the stale marker.
|
||||
pub(crate) async fn persist_marker_purge_receipt(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version: Uuid,
|
||||
purge: &rustfs_common::mrf_channel::MrfDeleteMarkerPurge,
|
||||
) -> bool {
|
||||
let path = marker_purge_receipt_path(bucket, object, version, purge);
|
||||
let api = Arc::new(self.clone());
|
||||
crate::config::com::save_config_with_opts(
|
||||
api,
|
||||
&path,
|
||||
b"rustfs-marker-purge-receipt-v1".to_vec(),
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.is_ok()
|
||||
}
|
||||
|
||||
pub(crate) async fn has_marker_purge_receipt(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version: Uuid,
|
||||
purge: &rustfs_common::mrf_channel::MrfDeleteMarkerPurge,
|
||||
) -> bool {
|
||||
let path = marker_purge_receipt_path(bucket, object, version, purge);
|
||||
crate::config::com::read_config_limited_preserve_empty(Arc::new(self.clone()), &path, 128)
|
||||
.await
|
||||
.is_ok()
|
||||
}
|
||||
|
||||
pub(crate) async fn consume_marker_purge_receipt(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version: Uuid,
|
||||
purge: &rustfs_common::mrf_channel::MrfDeleteMarkerPurge,
|
||||
) {
|
||||
let path = marker_purge_receipt_path(bucket, object, version, purge);
|
||||
let _ = crate::config::com::delete_config_no_lock(Arc::new(self.clone()), &path).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Read every physical copy before selecting a version quorum. A minority
|
||||
/// legacy record is still evidence and must not disappear behind a majority
|
||||
/// not-found result. Only an explicit file/volume absence produces `None`;
|
||||
@@ -4763,7 +4690,6 @@ impl SetDisks {
|
||||
ctx,
|
||||
capacity_scope_cache: Arc::new(std::sync::RwLock::new(CapacityScopeCache::default())),
|
||||
capacity_dirty_generation: Arc::new(AtomicU64::new(u64::MAX)),
|
||||
orphan_purge_backoff: Arc::new(std::sync::Mutex::new(HashMap::new())),
|
||||
#[cfg(test)]
|
||||
storage_class_config_override: Arc::new(std::sync::RwLock::new(None)),
|
||||
#[cfg(test)]
|
||||
@@ -4777,12 +4703,6 @@ impl SetDisks {
|
||||
&self.ctx
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn set_instance_ctx_for_test(&mut self, ctx: Arc<InstanceContext>) {
|
||||
self.local_lock_manager = ctx.lock_manager();
|
||||
self.ctx = ctx;
|
||||
}
|
||||
|
||||
/// Read the persisted bucket identity through this set's metadata owner.
|
||||
/// Missing or non-authoritative legacy identities remain errors.
|
||||
pub async fn bucket_incarnation_id_from_disk(&self, bucket: &str) -> Result<Uuid> {
|
||||
@@ -5300,24 +5220,6 @@ fn known_put_object_storage_size(data_size: i64) -> i64 {
|
||||
}
|
||||
}
|
||||
|
||||
/// Shard size the inline admission check evaluates for a single PUT.
|
||||
///
|
||||
/// A compressed or encrypted stream reports `SIZE_PRESERVE_LAYER` as its
|
||||
/// stored size because the transformed length is only known after the write.
|
||||
/// MinIO's `putObject` sizes such objects for inline admission by their
|
||||
/// plaintext `ActualSize`; without that fallback every transformed object,
|
||||
/// however small, lands in `part.1` files. A stream with neither size known
|
||||
/// yields a negative shard size, which `should_inline` rejects.
|
||||
fn inline_admission_shard_size(erasure: &coding::Erasure, stored_size: i64, actual_size: i64) -> i64 {
|
||||
if stored_size >= 0 {
|
||||
return erasure.shard_file_size(stored_size);
|
||||
}
|
||||
if actual_size > 0 {
|
||||
return erasure.shard_file_size(actual_size);
|
||||
}
|
||||
HashReader::SIZE_PRESERVE_LAYER
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn build_inline_bitrot_readers(
|
||||
files: &[FileInfo],
|
||||
@@ -6473,28 +6375,6 @@ async fn verify_inline_part_bitrot(meta: &FileInfo) -> disk::error::Result<()> {
|
||||
.map_err(|_| DiskError::FileCorrupt)
|
||||
}
|
||||
|
||||
fn validate_deep_scan_results(results: &[usize], expected_parts: usize) -> disk::error::Result<()> {
|
||||
if results.len() != expected_parts {
|
||||
return Err(DiskError::other(format!(
|
||||
"incomplete deep scan result: expected {expected_parts} parts, received {}",
|
||||
results.len()
|
||||
)));
|
||||
}
|
||||
for (part, status) in results.iter().enumerate() {
|
||||
match *status {
|
||||
CHECK_PART_SUCCESS | CHECK_PART_FILE_NOT_FOUND | CHECK_PART_FILE_CORRUPT => {}
|
||||
CHECK_PART_DISK_NOT_FOUND => return Err(DiskError::DiskNotFound),
|
||||
crate::disk::CHECK_PART_VOLUME_NOT_FOUND => return Err(DiskError::VolumeNotFound),
|
||||
_ => {
|
||||
return Err(DiskError::other(format!(
|
||||
"incomplete deep scan result: part {part} has unverified status {status}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// disks_with_all_partsv2 is a corrected version based on Go implementation.
|
||||
/// It sets partsMetadata and onlineDisks when xl.meta is inexistant/corrupted or outdated.
|
||||
/// It also checks if the status of each part (corrupted, missing, ok) in each drive.
|
||||
@@ -6715,13 +6595,9 @@ async fn disks_with_all_parts(
|
||||
// it needs healing too.
|
||||
match disk.verify_file(bucket, object, meta).await {
|
||||
Ok(v) => {
|
||||
validate_deep_scan_results(&v.results, latest_meta.parts.len())?;
|
||||
verify_resp = v;
|
||||
}
|
||||
Err(err) => {
|
||||
if !matches!(err, DiskError::FileNotFound | DiskError::FileVersionNotFound | DiskError::FileCorrupt) {
|
||||
return Err(err);
|
||||
}
|
||||
debug!(
|
||||
event = EVENT_SET_DISK_HEAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -9437,175 +9313,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
async fn write_committed_residue(object_dir: &std::path::Path) -> std::path::PathBuf {
|
||||
let residue = object_dir.join(Uuid::new_v4().to_string());
|
||||
fs::create_dir_all(&residue)
|
||||
.await
|
||||
.expect("committed data directory should be created");
|
||||
fs::write(residue.join("part.1"), b"stale")
|
||||
.await
|
||||
.expect("stale part should be written");
|
||||
fs::write(
|
||||
residue.join(format!("{}{}", crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX, Uuid::new_v4())),
|
||||
[],
|
||||
)
|
||||
.await
|
||||
.expect("committed delete marker should be written");
|
||||
residue
|
||||
}
|
||||
|
||||
// #6898: residue from a build that never wrote delete markers blocks only
|
||||
// its own ancestor chain; a committed sibling subtree is still reclaimed
|
||||
// and the call reports the partial purge.
|
||||
#[tokio::test]
|
||||
async fn purge_orphan_dir_object_reclaims_committed_subtree_beside_blocked_subtree() {
|
||||
let (dir0, disk0) = make_single_local_disk().await;
|
||||
let (dir1, disk1) = make_single_local_disk().await;
|
||||
|
||||
let committed = write_committed_residue(&dir0.path().join("bucket/pfx/a/obj")).await;
|
||||
let unmarked = dir1.path().join("bucket/pfx/b/obj").join(Uuid::new_v4().to_string());
|
||||
fs::create_dir_all(&unmarked)
|
||||
.await
|
||||
.expect("unmarked residue should be created");
|
||||
fs::write(unmarked.join("part.1"), b"stale")
|
||||
.await
|
||||
.expect("stale part should be written");
|
||||
fs::create_dir_all(dir0.path().join("bucket/pfx/b/obj"))
|
||||
.await
|
||||
.expect("disk0 copy of the blocked chain should be created");
|
||||
|
||||
let set = make_set_disks_with(vec![Some(disk0), Some(disk1)]).await;
|
||||
let purged = set
|
||||
.purge_orphan_dir_object("bucket", "pfx/")
|
||||
.await
|
||||
.expect("scan should succeed");
|
||||
|
||||
assert!(purged, "the committed subtree must be reported as purged");
|
||||
assert!(!committed.exists(), "committed residue beside a blocked subtree must be reclaimed");
|
||||
assert!(!dir0.path().join("bucket/pfx/a").exists(), "the reclaimed subtree's directories must go");
|
||||
assert!(unmarked.join("part.1").exists(), "unmarked residue must survive");
|
||||
assert!(
|
||||
dir0.path().join("bucket/pfx/b/obj").exists(),
|
||||
"a directory blocked on another disk must be left alone on this one"
|
||||
);
|
||||
assert!(dir0.path().join("bucket/pfx").exists() && dir1.path().join("bucket/pfx").exists());
|
||||
}
|
||||
|
||||
// A data dir that is committed residue on one disk but still holds an
|
||||
// object below it on another disk is blocked on every disk.
|
||||
#[tokio::test]
|
||||
async fn purge_orphan_dir_object_blocks_committed_files_by_other_disks_data() {
|
||||
let (dir0, disk0) = make_single_local_disk().await;
|
||||
let (dir1, disk1) = make_single_local_disk().await;
|
||||
|
||||
let data_dir = Uuid::new_v4();
|
||||
let committed = dir0.path().join("bucket/pfx/x").join(data_dir.to_string());
|
||||
fs::create_dir_all(&committed)
|
||||
.await
|
||||
.expect("committed data directory should be created");
|
||||
fs::write(committed.join("part.1"), b"stale")
|
||||
.await
|
||||
.expect("stale part should be written");
|
||||
fs::write(
|
||||
committed.join(format!("{}{}", crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX, Uuid::new_v4())),
|
||||
[],
|
||||
)
|
||||
.await
|
||||
.expect("committed delete marker should be written");
|
||||
let nested = dir1.path().join("bucket/pfx/x").join(data_dir.to_string()).join("nested");
|
||||
fs::create_dir_all(&nested)
|
||||
.await
|
||||
.expect("nested object dir should be created");
|
||||
fs::write(nested.join(STORAGE_FORMAT_FILE), b"meta")
|
||||
.await
|
||||
.expect("nested object metadata should be written");
|
||||
|
||||
let set = make_set_disks_with(vec![Some(disk0), Some(disk1)]).await;
|
||||
let purged = set
|
||||
.purge_orphan_dir_object("bucket", "pfx/")
|
||||
.await
|
||||
.expect("scan should succeed");
|
||||
|
||||
assert!(!purged);
|
||||
assert!(
|
||||
committed.join("part.1").exists(),
|
||||
"committed files under a dir blocked elsewhere must remain"
|
||||
);
|
||||
assert!(nested.join(STORAGE_FORMAT_FILE).exists());
|
||||
}
|
||||
|
||||
// An unreadable directory anywhere under the prefix aborts the purge on
|
||||
// every disk before anything is deleted.
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn purge_orphan_dir_object_aborts_when_a_directory_is_unreadable() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
let (dir, disk) = make_single_local_disk().await;
|
||||
let committed = write_committed_residue(&dir.path().join("bucket/pfx/b/obj")).await;
|
||||
let sealed = dir.path().join("bucket/pfx/a");
|
||||
fs::create_dir_all(&sealed).await.expect("sealed directory should be created");
|
||||
fs::set_permissions(&sealed, std::fs::Permissions::from_mode(0o000))
|
||||
.await
|
||||
.expect("sealed directory should become unreadable");
|
||||
|
||||
let set = make_set_disks_with(vec![Some(disk)]).await;
|
||||
let purged = set.purge_orphan_dir_object("bucket", "pfx/").await;
|
||||
fs::set_permissions(&sealed, std::fs::Permissions::from_mode(0o755))
|
||||
.await
|
||||
.expect("sealed directory should be restored");
|
||||
|
||||
assert!(matches!(purged, Ok(false)), "an unreadable directory must abort the purge");
|
||||
assert!(committed.join("part.1").exists(), "nothing may be deleted once classification failed");
|
||||
}
|
||||
|
||||
// After a scan met unpurgeable data the prefix is not rescanned for a
|
||||
// while, even when new committed residue appears under it.
|
||||
#[tokio::test]
|
||||
async fn purge_orphan_dir_object_backs_off_after_blocked_scan() {
|
||||
let (dir, disk) = make_single_local_disk().await;
|
||||
let unmarked = dir.path().join("bucket/pfx/b/obj").join(Uuid::new_v4().to_string());
|
||||
fs::create_dir_all(&unmarked)
|
||||
.await
|
||||
.expect("unmarked residue should be created");
|
||||
fs::write(unmarked.join("part.1"), b"stale")
|
||||
.await
|
||||
.expect("stale part should be written");
|
||||
|
||||
let set = make_set_disks_with(vec![Some(disk)]).await;
|
||||
assert!(
|
||||
!set.purge_orphan_dir_object("bucket", "pfx/")
|
||||
.await
|
||||
.expect("scan should succeed")
|
||||
);
|
||||
|
||||
let committed = write_committed_residue(&dir.path().join("bucket/pfx/a/obj")).await;
|
||||
assert!(
|
||||
!set.purge_orphan_dir_object("bucket", "pfx/")
|
||||
.await
|
||||
.expect("scan should succeed"),
|
||||
"a prefix in backoff is not rescanned"
|
||||
);
|
||||
assert!(committed.exists());
|
||||
assert!(
|
||||
set.purge_orphan_dir_object("bucket", "pfx/a/")
|
||||
.await
|
||||
.expect("scan should succeed"),
|
||||
"backoff is per prefix, a narrower prefix still purges"
|
||||
);
|
||||
assert!(!committed.exists());
|
||||
|
||||
let committed = write_committed_residue(&dir.path().join("bucket/pfx/a/obj")).await;
|
||||
set.clear_orphan_purge_backoff();
|
||||
assert!(
|
||||
set.purge_orphan_dir_object("bucket", "pfx/")
|
||||
.await
|
||||
.expect("scan should succeed")
|
||||
);
|
||||
assert!(!committed.exists());
|
||||
assert!(unmarked.join("part.1").exists());
|
||||
}
|
||||
|
||||
// Build an `xl.meta` under `object_dir` whose versions reference `data_dirs`
|
||||
// (one Object version per data dir, each with its own version id).
|
||||
async fn write_object_meta_with_data_dirs(object_dir: &std::path::Path, bucket: &str, object: &str, data_dirs: &[Uuid]) {
|
||||
@@ -11135,50 +10842,6 @@ mod tests {
|
||||
assert_eq!(conv_part_err_to_int(&Some(other_err)), CHECK_PART_UNKNOWN); // Other errors should return UNKNOWN, not SUCCESS
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deep_scan_results_accept_only_complete_verified_or_repairable_parts() {
|
||||
for (results, expected_parts) in [
|
||||
(vec![], 0),
|
||||
(vec![CHECK_PART_SUCCESS], 1),
|
||||
(vec![CHECK_PART_FILE_NOT_FOUND], 1),
|
||||
(vec![CHECK_PART_FILE_CORRUPT], 1),
|
||||
(vec![CHECK_PART_SUCCESS, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_FILE_CORRUPT], 3),
|
||||
] {
|
||||
validate_deep_scan_results(&results, expected_parts)
|
||||
.expect("complete results must allow healthy or repairable parts");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deep_scan_results_reject_unknown_and_incomplete_observations() {
|
||||
for (results, expected_parts) in [
|
||||
(vec![CHECK_PART_UNKNOWN], 1),
|
||||
(vec![usize::MAX], 1),
|
||||
(vec![CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN], 2),
|
||||
(vec![], 1),
|
||||
(vec![CHECK_PART_SUCCESS], 2),
|
||||
(vec![CHECK_PART_SUCCESS, CHECK_PART_SUCCESS], 1),
|
||||
(vec![CHECK_PART_SUCCESS], 0),
|
||||
] {
|
||||
let error =
|
||||
validate_deep_scan_results(&results, expected_parts).expect_err("unverified parts must not become healthy");
|
||||
assert!(matches!(error, DiskError::Io(_)), "an incomplete observation is not proven corruption");
|
||||
assert!(error.to_string().contains("incomplete deep scan result"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deep_scan_results_preserve_disk_and_volume_failures() {
|
||||
for (status, expected_error) in [
|
||||
(CHECK_PART_DISK_NOT_FOUND, DiskError::DiskNotFound),
|
||||
(CHECK_PART_VOLUME_NOT_FOUND, DiskError::VolumeNotFound),
|
||||
] {
|
||||
let error = validate_deep_scan_results(&[CHECK_PART_SUCCESS, status], 2)
|
||||
.expect_err("an unavailable part cannot certify a healthy disk");
|
||||
assert_eq!(error, expected_error);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_has_part_err() {
|
||||
// Test checking for part errors
|
||||
@@ -13922,29 +13585,6 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inline_admission_falls_back_to_actual_size_for_transformed_streams() {
|
||||
let erasure = coding::Erasure::new(2, 2, 1024 * 1024);
|
||||
let unknown = HashReader::SIZE_PRESERVE_LAYER;
|
||||
|
||||
// A known stored size is authoritative, whatever the plaintext size says.
|
||||
assert_eq!(
|
||||
inline_admission_shard_size(&erasure, 16 * 1024, 4 * 1024 * 1024),
|
||||
erasure.shard_file_size(16 * 1024)
|
||||
);
|
||||
assert_eq!(inline_admission_shard_size(&erasure, 0, 4 * 1024), 0);
|
||||
|
||||
// A transformed stream is sized by its plaintext length (MinIO parity).
|
||||
assert_eq!(
|
||||
inline_admission_shard_size(&erasure, unknown, 16 * 1024),
|
||||
erasure.shard_file_size(16 * 1024)
|
||||
);
|
||||
|
||||
// Neither size known, or an empty transformed stream, cannot be admitted.
|
||||
assert!(inline_admission_shard_size(&erasure, unknown, 0) < 0);
|
||||
assert!(inline_admission_shard_size(&erasure, unknown, unknown) < 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_object_part_fast_path_selection_matches_single_block_non_inline_rules() {
|
||||
assert!(should_use_single_block_non_inline_fast_path(false, 4096, 4096));
|
||||
|
||||
@@ -141,7 +141,6 @@ pub(in crate::set_disk::ops) async fn verify_written_bitrot_shards(
|
||||
mod tests {
|
||||
use super::super::object::hermetic_set_disks_support::hermetic_set_disks_for_pool_with_default_parity;
|
||||
use super::*;
|
||||
use crate::storage_api_contracts::object::ObjectIO as _;
|
||||
|
||||
async fn encode_streaming_shard(data: &[u8], shard_size: usize) -> Bytes {
|
||||
let mut writer = coding::BitrotWriter::new(Cursor::new(Vec::new()), shard_size, HashAlgorithm::HighwayHash256S);
|
||||
@@ -296,19 +295,4 @@ mod tests {
|
||||
assert!(err.to_string().contains("trailing data"));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn no_parity_put_round_trip_large_stream() {
|
||||
let (_temp_dirs, disks, set_disks) = hermetic_set_disks_for_pool_with_default_parity(1, 0, 0).await;
|
||||
let bucket = "no-parity-put-round-trip";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
}
|
||||
let payload = vec![0x5a; 13 * 1024 * 1024 + 727_213];
|
||||
let mut reader = crate::set_disk::PutObjReader::from_vec(payload);
|
||||
set_disks
|
||||
.put_object(bucket, "object", &mut reader, &crate::set_disk::ObjectOptions::default())
|
||||
.await
|
||||
.expect("single-disk no-parity PUT should pass self-verify");
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -2613,7 +2613,16 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
|
||||
fi.parts = Vec::with_capacity(uploaded_parts.len());
|
||||
|
||||
let quota_context = reservation::begin(&self.ctx, bucket, object, opts, self.pool_index, self.set_index).await?;
|
||||
let quota_context = reservation::begin(
|
||||
&self.ctx,
|
||||
bucket,
|
||||
object,
|
||||
opts.quota_admission,
|
||||
opts.data_movement,
|
||||
self.pool_index,
|
||||
self.set_index,
|
||||
)
|
||||
.await?;
|
||||
let quota_mutation_fence = quota_context.is_enforced() || opts.quota_admission.is_some();
|
||||
let preserve_replication_ciphertext = opts.replication_request
|
||||
&& contains_key_str(&fi.metadata, rustfs_utils::http::SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT);
|
||||
|
||||
@@ -49,16 +49,16 @@ use super::super::{
|
||||
disk, ensure_delete_commit_locks_held, error, explicit_delete_removed_marker, finish_set_disk_read_lock,
|
||||
get_codec_streaming_reader_gate_with_plan, get_object_body_cache_hook, get_raw_etag,
|
||||
get_small_object_direct_memory_decision_with_threshold_and_plan, get_small_object_direct_memory_threshold,
|
||||
get_stage_timer_if_enabled, get_str, get_transitioned_object_reader_with_tier_manager, inline_admission_shard_size,
|
||||
inline_erasure_shard_file_offset, inline_erasure_shard_size, insert_str, is_deadlock_detection_enabled,
|
||||
is_err_object_not_found, is_err_version_not_found, is_explicit_null_version, is_get_codec_streaming_base_enabled,
|
||||
is_get_small_object_direct_memory_enabled, is_lock_optimization_enabled, issue3031_diag_enabled, join_all,
|
||||
known_put_object_storage_size, path_join_buf, put_restore_opts, record_compression_total_memory,
|
||||
record_get_codec_streaming_gate_decision, record_get_direct_memory_decision, record_get_object_pipeline_failure,
|
||||
record_get_object_pipeline_failure_for_path, record_get_object_reader_path_observation, record_get_stage_duration_if_enabled,
|
||||
record_lock_acquire, reduce_write_quorum_errs, release_materialized_read_lock, replication_write_may_pass_worm_gate,
|
||||
require_restore_operation_id, resolve_delete_version_state, resolve_tiered_decommission_write_quorum_result,
|
||||
resolve_write_layout, restore_commit_operation_id_from_metadata, restore_operation_id_from_metadata, send_event,
|
||||
get_stage_timer_if_enabled, get_str, get_transitioned_object_reader_with_tier_manager, inline_erasure_shard_file_offset,
|
||||
inline_erasure_shard_size, insert_str, is_deadlock_detection_enabled, is_err_object_not_found, is_err_version_not_found,
|
||||
is_explicit_null_version, is_get_codec_streaming_base_enabled, is_get_small_object_direct_memory_enabled,
|
||||
is_lock_optimization_enabled, issue3031_diag_enabled, join_all, known_put_object_storage_size, path_join_buf,
|
||||
put_restore_opts, record_compression_total_memory, record_get_codec_streaming_gate_decision,
|
||||
record_get_direct_memory_decision, record_get_object_pipeline_failure, record_get_object_pipeline_failure_for_path,
|
||||
record_get_object_reader_path_observation, record_get_stage_duration_if_enabled, record_lock_acquire,
|
||||
reduce_write_quorum_errs, release_materialized_read_lock, replication_write_may_pass_worm_gate, require_restore_operation_id,
|
||||
resolve_delete_version_state, resolve_tiered_decommission_write_quorum_result, resolve_write_layout,
|
||||
restore_commit_operation_id_from_metadata, restore_operation_id_from_metadata, send_event,
|
||||
set_disk_delete_creates_delete_marker, should_force_delete_marker_for_missing_version,
|
||||
should_persist_encryption_original_size, should_preserve_delete_replication_state, should_use_inline_fast_path_with_plan,
|
||||
take_prepared_get_object_metadata, to_object_err, try_read_inline_data_shards_direct, warn,
|
||||
@@ -3427,8 +3427,8 @@ impl SetDisks {
|
||||
Ok(removed)
|
||||
}
|
||||
|
||||
async fn validate_bucket_incarnation(&self, bucket: &str, expected: Uuid, opts: &ObjectOptions) -> Result<()> {
|
||||
let current = metadata_sys::get_bucket_incarnation_id_for_options_in(&self.ctx, bucket, opts).await?;
|
||||
async fn validate_bucket_incarnation(&self, bucket: &str, expected: Uuid) -> Result<()> {
|
||||
let current = metadata_sys::get_bucket_incarnation_id_in(&self.ctx, bucket).await?;
|
||||
if current != expected {
|
||||
return Err(StorageError::BucketNotFound(bucket.to_string()));
|
||||
}
|
||||
@@ -3633,25 +3633,9 @@ impl SetDisks {
|
||||
|
||||
let put_object_size = known_put_object_storage_size(data.size());
|
||||
let shard_file_size_raw = erasure.shard_file_size(put_object_size);
|
||||
// Transformed streams (unknown stored size) are admitted by their
|
||||
// plaintext size and then take the streaming encode with inline
|
||||
// buffer writers, since the single-block fast path needs a known length.
|
||||
// Every inline candidate keeps the single-block bound on its admission
|
||||
// size: the inline buffer writer grows without a cap, so an explicit
|
||||
// INLINE_BLOCK budget above `block_size` must not admit multi-block
|
||||
// payloads into memory and xl.meta. Protected inline writes also need
|
||||
// a known stored length for their proof metadata, so they never fall
|
||||
// back to the plaintext size.
|
||||
let inline_admission_size = if put_object_size >= 0 || protect_write {
|
||||
put_object_size
|
||||
} else {
|
||||
data.actual_size()
|
||||
};
|
||||
let is_inline_buffer = storage_class_config.should_inline(
|
||||
inline_admission_shard_size(&erasure, put_object_size, data.actual_size()),
|
||||
erasure.data_shards,
|
||||
opts.versioned,
|
||||
) && usize::try_from(inline_admission_size).is_ok_and(|size| size <= erasure.block_size);
|
||||
let is_inline_buffer = storage_class_config.should_inline(shard_file_size_raw, erasure.data_shards, opts.versioned)
|
||||
&& put_object_size >= 0
|
||||
&& usize::try_from(put_object_size).is_ok_and(|size| size <= erasure.block_size);
|
||||
|
||||
let collect_stage_timing = rustfs_io_metrics::put_stage_metrics_enabled() || issue3031_diag_enabled();
|
||||
let shard_file_size = shard_file_size_raw;
|
||||
@@ -4243,7 +4227,16 @@ impl SetDisks {
|
||||
None
|
||||
};
|
||||
|
||||
let quota_context = reservation::begin(&self.ctx, bucket, object, opts, self.pool_index, self.set_index).await?;
|
||||
let quota_context = reservation::begin(
|
||||
&self.ctx,
|
||||
bucket,
|
||||
object,
|
||||
opts.quota_admission,
|
||||
opts.data_movement,
|
||||
self.pool_index,
|
||||
self.set_index,
|
||||
)
|
||||
.await?;
|
||||
let quota_mutation_fence = quota_context.is_enforced() || opts.quota_admission.is_some();
|
||||
let mut replication_quota_size = None;
|
||||
|
||||
@@ -6715,7 +6708,7 @@ fn remote_version_state_writer_enabled() -> bool {
|
||||
}
|
||||
|
||||
fn remote_version_state_writer_fleet_proof() -> Option<RemoteVersionStateFleetProofToken> {
|
||||
crate::services::notification_sys::acquire_remote_version_state_writer_fleet_proof()
|
||||
transaction_fencing_fleet_proof(remote_version_state_writer_requested())
|
||||
}
|
||||
|
||||
fn remote_version_state_writer_requested() -> bool {
|
||||
@@ -7571,11 +7564,7 @@ impl SetDisks {
|
||||
let fi = &transported;
|
||||
let disks = self.disk_inventory().await;
|
||||
let namespace_owner = (!is_meta_bucketname(bucket)).then(|| self.ctx.begin_namespace_commit());
|
||||
// Quorum is a property of the configured set, not of the current
|
||||
// online snapshot. A decommission/offline refresh may temporarily
|
||||
// shorten the snapshot; deriving quorum from it would turn a
|
||||
// quorum-minus-one delete into an apparent success.
|
||||
let write_quorum = self.set_drive_count / 2 + 1;
|
||||
let write_quorum = disks.len() / 2 + 1;
|
||||
let rollback_dir = Uuid::new_v4();
|
||||
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
@@ -7703,11 +7692,10 @@ impl SetDisks {
|
||||
drop(namespace_owner);
|
||||
if quorum_result.is_ok()
|
||||
&& errs.iter().any(Option::is_some)
|
||||
&& let Some(purge) = delete_marker_purge.as_ref()
|
||||
&& let Some(purge) = delete_marker_purge
|
||||
&& let Some(version) = fi.version_id.filter(|version| !version.is_nil())
|
||||
{
|
||||
let _ = self.persist_marker_purge_receipt(bucket, object, version, purge).await;
|
||||
let _ = self.persist_delete_marker_purge(bucket, object, version, purge.clone()).await;
|
||||
let _ = self.persist_delete_marker_purge(bucket, object, version, purge).await;
|
||||
}
|
||||
// An explicit purge can carry deleted=true for the existing marker.
|
||||
// It must not create a repair intent that could reintroduce that marker.
|
||||
@@ -8874,8 +8862,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
return Ok(ObjectInfo::default());
|
||||
}
|
||||
if let Some(expected_incarnation_id) = opts.expected_bucket_incarnation_id {
|
||||
self.validate_bucket_incarnation(bucket, expected_incarnation_id, &opts)
|
||||
.await?;
|
||||
self.validate_bucket_incarnation(bucket, expected_incarnation_id).await?;
|
||||
}
|
||||
ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?;
|
||||
begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?;
|
||||
@@ -9289,8 +9276,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
check_object_lock_retention_update(bucket, object, &obj_info, opts)?;
|
||||
|
||||
if let Some(expected_incarnation_id) = opts.expected_bucket_incarnation_id {
|
||||
self.validate_bucket_incarnation(bucket, expected_incarnation_id, opts)
|
||||
.await?;
|
||||
self.validate_bucket_incarnation(bucket, expected_incarnation_id).await?;
|
||||
}
|
||||
if _lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| opts
|
||||
@@ -9870,8 +9856,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
.await?
|
||||
.acquire_bucket_lifecycle_read_lock(bucket)
|
||||
.await?;
|
||||
self.validate_bucket_incarnation(bucket, expected_incarnation_id, opts)
|
||||
.await?;
|
||||
self.validate_bucket_incarnation(bucket, expected_incarnation_id).await?;
|
||||
Some(guard)
|
||||
} else {
|
||||
None
|
||||
@@ -11655,241 +11640,6 @@ mod inline_put_commit_path_tests {
|
||||
assert_eq!(restored, payload);
|
||||
}
|
||||
|
||||
/// Counts `part.*` files under every hermetic disk root.
|
||||
fn count_part_files(temp_dirs: &[tempfile::TempDir]) -> usize {
|
||||
fn walk(dir: &std::path::Path, hits: &mut usize) {
|
||||
let Ok(entries) = std::fs::read_dir(dir) else {
|
||||
return;
|
||||
};
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
if path.is_dir() {
|
||||
walk(&path, hits);
|
||||
} else if path
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.is_some_and(|name| name.starts_with("part."))
|
||||
{
|
||||
*hits += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut hits = 0;
|
||||
for dir in temp_dirs {
|
||||
walk(dir.path(), &mut hits);
|
||||
}
|
||||
hits
|
||||
}
|
||||
|
||||
/// The reader shape the app layer hands to `put_object` for a compressed
|
||||
/// single PUT: the stored size is unknown (`SIZE_PRESERVE_LAYER`), the
|
||||
/// plaintext size is known, and the metadata marks the object compressed so
|
||||
/// GET decompresses it. Encrypted PUTs present the same shape.
|
||||
fn transformed_put(plaintext: Vec<u8>) -> (PutObjReader, ObjectOptions) {
|
||||
let actual_size = plaintext.len() as i64;
|
||||
let plain = HashReader::from_stream(Cursor::new(plaintext), actual_size, actual_size, None, None, false)
|
||||
.expect("hash reader over plaintext");
|
||||
let compressed = crate::io_support::rio::compression_reader(plain, rustfs_utils::CompressionAlgorithm::default(), false);
|
||||
let stream = HashReader::from_reader(compressed, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false)
|
||||
.expect("hash reader over transformed stream");
|
||||
let mut user_defined = HashMap::new();
|
||||
insert_str(
|
||||
&mut user_defined,
|
||||
SUFFIX_COMPRESSION,
|
||||
crate::io_support::rio::compression_metadata_value(rustfs_utils::CompressionAlgorithm::default()),
|
||||
);
|
||||
insert_str(&mut user_defined, SUFFIX_ACTUAL_SIZE, actual_size.to_string());
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
user_defined,
|
||||
..Default::default()
|
||||
};
|
||||
(PutObjReader::new(stream), opts)
|
||||
}
|
||||
|
||||
fn compressible_payload(size: usize) -> Vec<u8> {
|
||||
b"inline transformed stream payload. "
|
||||
.iter()
|
||||
.copied()
|
||||
.cycle()
|
||||
.take(size)
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn read_back(set_disks: &Arc<SetDisks>, bucket: &str, object: &str) -> Vec<u8> {
|
||||
let mut object_reader = set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("committed object should be readable");
|
||||
let mut restored = Vec::new();
|
||||
object_reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("object should stream");
|
||||
restored
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn transformed_small_put_is_stored_inline_and_round_trips() {
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "inline-transformed-small";
|
||||
let object = "object.txt";
|
||||
// 16 KiB plaintext over EC 2+2 is 8 KiB per data shard, inside the
|
||||
// default 128 KiB inline budget; the stored size is unknown up front.
|
||||
let plaintext = compressible_payload(16 * 1024);
|
||||
make_bucket(&disk_stores, bucket).await;
|
||||
|
||||
let (mut reader, opts) = transformed_put(plaintext.clone());
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut reader, &opts)
|
||||
.await
|
||||
.expect("transformed PUT should commit");
|
||||
|
||||
let read_data = ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
};
|
||||
for (disk_index, disk) in disk_stores.iter().enumerate() {
|
||||
let file_info = disk
|
||||
.read_version("", bucket, object, "", &read_data)
|
||||
.await
|
||||
.unwrap_or_else(|err| panic!("disk {disk_index} should persist metadata: {err}"));
|
||||
assert!(file_info.inline_data(), "disk {disk_index} must mark the transformed shard inline");
|
||||
assert!(
|
||||
file_info.data.as_ref().is_some_and(|data| !data.is_empty()),
|
||||
"disk {disk_index} must embed the shard in xl.meta"
|
||||
);
|
||||
assert!(file_info.is_compressed(), "disk {disk_index} must keep the compression marker");
|
||||
}
|
||||
assert_eq!(count_part_files(&temp_dirs), 0, "an inline transformed object must not leave part files");
|
||||
|
||||
assert_eq!(read_back(&set_disks, bucket, object).await, plaintext);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn protected_puts_keep_unbounded_inline_candidates_external() {
|
||||
for transformed in [true, false] {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let storage_class =
|
||||
temp_env::with_var(INLINE_BLOCK_ENV, Some("16MiB"), || lookup_config_for_pools(&KVS::new(), &[4]))
|
||||
.expect("large inline budget should resolve");
|
||||
set_disks.set_test_storage_class_config(storage_class);
|
||||
let bucket = "protected-inline-candidates";
|
||||
let object = "object.bin";
|
||||
make_bucket(&disk_stores, bucket).await;
|
||||
let plaintext = compressible_payload(if transformed { 16 * 1024 } else { 1024 * 1024 + 17 });
|
||||
let (mut reader, mut opts) = if transformed {
|
||||
transformed_put(plaintext.clone())
|
||||
} else {
|
||||
(PutObjReader::from_vec(plaintext.clone()), ObjectOptions::default())
|
||||
};
|
||||
opts.shard_integrity_write_mode = Some(crate::object_api::ShardIntegrityWriteMode::Protected);
|
||||
temp_env::async_with_vars(
|
||||
[(
|
||||
crate::set_disk::core::io_primitives::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE,
|
||||
Some("false"),
|
||||
)],
|
||||
set_disks.put_object(bucket, object, &mut reader, &opts),
|
||||
)
|
||||
.await
|
||||
.expect("protected PUT should commit with an external integrity index");
|
||||
|
||||
for (disk_index, disk) in disk_stores.iter().enumerate() {
|
||||
let file_info = disk
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.unwrap_or_else(|err| panic!("disk {disk_index} should persist protected metadata: {err}"));
|
||||
assert!(!file_info.inline_data(), "unknown or multi-block protected shards must remain external");
|
||||
assert!(file_info.parts[0].integrity.is_some(), "protected PUT must retain its integrity proof");
|
||||
}
|
||||
assert_eq!(read_back(&set_disks, bucket, object).await, plaintext);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn unprotected_puts_keep_multi_block_inline_candidates_external() {
|
||||
for transformed in [true, false] {
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let storage_class =
|
||||
temp_env::with_var(INLINE_BLOCK_ENV, Some("16MiB"), || lookup_config_for_pools(&KVS::new(), &[4]))
|
||||
.expect("large inline budget should resolve");
|
||||
set_disks.set_test_storage_class_config(storage_class);
|
||||
let bucket = "unprotected-inline-candidates";
|
||||
let object = "object.bin";
|
||||
make_bucket(&disk_stores, bucket).await;
|
||||
// Above the single erasure block: an explicit INLINE_BLOCK budget must
|
||||
// not pull a multi-block payload into the unbounded inline buffer.
|
||||
let plaintext = compressible_payload(1024 * 1024 + 17);
|
||||
let (mut reader, opts) = if transformed {
|
||||
transformed_put(plaintext.clone())
|
||||
} else {
|
||||
(PutObjReader::from_vec(plaintext.clone()), ObjectOptions::default())
|
||||
};
|
||||
temp_env::async_with_vars(
|
||||
[(
|
||||
crate::set_disk::core::io_primitives::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE,
|
||||
Some("false"),
|
||||
)],
|
||||
set_disks.put_object(bucket, object, &mut reader, &opts),
|
||||
)
|
||||
.await
|
||||
.expect("multi-block PUT should commit with part files");
|
||||
|
||||
for (disk_index, disk) in disk_stores.iter().enumerate() {
|
||||
let file_info = disk
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.unwrap_or_else(|err| panic!("disk {disk_index} should persist metadata: {err}"));
|
||||
assert!(
|
||||
!file_info.inline_data(),
|
||||
"disk {disk_index}: a payload above block_size must stay external (transformed={transformed})"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
count_part_files(&temp_dirs) > 0,
|
||||
"a multi-block object must keep part files (transformed={transformed})"
|
||||
);
|
||||
assert_eq!(read_back(&set_disks, bucket, object).await, plaintext);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn transformed_put_above_inline_budget_keeps_part_files() {
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "inline-transformed-large";
|
||||
let object = "object.txt";
|
||||
// 512 KiB plaintext is 256 KiB per data shard, above the 128 KiB budget,
|
||||
// even though the compressed bytes would fit: the plaintext size is the
|
||||
// admission input, exactly like a known-size PUT of the same object.
|
||||
let plaintext = compressible_payload(512 * 1024);
|
||||
make_bucket(&disk_stores, bucket).await;
|
||||
|
||||
let (mut reader, opts) = transformed_put(plaintext.clone());
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut reader, &opts)
|
||||
.await
|
||||
.expect("transformed PUT should commit");
|
||||
|
||||
let read_data = ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
};
|
||||
for (disk_index, disk) in disk_stores.iter().enumerate() {
|
||||
let file_info = disk
|
||||
.read_version("", bucket, object, "", &read_data)
|
||||
.await
|
||||
.unwrap_or_else(|err| panic!("disk {disk_index} should persist metadata: {err}"));
|
||||
assert!(!file_info.inline_data(), "disk {disk_index} must keep the shard outside xl.meta");
|
||||
}
|
||||
assert_eq!(count_part_files(&temp_dirs), disk_stores.len(), "every disk must hold one part file");
|
||||
|
||||
assert_eq!(read_back(&set_disks, bucket, object).await, plaintext);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn get_object_reader_wires_mid_size_to_single_inflight() {
|
||||
|
||||
@@ -645,17 +645,6 @@ impl ECStore {
|
||||
.as_ref()
|
||||
.and_then(|info| info.created)
|
||||
.unwrap_or(OffsetDateTime::UNIX_EPOCH);
|
||||
// A sidecar left by an interrupted legacy migration is the
|
||||
// bucket's published incarnation; keep it rather than minting
|
||||
// one that would invalidate fences taken on the stored value.
|
||||
// A retired incarnation is residue of a deleted bucket and
|
||||
// must not come back.
|
||||
let store = metadata_sys::object_store_in(&self.ctx).await?;
|
||||
if let Some(stored) = crate::bucket::metadata::load_bucket_incarnation(store.clone(), bucket).await?
|
||||
&& !crate::bucket::retirement::is_retired(store, bucket, stored).await?
|
||||
{
|
||||
metadata.bucket_incarnation_id = stored;
|
||||
}
|
||||
} else if !metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() {
|
||||
return Err(Error::other(format!(
|
||||
"bucket incarnation sidecar is missing for new-format metadata: {bucket}"
|
||||
@@ -2892,57 +2881,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// rustfs/rustfs#8003: a force-create (site replication replay, admin
|
||||
/// import) against a bucket whose legacy migration stopped after the
|
||||
/// sidecar write must persist `.metadata.bin` under the stored incarnation
|
||||
/// rather than fail closed or replace the published identity.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn issue_8003_force_create_adopts_existing_incarnation_sidecar() {
|
||||
let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
|
||||
let bucket = format!("bucket-8003-sidecar-{}", Uuid::new_v4().simple());
|
||||
|
||||
ecstore
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let stored = metadata_sys::get_in(&ecstore.ctx, &bucket)
|
||||
.await
|
||||
.expect("metadata should load")
|
||||
.bucket_incarnation_id;
|
||||
assert!(!stored.is_nil());
|
||||
|
||||
let metadata_path = format!("{BUCKET_META_PREFIX}/{bucket}/{}", crate::bucket::metadata::BUCKET_METADATA_FILE);
|
||||
crate::config::com::delete_config(ecstore.clone(), &metadata_path)
|
||||
.await
|
||||
.expect("simulate the interrupted migration by removing only .metadata.bin");
|
||||
assert!(!any_disk_path_exists(&disk_paths, format!("{RUSTFS_META_BUCKET}/{metadata_path}")).await);
|
||||
metadata_sys::remove_bucket_metadata_in(&ecstore.ctx, &bucket)
|
||||
.await
|
||||
.expect("drop the cached copy so the next read hits disk");
|
||||
|
||||
ecstore
|
||||
.make_bucket(
|
||||
&bucket,
|
||||
&MakeBucketOptions {
|
||||
force_create: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("force create must repair the sidecar-only bucket");
|
||||
|
||||
let (repaired, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&ecstore.ctx, &bucket)
|
||||
.await
|
||||
.expect("repaired metadata should load");
|
||||
assert!(persisted, "force create must persist .metadata.bin again");
|
||||
assert!(repaired.bucket_incarnation_sidecar);
|
||||
assert_eq!(
|
||||
repaired.bucket_incarnation_id, stored,
|
||||
"the repair must keep the incarnation the sidecar already published"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn make_bucket_seeds_new_bucket_durability_override() {
|
||||
|
||||
@@ -58,26 +58,6 @@ pub(crate) fn bucket_heal_scope(bucket: &str) -> Option<Arc<BucketHealScope>> {
|
||||
.flatten()
|
||||
}
|
||||
|
||||
/// Resolve only the two canonical bucket records; other internal paths carry no bucket authority.
|
||||
pub(super) fn bucket_metadata_owner(object: &str) -> Option<&str> {
|
||||
use crate::bucket::metadata::{BUCKET_INCARNATION_FILE, BUCKET_METADATA_FILE};
|
||||
let (prefix, rest) = object.split_once('/')?;
|
||||
let (bucket, file) = rest.split_once('/')?;
|
||||
(prefix == disk::BUCKET_META_PREFIX
|
||||
&& matches!(file, BUCKET_METADATA_FILE | BUCKET_INCARNATION_FILE)
|
||||
&& check_valid_bucket_name_strict(bucket).is_ok())
|
||||
.then_some(bucket)
|
||||
}
|
||||
|
||||
pub(crate) fn bucket_heal_scope_for_object(bucket: &str, object: &str) -> Option<Arc<BucketHealScope>> {
|
||||
let owner = if bucket == RUSTFS_META_BUCKET {
|
||||
bucket_metadata_owner(object)?
|
||||
} else {
|
||||
bucket
|
||||
};
|
||||
bucket_heal_scope(owner)
|
||||
}
|
||||
|
||||
/// Storage-owned proof for the exact version and every selected erasure location.
|
||||
/// This is an in-process result, never reconstructed from admin drive telemetry.
|
||||
#[derive(Debug)]
|
||||
@@ -316,24 +296,6 @@ impl ECStore {
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn heal_mrf_object_at_incarnation(
|
||||
self: &Arc<Self>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: &str,
|
||||
expected: uuid::Uuid,
|
||||
opts: &HealOpts,
|
||||
) -> Result<HealObjectStorageResult> {
|
||||
let object = object.to_owned();
|
||||
let version_id = version_id.to_owned();
|
||||
self.run_bucket_heal_at_incarnation(bucket, expected, opts, |store, bucket, opts| async move {
|
||||
store
|
||||
.heal_object_with_authoritative_absence_proof(&bucket, &object, &version_id, &opts)
|
||||
.await
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn acquire_heal_format_fence(
|
||||
&self,
|
||||
) -> Result<(
|
||||
@@ -442,97 +404,6 @@ impl ECStore {
|
||||
Ok(Arc::ptr_eq(&target_set, &pool.get_disks_by_key(POOL_META_NAME)))
|
||||
}
|
||||
|
||||
/// Restore the bucket configuration and incarnation before retiring a replacement marker.
|
||||
pub async fn heal_replacement_bucket_metadata(
|
||||
self: &Arc<Self>,
|
||||
bucket: &str,
|
||||
opts: &HealOpts,
|
||||
targets: &[String],
|
||||
) -> Result<()> {
|
||||
use crate::bucket::metadata::{BUCKET_INCARNATION_FILE, BUCKET_METADATA_FILE};
|
||||
use crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_read_lock_in;
|
||||
|
||||
let (Some(pool_index), Some(set_index)) = (opts.pool, opts.set) else {
|
||||
return Err(Error::PreconditionFailed);
|
||||
};
|
||||
if targets.is_empty() || opts.dry_run || opts.no_lock || check_valid_bucket_name_strict(bucket).is_err() {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
let pool = self
|
||||
.pools
|
||||
.get(pool_index)
|
||||
.ok_or_else(|| invalid_heal_pool_index(pool_index, self.pools.len()))?;
|
||||
let target_set = pool.get_disks_for_heal_object(BUCKET_METADATA_FILE, opts)?;
|
||||
if targets.iter().any(|target| {
|
||||
target_set
|
||||
.set_endpoints
|
||||
.iter()
|
||||
.filter(|endpoint| endpoint.to_string() == *target)
|
||||
.count()
|
||||
!= 1
|
||||
}) {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
let objects: Vec<_> = [BUCKET_METADATA_FILE, BUCKET_INCARNATION_FILE]
|
||||
.into_iter()
|
||||
.map(|file| format!("{}/{bucket}/{file}", disk::BUCKET_META_PREFIX))
|
||||
.filter(|object| Arc::ptr_eq(&target_set, &pool.get_disks_by_key(object)))
|
||||
.collect();
|
||||
if objects.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Read the persisted identity without lazy migration: a replacement
|
||||
// must never turn lost configuration into newly fabricated defaults.
|
||||
let incarnation = self.bucket_incarnation_id_from_disk(bucket).await?;
|
||||
let targets = targets.to_vec();
|
||||
self.run_bucket_heal_at_incarnation(bucket, incarnation, opts, move |store, bucket, opts| async move {
|
||||
// Match config writers: lifecycle, metadata transaction, object locks.
|
||||
// Keep the transaction stable through target readback, including when
|
||||
// the caller is cancelled and the storage owner finishes its write.
|
||||
let transaction = acquire_bucket_metadata_transaction_read_lock_in(&store.ctx, &bucket).await?;
|
||||
for object in objects {
|
||||
let scope = bucket_heal_scope(&bucket).ok_or(Error::PreconditionFailed)?;
|
||||
scope.check()?;
|
||||
if transaction.is_lock_lost() {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
// Config objects occupy one pool, unlike pool.bin. Require a
|
||||
// successful all-pool lookup before accepting another pool's ownership.
|
||||
if !store.single_pool() {
|
||||
let (owner, _) = store
|
||||
.get_pool_info_for_delete_marker(RUSTFS_META_BUCKET, &object, &ObjectOptions::default())
|
||||
.await?;
|
||||
if owner.index != pool_index {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
let metadata_opts = HealOpts { remove: false, ..opts };
|
||||
let (result, error) = store.heal_object(RUSTFS_META_BUCKET, &object, "", &metadata_opts).await?;
|
||||
if let Some(error) = error {
|
||||
return Err(error);
|
||||
}
|
||||
for target in &targets {
|
||||
let mut drives = result.after.drives.iter().filter(|drive| &drive.endpoint == target);
|
||||
if !drives.next().is_some_and(|drive| drive.state == "ok") || drives.next().is_some() {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
}
|
||||
if !store
|
||||
.replacement_targets_have_version(RUSTFS_META_BUCKET, &object, "", pool_index, set_index, &targets)
|
||||
.await?
|
||||
{
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
}
|
||||
if transaction.is_lock_lost() {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Return every live erasure set selected by an object-heal scope.
|
||||
pub async fn heal_erasure_set_scopes(&self, opts: &HealOpts) -> Result<Vec<(usize, usize)>> {
|
||||
let pools = self.get_pools_for_heal_object(opts)?;
|
||||
@@ -813,18 +684,8 @@ impl ECStore {
|
||||
version_id: &str,
|
||||
opts: &HealOpts,
|
||||
) -> Result<(HealResultItem, Option<Error>)> {
|
||||
self.handle_heal_object_with_absence(
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
opts,
|
||||
&mut None,
|
||||
crate::set_disk::AbsenceProofRequest {
|
||||
retirement: None,
|
||||
allow_unversioned: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
self.handle_heal_object_with_absence(bucket, object, version_id, opts, &mut None, None)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn heal_object_with_proof(
|
||||
@@ -834,34 +695,7 @@ impl ECStore {
|
||||
version_id: &str,
|
||||
opts: &HealOpts,
|
||||
) -> Result<HealObjectStorageResult> {
|
||||
self.heal_object_with_absence_proof(bucket, object, version_id, opts, false)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn heal_object_with_authoritative_absence_proof(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: &str,
|
||||
opts: &HealOpts,
|
||||
) -> Result<HealObjectStorageResult> {
|
||||
self.heal_object_with_absence_proof(bucket, object, version_id, opts, true)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn heal_object_with_absence_proof(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: &str,
|
||||
opts: &HealOpts,
|
||||
allow_unversioned_absence: bool,
|
||||
) -> Result<HealObjectStorageResult> {
|
||||
if opts.dry_run
|
||||
|| opts.no_lock
|
||||
|| (!allow_unversioned_absence && version_id.is_empty())
|
||||
|| super::utils::is_reserved_or_invalid_bucket(bucket, false)
|
||||
{
|
||||
if opts.dry_run || opts.no_lock || version_id.is_empty() || super::utils::is_reserved_or_invalid_bucket(bucket, false) {
|
||||
let (item, error) = self.handle_heal_object(bucket, object, version_id, opts).await?;
|
||||
return Ok(HealObjectStorageResult {
|
||||
item,
|
||||
@@ -904,17 +738,7 @@ impl ECStore {
|
||||
};
|
||||
let mut proofs = None;
|
||||
let (item, mut error) = self
|
||||
.handle_heal_object_with_absence(
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
opts,
|
||||
&mut proofs,
|
||||
crate::set_disk::AbsenceProofRequest {
|
||||
retirement: Some(&retirement),
|
||||
allow_unversioned: allow_unversioned_absence,
|
||||
},
|
||||
)
|
||||
.handle_heal_object_with_absence(bucket, object, version_id, opts, &mut proofs, Some(&retirement))
|
||||
.await?;
|
||||
// Read the authoritative incarnation only for an absence candidate.
|
||||
// The lifecycle guard has pinned it throughout the storage operation.
|
||||
@@ -956,7 +780,7 @@ impl ECStore {
|
||||
version_id: &str,
|
||||
opts: &HealOpts,
|
||||
absence: &mut Option<Vec<crate::set_disk::HealedObjectAbsence>>,
|
||||
proof: crate::set_disk::AbsenceProofRequest<'_>,
|
||||
retirement: Option<&crate::bucket::retirement::MarkerRetirementContext<'_>>,
|
||||
) -> Result<(HealResultItem, Option<Error>)> {
|
||||
trace!(
|
||||
event = EVENT_HEAL_OBJECT_STARTED,
|
||||
@@ -1047,7 +871,7 @@ impl ECStore {
|
||||
}
|
||||
#[cfg(test)]
|
||||
crate::core::pools::notify_decommission_external_heal_operation_started(store_id);
|
||||
pool.heal_object_with_absence(bucket, &pool_object, version_id, &opts, proof)
|
||||
pool.heal_object_with_absence(bucket, &pool_object, version_id, &opts, retirement)
|
||||
.await
|
||||
}
|
||||
});
|
||||
@@ -1071,7 +895,7 @@ impl ECStore {
|
||||
move |opts| async move {
|
||||
#[cfg(test)]
|
||||
crate::core::pools::notify_decommission_external_heal_operation_started(store_id);
|
||||
pool.heal_object_with_absence(bucket, &pool_object, version_id, &opts, proof)
|
||||
pool.heal_object_with_absence(bucket, &pool_object, version_id, &opts, retirement)
|
||||
.await
|
||||
},
|
||||
));
|
||||
@@ -1456,16 +1280,6 @@ mod tests {
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
for file in [".metadata.bin", ".bucket-incarnation"] {
|
||||
let metadata = format!("buckets/{bucket}/{file}");
|
||||
assert!(
|
||||
store
|
||||
.rename_local_data_at_incarnation(&disk_ref, (&bucket, object), &fi, (RUSTFS_META_BUCKET, &metadata), old)
|
||||
.await
|
||||
.is_err(),
|
||||
"stale remote heal must not replace the successor's {file}"
|
||||
);
|
||||
}
|
||||
assert!(store.heal_bucket_at_incarnation(&bucket, old, &opts).await.is_err());
|
||||
assert!(
|
||||
store
|
||||
@@ -1864,194 +1678,6 @@ mod tests {
|
||||
(temp_dir, store, shutdown)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_metadata_heal_owner_rejects_path_aliases_and_unrelated_records() {
|
||||
for file in [".metadata.bin", ".bucket-incarnation"] {
|
||||
assert_eq!(bucket_metadata_owner(&format!("buckets/example/{file}")), Some("example"));
|
||||
}
|
||||
for path in [
|
||||
"/buckets/example/.metadata.bin",
|
||||
"buckets/../.metadata.bin",
|
||||
"buckets/example/../.metadata.bin",
|
||||
"buckets/example/.metadata.bin/extra",
|
||||
"buckets/example//.metadata.bin",
|
||||
"buckets/example/.metadata.bin.old",
|
||||
"config/iam/example/.metadata.bin",
|
||||
"buckets/example/usage.json",
|
||||
] {
|
||||
assert_eq!(bucket_metadata_owner(path), None, "unexpected bucket authority for {path}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replacement_bucket_metadata_respects_pool_ownership_and_required_records() {
|
||||
let (root, store, shutdown) = multi_pool_heal_store().await;
|
||||
let bucket = "replacement-bucket-metadata";
|
||||
for invalid_bucket in ["MixedCase", "../example", RUSTFS_META_BUCKET] {
|
||||
assert!(matches!(
|
||||
store
|
||||
.heal_replacement_bucket_metadata(
|
||||
invalid_bucket,
|
||||
&HealOpts {
|
||||
pool: Some(0),
|
||||
set: Some(0),
|
||||
..Default::default()
|
||||
},
|
||||
&[root.path().join("pool0-disk0").to_string_lossy().into_owned()],
|
||||
)
|
||||
.await,
|
||||
Err(Error::PreconditionFailed)
|
||||
));
|
||||
}
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket and its persisted records");
|
||||
let mut owned = 0;
|
||||
for pool in 0..2 {
|
||||
let target = root.path().join(format!("pool{pool}-disk0"));
|
||||
let records: Vec<_> = [".metadata.bin", ".bucket-incarnation"]
|
||||
.into_iter()
|
||||
.map(|file| {
|
||||
let path = target.join(RUSTFS_META_BUCKET).join("buckets").join(bucket).join(file);
|
||||
let exists = path.join("xl.meta").exists();
|
||||
if exists {
|
||||
std::fs::remove_dir_all(&path).expect("remove replacement metadata shard");
|
||||
owned += 1;
|
||||
}
|
||||
(path, exists)
|
||||
})
|
||||
.collect();
|
||||
store
|
||||
.heal_replacement_bucket_metadata(
|
||||
bucket,
|
||||
&HealOpts {
|
||||
pool: Some(pool),
|
||||
set: Some(0),
|
||||
recreate: true,
|
||||
..Default::default()
|
||||
},
|
||||
&[target.to_string_lossy().into_owned()],
|
||||
)
|
||||
.await
|
||||
.expect("repair owned records and accept authoritative ownership in the other pool");
|
||||
for (path, existed) in records {
|
||||
assert_eq!(path.join("xl.meta").exists(), existed, "wrong placement for {}", path.display());
|
||||
}
|
||||
}
|
||||
assert_eq!(owned, 2, "each required record must have exactly one owning pool");
|
||||
let object = format!("buckets/{bucket}/.metadata.bin");
|
||||
let owner = store
|
||||
.get_pool_idx_existing_with_opts(RUSTFS_META_BUCKET, &object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("persisted metadata owner");
|
||||
let disk = store.pools[owner].disk_set[0].disks.read().await[0]
|
||||
.clone()
|
||||
.expect("metadata target disk");
|
||||
let fi = disk
|
||||
.read_version(
|
||||
"",
|
||||
RUSTFS_META_BUCKET,
|
||||
&object,
|
||||
"",
|
||||
&disk::ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("inline metadata to repair through target-side admission");
|
||||
assert!(fi.data.is_some(), "fixture must contain an inline metadata shard");
|
||||
let target_path = root.path().join(format!("pool{owner}-disk0"));
|
||||
std::fs::remove_dir_all(target_path.join(RUSTFS_META_BUCKET).join(&object)).expect("remove remote target shard");
|
||||
let incarnation = store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("current incarnation");
|
||||
store
|
||||
.rename_local_data_at_incarnation(
|
||||
&disk.endpoint().to_string(),
|
||||
(disk::RUSTFS_META_TMP_BUCKET, "metadata-heal-control"),
|
||||
&fi,
|
||||
(RUSTFS_META_BUCKET, &object),
|
||||
incarnation,
|
||||
)
|
||||
.await
|
||||
.expect("the target-side RPC admission must accept current bucket metadata repair");
|
||||
assert!(target_path.join(RUSTFS_META_BUCKET).join(&object).join("xl.meta").exists());
|
||||
let (configuration, _) = read_config_no_lock_preserve_empty_with_metadata(store.clone(), &object)
|
||||
.await
|
||||
.expect("read the current bucket configuration");
|
||||
save_config(store.clone(), &object, configuration)
|
||||
.await
|
||||
.expect("publish a newer config revision in the same bucket incarnation");
|
||||
let committed =
|
||||
std::fs::read(target_path.join(RUSTFS_META_BUCKET).join(&object).join("xl.meta")).expect("new target configuration");
|
||||
assert!(
|
||||
store
|
||||
.rename_local_data_at_incarnation(
|
||||
&disk.endpoint().to_string(),
|
||||
(disk::RUSTFS_META_TMP_BUCKET, "delayed-metadata-heal"),
|
||||
&fi,
|
||||
(RUSTFS_META_BUCKET, &object),
|
||||
incarnation,
|
||||
)
|
||||
.await
|
||||
.is_err(),
|
||||
"a delayed repair must not overwrite a newer config in the same incarnation"
|
||||
);
|
||||
assert_eq!(
|
||||
std::fs::read(target_path.join(RUSTFS_META_BUCKET).join(&object).join("xl.meta")).expect("retained configuration"),
|
||||
committed,
|
||||
"rejected repair must leave the newer configuration unchanged"
|
||||
);
|
||||
for pool in 0..2 {
|
||||
for disk in 0..4 {
|
||||
let path = root
|
||||
.path()
|
||||
.join(format!("pool{pool}-disk{disk}"))
|
||||
.join(RUSTFS_META_BUCKET)
|
||||
.join("buckets")
|
||||
.join(bucket)
|
||||
.join(".metadata.bin");
|
||||
if path.exists() {
|
||||
std::fs::remove_dir_all(path).expect("remove every copy of required bucket configuration");
|
||||
}
|
||||
}
|
||||
}
|
||||
metadata_sys::remove_bucket_metadata_in(&store.ctx, bucket)
|
||||
.await
|
||||
.expect("remove the cached configuration so repair cannot rely on it");
|
||||
assert!(
|
||||
store
|
||||
.heal_replacement_bucket_metadata(
|
||||
bucket,
|
||||
&HealOpts {
|
||||
pool: Some(0),
|
||||
set: Some(0),
|
||||
recreate: true,
|
||||
..Default::default()
|
||||
},
|
||||
&[root.path().join("pool0-disk0").to_string_lossy().into_owned()],
|
||||
)
|
||||
.await
|
||||
.is_err(),
|
||||
"total metadata loss must not be accepted as ownership in another pool"
|
||||
);
|
||||
for pool in 0..2 {
|
||||
assert!(
|
||||
!root
|
||||
.path()
|
||||
.join(format!("pool{pool}-disk0"))
|
||||
.join(RUSTFS_META_BUCKET)
|
||||
.join(&object)
|
||||
.exists(),
|
||||
"missing required configuration must not be recreated with defaults"
|
||||
);
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn absence_proof_requires_every_selected_pool() {
|
||||
@@ -3150,16 +2776,7 @@ mod tests {
|
||||
|
||||
let mut active_reader = PutObjReader::from_vec(b"active owner".to_vec());
|
||||
store.pools[0]
|
||||
.put_object(
|
||||
&bucket,
|
||||
active_object,
|
||||
&mut active_reader,
|
||||
&ObjectOptions {
|
||||
// Drain the rename tail before removing a physical shard.
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.put_object(&bucket, active_object, &mut active_reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("active owner object should be written");
|
||||
let active_disks = store.pools[0].disk_set[0].disks.read().await.clone();
|
||||
@@ -3195,7 +2812,6 @@ mod tests {
|
||||
&mut duplicate_reader,
|
||||
&ObjectOptions {
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(mod_time)),
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
|
||||
@@ -2905,17 +2905,8 @@ mod tests {
|
||||
if !version.is_empty() {
|
||||
insert_str(&mut metadata, SUFFIX_TRANSITIONED_VERSION_ID, version.to_string());
|
||||
}
|
||||
// The commit path acknowledges at write quorum and drains
|
||||
// the remaining rename fan-out in the background. This
|
||||
// fixture "crashes" the store right after the overwrite,
|
||||
// so it must wait for that tail: a disk left with the old
|
||||
// live transitioned source makes exact cleanup fail closed
|
||||
// (a minority live owner still references the remote
|
||||
// tuple) until heal repairs it, which this fixture never
|
||||
// runs (rustfs#7921).
|
||||
let options = ObjectOptions {
|
||||
version_suspended: suspended,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
};
|
||||
store
|
||||
@@ -2987,7 +2978,6 @@ mod tests {
|
||||
assert_eq!(free.len(), 1, "{state:?}, suspended={suspended}, copy={self_copy}");
|
||||
assert_eq!(free[0].transitioned_objname, remote);
|
||||
assert_eq!(free[0].transition_version_state, state);
|
||||
assert_every_disk_holds_only_the_cleanup_owner(&set, &bucket, object, &remote).await;
|
||||
assert!(backend.contains(&remote).await, "commit must not delete remote bytes before cleanup");
|
||||
let removed_before = backend.remove_count().await;
|
||||
|
||||
@@ -3159,120 +3149,6 @@ mod tests {
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
/// rustfs/rustfs#7674: a bucket carrying a legacy snapshot-protocol quota
|
||||
/// (written before durable reservations existed, so `quota.json` has no
|
||||
/// `reservation_protocol`) must accept a cross-key CopyObject whose
|
||||
/// destination options carry the handler's quota admission. The storage
|
||||
/// layer fails closed with `PartMissingOrCorrupt` when a quota-enforced
|
||||
/// bucket sees a write without admission, so dropping the admission while
|
||||
/// rebuilding the destination options turns every copy into a
|
||||
/// deterministic "part missing or corrupt" failure even though the source
|
||||
/// object is perfectly readable.
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn copy_object_forwards_quota_admission_on_legacy_snapshot_quota_bucket() {
|
||||
let temp_dir = tempfile::tempdir().expect("create legacy quota copy store dir");
|
||||
let (ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-quota-copy", &[1])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
|
||||
let bucket = format!("legacy-quota-copy-{}", Uuid::new_v4());
|
||||
let source_object = "docker/registry/v2/repositories/example/_uploads/upload-id/data";
|
||||
let target_object = "docker/registry/v2/blobs/sha256/a5/digest/data";
|
||||
let payload = vec![0x5A; 8178];
|
||||
let quota_limit = 1u64 << 30;
|
||||
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket for legacy quota copy");
|
||||
// Legacy quota shape: no `reservation_protocol`, so the storage layer
|
||||
// requires the handler-supplied snapshot admission on every write.
|
||||
let legacy_quota = format!(r#"{{"quota":{quota_limit},"quota_type":"Hard"}}"#);
|
||||
crate::bucket::metadata_sys::update_in(
|
||||
&ctx,
|
||||
&bucket,
|
||||
crate::bucket::metadata::BUCKET_QUOTA_CONFIG_FILE,
|
||||
legacy_quota.into_bytes(),
|
||||
)
|
||||
.await
|
||||
.expect("persist legacy snapshot quota");
|
||||
let (quota, _, _) = crate::bucket::metadata_sys::get_quota_config_and_incarnation_from_disk_in(&ctx, &bucket)
|
||||
.await
|
||||
.expect("legacy quota should load");
|
||||
let quota = quota.expect("legacy quota must be persisted");
|
||||
assert_eq!(quota.quota, Some(quota_limit));
|
||||
assert!(!quota.uses_durable_reservations(), "fixture must stay on the snapshot protocol");
|
||||
|
||||
let mut write_opts = ObjectOptions::default();
|
||||
assert!(write_opts.set_quota_admission(0, quota_limit));
|
||||
|
||||
let upload = store
|
||||
.new_multipart_upload(&bucket, source_object, &write_opts)
|
||||
.await
|
||||
.expect("create source multipart upload");
|
||||
let mut part_reader = PutObjReader::from_vec(payload.clone());
|
||||
let part = store
|
||||
.put_object_part(&bucket, source_object, &upload.upload_id, 1, &mut part_reader, &write_opts)
|
||||
.await
|
||||
.expect("stage multipart source part");
|
||||
store
|
||||
.clone()
|
||||
.complete_multipart_upload(
|
||||
&bucket,
|
||||
source_object,
|
||||
&upload.upload_id,
|
||||
vec![crate::storage_api_contracts::multipart::CompletePart {
|
||||
part_num: part.part_num,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
}],
|
||||
&write_opts,
|
||||
)
|
||||
.await
|
||||
.expect("complete the multipart source under the legacy quota");
|
||||
|
||||
let source_reader = store
|
||||
.get_object_reader(&bucket, source_object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("completed multipart source should be readable");
|
||||
let mut copy_info = source_reader.object_info.clone();
|
||||
let actual_size = copy_info.get_actual_size().expect("copy source logical size should resolve");
|
||||
assert_eq!(actual_size, payload.len() as i64);
|
||||
let copy_reader = rustfs_rio::HashReader::from_stream(source_reader.stream, actual_size, actual_size, None, None, false)
|
||||
.expect("copy source hash reader should build");
|
||||
copy_info.put_object_reader = Some(PutObjReader::new(copy_reader));
|
||||
|
||||
let mut dst_opts = ObjectOptions::default();
|
||||
assert!(dst_opts.set_quota_admission(payload.len() as u64, quota_limit));
|
||||
store
|
||||
.copy_object(
|
||||
&bucket,
|
||||
source_object,
|
||||
&bucket,
|
||||
target_object,
|
||||
&mut copy_info,
|
||||
&ObjectOptions::default(),
|
||||
&dst_opts,
|
||||
)
|
||||
.await
|
||||
.expect("CopyObject must forward the handler quota admission to the destination write");
|
||||
|
||||
let mut target_reader = store
|
||||
.get_object_reader(&bucket, target_object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("copied target should be readable");
|
||||
let mut target_body = Vec::new();
|
||||
target_reader
|
||||
.stream
|
||||
.read_to_end(&mut target_body)
|
||||
.await
|
||||
.expect("target body should stream");
|
||||
assert_eq!(target_body, payload);
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
@@ -4090,7 +3966,7 @@ mod tests {
|
||||
.collect::<Vec<_>>();
|
||||
for disk in &disks {
|
||||
disk.close().await.expect("fault injection should stop per-disk monitoring");
|
||||
disk.force_offline_for_test();
|
||||
disk.force_runtime_state_for_test(crate::disk::health_state::RuntimeDriveHealthState::Offline);
|
||||
}
|
||||
|
||||
// Sets has an independent endpoint monitor that renews missing slots.
|
||||
@@ -4127,7 +4003,7 @@ mod tests {
|
||||
.collect::<Vec<_>>();
|
||||
for disk in &disks {
|
||||
disk.close().await.expect("fault injection should stop per-disk monitoring");
|
||||
disk.force_offline_for_test();
|
||||
disk.force_runtime_state_for_test(crate::disk::health_state::RuntimeDriveHealthState::Offline);
|
||||
}
|
||||
set.connect_disks().await;
|
||||
for disk in &disks {
|
||||
@@ -9836,7 +9712,14 @@ mod tests {
|
||||
.await
|
||||
.expect("manual task receipt path should resolve");
|
||||
let target_task_set = store.pools[1].get_disks_by_key(&manual_task_receipt_path);
|
||||
let offline_target_task_disks = force_set_disk_range_offline_for_test(&target_task_set, 0..2).await;
|
||||
let original_target_task_disks = {
|
||||
let mut disks = target_task_set.disks.write().await;
|
||||
let original = disks.clone();
|
||||
for disk in disks.iter_mut().take(2) {
|
||||
*disk = None;
|
||||
}
|
||||
original
|
||||
};
|
||||
let receipt_quorum_error = store
|
||||
.verify_and_cleanup_decommissioned_durable_ilm_record_for_test(
|
||||
0,
|
||||
@@ -9845,7 +9728,7 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect_err("target read quorum without receipt write quorum must retain the source");
|
||||
drop(offline_target_task_disks);
|
||||
*target_task_set.disks.write().await = original_target_task_disks;
|
||||
let receipt_quorum_error = receipt_quorum_error.to_string();
|
||||
assert!(receipt_quorum_error.contains("receipt"));
|
||||
assert!(receipt_quorum_error.contains(&manual_task_path));
|
||||
@@ -10052,18 +9935,9 @@ mod tests {
|
||||
recovered_transition_version.clone(),
|
||||
)))
|
||||
.await;
|
||||
let transaction_stats = {
|
||||
let _proof = crate::services::notification_sys::install_current_remote_version_state_fleet_proof_for_test();
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_WRITE, Some("true")),
|
||||
(rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED, Some("true")),
|
||||
],
|
||||
recover_transition_transaction_records(store.clone(), 100, None),
|
||||
)
|
||||
let transaction_stats = recover_transition_transaction_records(store.clone(), 100, None)
|
||||
.await
|
||||
.expect("transition recovery should advance and consume the migrated transaction before completion")
|
||||
};
|
||||
.expect("transition recovery should advance and consume the migrated transaction before completion");
|
||||
backend.set_transition_candidate_probe_override(None).await;
|
||||
assert_eq!(
|
||||
(
|
||||
@@ -12774,45 +12648,10 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// Every physical copy must carry the replacement plus its tier
|
||||
/// free-version owner, and none may still hold the pre-overwrite live
|
||||
/// transitioned source. A stale minority copy is exactly what a lost
|
||||
/// early-ACK rename tail leaves behind, and exact cleanup refuses to
|
||||
/// delete remote bytes while such a live reference exists.
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn assert_every_disk_holds_only_the_cleanup_owner(
|
||||
set: &crate::set_disk::SetDisks,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
remote: &str,
|
||||
) {
|
||||
use crate::disk::DiskAPI as _;
|
||||
|
||||
let disk_object = rustfs_utils::path::encode_dir_object(object);
|
||||
for (index, disk) in set.disk_inventory().await.into_iter().enumerate() {
|
||||
let disk = disk.unwrap_or_else(|| panic!("disk{index} should be online"));
|
||||
let raw = disk
|
||||
.read_xl(bucket, &disk_object, false)
|
||||
.await
|
||||
.unwrap_or_else(|err| panic!("disk{index} xl.meta should be readable after the overwrite: {err:?}"));
|
||||
let versions = rustfs_filemeta::FileMeta::load(&raw.buf)
|
||||
.and_then(|meta| meta.get_all_file_info_versions(bucket, object, true))
|
||||
.unwrap_or_else(|err| panic!("disk{index} xl.meta should decode: {err:?}"));
|
||||
let all: Vec<_> = versions.versions.iter().chain(versions.free_versions.iter()).collect();
|
||||
let owners = all.iter().filter(|fi| fi.tier_free_version()).count();
|
||||
let live_sources = all
|
||||
.iter()
|
||||
.filter(|fi| !fi.tier_free_version() && fi.transitioned_objname == remote)
|
||||
.count();
|
||||
assert_eq!(owners, 1, "disk{index} must hold exactly one cleanup owner after the drained overwrite");
|
||||
assert_eq!(live_sources, 0, "disk{index} must not retain the pre-overwrite live transitioned source");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn wait_for_expiry_workers_idle(store: &crate::store::ECStore) {
|
||||
let expiry_state = store.ctx.expiry_state();
|
||||
let idle = tokio::time::timeout(Duration::from_secs(30), async {
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
let idle = {
|
||||
let state = expiry_state.read().await;
|
||||
@@ -12832,15 +12671,8 @@ mod tests {
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
if idle.is_err() {
|
||||
let state = expiry_state.read().await;
|
||||
panic!(
|
||||
"lifecycle expiry workers should become idle: pending={} active={}",
|
||||
state.pending_tasks(),
|
||||
state.active_tasks()
|
||||
);
|
||||
}
|
||||
.await
|
||||
.expect("lifecycle expiry workers should become idle");
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
@@ -21947,18 +21779,9 @@ mod tests {
|
||||
expected_removes.push((transaction.remote_object, put_version));
|
||||
}
|
||||
|
||||
let stats = {
|
||||
let _proof = crate::services::notification_sys::install_current_remote_version_state_fleet_proof_for_test();
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_WRITE, Some("true")),
|
||||
(rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED, Some("true")),
|
||||
],
|
||||
recover_transition_transaction_records(store.clone(), 100, None),
|
||||
)
|
||||
let stats = recover_transition_transaction_records(store.clone(), 100, None)
|
||||
.await
|
||||
.expect("transition transaction recovery should run")
|
||||
};
|
||||
.expect("transition transaction recovery should run");
|
||||
|
||||
assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (3, 3, 0, 0));
|
||||
assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
|
||||
|
||||
@@ -66,7 +66,7 @@ use std::future::Future;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, AtomicU64, Ordering},
|
||||
atomic::{AtomicU64, Ordering},
|
||||
};
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
use tokio::io::duplex;
|
||||
@@ -341,18 +341,6 @@ pub struct ListPathOptions {
|
||||
pub cursor_generation: Option<String>,
|
||||
pub walkdir_timeout: Option<Duration>,
|
||||
pub walkdir_stall_timeout: Option<Duration>,
|
||||
/// Shared by the collector and raw producers to preserve bounded-walk
|
||||
/// completion when filtering removes all entries from a batch.
|
||||
pub producer_limit_reached: Option<Arc<AtomicBool>>,
|
||||
}
|
||||
|
||||
fn ensure_producer_limit_state(options: &mut ListPathOptions) -> Arc<AtomicBool> {
|
||||
let state = options
|
||||
.producer_limit_reached
|
||||
.clone()
|
||||
.unwrap_or_else(|| Arc::new(AtomicBool::new(false)));
|
||||
options.producer_limit_reached = Some(state.clone());
|
||||
state
|
||||
}
|
||||
|
||||
async fn can_skip_hidden_prefix_check(options: &ListPathOptions) -> bool {
|
||||
@@ -4316,7 +4304,6 @@ impl ECStore {
|
||||
// cancel channel
|
||||
let cancel = CancellationToken::new();
|
||||
let _cancel_guard = cancel.clone().drop_guard();
|
||||
ensure_producer_limit_state(&mut o);
|
||||
|
||||
let (err_tx, mut err_rx) = broadcast::channel::<Arc<Error>>(1);
|
||||
|
||||
@@ -5060,15 +5047,6 @@ async fn gather_results(
|
||||
}
|
||||
}
|
||||
|
||||
// A producer can close its stream after exhausting its own scan budget
|
||||
// before this collector reaches its output limit. In that case the input
|
||||
// channel closing is not authoritative EOF: the caller must advertise a
|
||||
// continuation page even when every remaining entry was filtered out.
|
||||
let producer_limit_reached = opts
|
||||
.producer_limit_reached
|
||||
.as_ref()
|
||||
.is_some_and(|reached| reached.load(Ordering::Acquire));
|
||||
|
||||
// finish not full, return eof
|
||||
let filtered = scanned_entries.saturating_sub(candidate_entries);
|
||||
if let Some(started) = gather_started {
|
||||
@@ -5105,7 +5083,7 @@ async fn gather_results(
|
||||
o: MetaCacheEntries(entries),
|
||||
..Default::default()
|
||||
}),
|
||||
err: (!producer_limit_reached).then_some(rustfs_filemeta::Error::Unexpected),
|
||||
err: Some(rustfs_filemeta::Error::Unexpected),
|
||||
})
|
||||
.await
|
||||
.is_err()
|
||||
@@ -5756,7 +5734,6 @@ impl Sets {
|
||||
let (err_tx, mut err_rx) = broadcast::channel::<Arc<Error>>(1);
|
||||
let (sender, recv) = mpsc::channel(o.limit as usize);
|
||||
|
||||
ensure_producer_limit_state(&mut o);
|
||||
let sets = self.clone();
|
||||
let opts = o.clone();
|
||||
let cancel_rx1 = cancel.clone();
|
||||
@@ -6800,7 +6777,6 @@ impl SetDisks {
|
||||
let (err_tx, mut err_rx) = broadcast::channel::<Arc<Error>>(1);
|
||||
let (sender, recv) = mpsc::channel(o.limit as usize);
|
||||
|
||||
ensure_producer_limit_state(&mut o);
|
||||
let set = self.clone();
|
||||
let opts = o.clone();
|
||||
let cancel_rx1 = cancel.clone();
|
||||
@@ -7122,7 +7098,6 @@ impl SetDisks {
|
||||
forward_to: opts.marker,
|
||||
min_disks: raw_min_disks,
|
||||
per_disk_limit: limit,
|
||||
producer_limit_reached: opts.producer_limit_reached.clone(),
|
||||
// A foreground listing is bounded by lack of drive progress (the walk
|
||||
// stall timeout) and by the page limit, never by how long a healthy
|
||||
// walk takes — a large prefix on slow media is not a fault (#4644).
|
||||
@@ -7353,7 +7328,6 @@ mod test {
|
||||
};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
use tokio::sync::mpsc;
|
||||
@@ -8373,45 +8347,6 @@ mod test {
|
||||
assert!(!cancel.is_cancelled());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_gather_results_preserves_bounded_producer_after_filtering() {
|
||||
let (entry_tx, entry_rx) = mpsc::channel(4);
|
||||
let (result_tx, mut result_rx) = mpsc::channel(1);
|
||||
let cancel = CancellationToken::new();
|
||||
let producer_limit_reached = Arc::new(AtomicBool::new(true));
|
||||
|
||||
entry_tx
|
||||
.send(test_meta_entry("outside-prefix"))
|
||||
.await
|
||||
.expect("filtered test entry should be queued");
|
||||
drop(entry_tx);
|
||||
|
||||
let handle = tokio::spawn(gather_results(
|
||||
cancel,
|
||||
ListPathOptions {
|
||||
bucket: "bucket".to_owned(),
|
||||
prefix: "requested/".to_owned(),
|
||||
limit: 8,
|
||||
incl_deleted: true,
|
||||
producer_limit_reached: Some(producer_limit_reached),
|
||||
..Default::default()
|
||||
},
|
||||
entry_rx,
|
||||
result_tx,
|
||||
));
|
||||
|
||||
let result = result_rx.recv().await.expect("bounded producer result should be delivered");
|
||||
assert!(result.entries.expect("entries should be present").entries().is_empty());
|
||||
assert!(result.err.is_none(), "bounded producer must not be reported as EOF");
|
||||
assert_eq!(
|
||||
handle
|
||||
.await
|
||||
.expect("gather task should not panic")
|
||||
.expect("gather should succeed"),
|
||||
GatherResultsState::InputClosed
|
||||
);
|
||||
}
|
||||
|
||||
/// A-1 guard (rustfs/backlog#1306): pin that a *successful* send is never
|
||||
/// misclassified as `ConsumerGone`. With the receiver alive, gather_results
|
||||
/// must return the real drain state and deliver the payload carrying the
|
||||
@@ -9743,131 +9678,6 @@ mod test {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn empty_delimiter_listing_of_residue_ancestor_hides_and_purges_whole_tree() {
|
||||
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
let bucket = "listing-purge-ancestor-bucket";
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created with authoritative metadata");
|
||||
let data_dir = uuid::Uuid::new_v4();
|
||||
let transaction = uuid::Uuid::new_v4();
|
||||
for dir in &dirs {
|
||||
let residue = dir
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join("metrics/kubelet/2026/08/28/23/74992556388248657933757.parquet")
|
||||
.join(data_dir.to_string());
|
||||
tokio::fs::create_dir_all(&residue)
|
||||
.await
|
||||
.expect("committed delete residue should be created");
|
||||
tokio::fs::write(residue.join("part.1"), b"stale")
|
||||
.await
|
||||
.expect("stale part should be written");
|
||||
tokio::fs::write(
|
||||
residue.join(format!("{}{}", crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX, transaction)),
|
||||
[],
|
||||
)
|
||||
.await
|
||||
.expect("committed delete marker should be written");
|
||||
}
|
||||
|
||||
// Browsing an ancestor of the deleted key must not show the empty
|
||||
// date folders, and the empty result reclaims the whole residue tree
|
||||
// in one pass instead of one level per listing.
|
||||
let result = store
|
||||
.clone()
|
||||
.list_objects_generic(bucket, "metrics/", None, Some("/".to_owned()), 1000, false)
|
||||
.await
|
||||
.expect("delimiter listing should succeed");
|
||||
assert!(result.objects.is_empty());
|
||||
assert!(result.prefixes.is_empty(), "ancestors of delete residue must not surface as prefixes");
|
||||
for dir in &dirs {
|
||||
assert!(
|
||||
!dir.path().join(bucket).join("metrics").exists(),
|
||||
"the empty delimiter listing should reclaim the committed delete residue tree under it"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn empty_delimiter_listing_of_mixed_residue_ancestor_reclaims_only_committed_subtree() {
|
||||
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
let bucket = "listing-purge-mixed-bucket";
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created with authoritative metadata");
|
||||
let committed_dir = uuid::Uuid::new_v4();
|
||||
let unmarked_dir = uuid::Uuid::new_v4();
|
||||
let transaction = uuid::Uuid::new_v4();
|
||||
for dir in &dirs {
|
||||
let committed = dir
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join("metrics/cpu/2026/08/28/22/a.parquet")
|
||||
.join(committed_dir.to_string());
|
||||
tokio::fs::create_dir_all(&committed)
|
||||
.await
|
||||
.expect("committed delete residue should be created");
|
||||
tokio::fs::write(committed.join("part.1"), b"stale")
|
||||
.await
|
||||
.expect("stale part should be written");
|
||||
tokio::fs::write(
|
||||
committed.join(format!("{}{}", crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX, transaction)),
|
||||
[],
|
||||
)
|
||||
.await
|
||||
.expect("committed delete marker should be written");
|
||||
|
||||
// Residue left by a build that never wrote delete markers, or a
|
||||
// PUT still streaming its parts: indistinguishable, so never purged.
|
||||
let unmarked = dir
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join("metrics/kubelet/2026/08/28/23/74992556388248657933757.parquet")
|
||||
.join(unmarked_dir.to_string());
|
||||
tokio::fs::create_dir_all(&unmarked)
|
||||
.await
|
||||
.expect("unmarked residue should be created");
|
||||
tokio::fs::write(unmarked.join("part.1"), b"stale")
|
||||
.await
|
||||
.expect("stale part should be written");
|
||||
}
|
||||
|
||||
let result = store
|
||||
.clone()
|
||||
.list_objects_generic(bucket, "metrics/", None, Some("/".to_owned()), 1000, false)
|
||||
.await
|
||||
.expect("delimiter listing should succeed");
|
||||
assert!(result.objects.is_empty());
|
||||
assert!(result.prefixes.is_empty(), "neither residue subtree may surface as a prefix");
|
||||
for dir in &dirs {
|
||||
let metrics = dir.path().join(bucket).join("metrics");
|
||||
assert!(
|
||||
!metrics.join("cpu").exists(),
|
||||
"the committed subtree must be reclaimed even though a sibling subtree is blocked"
|
||||
);
|
||||
assert!(
|
||||
metrics
|
||||
.join("kubelet/2026/08/28/23/74992556388248657933757.parquet")
|
||||
.join(unmarked_dir.to_string())
|
||||
.join("part.1")
|
||||
.exists(),
|
||||
"residue without a committed marker must survive the purge"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_objects_index_provider_state_uses_lifecycle_active_generation() {
|
||||
let provider = ListObjectsIndexProviderState::walker_key_only();
|
||||
|
||||
@@ -395,19 +395,12 @@ async fn scan_metadata_less_residue_with_budget(
|
||||
Ok(scan)
|
||||
}
|
||||
|
||||
async fn enqueue_transition_after_write(
|
||||
store: &ECStore,
|
||||
result: Result<ObjectInfo>,
|
||||
src: LcEventSrc,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
async fn enqueue_transition_after_write(result: Result<ObjectInfo>, src: LcEventSrc) -> Result<ObjectInfo> {
|
||||
match result {
|
||||
Ok(oi) => {
|
||||
if should_enqueue_transition_immediately(&oi) {
|
||||
enqueue_transition_immediate(&oi, src.clone()).await;
|
||||
if let Ok(api) = metadata_sys::object_store_in(&store.ctx).await {
|
||||
enqueue_immediate_expiry(api, &oi, src, opts).await;
|
||||
}
|
||||
enqueue_immediate_expiry(&oi, src).await;
|
||||
}
|
||||
Ok(oi)
|
||||
}
|
||||
@@ -426,8 +419,8 @@ mod bucket_fence;
|
||||
pub(crate) use bucket::await_bucket_namespace_operation;
|
||||
pub use bucket_fence::BucketIncarnationFenceGuard;
|
||||
mod heal;
|
||||
pub(crate) use heal::bucket_heal_scope;
|
||||
pub use heal::{HealObjectAbsenceProof, HealObjectStorageResult};
|
||||
pub(crate) use heal::{bucket_heal_scope, bucket_heal_scope_for_object};
|
||||
mod heal_walk;
|
||||
pub use heal_walk::HealWalkVersion;
|
||||
mod init;
|
||||
@@ -1288,11 +1281,9 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(ObjectInfo, Option<crate::disk::OldCurrentSize>)> {
|
||||
let result = match self.handle_put_object(bucket, object, data, opts).await {
|
||||
Ok((object_info, old_current_size)) => {
|
||||
enqueue_transition_after_write(self, Ok(object_info), LcEventSrc::S3PutObject, opts)
|
||||
.await
|
||||
.map(|object_info| (object_info, old_current_size))
|
||||
}
|
||||
Ok((object_info, old_current_size)) => enqueue_transition_after_write(Ok(object_info), LcEventSrc::S3PutObject)
|
||||
.await
|
||||
.map(|object_info| (object_info, old_current_size)),
|
||||
Err(err) => Err(err),
|
||||
};
|
||||
if result.is_ok() {
|
||||
@@ -1365,11 +1356,9 @@ impl crate::storage_api_contracts::object::ObjectOperations for ECStore {
|
||||
dst_opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
let result = enqueue_transition_after_write(
|
||||
self,
|
||||
self.handle_copy_object(src_bucket, src_object, dst_bucket, dst_object, src_info, src_opts, dst_opts)
|
||||
.await,
|
||||
LcEventSrc::S3CopyObject,
|
||||
dst_opts,
|
||||
)
|
||||
.await;
|
||||
if result.is_ok() {
|
||||
@@ -1636,12 +1625,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
let result = enqueue_transition_after_write(
|
||||
self.as_ref(),
|
||||
self.clone()
|
||||
.handle_complete_multipart_upload(bucket, object, upload_id, uploaded_parts, opts)
|
||||
.await,
|
||||
LcEventSrc::S3CompleteMultipartUpload,
|
||||
opts,
|
||||
)
|
||||
.await;
|
||||
if result.is_ok() {
|
||||
|
||||
@@ -355,7 +355,7 @@ impl ECStore {
|
||||
if let Some(guard) = guard.as_ref() {
|
||||
opts.add_bucket_lifecycle_lock_guard(guard);
|
||||
}
|
||||
let current = crate::bucket::metadata_sys::get_bucket_incarnation_id_for_options_in(&self.ctx, bucket, &opts).await?;
|
||||
let current = crate::bucket::metadata_sys::get_bucket_incarnation_id_in(&self.ctx, bucket).await?;
|
||||
if opts.expected_bucket_incarnation_id != Some(current) {
|
||||
return Err(StorageError::BucketNotFound(bucket.to_string()));
|
||||
}
|
||||
@@ -530,11 +530,7 @@ impl ECStore {
|
||||
return Ok((result, 0, opts.expected_bucket_incarnation_id));
|
||||
}
|
||||
|
||||
let capacity_owner = DecommissionCapacityOwner::from_options(&opts);
|
||||
if opts.data_movement && (opts.version_id.is_some() || capacity_owner.is_some()) {
|
||||
// Capacity-owned decommission writes must remain on the target
|
||||
// selected by the durable reservation, including unversioned
|
||||
// objects whose ObjectOptions carry no version ID.
|
||||
if opts.data_movement && opts.version_id.is_some() {
|
||||
let idx = self.select_data_movement_pool_idx(bucket, object, -1, &opts, false).await?;
|
||||
if idx == opts.src_pool_idx {
|
||||
return Err(StorageError::DataMovementOverwriteErr(
|
||||
@@ -546,9 +542,12 @@ impl ECStore {
|
||||
self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence)
|
||||
.await;
|
||||
let res = self
|
||||
.run_decommission_capacity_temporary_mutation(idx, capacity_owner, None, || async {
|
||||
self.pools[idx].new_multipart_upload(bucket, object, &opts).await
|
||||
})
|
||||
.run_decommission_capacity_temporary_mutation(
|
||||
idx,
|
||||
DecommissionCapacityOwner::from_options(&opts),
|
||||
None,
|
||||
|| async { self.pools[idx].new_multipart_upload(bucket, object, &opts).await },
|
||||
)
|
||||
.await?;
|
||||
return Ok((res, idx, opts.expected_bucket_incarnation_id));
|
||||
}
|
||||
@@ -1047,7 +1046,6 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
publication_fence: Option<RemoteTuplePublicationFence>,
|
||||
) -> Result<ObjectInfo> {
|
||||
let request_opts = opts;
|
||||
let (target_pool_idx, mutation_fence) = target;
|
||||
check_complete_multipart_args(bucket, object, upload_id)?;
|
||||
if !opts.data_movement {
|
||||
@@ -1133,8 +1131,7 @@ impl ECStore {
|
||||
)
|
||||
.await;
|
||||
drop(publication_guard);
|
||||
let result =
|
||||
enqueue_transition_after_write(self.as_ref(), result, LcEventSrc::S3CompleteMultipartUpload, request_opts).await;
|
||||
let result = enqueue_transition_after_write(result, LcEventSrc::S3CompleteMultipartUpload).await;
|
||||
if result.is_ok() {
|
||||
list_objects::observe_list_objects_mutation(self.as_ref(), bucket).await;
|
||||
}
|
||||
|
||||
@@ -29,8 +29,8 @@ use crate::bucket::lifecycle::{
|
||||
},
|
||||
};
|
||||
use crate::bucket::metadata_sys::{
|
||||
acquire_bucket_metadata_transaction_read_lock_in, get_bucket_incarnation_id_for_options_in,
|
||||
get_cached_bucket_incarnation_id_in, get_object_lock_config_and_incarnation_from_disk_in,
|
||||
acquire_bucket_metadata_transaction_read_lock_in, get_bucket_incarnation_id_in, get_cached_bucket_incarnation_id_in,
|
||||
get_object_lock_config_and_incarnation_from_disk_in,
|
||||
};
|
||||
use crate::bucket::object_lock::objectlock_sys::{
|
||||
check_object_lock_for_deletion_with_state, ensure_recursive_force_delete_allowed_for_state,
|
||||
@@ -2837,7 +2837,7 @@ impl ECStore {
|
||||
config_revision,
|
||||
state,
|
||||
lifecycle_fence.clone(),
|
||||
Arc::new(metadata_guard),
|
||||
metadata_guard,
|
||||
)))
|
||||
}
|
||||
|
||||
@@ -4296,7 +4296,6 @@ impl ECStore {
|
||||
if !opts.data_movement {
|
||||
return Err(Error::other("data movement PUT requires data_movement options"));
|
||||
}
|
||||
let request_opts = opts;
|
||||
let (object, mut opts) = self.prepare_put_object(bucket, object, opts).await?;
|
||||
ensure_decommission_capacity_mutation_id(bucket, &object, &mut opts);
|
||||
let idx = self
|
||||
@@ -4339,7 +4338,7 @@ impl ECStore {
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let result = enqueue_transition_after_write(self, result, LcEventSrc::S3PutObject, request_opts).await;
|
||||
let result = enqueue_transition_after_write(result, LcEventSrc::S3PutObject).await;
|
||||
if result.is_ok() {
|
||||
list_objects::observe_list_objects_mutation(self, bucket).await;
|
||||
}
|
||||
@@ -4446,7 +4445,7 @@ impl ECStore {
|
||||
};
|
||||
let current_bucket_incarnation_id = if let Some(guard) = _bucket_lifecycle_guard.as_ref() {
|
||||
dst_opts.add_bucket_lifecycle_lock_guard(guard);
|
||||
let current_incarnation_id = get_bucket_incarnation_id_for_options_in(&self.ctx, dst_bucket, &dst_opts).await?;
|
||||
let current_incarnation_id = get_bucket_incarnation_id_in(&self.ctx, dst_bucket).await?;
|
||||
if dst_opts
|
||||
.expected_bucket_incarnation_id
|
||||
.is_some_and(|expected| expected != current_incarnation_id)
|
||||
@@ -4565,7 +4564,6 @@ impl ECStore {
|
||||
namespace_lock_fence: dst_opts.namespace_lock_fence.clone(),
|
||||
bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
|
||||
object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
|
||||
quota_admission: dst_opts.quota_admission,
|
||||
..Default::default()
|
||||
};
|
||||
if !self.single_pool() {
|
||||
@@ -4608,7 +4606,6 @@ impl ECStore {
|
||||
namespace_lock_fence: dst_opts.namespace_lock_fence.clone(),
|
||||
bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
|
||||
object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
|
||||
quota_admission: dst_opts.quota_admission,
|
||||
..Default::default()
|
||||
};
|
||||
if !self.single_pool() {
|
||||
@@ -4661,7 +4658,6 @@ impl ECStore {
|
||||
namespace_lock_fence: dst_opts.namespace_lock_fence.clone(),
|
||||
bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
|
||||
object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
|
||||
quota_admission: dst_opts.quota_admission,
|
||||
..Default::default()
|
||||
};
|
||||
if !self.single_pool() {
|
||||
@@ -4827,26 +4823,22 @@ impl ECStore {
|
||||
get_cached_bucket_incarnation_id_in(&self.ctx, bucket).await?;
|
||||
}
|
||||
let _object_lock_metadata_guard = if !is_meta_bucketname(bucket) {
|
||||
Some(Arc::new(acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?))
|
||||
Some(acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(guard) = _object_lock_metadata_guard.as_ref() {
|
||||
opts.add_namespace_lock_guard(guard);
|
||||
}
|
||||
let current_bucket_incarnation_id = if let Some(metadata_guard) = _object_lock_metadata_guard.as_ref() {
|
||||
let current_bucket_incarnation_id = if _object_lock_metadata_guard.is_some() {
|
||||
let (state, incarnation_id, config_revision) =
|
||||
get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await?;
|
||||
opts.object_lock_config_snapshot = Some(Arc::new(ObjectLockConfigSnapshot::for_store_bucket_under_lifecycle_fence(
|
||||
opts.object_lock_config_snapshot = Some(Arc::new(ObjectLockConfigSnapshot::for_store_bucket(
|
||||
self.id,
|
||||
bucket,
|
||||
incarnation_id,
|
||||
config_revision,
|
||||
state,
|
||||
opts.bucket_lifecycle_lock_fence
|
||||
.clone()
|
||||
.unwrap_or_else(NamespaceLockFence::new),
|
||||
Arc::clone(metadata_guard),
|
||||
)));
|
||||
Some(incarnation_id)
|
||||
} else {
|
||||
@@ -5265,32 +5257,26 @@ impl ECStore {
|
||||
let _object_lock_metadata_guard = if is_meta_bucketname(bucket) {
|
||||
None
|
||||
} else {
|
||||
Some(Arc::new(
|
||||
match acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
|
||||
},
|
||||
))
|
||||
Some(match acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
|
||||
})
|
||||
};
|
||||
if let Some(guard) = _object_lock_metadata_guard.as_ref() {
|
||||
opts.add_namespace_lock_guard(guard);
|
||||
}
|
||||
let current_bucket_incarnation_id = if let Some(metadata_guard) = _object_lock_metadata_guard.as_ref() {
|
||||
let current_bucket_incarnation_id = if _object_lock_metadata_guard.is_some() {
|
||||
let (state, incarnation_id, config_revision) =
|
||||
match get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await {
|
||||
Ok(snapshot) => snapshot,
|
||||
Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
|
||||
};
|
||||
opts.object_lock_config_snapshot = Some(Arc::new(ObjectLockConfigSnapshot::for_store_bucket_under_lifecycle_fence(
|
||||
opts.object_lock_config_snapshot = Some(Arc::new(ObjectLockConfigSnapshot::for_store_bucket(
|
||||
self.id,
|
||||
bucket,
|
||||
incarnation_id,
|
||||
config_revision,
|
||||
state,
|
||||
opts.bucket_lifecycle_lock_fence
|
||||
.clone()
|
||||
.unwrap_or_else(NamespaceLockFence::new),
|
||||
Arc::clone(metadata_guard),
|
||||
)));
|
||||
Some(incarnation_id)
|
||||
} else {
|
||||
@@ -5631,7 +5617,7 @@ impl ECStore {
|
||||
opts.add_bucket_lifecycle_lock_guard(guard);
|
||||
}
|
||||
if !is_meta_bucketname(bucket) {
|
||||
let current_incarnation_id = get_bucket_incarnation_id_for_options_in(&self.ctx, bucket, &opts).await?;
|
||||
let current_incarnation_id = get_bucket_incarnation_id_in(&self.ctx, bucket).await?;
|
||||
if opts.expected_bucket_incarnation_id != Some(current_incarnation_id) {
|
||||
return Err(StorageError::BucketNotFound(bucket.to_string()));
|
||||
}
|
||||
@@ -5704,7 +5690,7 @@ impl ECStore {
|
||||
None
|
||||
} else {
|
||||
let guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
|
||||
let current_incarnation_id = get_bucket_incarnation_id_for_options_in(&self.ctx, bucket, &opts).await?;
|
||||
let current_incarnation_id = get_bucket_incarnation_id_in(&self.ctx, bucket).await?;
|
||||
if opts
|
||||
.expected_bucket_incarnation_id
|
||||
.is_some_and(|expected| expected != current_incarnation_id)
|
||||
@@ -9124,100 +9110,6 @@ mod tests {
|
||||
assert_eq!(dirs.len(), 16);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn current_marker_delete_ack_removes_stale_rejoined_copy() {
|
||||
let bucket = "current-marker-stale-rejoin";
|
||||
let object = "marker.bin";
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let (_dirs, original) = make_local_set_disks_with_ctx(16, 4, ctx.clone()).await;
|
||||
let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], ctx).await);
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
store
|
||||
.handle_make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create versioned bucket");
|
||||
let marker = store
|
||||
.delete_object(
|
||||
bucket,
|
||||
object,
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create current marker")
|
||||
.version_id
|
||||
.expect("marker has explicit version");
|
||||
let current = store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("bucket incarnation");
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
let before = set.disks.read().await.clone();
|
||||
for slot in 12..16 {
|
||||
set.disks.write().await[slot] = None;
|
||||
}
|
||||
store
|
||||
.delete_object(
|
||||
bucket,
|
||||
object,
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(marker.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("acknowledged delete should remove the marker from the online quorum");
|
||||
*set.disks.write().await = before.clone();
|
||||
|
||||
for disk in before.iter().take(12).flatten() {
|
||||
assert!(matches!(
|
||||
disk.read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
|
||||
.await,
|
||||
Err(crate::disk::error::DiskError::FileVersionNotFound | crate::disk::error::DiskError::FileNotFound)
|
||||
));
|
||||
}
|
||||
for disk in before.iter().skip(12).flatten() {
|
||||
assert!(
|
||||
disk.read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
|
||||
.await
|
||||
.is_ok(),
|
||||
"offline member must retain the stale current marker"
|
||||
);
|
||||
}
|
||||
let healed = store
|
||||
.heal_object_at_incarnation(
|
||||
bucket,
|
||||
object,
|
||||
&marker.to_string(),
|
||||
current,
|
||||
&rustfs_heal_contracts::heal_channel::HealOpts {
|
||||
remove: true,
|
||||
scan_mode: rustfs_heal_contracts::heal_channel::HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("stale current marker heal should complete");
|
||||
assert!(healed.error.is_none(), "stale current marker heal: {:?}", healed.error);
|
||||
assert!(healed.absence.as_ref().is_some_and(|proof| proof.removed));
|
||||
for disk in before.iter().flatten() {
|
||||
assert!(matches!(
|
||||
disk.read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
|
||||
.await,
|
||||
Err(crate::disk::error::DiskError::FileVersionNotFound | crate::disk::error::DiskError::FileNotFound)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipool_delete_marker_stays_with_existing_versions() {
|
||||
let bucket = "multipool-marker-routing";
|
||||
|
||||
@@ -161,52 +161,14 @@ impl ECStore {
|
||||
"incarnation-bound rename requires namespace locking and a non-nil identity",
|
||||
));
|
||||
}
|
||||
let bucket = if destination.0 == RUSTFS_META_BUCKET {
|
||||
super::heal::bucket_metadata_owner(destination.1).ok_or(DiskError::FileAccessDenied)?
|
||||
} else {
|
||||
destination.0
|
||||
};
|
||||
let guard = Arc::new(
|
||||
self.acquire_bucket_incarnation_fence(bucket, expected)
|
||||
self.acquire_bucket_incarnation_fence(destination.0, expected)
|
||||
.await
|
||||
.map_err(|error| DiskError::other(error.to_string()))?,
|
||||
);
|
||||
if guard.is_lock_lost() {
|
||||
return Err(DiskError::other("bucket heal incarnation fence was lost"));
|
||||
}
|
||||
let transaction = if destination.0 == RUSTFS_META_BUCKET {
|
||||
// The receiver must retain the config fence after an RPC timeout.
|
||||
// A delayed request must also reject a newer config in the same
|
||||
// bucket incarnation before publishing its older repair payload.
|
||||
let transaction = crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket)
|
||||
.await
|
||||
.map_err(|error| DiskError::other(error.to_string()))?;
|
||||
let (current, _) = self
|
||||
.get_pool_info_for_delete_marker(
|
||||
destination.0,
|
||||
destination.1,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(|error| DiskError::other(error.to_string()))?;
|
||||
let current = current.object_info;
|
||||
if transaction.is_lock_lost()
|
||||
|| guard.is_lock_lost()
|
||||
|| fi.mod_time.is_none()
|
||||
|| current.mod_time != fi.mod_time
|
||||
|| current.data_dir != fi.data_dir
|
||||
|| current.size != fi.size
|
||||
|| current.etag != fi.get_etag()
|
||||
{
|
||||
return Err(DiskError::FileAccessDenied);
|
||||
}
|
||||
Some(transaction)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
rename_local_data_with_ctx(
|
||||
&self.ctx,
|
||||
disk_ref,
|
||||
@@ -214,7 +176,7 @@ impl ECStore {
|
||||
fi,
|
||||
destination,
|
||||
RenameDataGuards {
|
||||
external_guard: Some(Arc::new((guard, transaction))),
|
||||
external_guard: Some(guard),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
|
||||
@@ -1,288 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#![cfg(feature = "test-util")]
|
||||
|
||||
mod storage_api;
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use storage_api::metadata_lock::{
|
||||
BucketOperations, CompletePart, Error, MakeBucketOptions, MultipartOperations, NamespaceLocking, ObjectIO, ObjectOperations,
|
||||
ObjectOptions, PutObjReader, PutObjectCommitBarrier, PutObjectCommitPause, init_bucket_metadata_sys,
|
||||
isolated_store_over_temp_disks,
|
||||
};
|
||||
use tokio::io::AsyncReadExt;
|
||||
use tokio::time::timeout;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[tokio::test]
|
||||
async fn replica_put_completes_while_metadata_writer_waits_for_its_snapshot() {
|
||||
replica_write_with_waiting_metadata_writer(false).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replica_multipart_completes_while_metadata_writer_waits_for_its_snapshot() {
|
||||
replica_write_with_waiting_metadata_writer(true).await;
|
||||
}
|
||||
|
||||
async fn replica_write_with_waiting_metadata_writer(multipart: bool) {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "replica-metadata-read-reuse";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
store
|
||||
.update_bucket_metadata_config(
|
||||
bucket,
|
||||
"lifecycle.xml",
|
||||
br#"<LifecycleConfiguration><Rule><ID>expire-old-versions</ID><Status>Enabled</Status><Filter><Prefix></Prefix></Filter><NoncurrentVersionExpiration><NoncurrentDays>1</NoncurrentDays></NoncurrentVersionExpiration></Rule></LifecycleConfiguration>"#.to_vec(),
|
||||
)
|
||||
.await
|
||||
.expect("configure post-write lifecycle evaluation");
|
||||
let opts = ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(Uuid::new_v4().to_string()),
|
||||
replication_request: true,
|
||||
expected_bucket_incarnation_id: Some(store.bucket_incarnation_id(bucket).await.expect("load incarnation")),
|
||||
object_lock_config_snapshot: Some(
|
||||
store
|
||||
.object_lock_config_snapshot(bucket)
|
||||
.await
|
||||
.expect("capture Object Lock snapshot"),
|
||||
),
|
||||
..Default::default()
|
||||
};
|
||||
let upload = if multipart {
|
||||
let upload = store
|
||||
.new_multipart_upload(bucket, "object", &opts)
|
||||
.await
|
||||
.expect("stage upload");
|
||||
let mut data = PutObjReader::from_vec(b"replicated version".to_vec());
|
||||
let part = store
|
||||
.put_object_part(bucket, "object", &upload.upload_id, 1, &mut data, &opts)
|
||||
.await
|
||||
.expect("stage part");
|
||||
Some((
|
||||
upload.upload_id,
|
||||
vec![CompletePart {
|
||||
part_num: 1,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
}],
|
||||
))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let lock = store
|
||||
.new_ns_lock(".rustfs.sys", &format!("bucket-targets/{bucket}/transaction.lock"))
|
||||
.await
|
||||
.expect("create metadata writer");
|
||||
let writer = lock.get_write_lock(Duration::from_secs(5));
|
||||
tokio::pin!(writer);
|
||||
// Leave the writer registered while the replica continues. A second read
|
||||
// acquisition would queue behind this writer, which needs our first read.
|
||||
assert!(futures::poll!(&mut writer).is_pending());
|
||||
timeout(Duration::from_secs(2), async {
|
||||
if let Some((upload_id, parts)) = upload {
|
||||
store
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, "object", &upload_id, parts, &opts)
|
||||
.await
|
||||
} else {
|
||||
let mut data = PutObjReader::from_vec(b"replicated version".to_vec());
|
||||
store.put_object(bucket, "object", &mut data, &opts).await
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("replica must reuse its metadata read lock instead of waiting behind the writer")
|
||||
.expect("replica PUT should commit");
|
||||
assert!(futures::poll!(&mut writer).is_pending(), "the snapshot must still fence metadata updates");
|
||||
let read_opts = ObjectOptions {
|
||||
version_id: opts.version_id.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
drop(opts);
|
||||
timeout(Duration::from_secs(2), writer)
|
||||
.await
|
||||
.expect("metadata writer must proceed after the snapshot drops")
|
||||
.expect("acquire metadata writer");
|
||||
let mut reader = store
|
||||
.get_object_reader(bucket, "object", None, Default::default(), &read_opts)
|
||||
.await
|
||||
.expect("read committed replica version");
|
||||
let mut bytes = Vec::new();
|
||||
reader.stream.read_to_end(&mut bytes).await.expect("read replica body");
|
||||
assert_eq!(bytes, b"replicated version");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reused_metadata_lock_still_enforces_quota() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "metadata-read-reuse-quota";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
store
|
||||
.update_bucket_metadata_config(bucket, "quota.json", br#"{"quota":1}"#.to_vec())
|
||||
.await
|
||||
.expect("set quota");
|
||||
let mut opts = ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(Uuid::new_v4().to_string()),
|
||||
expected_bucket_incarnation_id: Some(store.bucket_incarnation_id(bucket).await.expect("load incarnation")),
|
||||
object_lock_config_snapshot: Some(store.object_lock_config_snapshot(bucket).await.expect("capture snapshot")),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(opts.set_quota_admission(0, 1));
|
||||
let lock = store
|
||||
.new_ns_lock(".rustfs.sys", &format!("bucket-targets/{bucket}/transaction.lock"))
|
||||
.await
|
||||
.expect("create writer");
|
||||
let writer = lock.get_write_lock(Duration::from_secs(5));
|
||||
tokio::pin!(writer);
|
||||
assert!(futures::poll!(&mut writer).is_pending());
|
||||
let mut data = PutObjReader::from_vec(b"too large".to_vec());
|
||||
let error = timeout(Duration::from_secs(2), store.put_object(bucket, "object", &mut data, &opts))
|
||||
.await
|
||||
.expect("quota admission must not reacquire the lock")
|
||||
.expect_err("quota must reject growth");
|
||||
assert!(matches!(error, Error::QuotaExceeded { limit: 1, .. }), "{error:?}");
|
||||
drop(opts);
|
||||
timeout(Duration::from_secs(2), writer)
|
||||
.await
|
||||
.expect("release metadata reader after denial")
|
||||
.expect("acquire writer");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_put_releases_shared_metadata_guard() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "metadata-read-reuse-cancel";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let opts = ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(Uuid::new_v4().to_string()),
|
||||
object_lock_config_snapshot: Some(store.object_lock_config_snapshot(bucket).await.expect("capture snapshot")),
|
||||
..Default::default()
|
||||
};
|
||||
let barrier = PutObjectCommitBarrier::install(bucket, "object", PutObjectCommitPause::AfterQuotaReservation);
|
||||
let put_store = Arc::clone(&store);
|
||||
let put = tokio::spawn(async move {
|
||||
let mut data = PutObjReader::from_vec(b"cancel before commit".to_vec());
|
||||
put_store.put_object(bucket, "object", &mut data, &opts).await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
let lock = store
|
||||
.new_ns_lock(".rustfs.sys", &format!("bucket-targets/{bucket}/transaction.lock"))
|
||||
.await
|
||||
.expect("create writer");
|
||||
let writer = lock.get_write_lock(Duration::from_secs(5));
|
||||
tokio::pin!(writer);
|
||||
assert!(futures::poll!(&mut writer).is_pending());
|
||||
put.abort();
|
||||
assert!(put.await.expect_err("PUT should be cancelled").is_cancelled());
|
||||
timeout(Duration::from_secs(2), writer)
|
||||
.await
|
||||
.expect("all metadata reader owners must drop after cancellation")
|
||||
.expect("acquire writer");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_rejects_metadata_snapshots_from_another_scope() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let (_other_dirs, other_store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&other_store), Vec::new()).await;
|
||||
let bucket = "metadata-read-reuse-scope";
|
||||
for target in [&store, &other_store] {
|
||||
target
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
}
|
||||
store
|
||||
.make_bucket("other-bucket", &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create other bucket");
|
||||
let other_store_incarnation = other_store
|
||||
.bucket_incarnation_id(bucket)
|
||||
.await
|
||||
.expect("load other store incarnation");
|
||||
let other_bucket_incarnation = store
|
||||
.bucket_incarnation_id("other-bucket")
|
||||
.await
|
||||
.expect("load other bucket incarnation");
|
||||
for (snapshot_store, snapshot_bucket, expected) in [
|
||||
(&other_store, bucket, other_store_incarnation),
|
||||
(&store, "other-bucket", other_bucket_incarnation),
|
||||
(&store, bucket, Uuid::new_v4()),
|
||||
] {
|
||||
let opts = ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(expected),
|
||||
object_lock_config_snapshot: Some(
|
||||
snapshot_store
|
||||
.object_lock_config_snapshot(snapshot_bucket)
|
||||
.await
|
||||
.expect("capture snapshot"),
|
||||
),
|
||||
..Default::default()
|
||||
};
|
||||
let error = store
|
||||
.new_multipart_upload(bucket, "object", &opts)
|
||||
.await
|
||||
.expect_err("foreign snapshot must be rejected");
|
||||
assert!(error.to_string().contains("valid metadata transaction fence"), "{error:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_prefix_reuses_its_metadata_snapshot_for_generation_validation() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "metadata-read-reuse-delete";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let mut data = PutObjReader::from_vec(b"remove prefix".to_vec());
|
||||
store
|
||||
.put_object(bucket, "prefix/object", &mut data, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("seed object");
|
||||
store
|
||||
.delete_object(
|
||||
bucket,
|
||||
"prefix/",
|
||||
ObjectOptions {
|
||||
delete_prefix: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("delete prefix with an internally captured metadata snapshot");
|
||||
let error = store
|
||||
.get_object_info(bucket, "prefix/object", &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("prefix object must be deleted");
|
||||
assert!(matches!(error, Error::ObjectNotFound(..)), "{error:?}");
|
||||
}
|
||||
@@ -6,17 +6,6 @@ pub(crate) use rustfs_ecstore::api::erasure::Erasure;
|
||||
pub(crate) use rustfs_ecstore::api::object::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
|
||||
pub(crate) use rustfs_ecstore::api::{error::Error, set_disk::SetDisks, storage::ECStore};
|
||||
use rustfs_storage_api as storage_contracts;
|
||||
#[cfg(feature = "test-util")]
|
||||
pub(crate) mod metadata_lock {
|
||||
pub(crate) use super::storage_contracts::{
|
||||
BucketOperations, CompletePart, MakeBucketOptions, MultipartOperations, NamespaceLocking, ObjectIO, ObjectOperations,
|
||||
};
|
||||
pub(crate) use super::{Error, ObjectOptions, PutObjReader};
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{
|
||||
init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::set_disk::test_util::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
}
|
||||
|
||||
pub(crate) mod contract_compat {
|
||||
pub(crate) use super::storage_contracts::{
|
||||
|
||||
@@ -994,15 +994,6 @@ impl FileMeta {
|
||||
Uuid::nil()
|
||||
}
|
||||
};
|
||||
// A nil selector denotes the null version slot, which legacy metadata
|
||||
// may encode as either an absent UUID or an explicit nil UUID.
|
||||
let matches_version = |actual: Option<Uuid>| {
|
||||
if vid.is_nil() {
|
||||
actual.is_none_or(|version_id| version_id.is_nil())
|
||||
} else {
|
||||
actual == Some(vid)
|
||||
}
|
||||
};
|
||||
|
||||
let mut is_latest = true;
|
||||
let mut succ_mod_time = None;
|
||||
@@ -1040,7 +1031,7 @@ impl FileMeta {
|
||||
}
|
||||
}
|
||||
|
||||
if !matches_version(header.version_id) {
|
||||
if header.version_id != Some(vid) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
@@ -1049,7 +1040,7 @@ impl FileMeta {
|
||||
continue;
|
||||
}
|
||||
|
||||
if !version_id.is_empty() && !matches_version(header.version_id) {
|
||||
if !version_id.is_empty() && header.version_id != Some(vid) {
|
||||
is_latest = false;
|
||||
succ_mod_time = header.mod_time;
|
||||
continue;
|
||||
@@ -2694,48 +2685,6 @@ mod test {
|
||||
);
|
||||
assert!(fi.uses_legacy_checksum);
|
||||
assert!(fi.is_latest);
|
||||
|
||||
let selected = fm
|
||||
.into_fileinfo(".rustfs.sys", "pool.bin", &Uuid::nil().to_string(), true, false, true)
|
||||
.expect("nil selector should match the legacy null version");
|
||||
assert_eq!(selected.version_id, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nil_version_selector_matches_only_the_null_slot() {
|
||||
let mod_time = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid test timestamp");
|
||||
let version_id = Uuid::new_v4();
|
||||
let versioned = version_for_ordering(VersionType::Object, version_id, mod_time + time::Duration::seconds(1), 2);
|
||||
let mut null = version_for_ordering(VersionType::Object, Uuid::nil(), mod_time, 1);
|
||||
null.object
|
||||
.as_mut()
|
||||
.expect("ordering helper should construct an object")
|
||||
.version_id = None;
|
||||
|
||||
let mut fm = FileMeta::new();
|
||||
fm.add_version_filemata(versioned.clone()).expect("add versioned object");
|
||||
fm.add_version_filemata(null).expect("add legacy null object");
|
||||
|
||||
let selected_null = fm
|
||||
.into_fileinfo("bucket", "object", &Uuid::nil().to_string(), false, false, true)
|
||||
.expect("nil selector should match only the legacy null slot");
|
||||
assert_eq!(selected_null.version_id, None);
|
||||
|
||||
let selected_versioned = fm
|
||||
.into_fileinfo("bucket", "object", &version_id.to_string(), false, false, true)
|
||||
.expect("non-nil selector should still match its exact version");
|
||||
assert_eq!(selected_versioned.version_id, Some(version_id));
|
||||
|
||||
let mut versioned_only = FileMeta::new();
|
||||
versioned_only
|
||||
.add_version_filemata(versioned)
|
||||
.expect("add versioned object without a null slot");
|
||||
assert!(
|
||||
versioned_only
|
||||
.into_fileinfo("bucket", "object", &Uuid::nil().to_string(), false, false, true)
|
||||
.is_err(),
|
||||
"nil selector must not fall back to a non-nil latest version"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -907,21 +907,6 @@ impl ErasureSetHealer {
|
||||
}
|
||||
|
||||
if failed_objects == 0 && skipped_objects == 0 && failed_buckets == 0 {
|
||||
let targets = if self.pool_metadata_target_endpoints.is_empty() {
|
||||
self.target_endpoints.as_ref()
|
||||
} else {
|
||||
self.pool_metadata_target_endpoints.as_ref()
|
||||
};
|
||||
if self.replacement_task_id.is_some() || (!self.heal_opts.dry_run && self.heal_opts.recreate && !targets.is_empty()) {
|
||||
self.verify_replacement_identity_fence("bucket metadata").await?;
|
||||
// Recheck even resumed buckets: their user-object cursor does not
|
||||
// prove that the replacement holds the internal bucket records.
|
||||
for bucket in buckets {
|
||||
self.storage
|
||||
.heal_replacement_bucket_metadata(bucket, &self.heal_opts, targets)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
self.heal_replacement_pool_metadata(
|
||||
set_disk_id,
|
||||
&mut ErasureSetPassCounters {
|
||||
@@ -2282,8 +2267,6 @@ mod resume_loop_tests {
|
||||
list_include_lifecycle_object_info: Mutex<Vec<bool>>,
|
||||
replacement_target_identity_sequences: Mutex<VecDeque<Vec<ReplacementTargetIdentity>>>,
|
||||
pool_metadata_placement: Mutex<Option<ReplacementCommitEvidence>>,
|
||||
bucket_metadata_calls: Mutex<Vec<String>>,
|
||||
bucket_metadata_failure: AtomicBool,
|
||||
fail_listing: AtomicBool,
|
||||
fail_listing_buckets: Mutex<HashSet<String>>,
|
||||
}
|
||||
@@ -2479,13 +2462,6 @@ mod resume_loop_tests {
|
||||
None => Ok(true),
|
||||
}
|
||||
}
|
||||
async fn heal_replacement_bucket_metadata(&self, bucket: &str, _opts: &HealOpts, _targets: &[String]) -> Result<()> {
|
||||
self.bucket_metadata_calls.lock().unwrap().push(bucket.to_owned());
|
||||
if self.bucket_metadata_failure.load(Ordering::SeqCst) {
|
||||
return Err(Error::Storage(EcstoreError::PreconditionFailed));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
async fn replacement_targets_have_version(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
@@ -3082,18 +3058,6 @@ mod resume_loop_tests {
|
||||
)
|
||||
.with_replacement_targets(vec!["replacement-a".to_string()], Some(replacement_task_id.clone()));
|
||||
|
||||
env.storage.bucket_metadata_failure.store(true, Ordering::SeqCst);
|
||||
assert!(healer.heal_erasure_set(&["b".to_string()], "pool_0_set_0").await.is_err());
|
||||
let incomplete = ResumeManager::load_replacement_intent(env.healer.disk.clone(), &replacement_task_id)
|
||||
.await
|
||||
.expect("failed metadata repair must retain replacement intent")
|
||||
.get_state()
|
||||
.await;
|
||||
assert!(!incomplete.completed);
|
||||
assert_ne!(incomplete.replacement_phase, crate::heal::resume::ReplacementPhase::Verified);
|
||||
assert!(env.storage.calls().is_empty(), "metadata failure must stop completion before pool.bin");
|
||||
env.storage.bucket_metadata_failure.store(false, Ordering::SeqCst);
|
||||
|
||||
let error = healer
|
||||
.heal_erasure_set(&["b".to_string()], "pool_0_set_0")
|
||||
.await
|
||||
@@ -3109,11 +3073,6 @@ mod resume_loop_tests {
|
||||
assert_eq!(state.replacement_phase, crate::heal::resume::ReplacementPhase::Intent);
|
||||
assert_eq!(state.retry_count, 1);
|
||||
assert_eq!(env.storage.calls(), vec![(POOL_META_NAME.to_string(), None)]);
|
||||
assert_eq!(
|
||||
*env.storage.bucket_metadata_calls.lock().unwrap(),
|
||||
vec!["b", "b"],
|
||||
"resuming a completed user scan must retry bucket metadata"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -72,41 +72,6 @@ const LEGACY_ROOT_HEAL_PATH: &str = ".";
|
||||
const MAX_RECOVERABLE_HEAL_RETRIES: u32 = 3;
|
||||
const MAX_RECOVERABLE_HEAL_RETRY_DELAY: Duration = Duration::from_secs(30);
|
||||
const RESUME_GC_INTERVAL: Duration = Duration::from_secs(60 * 60);
|
||||
const BEST_EFFORT_QUEUE_RESERVE_PERCENT: usize = 10;
|
||||
const BEST_EFFORT_SOURCE_COUNT: usize = 3;
|
||||
|
||||
fn is_best_effort_source(source: HealRequestSource) -> bool {
|
||||
matches!(
|
||||
source,
|
||||
HealRequestSource::Scanner | HealRequestSource::AutoHeal | HealRequestSource::ReadRepair
|
||||
)
|
||||
}
|
||||
|
||||
fn best_effort_source_quota_index(source: HealRequestSource) -> Option<usize> {
|
||||
match source {
|
||||
HealRequestSource::Scanner => Some(0),
|
||||
HealRequestSource::ReadRepair => Some(1),
|
||||
HealRequestSource::AutoHeal => Some(2),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn best_effort_queue_reserve(queue_capacity: usize) -> usize {
|
||||
match queue_capacity {
|
||||
0..=2 => 0,
|
||||
capacity => (capacity.saturating_mul(BEST_EFFORT_QUEUE_RESERVE_PERCENT) / 100).max(BEST_EFFORT_SOURCE_COUNT),
|
||||
}
|
||||
}
|
||||
|
||||
fn best_effort_source_reserves(queue_capacity: usize) -> [usize; BEST_EFFORT_SOURCE_COUNT] {
|
||||
let reserve = best_effort_queue_reserve(queue_capacity);
|
||||
if reserve == 0 {
|
||||
return [0; BEST_EFFORT_SOURCE_COUNT];
|
||||
}
|
||||
let base = reserve / BEST_EFFORT_SOURCE_COUNT;
|
||||
let remainder = reserve % BEST_EFFORT_SOURCE_COUNT;
|
||||
[base + usize::from(remainder > 0), base + usize::from(remainder > 1), base]
|
||||
}
|
||||
|
||||
// Admission/scheduler outcomes for per-object requests (Object/Metadata/
|
||||
// ECDecode) log via demote_to_debug_when! — MRF, autoheal, and scanner
|
||||
@@ -1000,7 +965,11 @@ struct HealQueueContext<'a> {
|
||||
|
||||
impl HealManager {
|
||||
fn classify_full_admission(request: &HealRequest, config: &HealConfig) -> HealAdmissionResult {
|
||||
if request.priority == HealPriority::Low && config.low_priority_drop_when_full {
|
||||
let best_effort_source = matches!(
|
||||
request.source,
|
||||
HealRequestSource::Scanner | HealRequestSource::AutoHeal | HealRequestSource::ReadRepair
|
||||
);
|
||||
if best_effort_source || (request.priority == HealPriority::Low && config.low_priority_drop_when_full) {
|
||||
HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull)
|
||||
} else {
|
||||
HealAdmissionResult::Full
|
||||
@@ -1011,6 +980,32 @@ impl HealManager {
|
||||
queue_len.saturating_mul(100).checked_div(queue_capacity).unwrap_or(0)
|
||||
}
|
||||
|
||||
fn classify_pressure_admission(
|
||||
request: &HealRequest,
|
||||
queue_len: usize,
|
||||
queue_capacity: usize,
|
||||
) -> Option<HealAdmissionResult> {
|
||||
if request.force_start || queue_capacity == 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let queue_usage_pct = Self::queue_usage_pct(queue_len, queue_capacity);
|
||||
if queue_usage_pct < 80 {
|
||||
return None;
|
||||
}
|
||||
|
||||
match request.source {
|
||||
HealRequestSource::ReadRepair => Some(HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped)),
|
||||
HealRequestSource::Scanner if request.priority == HealPriority::Low => {
|
||||
Some(HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped))
|
||||
}
|
||||
HealRequestSource::AutoHeal if queue_usage_pct >= 95 => {
|
||||
Some(HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped))
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn duplicate_admission_for_request(request: &HealRequest, config: &HealConfig) -> HealAdmissionResult {
|
||||
if request.priority == HealPriority::Low && !config.low_priority_merge_enable {
|
||||
HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped)
|
||||
@@ -1161,8 +1156,6 @@ impl HealManager {
|
||||
publish_heal_queue_length(queue);
|
||||
let queue_capacity = config.queue_size;
|
||||
let per_object_request = request.heal_type.is_per_object();
|
||||
let best_effort_reserve = best_effort_queue_reserve(queue_capacity);
|
||||
let best_effort_source_reserves = best_effort_source_reserves(queue_capacity);
|
||||
|
||||
if queue_len >= queue_capacity && !request.force_start {
|
||||
if Self::can_displace_queued_work(&request)
|
||||
@@ -1250,55 +1243,27 @@ impl HealManager {
|
||||
return HealAdmissionDecision::new(admission);
|
||||
}
|
||||
|
||||
if !is_best_effort_source(request.source)
|
||||
&& request.priority <= HealPriority::Normal
|
||||
&& !request.force_start
|
||||
&& best_effort_reserve > 0
|
||||
&& queue_len >= queue_capacity.saturating_sub(best_effort_reserve)
|
||||
{
|
||||
Self::record_admission_metric(request.source, HealAdmissionResult::Full, context);
|
||||
demote_to_debug_when!(per_object_request, warn, target: "rustfs::heal::manager", {
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
request_id = %request.id,
|
||||
priority = ?request.priority,
|
||||
source = request.source.as_str(),
|
||||
context,
|
||||
queue_len,
|
||||
queue_capacity,
|
||||
reserved_for_best_effort = best_effort_reserve,
|
||||
result = "rejected_for_reserved_capacity",
|
||||
"Normal heal request deferred to preserve best-effort capacity"
|
||||
});
|
||||
return HealAdmissionDecision::new(HealAdmissionResult::Full);
|
||||
}
|
||||
|
||||
if is_best_effort_source(request.source)
|
||||
&& request.priority <= HealPriority::Normal
|
||||
&& !request.force_start
|
||||
&& best_effort_reserve > 0
|
||||
&& queue_len >= queue_capacity.saturating_sub(best_effort_reserve)
|
||||
&& let Some(source_index) = best_effort_source_quota_index(request.source)
|
||||
&& queue.best_effort_source_count(request.source) >= best_effort_source_reserves[source_index]
|
||||
{
|
||||
Self::record_admission_metric(request.source, HealAdmissionResult::Full, context);
|
||||
demote_to_debug_when!(per_object_request, warn, target: "rustfs::heal::manager", {
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
request_id = %request.id,
|
||||
priority = ?request.priority,
|
||||
source = request.source.as_str(),
|
||||
context,
|
||||
queue_len,
|
||||
queue_capacity,
|
||||
reserved_for_best_effort = best_effort_reserve,
|
||||
source_quota = best_effort_source_reserves[source_index],
|
||||
result = "rejected_for_source_quota",
|
||||
"Best-effort source deferred to preserve per-source capacity"
|
||||
});
|
||||
return HealAdmissionDecision::new(HealAdmissionResult::Full);
|
||||
if let Some(admission) = Self::classify_pressure_admission(&request, queue_len, queue_capacity) {
|
||||
Self::record_admission_metric(request.source, admission, context);
|
||||
if let HealAdmissionResult::Dropped(reason) = admission {
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
request_id = %request.id,
|
||||
priority = ?request.priority,
|
||||
source = request.source.as_str(),
|
||||
context,
|
||||
queue_len,
|
||||
queue_capacity,
|
||||
queue_usage_pct = Self::queue_usage_pct(queue_len, queue_capacity),
|
||||
reason = reason.as_str(),
|
||||
result = "dropped_pressure",
|
||||
"Heal queue request dropped under pressure"
|
||||
);
|
||||
}
|
||||
return HealAdmissionDecision::new(admission);
|
||||
}
|
||||
|
||||
if queue_capacity > 0 {
|
||||
|
||||
@@ -34,33 +34,6 @@ pub(super) struct PriorityHealQueue {
|
||||
pub(super) sequence: u64,
|
||||
/// Deduplication index for queued requests
|
||||
pub(super) dedup_keys: HashMap<String, DedupKeyEntry>,
|
||||
pub(super) best_effort_queued: usize,
|
||||
pub(super) best_effort_queued_by_source: [usize; 3],
|
||||
/// Number of MRF/Normal tasks dispatched since a best-effort task ran.
|
||||
/// This is kept with the queue so event-driven scheduler wakeups cannot
|
||||
/// reset the fairness budget between cycles.
|
||||
pub(super) mrf_normal_dispatches_since_best_effort: usize,
|
||||
pub(super) last_best_effort_source: Option<HealRequestSource>,
|
||||
}
|
||||
|
||||
const MRF_NORMAL_DISPATCH_QUANTUM: usize = 4;
|
||||
|
||||
fn best_effort_source_index(source: HealRequestSource) -> Option<usize> {
|
||||
match source {
|
||||
HealRequestSource::Scanner => Some(0),
|
||||
HealRequestSource::ReadRepair => Some(1),
|
||||
HealRequestSource::AutoHeal => Some(2),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn best_effort_source_order(last: Option<HealRequestSource>) -> [usize; 3] {
|
||||
match last {
|
||||
Some(HealRequestSource::Scanner) => [1, 2, 0],
|
||||
Some(HealRequestSource::ReadRepair) => [2, 0, 1],
|
||||
Some(HealRequestSource::AutoHeal) => [0, 1, 2],
|
||||
_ => [0, 1, 2],
|
||||
}
|
||||
}
|
||||
|
||||
/// Wrapper for heap items to implement proper ordering
|
||||
@@ -291,10 +264,6 @@ impl PriorityHealQueue {
|
||||
heap: BinaryHeap::new(),
|
||||
sequence: 0,
|
||||
dedup_keys: HashMap::new(),
|
||||
best_effort_queued: 0,
|
||||
best_effort_queued_by_source: [0; 3],
|
||||
mrf_normal_dispatches_since_best_effort: 0,
|
||||
last_best_effort_source: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -302,11 +271,9 @@ impl PriorityHealQueue {
|
||||
self.heap.len()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn pop_next(&mut self) -> Option<HealRequest> {
|
||||
self.heap.pop().map(|item| {
|
||||
Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item.dedup_key);
|
||||
self.remove_best_effort_count(item.request.source);
|
||||
item.request
|
||||
})
|
||||
}
|
||||
@@ -334,19 +301,12 @@ impl PriorityHealQueue {
|
||||
})
|
||||
.refcount += 1;
|
||||
self.sequence += 1;
|
||||
let source = request.source;
|
||||
self.heap.push(PriorityQueueItem {
|
||||
priority: request.priority,
|
||||
sequence: self.sequence,
|
||||
dedup_key: key,
|
||||
request,
|
||||
});
|
||||
if is_best_effort_source(source) {
|
||||
self.best_effort_queued += 1;
|
||||
if let Some(index) = best_effort_source_index(source) {
|
||||
self.best_effort_queued_by_source[index] += 1;
|
||||
}
|
||||
}
|
||||
QueuePushOutcome::Accepted
|
||||
}
|
||||
|
||||
@@ -396,7 +356,6 @@ impl PriorityHealQueue {
|
||||
|
||||
let displaced = displaced.map(|item| {
|
||||
Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item.dedup_key);
|
||||
self.remove_best_effort_count(item.request.source);
|
||||
self.refresh_dedup_representative(&item.dedup_key);
|
||||
item.request
|
||||
});
|
||||
@@ -441,7 +400,6 @@ impl PriorityHealQueue {
|
||||
pub(super) fn pop(&mut self) -> Option<HealRequest> {
|
||||
self.heap.pop().map(|item| {
|
||||
Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item.dedup_key);
|
||||
self.remove_best_effort_count(item.request.source);
|
||||
item.request
|
||||
})
|
||||
}
|
||||
@@ -471,115 +429,12 @@ impl PriorityHealQueue {
|
||||
(
|
||||
selected.map(|item| {
|
||||
Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item.dedup_key);
|
||||
self.remove_best_effort_count(item.request.source);
|
||||
item.request
|
||||
}),
|
||||
skipped,
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn pop_runnable_with_fairness<F, G>(&mut self, can_run: F, skip_label: G) -> (Option<HealRequest>, Vec<String>)
|
||||
where
|
||||
F: Fn(&HealRequest) -> bool,
|
||||
G: Fn(&HealRequest) -> Option<String>,
|
||||
{
|
||||
if self.mrf_normal_dispatches_since_best_effort < MRF_NORMAL_DISPATCH_QUANTUM {
|
||||
let (chosen, skipped) = self.pop_runnable_with_skips(can_run, skip_label);
|
||||
if let Some(request) = chosen.as_ref() {
|
||||
self.record_dispatch(request);
|
||||
}
|
||||
return (chosen, skipped);
|
||||
}
|
||||
|
||||
if self.best_effort_queued == 0 {
|
||||
self.mrf_normal_dispatches_since_best_effort = 0;
|
||||
let (chosen, skipped) = self.pop_runnable_with_skips(can_run, skip_label);
|
||||
if let Some(request) = chosen.as_ref() {
|
||||
self.record_dispatch(request);
|
||||
}
|
||||
return (chosen, skipped);
|
||||
}
|
||||
|
||||
let mut items = std::mem::take(&mut self.heap).into_vec();
|
||||
let mut skipped = Vec::new();
|
||||
let mut first_runnable: Option<usize> = None;
|
||||
let mut best_effort_candidates: [Option<usize>; 3] = [None; 3];
|
||||
for (index, item) in items.iter().enumerate() {
|
||||
if !can_run(&item.request) {
|
||||
if let Some(label) = skip_label(&item.request) {
|
||||
skipped.push(label);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if first_runnable.is_none_or(|current| item.cmp(&items[current]).is_gt()) {
|
||||
first_runnable = Some(index);
|
||||
}
|
||||
let Some(source_index) = best_effort_source_index(item.request.source) else {
|
||||
continue;
|
||||
};
|
||||
if best_effort_candidates[source_index].is_none_or(|current| {
|
||||
item.priority > items[current].priority
|
||||
|| (item.priority == items[current].priority && item.sequence < items[current].sequence)
|
||||
}) {
|
||||
best_effort_candidates[source_index] = Some(index);
|
||||
}
|
||||
}
|
||||
|
||||
let chosen_index = first_runnable.filter(|index| {
|
||||
let item = &items[*index];
|
||||
item.priority == HealPriority::Normal
|
||||
&& matches!(item.request.source, HealRequestSource::Mrf | HealRequestSource::Internal)
|
||||
});
|
||||
let best_effort_priority = best_effort_candidates
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.map(|index| items[index].priority)
|
||||
.max();
|
||||
let chosen_index = chosen_index
|
||||
.and_then(|_| {
|
||||
let priority = best_effort_priority?;
|
||||
best_effort_source_order(self.last_best_effort_source)
|
||||
.into_iter()
|
||||
.find_map(|source| best_effort_candidates[source].filter(|index| items[*index].priority == priority))
|
||||
})
|
||||
.or(first_runnable);
|
||||
let chosen = chosen_index.map(|index| items.swap_remove(index));
|
||||
self.heap = BinaryHeap::from(items);
|
||||
|
||||
let chosen = chosen.map(|item| {
|
||||
let request = item.request;
|
||||
Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item.dedup_key);
|
||||
self.remove_best_effort_count(request.source);
|
||||
self.record_dispatch(&request);
|
||||
request
|
||||
});
|
||||
(chosen, skipped)
|
||||
}
|
||||
|
||||
fn record_dispatch(&mut self, request: &HealRequest) {
|
||||
if is_best_effort_source(request.source) {
|
||||
self.mrf_normal_dispatches_since_best_effort = 0;
|
||||
self.last_best_effort_source = Some(request.source);
|
||||
} else if matches!(request.source, HealRequestSource::Mrf | HealRequestSource::Internal)
|
||||
&& request.priority == HealPriority::Normal
|
||||
{
|
||||
self.mrf_normal_dispatches_since_best_effort = self.mrf_normal_dispatches_since_best_effort.saturating_add(1);
|
||||
}
|
||||
}
|
||||
|
||||
fn remove_best_effort_count(&mut self, source: HealRequestSource) {
|
||||
if is_best_effort_source(source) {
|
||||
self.best_effort_queued = self.best_effort_queued.saturating_sub(1);
|
||||
if let Some(index) = best_effort_source_index(source) {
|
||||
self.best_effort_queued_by_source[index] = self.best_effort_queued_by_source[index].saturating_sub(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn best_effort_source_count(&self, source: HealRequestSource) -> usize {
|
||||
best_effort_source_index(source).map_or(0, |index| self.best_effort_queued_by_source[index])
|
||||
}
|
||||
|
||||
fn restore_deferred_items(&mut self, deferred: Vec<PriorityQueueItem>) {
|
||||
if deferred.is_empty() {
|
||||
return;
|
||||
@@ -735,7 +590,6 @@ impl PriorityHealQueue {
|
||||
if removed.is_none() && item.request.id == request_id {
|
||||
let key = item.dedup_key.clone();
|
||||
Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &key);
|
||||
self.remove_best_effort_count(item.request.source);
|
||||
affected_key = Some(key);
|
||||
removed = Some(item.request);
|
||||
} else {
|
||||
@@ -761,7 +615,6 @@ impl PriorityHealQueue {
|
||||
while let Some(item) = self.heap.pop() {
|
||||
if should_remove(&item.request) {
|
||||
Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item.dedup_key);
|
||||
self.remove_best_effort_count(item.request.source);
|
||||
affected_keys.push(item.dedup_key);
|
||||
removed.push(item.request);
|
||||
} else {
|
||||
|
||||
@@ -150,7 +150,7 @@ impl HealManager {
|
||||
for _ in 0..available_slots {
|
||||
let selected_request = if config.set_bulkhead_enable || mainline_pressure.is_some() {
|
||||
let max_concurrent_per_set = config.max_concurrent_per_set;
|
||||
let (selected_request, skipped_sets) = queue.pop_runnable_with_fairness(
|
||||
let (selected_request, skipped_sets) = queue.pop_runnable_with_skips(
|
||||
|request| {
|
||||
let set_allowed = !config.set_bulkhead_enable
|
||||
|| can_schedule_request(request, &running_per_set, max_concurrent_per_set);
|
||||
@@ -164,7 +164,7 @@ impl HealManager {
|
||||
}
|
||||
selected_request
|
||||
} else {
|
||||
queue.pop_runnable_with_fairness(|_| true, |_| None).0
|
||||
queue.pop_next()
|
||||
};
|
||||
|
||||
if let Some(mut request) = selected_request {
|
||||
@@ -811,9 +811,6 @@ pub(super) fn mrf_verified_repair_event_for_target(
|
||||
let disposition = match outcome.disposition {
|
||||
HealObjectDisposition::Repaired => MrfVerifiedRepairDisposition::Repaired,
|
||||
HealObjectDisposition::VerifiedHealthy => MrfVerifiedRepairDisposition::VerifiedHealthy,
|
||||
HealObjectDisposition::MetadataHealthy if target.kind == MrfKind::MetadataCorruption => {
|
||||
MrfVerifiedRepairDisposition::VerifiedHealthy
|
||||
}
|
||||
HealObjectDisposition::AuthoritativelyAbsent => MrfVerifiedRepairDisposition::AuthoritativelyAbsent,
|
||||
_ => return None,
|
||||
};
|
||||
|
||||
@@ -1488,164 +1488,6 @@ fn test_priority_queue_ordering() {
|
||||
assert_eq!(queue.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_priority_queue_fairness_gives_best_effort_bounded_service() {
|
||||
let mut queue = PriorityHealQueue::new();
|
||||
for index in 0..5 {
|
||||
assert_eq!(
|
||||
queue.push(bucket_request(&format!("mrf-{index}"), HealPriority::Normal, HealRequestSource::Mrf)),
|
||||
QueuePushOutcome::Accepted
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
queue.push(bucket_request("scanner", HealPriority::Low, HealRequestSource::Scanner)),
|
||||
QueuePushOutcome::Accepted
|
||||
);
|
||||
|
||||
for _ in 0..4 {
|
||||
let (request, skipped) = queue.pop_runnable_with_fairness(|_| true, |_| None);
|
||||
assert!(skipped.is_empty());
|
||||
assert_eq!(
|
||||
request.expect("MRF request should run before the fairness quantum").source,
|
||||
HealRequestSource::Mrf
|
||||
);
|
||||
}
|
||||
|
||||
let (request, skipped) = queue.pop_runnable_with_fairness(|_| true, |_| None);
|
||||
assert!(skipped.is_empty());
|
||||
assert_eq!(
|
||||
request
|
||||
.expect("best-effort work must get a bounded service opportunity")
|
||||
.source,
|
||||
HealRequestSource::Scanner
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_priority_queue_fairness_covers_internal_normal_background_work() {
|
||||
let mut queue = PriorityHealQueue::new();
|
||||
for index in 0..5 {
|
||||
assert_eq!(
|
||||
queue.push(bucket_request(
|
||||
&format!("internal-{index}"),
|
||||
HealPriority::Normal,
|
||||
HealRequestSource::Internal
|
||||
)),
|
||||
QueuePushOutcome::Accepted
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
queue.push(bucket_request("scanner", HealPriority::Low, HealRequestSource::Scanner)),
|
||||
QueuePushOutcome::Accepted
|
||||
);
|
||||
|
||||
for _ in 0..4 {
|
||||
let (request, _) = queue.pop_runnable_with_fairness(|_| true, |_| None);
|
||||
assert_eq!(request.expect("internal background work should run").source, HealRequestSource::Internal);
|
||||
}
|
||||
let (request, _) = queue.pop_runnable_with_fairness(|_| true, |_| None);
|
||||
assert_eq!(
|
||||
request.expect("Scanner should get a service opportunity").source,
|
||||
HealRequestSource::Scanner
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_priority_queue_fairness_round_robins_best_effort_sources() {
|
||||
let mut queue = PriorityHealQueue::new();
|
||||
for index in 0..8 {
|
||||
assert_eq!(
|
||||
queue.push(bucket_request(&format!("mrf-{index}"), HealPriority::Normal, HealRequestSource::Mrf)),
|
||||
QueuePushOutcome::Accepted
|
||||
);
|
||||
}
|
||||
for (bucket, source) in [
|
||||
("scanner", HealRequestSource::Scanner),
|
||||
("read-repair", HealRequestSource::ReadRepair),
|
||||
("auto-heal", HealRequestSource::AutoHeal),
|
||||
] {
|
||||
assert_eq!(queue.push(bucket_request(bucket, HealPriority::Low, source)), QueuePushOutcome::Accepted);
|
||||
}
|
||||
|
||||
for _ in 0..4 {
|
||||
let (request, _) = queue.pop_runnable_with_fairness(|_| true, |_| None);
|
||||
assert_eq!(request.expect("first MRF quantum should run").source, HealRequestSource::Mrf);
|
||||
}
|
||||
let (request, _) = queue.pop_runnable_with_fairness(|_| true, |_| None);
|
||||
assert_eq!(
|
||||
request.expect("Scanner should get the first best-effort turn").source,
|
||||
HealRequestSource::Scanner
|
||||
);
|
||||
|
||||
for _ in 0..4 {
|
||||
let (request, _) = queue.pop_runnable_with_fairness(|_| true, |_| None);
|
||||
assert_eq!(request.expect("second MRF quantum should run").source, HealRequestSource::Mrf);
|
||||
}
|
||||
let (request, _) = queue.pop_runnable_with_fairness(|_| true, |_| None);
|
||||
assert_eq!(
|
||||
request.expect("ReadRepair should get the next best-effort turn").source,
|
||||
HealRequestSource::ReadRepair
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_priority_queue_fairness_keeps_best_effort_priority_order() {
|
||||
let mut queue = PriorityHealQueue::new();
|
||||
for index in 0..4 {
|
||||
assert_eq!(
|
||||
queue.push(bucket_request(&format!("mrf-{index}"), HealPriority::Normal, HealRequestSource::Mrf)),
|
||||
QueuePushOutcome::Accepted
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
queue.push(bucket_request("scanner", HealPriority::Low, HealRequestSource::Scanner)),
|
||||
QueuePushOutcome::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
queue.push(bucket_request("read-repair", HealPriority::Normal, HealRequestSource::ReadRepair)),
|
||||
QueuePushOutcome::Accepted
|
||||
);
|
||||
|
||||
for _ in 0..4 {
|
||||
let (request, _) = queue.pop_runnable_with_fairness(|_| true, |_| None);
|
||||
assert_eq!(request.expect("MRF request should run").source, HealRequestSource::Mrf);
|
||||
}
|
||||
let (request, _) = queue.pop_runnable_with_fairness(|_| true, |_| None);
|
||||
assert_eq!(
|
||||
request.expect("higher-priority best-effort work should run first").source,
|
||||
HealRequestSource::ReadRepair
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_priority_queue_fairness_skips_blocked_best_effort_source() {
|
||||
let mut queue = PriorityHealQueue::new();
|
||||
for index in 0..4 {
|
||||
assert_eq!(
|
||||
queue.push(bucket_request(&format!("mrf-{index}"), HealPriority::Normal, HealRequestSource::Mrf)),
|
||||
QueuePushOutcome::Accepted
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
queue.push(bucket_request("scanner", HealPriority::Low, HealRequestSource::Scanner)),
|
||||
QueuePushOutcome::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
queue.push(bucket_request("read-repair", HealPriority::Low, HealRequestSource::ReadRepair)),
|
||||
QueuePushOutcome::Accepted
|
||||
);
|
||||
|
||||
for _ in 0..4 {
|
||||
let (request, _) = queue.pop_runnable_with_fairness(|_| true, |_| None);
|
||||
assert_eq!(request.expect("MRF request should run").source, HealRequestSource::Mrf);
|
||||
}
|
||||
let (request, _) = queue.pop_runnable_with_fairness(|request| request.source != HealRequestSource::Scanner, |_| None);
|
||||
assert_eq!(
|
||||
request.expect("runnable ReadRepair should bypass blocked Scanner").source,
|
||||
HealRequestSource::ReadRepair
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_priority_queue_fifo_same_priority() {
|
||||
let mut queue = PriorityHealQueue::new();
|
||||
@@ -4658,7 +4500,7 @@ async fn test_displacing_registered_mrf_task_drops_notice_ownership() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_submit_heal_request_admits_read_repair_under_pressure() {
|
||||
async fn test_submit_heal_request_drops_read_repair_under_pressure() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
storage,
|
||||
@@ -4668,7 +4510,7 @@ async fn test_submit_heal_request_admits_read_repair_under_pressure() {
|
||||
}),
|
||||
);
|
||||
|
||||
for index in 0..7 {
|
||||
for index in 0..8 {
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request(
|
||||
@@ -4687,12 +4529,12 @@ async fn test_submit_heal_request_admits_read_repair_under_pressure() {
|
||||
.await
|
||||
.expect("read repair admission should return a result");
|
||||
|
||||
assert_eq!(admission, HealAdmissionResult::Accepted);
|
||||
assert_eq!(admission, HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped));
|
||||
assert_eq!(manager.get_queue_length().await, 8);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_submit_heal_request_admits_low_scanner_under_pressure() {
|
||||
async fn test_submit_heal_request_drops_low_scanner_under_pressure() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
storage,
|
||||
@@ -4702,7 +4544,7 @@ async fn test_submit_heal_request_admits_low_scanner_under_pressure() {
|
||||
}),
|
||||
);
|
||||
|
||||
for index in 0..7 {
|
||||
for index in 0..8 {
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request(
|
||||
@@ -4721,220 +4563,10 @@ async fn test_submit_heal_request_admits_low_scanner_under_pressure() {
|
||||
.await
|
||||
.expect("scanner admission should return a result");
|
||||
|
||||
assert_eq!(admission, HealAdmissionResult::Accepted);
|
||||
assert_eq!(admission, HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped));
|
||||
assert_eq!(manager.get_queue_length().await, 8);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mrf_admission_preserves_best_effort_queue_reserve() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
Arc::new(MockStorage),
|
||||
Some(HealConfig {
|
||||
queue_size: 10,
|
||||
..HealConfig::default()
|
||||
}),
|
||||
);
|
||||
|
||||
for index in 0..7 {
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request(&format!("mrf-{index}"), HealPriority::Normal, HealRequestSource::Mrf,))
|
||||
.await
|
||||
.expect("MRF request should be admitted before the reserve threshold"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("mrf-deferred", HealPriority::Normal, HealRequestSource::Mrf))
|
||||
.await
|
||||
.expect("MRF admission should return a typed result"),
|
||||
HealAdmissionResult::Full
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("internal-deferred", HealPriority::Normal, HealRequestSource::Internal))
|
||||
.await
|
||||
.expect("ordinary background admission should return a typed result"),
|
||||
HealAdmissionResult::Full
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("scanner-reserved", HealPriority::Low, HealRequestSource::Scanner))
|
||||
.await
|
||||
.expect("Scanner should consume the reserved slot"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("read-repair-reserved", HealPriority::Low, HealRequestSource::ReadRepair))
|
||||
.await
|
||||
.expect("ReadRepair should consume its reserved slot"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("auto-heal-reserved", HealPriority::Low, HealRequestSource::AutoHeal))
|
||||
.await
|
||||
.expect("AutoHeal should consume its reserved slot"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(manager.get_queue_length().await, 10);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_small_queue_keeps_per_source_best_effort_slots() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
Arc::new(MockStorage),
|
||||
Some(HealConfig {
|
||||
queue_size: 9,
|
||||
..HealConfig::default()
|
||||
}),
|
||||
);
|
||||
|
||||
for index in 0..6 {
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request(&format!("mrf-{index}"), HealPriority::Normal, HealRequestSource::Mrf))
|
||||
.await
|
||||
.expect("MRF request should be admitted before the per-source reserve"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("mrf-deferred", HealPriority::Normal, HealRequestSource::Mrf))
|
||||
.await
|
||||
.expect("MRF admission should return a typed result"),
|
||||
HealAdmissionResult::Full
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("scanner-reserved", HealPriority::Low, HealRequestSource::Scanner))
|
||||
.await
|
||||
.expect("Scanner should consume its reserved slot"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("read-repair-reserved", HealPriority::Low, HealRequestSource::ReadRepair))
|
||||
.await
|
||||
.expect("ReadRepair should consume its reserved slot"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("auto-heal-reserved", HealPriority::Low, HealRequestSource::AutoHeal))
|
||||
.await
|
||||
.expect("AutoHeal should consume its reserved slot"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_queue_size_four_keeps_mrf_and_each_best_effort_source() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
Arc::new(MockStorage),
|
||||
Some(HealConfig {
|
||||
queue_size: 4,
|
||||
..HealConfig::default()
|
||||
}),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("mrf", HealPriority::Normal, HealRequestSource::Mrf))
|
||||
.await
|
||||
.expect("MRF should retain one queue slot"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("scanner", HealPriority::Low, HealRequestSource::Scanner))
|
||||
.await
|
||||
.expect("Scanner should use its source slot"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("read-repair", HealPriority::Low, HealRequestSource::ReadRepair))
|
||||
.await
|
||||
.expect("ReadRepair should use its source slot"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("auto-heal", HealPriority::Low, HealRequestSource::AutoHeal))
|
||||
.await
|
||||
.expect("AutoHeal should use its source slot"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_high_priority_mrf_bypasses_best_effort_reserve() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
Arc::new(MockStorage),
|
||||
Some(HealConfig {
|
||||
queue_size: 10,
|
||||
..HealConfig::default()
|
||||
}),
|
||||
);
|
||||
|
||||
for index in 0..7 {
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request(&format!("mrf-{index}"), HealPriority::Normal, HealRequestSource::Mrf))
|
||||
.await
|
||||
.expect("normal MRF request should be admitted before the reserve"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("urgent-mrf", HealPriority::Urgent, HealRequestSource::Mrf))
|
||||
.await
|
||||
.expect("urgent MRF admission should return a typed result"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_high_priority_best_effort_bypasses_source_quota() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
Arc::new(MockStorage),
|
||||
Some(HealConfig {
|
||||
queue_size: 10,
|
||||
..HealConfig::default()
|
||||
}),
|
||||
);
|
||||
|
||||
for index in 0..7 {
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request(&format!("mrf-{index}"), HealPriority::Normal, HealRequestSource::Mrf))
|
||||
.await
|
||||
.expect("normal MRF request should be admitted before the reserve"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("scanner-low", HealPriority::Low, HealRequestSource::Scanner))
|
||||
.await
|
||||
.expect("low Scanner request should be admitted into its source slot"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("scanner-high", HealPriority::High, HealRequestSource::Scanner))
|
||||
.await
|
||||
.expect("high Scanner retry should bypass its low-priority quota"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_submit_heal_request_accepts_admin_high_under_pressure() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
@@ -4946,7 +4578,7 @@ async fn test_submit_heal_request_accepts_admin_high_under_pressure() {
|
||||
}),
|
||||
);
|
||||
|
||||
for index in 0..7 {
|
||||
for index in 0..8 {
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request(
|
||||
@@ -4966,56 +4598,7 @@ async fn test_submit_heal_request_accepts_admin_high_under_pressure() {
|
||||
.expect("admin admission should return a result");
|
||||
|
||||
assert_eq!(admission, HealAdmissionResult::Accepted);
|
||||
assert_eq!(manager.get_queue_length().await, 8);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_admin_normal_displaces_best_effort_when_queue_is_full() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
Arc::new(MockStorage),
|
||||
Some(HealConfig {
|
||||
queue_size: 10,
|
||||
..HealConfig::default()
|
||||
}),
|
||||
);
|
||||
|
||||
for index in 0..7 {
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request(&format!("mrf-{index}"), HealPriority::Normal, HealRequestSource::Mrf))
|
||||
.await
|
||||
.expect("MRF request should be queued"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("scanner", HealPriority::Low, HealRequestSource::Scanner))
|
||||
.await
|
||||
.expect("Scanner request should be queued"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("read-repair", HealPriority::Low, HealRequestSource::ReadRepair))
|
||||
.await
|
||||
.expect("ReadRepair request should be queued"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("auto-heal", HealPriority::Low, HealRequestSource::AutoHeal))
|
||||
.await
|
||||
.expect("AutoHeal request should be queued"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("admin", HealPriority::Normal, HealRequestSource::Admin))
|
||||
.await
|
||||
.expect("admin admission should return a typed result"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(manager.get_queue_length().await, 9);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -76,9 +76,6 @@ pub enum HealObjectDisposition {
|
||||
Unknown,
|
||||
Repaired,
|
||||
VerifiedHealthy,
|
||||
/// Authoritative metadata/presence proof only. This does not certify
|
||||
/// payload integrity.
|
||||
MetadataHealthy,
|
||||
AuthoritativelyAbsent,
|
||||
Deferred {
|
||||
reason: HealDeferredReason,
|
||||
@@ -105,17 +102,12 @@ pub struct HealObjectReceipt {
|
||||
|
||||
impl HealObjectReceipt {
|
||||
pub(crate) fn verified_for(&self, expected: &HealObjectIdentity) -> bool {
|
||||
let disposition_verifies = match self.disposition {
|
||||
matches!(
|
||||
self.disposition,
|
||||
HealObjectDisposition::Repaired
|
||||
| HealObjectDisposition::VerifiedHealthy
|
||||
| HealObjectDisposition::AuthoritativelyAbsent => true,
|
||||
// A metadata/presence proof never certifies payload bytes, so it
|
||||
// cannot discharge a request that exists to decode the payload.
|
||||
HealObjectDisposition::MetadataHealthy => expected.kind != HealObjectKind::Decode,
|
||||
_ => false,
|
||||
};
|
||||
disposition_verifies
|
||||
&& self.identity.kind == expected.kind
|
||||
| HealObjectDisposition::VerifiedHealthy
|
||||
| HealObjectDisposition::AuthoritativelyAbsent
|
||||
) && self.identity.kind == expected.kind
|
||||
&& self.identity.bucket == expected.bucket
|
||||
&& self.identity.object == expected.object
|
||||
&& self.identity.version_id == expected.version_id
|
||||
@@ -375,9 +367,7 @@ impl HealTaskOutcome {
|
||||
counters.overflowed |= !increment_counter(&mut counters.processed);
|
||||
let counter = match item.disposition {
|
||||
HealObjectDisposition::Repaired => &mut counters.healed,
|
||||
HealObjectDisposition::VerifiedHealthy
|
||||
| HealObjectDisposition::MetadataHealthy
|
||||
| HealObjectDisposition::AuthoritativelyAbsent => &mut counters.unchanged,
|
||||
HealObjectDisposition::VerifiedHealthy | HealObjectDisposition::AuthoritativelyAbsent => &mut counters.unchanged,
|
||||
HealObjectDisposition::Failed(_) => &mut counters.failed,
|
||||
HealObjectDisposition::Unknown => {
|
||||
counters.overflowed |= !increment_counter(&mut counters.unknown);
|
||||
@@ -534,32 +524,6 @@ mod canonical_outcome_tests {
|
||||
assert!(serde_json::from_value::<HealTaskOutcome>(value).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_health_receipt_never_discharges_a_payload_decode_request() {
|
||||
let incarnation = Uuid::new_v4();
|
||||
let mut expected = HealObjectIdentity {
|
||||
bucket_incarnation_id: Some(incarnation),
|
||||
..item(HealObjectDisposition::Unknown).identity
|
||||
};
|
||||
let mut receipt = HealObjectReceipt {
|
||||
identity: expected.clone(),
|
||||
disposition: HealObjectDisposition::MetadataHealthy,
|
||||
};
|
||||
assert!(
|
||||
receipt.verified_for(&expected),
|
||||
"a presence proof still settles a metadata-level object request"
|
||||
);
|
||||
|
||||
expected.kind = HealObjectKind::Decode;
|
||||
receipt.identity.kind = HealObjectKind::Decode;
|
||||
assert!(
|
||||
!receipt.verified_for(&expected),
|
||||
"a presence-only proof must not clear a payload decode responsibility"
|
||||
);
|
||||
receipt.disposition = HealObjectDisposition::VerifiedHealthy;
|
||||
assert!(receipt.verified_for(&expected));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_outcome_categories_have_one_terminal_count() {
|
||||
let mut outcome = HealTaskOutcome::default();
|
||||
@@ -567,7 +531,6 @@ mod canonical_outcome_tests {
|
||||
HealObjectDisposition::Unknown,
|
||||
HealObjectDisposition::Repaired,
|
||||
HealObjectDisposition::VerifiedHealthy,
|
||||
HealObjectDisposition::MetadataHealthy,
|
||||
HealObjectDisposition::AuthoritativelyAbsent,
|
||||
HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::DanglingDeleteGrace,
|
||||
@@ -580,7 +543,7 @@ mod canonical_outcome_tests {
|
||||
outcome.record(item(disposition));
|
||||
}
|
||||
let c = &outcome.counters;
|
||||
assert_eq!((c.processed, c.healed, c.unchanged, c.skipped, c.failed, c.unknown), (9, 1, 3, 4, 1, 1));
|
||||
assert_eq!((c.processed, c.healed, c.unchanged, c.skipped, c.failed, c.unknown), (8, 1, 2, 4, 1, 1));
|
||||
assert_eq!(c.processed, c.healed + c.unchanged + c.skipped + c.failed);
|
||||
}
|
||||
|
||||
|
||||
@@ -109,9 +109,7 @@ fn verified_object_receipt(
|
||||
item: &HealResultItem,
|
||||
bucket_incarnation_id: Uuid,
|
||||
) -> Option<HealObjectReceipt> {
|
||||
if opts.dry_run
|
||||
|| (!item.integrity_verified && !item.repair_verified && !item.metadata_verified && !item.metadata_repair_verified)
|
||||
{
|
||||
if opts.dry_run || (!item.integrity_verified && !item.repair_verified) {
|
||||
return None;
|
||||
}
|
||||
let resolved_version = Uuid::from_bytes(item.resolved_version_id?);
|
||||
@@ -122,7 +120,7 @@ fn verified_object_receipt(
|
||||
}
|
||||
item.drives_reported()?;
|
||||
let drives_healed = item.drives_healed()?;
|
||||
if (item.repair_verified || item.metadata_repair_verified) && drives_healed == 0 {
|
||||
if item.repair_verified && drives_healed == 0 {
|
||||
return None;
|
||||
}
|
||||
let ok_drive_state = DriveState::Ok.to_string();
|
||||
@@ -144,12 +142,10 @@ fn verified_object_receipt(
|
||||
pool_index: opts.pool,
|
||||
set_index: opts.set,
|
||||
},
|
||||
disposition: if drives_healed > 0 && (item.integrity_verified || item.repair_verified || item.metadata_repair_verified) {
|
||||
disposition: if drives_healed > 0 {
|
||||
HealObjectDisposition::Repaired
|
||||
} else if drives_healed == 0 && item.integrity_verified {
|
||||
} else if item.integrity_verified {
|
||||
HealObjectDisposition::VerifiedHealthy
|
||||
} else if drives_healed == 0 && item.metadata_verified {
|
||||
HealObjectDisposition::MetadataHealthy
|
||||
} else {
|
||||
return None;
|
||||
},
|
||||
@@ -532,19 +528,6 @@ pub trait HealStorageAPI: Send + Sync {
|
||||
Err(Error::other("storage does not support incarnation-bound object healing"))
|
||||
}
|
||||
|
||||
/// Durable MRF repair may request an authoritative unversioned absence proof.
|
||||
async fn heal_mrf_object_at_incarnation(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<&str>,
|
||||
expected: Uuid,
|
||||
opts: &HealOpts,
|
||||
) -> Result<HealStorageObjectResult> {
|
||||
self.heal_object_at_incarnation(bucket, object, version_id, expected, opts)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Heal object using ecstore
|
||||
async fn heal_object(
|
||||
&self,
|
||||
@@ -591,11 +574,6 @@ pub trait HealStorageAPI: Send + Sync {
|
||||
Err(Error::other("replacement pool metadata placement is unsupported"))
|
||||
}
|
||||
|
||||
/// Heal and physically verify bucket configuration on its replacement targets.
|
||||
async fn heal_replacement_bucket_metadata(&self, _bucket: &str, _opts: &HealOpts, _targets: &[String]) -> Result<()> {
|
||||
Err(Error::Storage(StorageError::PreconditionFailed))
|
||||
}
|
||||
|
||||
/// Read target-specific physical evidence for one replacement version.
|
||||
///
|
||||
/// This is only used by automatic replacement healing after the normal
|
||||
@@ -774,10 +752,7 @@ impl ECStoreHealStorage {
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else if error.is_none()
|
||||
&& !opts.dry_run
|
||||
&& (item.integrity_verified || item.repair_verified || item.metadata_verified || item.metadata_repair_verified)
|
||||
{
|
||||
} else if error.is_none() && !opts.dry_run && (item.integrity_verified || item.repair_verified) {
|
||||
let bucket_incarnation_id = match expected {
|
||||
Some(expected) => Some(expected),
|
||||
None => self.ecstore.bucket_incarnation_id(bucket).await.ok(),
|
||||
@@ -985,24 +960,6 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
.await)
|
||||
}
|
||||
|
||||
async fn heal_mrf_object_at_incarnation(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<&str>,
|
||||
expected: Uuid,
|
||||
opts: &HealOpts,
|
||||
) -> Result<HealStorageObjectResult> {
|
||||
let result = self
|
||||
.ecstore
|
||||
.heal_mrf_object_at_incarnation(bucket, object, version_id.unwrap_or_default(), expected, opts)
|
||||
.await
|
||||
.map_err(|error| incarnation_storage_error(bucket, expected, error))?;
|
||||
Ok(self
|
||||
.object_result_with_receipt(bucket, object, version_id, opts, result, Some(expected))
|
||||
.await)
|
||||
}
|
||||
|
||||
async fn get_object_meta(&self, bucket: &str, object: &str) -> Result<Option<HealObjectInfo>> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
@@ -1600,13 +1557,6 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
.map_err(Error::Storage)
|
||||
}
|
||||
|
||||
async fn heal_replacement_bucket_metadata(&self, bucket: &str, opts: &HealOpts, targets: &[String]) -> Result<()> {
|
||||
self.ecstore
|
||||
.heal_replacement_bucket_metadata(bucket, opts, targets)
|
||||
.await
|
||||
.map_err(Error::Storage)
|
||||
}
|
||||
|
||||
async fn replacement_targets_have_version(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -2132,51 +2082,6 @@ mod tests {
|
||||
assert!(verified_object_receipt("bucket", "object", None, &options, &item, incarnation).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_health_and_marker_repair_have_distinct_receipts() {
|
||||
use super::{HealObjectDisposition, HealOpts, HealResultItem, Uuid, verified_object_receipt};
|
||||
use rustfs_madmin::heal_commands::HealDriveInfo;
|
||||
|
||||
let incarnation = Uuid::new_v4();
|
||||
let version = Uuid::new_v4().to_string();
|
||||
let options = HealOpts::default();
|
||||
let healthy = HealDriveInfo {
|
||||
state: "ok".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let mut item = HealResultItem {
|
||||
metadata_verified: true,
|
||||
version_id: version.clone(),
|
||||
resolved_version_id: Some(*Uuid::parse_str(&version).expect("version UUID").as_bytes()),
|
||||
..Default::default()
|
||||
};
|
||||
item.before.drives.push(healthy.clone());
|
||||
item.after.drives.push(healthy);
|
||||
|
||||
let receipt = verified_object_receipt("bucket", "object", Some(&version), &options, &item, incarnation)
|
||||
.expect("metadata quorum should certify metadata health");
|
||||
assert_eq!(receipt.disposition, HealObjectDisposition::MetadataHealthy);
|
||||
|
||||
item.integrity_verified = true;
|
||||
let receipt = verified_object_receipt("bucket", "object", Some(&version), &options, &item, incarnation)
|
||||
.expect("stronger integrity proof should remain available");
|
||||
assert_eq!(receipt.disposition, HealObjectDisposition::VerifiedHealthy);
|
||||
|
||||
item.integrity_verified = false;
|
||||
item.metadata_verified = false;
|
||||
item.metadata_repair_verified = true;
|
||||
item.before.drives[0].state = "missing".to_string();
|
||||
let receipt = verified_object_receipt("bucket", "object", Some(&version), &options, &item, incarnation)
|
||||
.expect("a committed metadata repair should certify the marker/version");
|
||||
assert_eq!(receipt.disposition, HealObjectDisposition::Repaired);
|
||||
|
||||
item.before.drives[0].state = "ok".to_string();
|
||||
assert!(
|
||||
verified_object_receipt("bucket", "object", Some(&version), &options, &item, incarnation).is_none(),
|
||||
"repair proof without a repaired drive must fail closed"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn next_heal_listing_token_returns_none_for_complete_page() {
|
||||
assert_eq!(
|
||||
|
||||
@@ -298,18 +298,6 @@ impl HealTask {
|
||||
"Heal object stage entered"
|
||||
);
|
||||
let object_size = result.object_size as u64;
|
||||
let ok_drive_state = DriveState::Ok.to_string();
|
||||
if self.source == HealRequestSource::Admin
|
||||
&& !self.options.dry_run
|
||||
&& !result.after.drives.is_empty()
|
||||
&& result.after.drives.iter().any(|drive| drive.state != ok_drive_state)
|
||||
{
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!(
|
||||
"Heal left one or more drives unhealthy for {bucket}/{object}; retry after the missing drives are restored"
|
||||
),
|
||||
});
|
||||
}
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_RESULT,
|
||||
@@ -429,10 +417,6 @@ impl HealTask {
|
||||
|
||||
/// Recreate missing object (for EC decode scenarios)
|
||||
async fn recreate_missing_object(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> {
|
||||
if self.source == HealRequestSource::Mrf {
|
||||
return self.recreate_missing_mrf_object(bucket, object, version_id).await;
|
||||
}
|
||||
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_STAGE,
|
||||
@@ -533,65 +517,4 @@ impl HealTask {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Durable MRF responsibilities may complete only with an exact storage proof.
|
||||
async fn recreate_missing_mrf_object(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> {
|
||||
let heal_opts = HealOpts {
|
||||
recursive: false,
|
||||
dry_run: self.options.dry_run,
|
||||
remove: false,
|
||||
recreate: true,
|
||||
scan_mode: HealScanMode::Deep,
|
||||
update_parity: true,
|
||||
no_lock: self.options.no_lock,
|
||||
read_repair: false,
|
||||
pool: self.options.pool_index,
|
||||
set: self.options.set_index,
|
||||
};
|
||||
let mut expected = self.outcome_identity(bucket, object, version_id, self.options.pool_index, self.options.set_index);
|
||||
let bucket_incarnation_id = self
|
||||
.outcome_bucket_incarnation_id(bucket, self.options.dry_run)
|
||||
.await?
|
||||
.ok_or_else(|| Error::TaskExecutionFailed {
|
||||
message: format!("Missing bucket incarnation for durable MRF repair {bucket}/{object}"),
|
||||
})?;
|
||||
expected.bucket_incarnation_id = Some(bucket_incarnation_id);
|
||||
|
||||
let storage_result = self
|
||||
.await_with_control(self.storage.heal_mrf_object_at_incarnation(
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
bucket_incarnation_id,
|
||||
&heal_opts,
|
||||
))
|
||||
.await?;
|
||||
if let Some(error) = storage_result.error {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to recreate missing object {bucket}/{object}: {error}"),
|
||||
});
|
||||
}
|
||||
|
||||
let object_size = storage_result.item.object_size as u64;
|
||||
let authoritatively_absent = matches!(
|
||||
storage_result.receipt.as_ref().map(|receipt| &receipt.disposition),
|
||||
Some(HealObjectDisposition::AuthoritativelyAbsent)
|
||||
);
|
||||
if !self.record_verified_storage_receipt(expected, storage_result.receipt).await {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Missing exact storage proof for durable MRF repair {bucket}/{object}"),
|
||||
});
|
||||
}
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
if authoritatively_absent {
|
||||
progress.update_object_progress(1, 0, 0, 1, 0);
|
||||
} else {
|
||||
progress.update_object_progress(1, 1, 0, 0, object_size);
|
||||
}
|
||||
}
|
||||
self.record_result_item(storage_result.item).await;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3471,40 +3471,6 @@ async fn test_prefix_heal_lists_and_repairs_objects_under_prefix() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_object_heal_fails_when_drive_remains_unhealthy() {
|
||||
let storage = Arc::new(MockStorage::default());
|
||||
storage.heal_object_outcomes.lock().expect("outcomes").insert(
|
||||
"object-a".to_string(),
|
||||
VecDeque::from([MockHealObjectOutcome::UnavailableDrive(DriveState::Missing)]),
|
||||
);
|
||||
let mut request = HealRequest::new(
|
||||
HealType::Object {
|
||||
bucket: "bucket-a".to_string(),
|
||||
object: "object-a".to_string(),
|
||||
version_id: None,
|
||||
},
|
||||
HealOptions {
|
||||
recreate_missing: false,
|
||||
timeout: None,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
);
|
||||
request.source = HealRequestSource::Admin;
|
||||
let task = HealTask::from_request(request, storage);
|
||||
|
||||
let error = task
|
||||
.execute()
|
||||
.await
|
||||
.expect_err("admin heal must not finish with an unhealthy drive");
|
||||
assert!(error.to_string().contains("left one or more drives unhealthy"));
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.failed, 1);
|
||||
assert_eq!(outcome.counters.unknown, 0);
|
||||
assert_eq!(outcome.execution, HealExecutionOutcome::CompletedWithErrors);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_data_usage_cache_lock_timeout_does_not_fail_object_heal() {
|
||||
let storage = Arc::new(MockStorage::default());
|
||||
@@ -3889,67 +3855,6 @@ async fn test_heal_recreate_scanner_non_dir_not_found_fails() {
|
||||
assert!(matches!(task.get_status().await, HealTaskStatus::Failed { .. }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_recreate_missing_object_records_exact_absence_receipt_with_scope() {
|
||||
let incarnation = Uuid::new_v4();
|
||||
let storage = Arc::new(MockStorage {
|
||||
object_exists: Mutex::new(Some(false)),
|
||||
bucket_incarnation_id: Mutex::new(Some(incarnation)),
|
||||
heal_object_receipts: Mutex::new(HashMap::from([(
|
||||
"deleted.bin".to_string(),
|
||||
VecDeque::from([HealObjectReceipt {
|
||||
identity: HealObjectIdentity {
|
||||
kind: HealObjectKind::Object,
|
||||
bucket: "bucket-a".to_string(),
|
||||
object: "deleted.bin".to_string(),
|
||||
version_id: None,
|
||||
bucket_incarnation_id: Some(incarnation),
|
||||
pool_index: Some(2),
|
||||
set_index: Some(3),
|
||||
},
|
||||
disposition: HealObjectDisposition::AuthoritativelyAbsent,
|
||||
}]),
|
||||
)])),
|
||||
..Default::default()
|
||||
});
|
||||
let mut request = HealRequest::new(
|
||||
HealType::Object {
|
||||
bucket: "bucket-a".to_string(),
|
||||
object: "deleted.bin".to_string(),
|
||||
version_id: None,
|
||||
},
|
||||
HealOptions {
|
||||
recreate_missing: true,
|
||||
pool_index: Some(2),
|
||||
set_index: Some(3),
|
||||
timeout: None,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
);
|
||||
request.source = HealRequestSource::Mrf;
|
||||
let task = HealTask::from_request(request, storage.clone());
|
||||
|
||||
task.execute()
|
||||
.await
|
||||
.expect("a complete MRF absence receipt must complete the task");
|
||||
|
||||
let opts = storage.object_heal_opts.lock().unwrap()[0];
|
||||
assert_eq!(opts.pool, Some(2));
|
||||
assert_eq!(opts.set, Some(3));
|
||||
assert!(opts.recreate);
|
||||
assert_eq!(opts.scan_mode, HealScanMode::Deep);
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.unchanged, 1);
|
||||
assert_eq!(outcome.counters.unknown, 0);
|
||||
assert_eq!(outcome.objects.len(), 1);
|
||||
assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::AuthoritativelyAbsent);
|
||||
assert_eq!(
|
||||
(outcome.objects[0].identity.pool_index, outcome.objects[0].identity.set_index),
|
||||
(Some(2), Some(3))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_heal_scanner_missing_object_without_recreate_probes_storage() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
|
||||
@@ -28,7 +28,6 @@ use http::HeaderMap;
|
||||
use rustfs_filemeta::{FileInfo, FileMeta};
|
||||
use rustfs_heal::heal::{
|
||||
manager::{HealConfig, HealManager},
|
||||
outcome::HealObjectDisposition,
|
||||
storage::{
|
||||
ECStoreHealStorage, HealListItem, HealObjectOptions as ObjectOptions, HealPutObjReader as PutObjReader, HealStorageAPI,
|
||||
},
|
||||
@@ -36,7 +35,6 @@ use rustfs_heal::heal::{
|
||||
};
|
||||
use rustfs_heal_contracts::heal_channel::{HealOpts, HealScanMode};
|
||||
use serial_test::serial;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::{
|
||||
path::{Path, PathBuf},
|
||||
sync::Arc,
|
||||
@@ -367,12 +365,7 @@ mod serial_tests {
|
||||
assert!(healed.error.is_none(), "exact version repair failed: {:?}", healed.error);
|
||||
if item.is_delete_marker {
|
||||
assert!(!healed.item.integrity_verified);
|
||||
let receipt = healed.receipt.expect("delete markers require an exact metadata receipt");
|
||||
assert_eq!(receipt.identity.version_id, item.version_id);
|
||||
assert!(matches!(
|
||||
receipt.disposition,
|
||||
HealObjectDisposition::Repaired | HealObjectDisposition::MetadataHealthy
|
||||
));
|
||||
assert!(healed.receipt.is_none(), "delete markers carry no shard-integrity proof");
|
||||
} else {
|
||||
assert!(healed.item.integrity_verified);
|
||||
let receipt = healed
|
||||
@@ -442,7 +435,7 @@ mod serial_tests {
|
||||
let mut latest_data = versioned_test_data(6);
|
||||
latest_data.extend_from_slice(b"new-uuid");
|
||||
let latest = put_versioned(&ecstore, &bucket, object, &latest_data).await;
|
||||
versions.push((latest.clone(), latest_data.clone()));
|
||||
versions.push((latest, latest_data.clone()));
|
||||
|
||||
let target = disk_paths
|
||||
.iter()
|
||||
@@ -526,17 +519,7 @@ mod serial_tests {
|
||||
.expect("an omitted selector must still heal latest");
|
||||
assert!(latest_result.error.is_none());
|
||||
assert_eq!(latest_result.item.object_size, latest_data.len());
|
||||
assert!(
|
||||
latest_result.receipt.is_none(),
|
||||
"an omitted selector certifies metadata health only for the null identity"
|
||||
);
|
||||
let latest_receipt = storage
|
||||
.heal_object_with_receipt(&bucket, object, Some(latest.as_str()), &HealOpts::default())
|
||||
.await
|
||||
.expect("a normal scan of the exact latest version")
|
||||
.receipt
|
||||
.expect("a normal scan should certify metadata health");
|
||||
assert_eq!(latest_receipt.disposition, HealObjectDisposition::MetadataHealthy);
|
||||
assert!(latest_result.receipt.is_none(), "a normal scan cannot certify payload integrity");
|
||||
let verified_latest = storage
|
||||
.heal_object_with_receipt(
|
||||
&bucket,
|
||||
@@ -723,131 +706,6 @@ mod serial_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn test_recursive_deep_heal_reports_stale_delete_marker_repair() {
|
||||
let (disk_paths, ecstore, storage) = heal_env().await;
|
||||
let manager = HealManager::new(
|
||||
storage.clone(),
|
||||
Some(HealConfig {
|
||||
heal_interval: Duration::from_millis(1),
|
||||
..Default::default()
|
||||
}),
|
||||
);
|
||||
manager.start().await.expect("heal manager should start");
|
||||
|
||||
let bucket = "b5-stale-delete-marker-recursive";
|
||||
let object = "obj.bin";
|
||||
create_versioned_bucket(&ecstore, bucket).await;
|
||||
put_unversioned(&ecstore, bucket, "control/object.bin", &versioned_test_data(42)).await;
|
||||
let historical_data = versioned_test_data(43);
|
||||
let historical = put_versioned(&ecstore, bucket, object, &historical_data).await;
|
||||
let target = &disk_paths[1];
|
||||
let target_meta = xl_meta_path(&object_dir(target, bucket, object));
|
||||
let stale_meta = std::fs::read(&target_meta).expect("target xl.meta must exist before marker creation");
|
||||
let stale_digest = Sha256::digest(&stale_meta);
|
||||
let marker = put_delete_marker(&ecstore, bucket, object).await;
|
||||
std::fs::write(&target_meta, &stale_meta).expect("restore stale target xl.meta");
|
||||
|
||||
let request = HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: bucket.to_string(),
|
||||
},
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
scan_mode: HealScanMode::Deep,
|
||||
pool_index: Some(0),
|
||||
set_index: Some(0),
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
);
|
||||
let task_id = request.id.clone();
|
||||
assert!(
|
||||
manager
|
||||
.submit_heal_request(request)
|
||||
.await
|
||||
.expect("submit recursive heal")
|
||||
.is_admitted()
|
||||
);
|
||||
wait_for_task(&manager, &task_id, Duration::from_secs(60)).await;
|
||||
|
||||
let report = manager.get_task_report(&task_id).await.expect("completed task report");
|
||||
let outcome = report.outcome.expect("recursive task must have an outcome");
|
||||
assert_eq!(outcome.counters.failed, 0);
|
||||
assert_eq!(outcome.counters.unknown, 0);
|
||||
assert_eq!(outcome.counters.processed, 3);
|
||||
assert_eq!(outcome.counters.healed, 1, "delete marker repair must be counted as healed");
|
||||
assert_eq!(outcome.counters.unchanged, 2);
|
||||
|
||||
let repaired_meta = std::fs::read(&target_meta).expect("repaired target xl.meta must exist");
|
||||
assert_ne!(
|
||||
Sha256::digest(&repaired_meta),
|
||||
stale_digest,
|
||||
"deep heal must replace stale rejoined metadata"
|
||||
);
|
||||
|
||||
let marker_info = physical_version(target, bucket, object, &marker);
|
||||
assert!(
|
||||
marker_info.deleted && marker_info.is_latest,
|
||||
"target metadata must converge to delete-marker latest"
|
||||
);
|
||||
let historical_info = physical_version(target, bucket, object, &historical);
|
||||
assert!(!historical_info.is_latest, "historical version must no longer be marked latest");
|
||||
assert_eq!(
|
||||
read_version(&ecstore, bucket, object, &historical).await,
|
||||
historical_data,
|
||||
"historical version must remain readable after marker convergence"
|
||||
);
|
||||
let marker_receipt = outcome
|
||||
.objects
|
||||
.iter()
|
||||
.find(|item| item.identity.version_id.as_deref() == Some(marker.as_str()))
|
||||
.expect("deep heal must emit a receipt for the repaired delete marker");
|
||||
assert_eq!(marker_receipt.disposition, HealObjectDisposition::Repaired);
|
||||
manager.stop().await.expect("heal manager should stop");
|
||||
}
|
||||
|
||||
/// A missing xl.meta must produce an exact marker repair receipt, while a
|
||||
/// healthy replay must prove metadata health without claiming payload integrity.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn test_delete_marker_receipt_repair_then_metadata_healthy() {
|
||||
let (disk_paths, ecstore, heal_storage) = heal_env().await;
|
||||
let bucket = "b5-dm-receipt";
|
||||
let object = "marker.bin";
|
||||
create_versioned_bucket(&ecstore, bucket).await;
|
||||
|
||||
let _data = put_versioned(&ecstore, bucket, object, &versioned_test_data(41)).await;
|
||||
let marker = put_delete_marker(&ecstore, bucket, object).await;
|
||||
let obj_dir = object_dir(&disk_paths[0], bucket, object);
|
||||
tokio::fs::remove_file(xl_meta_path(&obj_dir))
|
||||
.await
|
||||
.expect("remove one xl.meta replica");
|
||||
|
||||
let repaired = heal_storage
|
||||
.heal_object_with_receipt(bucket, object, Some(&marker), &recreate_heal_opts())
|
||||
.await
|
||||
.expect("marker repair request");
|
||||
assert!(repaired.error.is_none(), "{:?}", repaired.error);
|
||||
assert!(repaired.item.metadata_repair_verified);
|
||||
let receipt = repaired.receipt.expect("committed marker repair receipt");
|
||||
assert_eq!(receipt.disposition, HealObjectDisposition::Repaired);
|
||||
assert_eq!(receipt.identity.version_id.as_deref(), Some(marker.as_str()));
|
||||
|
||||
let healthy = heal_storage
|
||||
.heal_object_with_receipt(bucket, object, Some(&marker), &recreate_heal_opts())
|
||||
.await
|
||||
.expect("healthy marker replay");
|
||||
assert!(healthy.error.is_none(), "{:?}", healthy.error);
|
||||
assert_eq!(healthy.item.drives_healed(), Some(0));
|
||||
assert!(healthy.item.metadata_verified);
|
||||
assert!(!healthy.item.integrity_verified);
|
||||
let receipt = healthy.receipt.expect("metadata health receipt");
|
||||
assert_eq!(receipt.disposition, HealObjectDisposition::MetadataHealthy);
|
||||
assert_eq!(receipt.identity.version_id.as_deref(), Some(marker.as_str()));
|
||||
}
|
||||
|
||||
/// Enumerated unversioned objects select the exact null slot once each.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
|
||||
@@ -717,49 +717,6 @@ mod absence_receipt_regressions {
|
||||
assert_versions(&store, bucket, &old, ¤t).await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn unversioned_missing_object_produces_fenced_absence_receipt() {
|
||||
let bucket = "absence-receipt-unversioned";
|
||||
let object = "deleted.bin";
|
||||
let (_paths, store, storage) = heal_env_n(16).await;
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create unversioned bucket");
|
||||
let incarnation = storage
|
||||
.admit_bucket_incarnation(bucket)
|
||||
.await
|
||||
.expect("admit bucket incarnation for the durable proof");
|
||||
|
||||
let result = storage
|
||||
.heal_mrf_object_at_incarnation(
|
||||
bucket,
|
||||
object,
|
||||
None,
|
||||
incarnation,
|
||||
&HealOpts {
|
||||
pool: Some(0),
|
||||
set: Some(0),
|
||||
..deep_heal_opts()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("fenced absence probe should return a storage result");
|
||||
assert!(
|
||||
result.error.is_none(),
|
||||
"complete unversioned absence must not remain an error: {:?}",
|
||||
result.error
|
||||
);
|
||||
let receipt = result.receipt.expect("complete unversioned absence needs a receipt");
|
||||
assert_eq!(receipt.disposition, HealObjectDisposition::AuthoritativelyAbsent);
|
||||
assert_eq!(receipt.identity.bucket, bucket);
|
||||
assert_eq!(receipt.identity.object, object);
|
||||
assert!(receipt.identity.version_id.is_none());
|
||||
assert_eq!((receipt.identity.pool_index, receipt.identity.set_index), (Some(0), Some(0)));
|
||||
assert_eq!(receipt.identity.bucket_incarnation_id, Some(incarnation));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn historical_absence_receipt_bucket_outcome_matches_c06() {
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Replacement healing must restore internal records only on their owning sets.
|
||||
//! Regression for replacement healing requiring pool.bin on a non-owning set.
|
||||
#![recursion_limit = "256"]
|
||||
|
||||
use http::HeaderMap;
|
||||
@@ -101,31 +101,6 @@ async fn replacement_pool_metadata_follows_real_two_set_placement() {
|
||||
storage.replacement_pool_metadata_required(&opts).is_err(),
|
||||
"invalid scope must fail closed"
|
||||
);
|
||||
assert!(
|
||||
storage
|
||||
.heal_replacement_bucket_metadata(bucket, &opts, &[paths[0].to_string_lossy().into_owned()])
|
||||
.await
|
||||
.is_err(),
|
||||
"bucket metadata repair must reject an incomplete or invalid replacement scope"
|
||||
);
|
||||
}
|
||||
|
||||
for targets in [vec![], vec![paths[4].to_string_lossy().into_owned()]] {
|
||||
assert!(
|
||||
storage
|
||||
.heal_replacement_bucket_metadata(
|
||||
bucket,
|
||||
&HealOpts {
|
||||
pool: Some(0),
|
||||
set: Some(0),
|
||||
..Default::default()
|
||||
},
|
||||
&targets,
|
||||
)
|
||||
.await
|
||||
.is_err(),
|
||||
"empty or wrong-set targets must be rejected before repairing metadata"
|
||||
);
|
||||
}
|
||||
|
||||
let mut owning_set = None;
|
||||
@@ -145,17 +120,6 @@ async fn replacement_pool_metadata_follows_real_two_set_placement() {
|
||||
assert!(owning_set.replace(set).is_none(), "only one set owns pool.bin");
|
||||
std::fs::remove_dir_all(&metadata_path).unwrap();
|
||||
}
|
||||
let bucket_records: Vec<_> = [".metadata.bin", ".bucket-incarnation"]
|
||||
.into_iter()
|
||||
.map(|file| {
|
||||
let path = target_path.join(RUSTFS_META_BUCKET).join("buckets").join(bucket).join(file);
|
||||
let existed = path.join("xl.meta").exists();
|
||||
if existed {
|
||||
std::fs::remove_dir_all(&path).expect("remove bucket metadata shard from replacement");
|
||||
}
|
||||
(path, existed)
|
||||
})
|
||||
.collect();
|
||||
// Select a user key using the real placement algorithm, independently
|
||||
// of the metadata-scope decision under test.
|
||||
let key = (0..1000)
|
||||
@@ -227,14 +191,6 @@ async fn replacement_pool_metadata_follows_real_two_set_placement() {
|
||||
"completed repair must clear its healing marker"
|
||||
);
|
||||
assert_eq!(metadata_path.join("xl.meta").exists(), owns_metadata);
|
||||
for (path, owned) in bucket_records {
|
||||
assert_eq!(
|
||||
path.join("xl.meta").exists(),
|
||||
owned,
|
||||
"replacement must restore each owned bucket record without creating a wrong-set copy: {}",
|
||||
path.display()
|
||||
);
|
||||
}
|
||||
for (relative, bytes) in original_parts {
|
||||
assert_eq!(std::fs::read(object_path.join(relative)).unwrap(), bytes, "reconstructed shard differs");
|
||||
}
|
||||
|
||||
@@ -90,82 +90,6 @@ async fn partial_write_persistence_failure_is_reported_and_retained_for_retry()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unversioned_deleted_partial_write_is_discharged_by_an_absence_proof() {
|
||||
const STACK_SIZE: usize = 8 * 1024 * 1024;
|
||||
std::thread::Builder::new()
|
||||
.name("mrf-partial-write-absence".to_owned())
|
||||
.stack_size(STACK_SIZE)
|
||||
.spawn(|| {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.thread_stack_size(STACK_SIZE)
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("partial-write absence runtime should build");
|
||||
runtime.block_on(unversioned_deleted_partial_write_is_discharged_by_an_absence_proof_inner());
|
||||
})
|
||||
.expect("partial-write absence test thread should spawn")
|
||||
.join()
|
||||
.expect("partial-write absence test thread should finish");
|
||||
}
|
||||
|
||||
async fn unversioned_deleted_partial_write_is_discharged_by_an_absence_proof_inner() {
|
||||
use rustfs_common::mrf_channel::{MrfScope, persist_partial_write_intent};
|
||||
|
||||
temp_env::async_with_vars([("RUSTFS_HEAL_MRF_ENABLE", Some("true"))], async {
|
||||
let root = tempfile::tempdir().expect("partial-write absence fixture directory");
|
||||
let env = TestECStoreEnv::builder().disk_count(16).base_dir(root.path()).build().await;
|
||||
env.make_bucket("partial-absence", false).await;
|
||||
let mut coordinator_pool = env.endpoint_pools.as_ref()[0].clone();
|
||||
let mut endpoints = coordinator_pool.endpoints.as_ref().to_vec();
|
||||
for endpoint in endpoints.iter_mut().skip(4) {
|
||||
endpoint.is_local = false;
|
||||
}
|
||||
coordinator_pool.endpoints = Endpoints::from(endpoints);
|
||||
init_local_disks(EndpointServerPools::from(vec![coordinator_pool]))
|
||||
.await
|
||||
.expect("coordinator journal disks");
|
||||
|
||||
let manager = manager(&env);
|
||||
mrf_queue::spawn_mrf_consumer(manager.clone());
|
||||
for (object, version_id) in [
|
||||
("deleted-unversioned.bin", None),
|
||||
("deleted-versioned.bin", Some(uuid::Uuid::new_v4())),
|
||||
] {
|
||||
persist_partial_write_intent(
|
||||
"partial-absence",
|
||||
object,
|
||||
version_id,
|
||||
MrfScope {
|
||||
pool_index: 0,
|
||||
set_index: 0,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("durable partial-write responsibility must commit before scheduling");
|
||||
assert!(snapshot_contains(object).await, "committed responsibility must exist before repair runs");
|
||||
}
|
||||
|
||||
manager.start().await.expect("MRF scheduler should start");
|
||||
for object in ["deleted-unversioned.bin", "deleted-versioned.bin"] {
|
||||
assert!(
|
||||
wait_until(|| async { !snapshot_contains(object).await }).await,
|
||||
"complete absence proof must discharge the durable responsibility"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
wait_until(|| async {
|
||||
let snapshot = manager.operations_snapshot().await;
|
||||
snapshot.queue_length == 0 && snapshot.active_tasks == 0
|
||||
})
|
||||
.await,
|
||||
"discharged absence repair must leave no queued work"
|
||||
);
|
||||
manager.stop().await.expect("absence manager should stop");
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn wait_until<F: FnMut() -> Fut, Fut: Future<Output = bool>>(mut probe: F) -> bool {
|
||||
tokio::time::timeout(Duration::from_secs(60), async {
|
||||
loop {
|
||||
|
||||
@@ -24,9 +24,7 @@ use serial_test::serial;
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
|
||||
mod storage_api;
|
||||
use storage_api::integration::{
|
||||
DiskAPI, ObjectIO, ObjectOptions, PutObjReader, ReadOptions, ShardIntegrityWriteMode, WriteCompletion,
|
||||
};
|
||||
use storage_api::integration::{DiskAPI, ObjectIO, ObjectOptions, PutObjReader, ReadOptions, ShardIntegrityWriteMode};
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
@@ -48,8 +46,6 @@ async fn legacy_repair_reports_execution_without_strong_receipt() {
|
||||
&mut PutObjReader::from_vec(expected.clone()),
|
||||
&ObjectOptions {
|
||||
shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Legacy),
|
||||
// Disk inspection and fault injection require every shard rename to finish.
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
@@ -200,12 +196,7 @@ async fn receipt_requires_independent_deep_verification() {
|
||||
)
|
||||
.await
|
||||
.expect("normal presence scan");
|
||||
let normal_receipt = normal.receipt.expect("a normal presence scan should certify metadata health");
|
||||
assert_eq!(
|
||||
normal_receipt.disposition,
|
||||
HealObjectDisposition::MetadataHealthy,
|
||||
"a presence scan must not be promoted to payload VerifiedHealthy"
|
||||
);
|
||||
assert!(normal.receipt.is_none(), "a presence scan cannot certify payload integrity");
|
||||
let result = storage
|
||||
.heal_object_with_receipt(
|
||||
bucket,
|
||||
|
||||
@@ -69,15 +69,6 @@ pub fn record_remote_lock_rpc_timeout(peer: &str, op: &'static str) {
|
||||
counter!("rustfs_remote_lock_rpc_timeouts_total", "peer" => peer.to_string(), "op" => op).increment(1);
|
||||
}
|
||||
|
||||
/// Record a lock acquisition rejected by the per-peer request breaker or
|
||||
/// in-flight admission limit.
|
||||
#[inline(always)]
|
||||
pub fn record_remote_lock_request_suppressed(peer: &str, op: &'static str, reason: &'static str) {
|
||||
use metrics::counter;
|
||||
counter!("rustfs_remote_lock_request_suppressed_total", "peer" => peer.to_string(), "op" => op, "reason" => reason)
|
||||
.increment(1);
|
||||
}
|
||||
|
||||
/// Record the cached lock channel to `peer` being evicted after an RPC failure.
|
||||
#[inline(always)]
|
||||
pub fn record_remote_lock_channel_eviction(peer: &str, trigger: &'static str) {
|
||||
@@ -238,7 +229,6 @@ mod tests {
|
||||
record_early_release();
|
||||
record_contention_event();
|
||||
record_remote_lock_rpc_timeout("http://peer:9000", "lock");
|
||||
record_remote_lock_request_suppressed("http://peer:9000", "lock", "breaker_open");
|
||||
record_remote_lock_channel_eviction("http://peer:9000", "timeout");
|
||||
record_remote_lock_channel_eviction_suppressed("http://peer:9000", "peer_recently_served");
|
||||
record_remote_lock_rpc_detached("lock", "detached");
|
||||
@@ -261,7 +251,6 @@ mod tests {
|
||||
"rustfs_lock_early_releases",
|
||||
"rustfs_lock_contentions",
|
||||
"rustfs_remote_lock_rpc_timeouts_total",
|
||||
"rustfs_remote_lock_request_suppressed_total",
|
||||
"rustfs_remote_lock_channel_evictions_total",
|
||||
"rustfs_remote_lock_channel_evictions_suppressed_total",
|
||||
"rustfs_remote_lock_rpc_detached_total",
|
||||
|
||||
@@ -241,9 +241,7 @@ impl RuleValidate for LifecycleRule {
|
||||
return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_DEL_MARKER_EXPIRATION_DAYS));
|
||||
}
|
||||
// Rule must have at least one action
|
||||
let has_expiration = self.expiration.as_ref().is_some_and(|expiration| {
|
||||
expiration.days.is_some() || expiration.date.is_some() || expiration.expired_object_delete_marker.is_some()
|
||||
});
|
||||
let has_expiration = self.expiration.is_some();
|
||||
let has_transition = self.transitions.as_ref().is_some_and(|t| !t.is_empty());
|
||||
// `NewerNoncurrentVersions` on its own is a MinIO extension, not an AWS
|
||||
// form: it keeps the newest N noncurrent versions and expires the rest
|
||||
@@ -4457,19 +4455,6 @@ mod tests {
|
||||
.expect_err("a negative retention count on a transition must be rejected");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn validate_rejects_empty_expiration_action() {
|
||||
let mut rule = rule_with_filter(LifecycleRuleFilter::default());
|
||||
rule.expiration = Some(LifecycleExpiration::default());
|
||||
|
||||
let err = config_with_rules(vec![rule])
|
||||
.validate(&ObjectLockConfiguration::default())
|
||||
.await
|
||||
.expect_err("an empty Expiration object must not count as an action");
|
||||
|
||||
assert_eq!(err.to_string(), ERR_LIFECYCLE_RULE_MUST_HAVE_ACTION);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn zero_newer_noncurrent_versions_means_no_count_constraint() {
|
||||
// Zero carries no constraint, matching how the batch limit path has
|
||||
|
||||
+24
-146
@@ -13,11 +13,12 @@
|
||||
// limitations under the License.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::hash::{Hash, Hasher};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::{Duration, SystemTime};
|
||||
use tokio::sync::RwLock;
|
||||
use tokio::time::{Instant, timeout};
|
||||
use tokio::time::Instant;
|
||||
|
||||
use crate::{
|
||||
FastLockGuard, GlobalLockManager, LockClient, LockId, LockInfo, LockManager, LockMetadata, LockPriority, LockRequest,
|
||||
@@ -130,52 +131,35 @@ impl LocalClient {
|
||||
self.manager.clone().unwrap_or_else(crate::get_global_lock_manager)
|
||||
}
|
||||
|
||||
/// Guard entries for one resource share a shard so expiration checks do not
|
||||
/// need to take every shard's write lock on the acquisition hot path.
|
||||
fn get_resource_shard_index(&self, resource: &crate::ObjectKey) -> usize {
|
||||
resource.shard_index(self.shard_mask)
|
||||
/// Get the shard index for a given lock ID
|
||||
fn get_shard_index(&self, lock_id: &LockId) -> usize {
|
||||
let mut hasher = std::collections::hash_map::DefaultHasher::new();
|
||||
lock_id.hash(&mut hasher);
|
||||
(hasher.finish() as usize) & self.shard_mask
|
||||
}
|
||||
|
||||
/// Get the shard for a given lock ID
|
||||
fn get_shard(&self, lock_id: &LockId) -> &Arc<RwLock<HashMap<LockId, LocalGuardEntry>>> {
|
||||
let index = self.get_resource_shard_index(&lock_id.resource);
|
||||
let index = self.get_shard_index(lock_id);
|
||||
&self.guard_storage[index]
|
||||
}
|
||||
|
||||
async fn reclaim_expired_guards_for_resource(&self, resource: &crate::ObjectKey) -> usize {
|
||||
let shard = &self.guard_storage[self.get_resource_shard_index(resource)];
|
||||
let has_expired = {
|
||||
let guards = shard.read().await;
|
||||
guards
|
||||
.iter()
|
||||
.any(|(lock_id, entry)| &lock_id.resource == resource && entry.is_expired())
|
||||
};
|
||||
if !has_expired {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let expired_entries = {
|
||||
let mut guards = shard.write().await;
|
||||
guards
|
||||
.extract_if(|lock_id, entry| &lock_id.resource == resource && entry.is_expired())
|
||||
.map(|(_, entry)| entry)
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
let expired_entries = Self::extract_expired_guards(&self.guard_storage, Some(resource)).await;
|
||||
Self::release_reclaimed_guards(expired_entries, Some(resource))
|
||||
}
|
||||
|
||||
async fn extract_expired_guards(storage: &GuardStorage) -> Vec<LocalGuardEntry> {
|
||||
async fn extract_expired_guards(storage: &GuardStorage, resource: Option<&crate::ObjectKey>) -> Vec<LocalGuardEntry> {
|
||||
let mut expired_entries = Vec::new();
|
||||
for shard in storage.iter() {
|
||||
let has_expired = {
|
||||
let guards = shard.read().await;
|
||||
guards.values().any(LocalGuardEntry::is_expired)
|
||||
};
|
||||
if !has_expired {
|
||||
continue;
|
||||
}
|
||||
let mut guards = shard.write().await;
|
||||
expired_entries.extend(guards.extract_if(|_, entry| entry.is_expired()).map(|(_, entry)| entry));
|
||||
expired_entries.extend(
|
||||
guards
|
||||
.extract_if(|lock_id, entry| {
|
||||
resource.is_none_or(|resource| &lock_id.resource == resource) && entry.is_expired()
|
||||
})
|
||||
.map(|(_, entry)| entry),
|
||||
);
|
||||
}
|
||||
expired_entries
|
||||
}
|
||||
@@ -213,7 +197,7 @@ impl LocalClient {
|
||||
let Some(storage) = storage.upgrade() else {
|
||||
break;
|
||||
};
|
||||
let expired_entries = Self::extract_expired_guards(&storage).await;
|
||||
let expired_entries = Self::extract_expired_guards(&storage, None).await;
|
||||
Self::release_reclaimed_guards(expired_entries, None);
|
||||
}
|
||||
});
|
||||
@@ -230,24 +214,11 @@ impl Default for LocalClient {
|
||||
impl LockClient for LocalClient {
|
||||
async fn acquire_lock(&self, request: &LockRequest) -> Result<LockResponse> {
|
||||
self.ensure_reaper();
|
||||
if request.lock_id.resource != request.resource {
|
||||
return Ok(LockResponse::failure(
|
||||
"Lock id resource does not match the requested resource",
|
||||
Duration::ZERO,
|
||||
));
|
||||
}
|
||||
let lock_manager = self.get_lock_manager();
|
||||
let reclaimed_before_acquire = self.reclaim_expired_guards_for_resource(&request.resource).await;
|
||||
let acquire_deadline = Instant::now()
|
||||
.checked_add(request.acquire_timeout)
|
||||
.unwrap_or_else(Instant::now);
|
||||
let lock_manager = self.get_lock_manager();
|
||||
let initial_remaining = acquire_deadline.saturating_duration_since(Instant::now());
|
||||
let reclaimed_before_acquire = if initial_remaining.is_zero() {
|
||||
0
|
||||
} else {
|
||||
timeout(initial_remaining, self.reclaim_expired_guards_for_resource(&request.resource))
|
||||
.await
|
||||
.unwrap_or(0)
|
||||
};
|
||||
|
||||
let build_lock_request = |acquire_timeout| match request.lock_type {
|
||||
LockType::Exclusive => crate::ObjectLockRequest::new_write(request.resource.clone(), request.owner.clone())
|
||||
@@ -291,11 +262,7 @@ impl LockClient for LocalClient {
|
||||
return Ok(LockResponse::success(lock_info, Duration::ZERO));
|
||||
}
|
||||
Err(crate::fast_lock::LockResult::Timeout) => {
|
||||
if !retried_after_reclaim {
|
||||
let remaining = acquire_deadline.saturating_duration_since(Instant::now());
|
||||
if !remaining.is_zero() {
|
||||
let _ = timeout(remaining, self.reclaim_expired_guards_for_resource(&request.resource)).await;
|
||||
}
|
||||
if !retried_after_reclaim && self.reclaim_expired_guards_for_resource(&request.resource).await > 0 {
|
||||
retried_after_reclaim = true;
|
||||
continue;
|
||||
}
|
||||
@@ -305,11 +272,7 @@ impl LockClient for LocalClient {
|
||||
current_owner,
|
||||
current_mode,
|
||||
}) => {
|
||||
if !retried_after_reclaim {
|
||||
let remaining = acquire_deadline.saturating_duration_since(Instant::now());
|
||||
if !remaining.is_zero() {
|
||||
let _ = timeout(remaining, self.reclaim_expired_guards_for_resource(&request.resource)).await;
|
||||
}
|
||||
if !retried_after_reclaim && self.reclaim_expired_guards_for_resource(&request.resource).await > 0 {
|
||||
retried_after_reclaim = true;
|
||||
continue;
|
||||
}
|
||||
@@ -327,11 +290,8 @@ impl LockClient for LocalClient {
|
||||
|
||||
async fn release(&self, lock_id: &LockId) -> Result<bool> {
|
||||
let shard = self.get_shard(lock_id);
|
||||
let guard = {
|
||||
let mut guards = shard.write().await;
|
||||
guards.remove(lock_id)
|
||||
};
|
||||
if let Some(guard) = guard {
|
||||
let mut guards = shard.write().await;
|
||||
if let Some(guard) = guards.remove(lock_id) {
|
||||
// Guard automatically releases the lock when dropped
|
||||
drop(guard.guard);
|
||||
Ok(true)
|
||||
@@ -458,88 +418,6 @@ mod tests {
|
||||
panic!("lock guard was not reaped before test deadline");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn acquire_does_not_wait_on_unrelated_guard_shards() {
|
||||
let manager = Arc::new(GlobalLockManager::new());
|
||||
let client = LocalClient::with_manager_and_reaper_interval(manager, Duration::from_secs(60));
|
||||
client.reaper_started.store(true, Ordering::Release);
|
||||
let resource = crate::ObjectKey::new("bucket", "resource-shard-hot-path");
|
||||
let target_shard = client.get_resource_shard_index(&resource);
|
||||
|
||||
let mut unrelated_writes = Vec::new();
|
||||
for (index, shard) in client.guard_storage.iter().enumerate() {
|
||||
if index != target_shard {
|
||||
unrelated_writes.push(shard.write().await);
|
||||
}
|
||||
}
|
||||
|
||||
let lock_request = request(resource, "owner", Duration::from_secs(30));
|
||||
let response = tokio::time::timeout(Duration::from_millis(100), client.acquire_lock(&lock_request))
|
||||
.await
|
||||
.expect("an acquire must not sweep unrelated guard shards")
|
||||
.expect("local acquire should return a response");
|
||||
assert!(response.success);
|
||||
|
||||
drop(unrelated_writes);
|
||||
assert!(client.release(&lock_request.lock_id).await.unwrap());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reclaim_expired_guard_only_writes_the_resource_shard() {
|
||||
let manager = Arc::new(GlobalLockManager::new());
|
||||
let client = LocalClient::with_manager_and_reaper_interval(manager, Duration::from_secs(60));
|
||||
client.reaper_started.store(true, Ordering::Release);
|
||||
let resource = crate::ObjectKey::new("bucket", "resource-shard-expired");
|
||||
let target_shard = client.get_resource_shard_index(&resource);
|
||||
let expired = request(resource.clone(), "expired-owner", Duration::ZERO);
|
||||
assert!(client.acquire_lock(&expired).await.unwrap().success);
|
||||
|
||||
let mut unrelated_writes = Vec::new();
|
||||
for (index, shard) in client.guard_storage.iter().enumerate() {
|
||||
if index != target_shard {
|
||||
unrelated_writes.push(shard.write().await);
|
||||
}
|
||||
}
|
||||
|
||||
let replacement = request(resource, "replacement-owner", Duration::from_secs(30));
|
||||
let response = tokio::time::timeout(Duration::from_millis(100), client.acquire_lock(&replacement))
|
||||
.await
|
||||
.expect("expired guard reclaim must only write the resource shard")
|
||||
.expect("local acquire should return a response");
|
||||
assert!(response.success);
|
||||
|
||||
drop(unrelated_writes);
|
||||
assert!(client.release(&replacement.lock_id).await.unwrap());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn acquire_deadline_includes_resource_shard_reclaim_wait() {
|
||||
let manager = Arc::new(GlobalLockManager::new());
|
||||
let client = LocalClient::with_manager_and_reaper_interval(manager, Duration::from_secs(60));
|
||||
client.reaper_started.store(true, Ordering::Release);
|
||||
let resource = crate::ObjectKey::new("bucket", "resource-shard-deadline");
|
||||
let target_shard = client.get_resource_shard_index(&resource);
|
||||
let _target_write = client.guard_storage[target_shard].write().await;
|
||||
let lock_request = request(resource, "owner", Duration::from_secs(30)).with_acquire_timeout(Duration::from_millis(10));
|
||||
|
||||
let response = tokio::time::timeout(Duration::from_millis(100), client.acquire_lock(&lock_request))
|
||||
.await
|
||||
.expect("acquire should return after its reclaim budget expires")
|
||||
.expect("local acquire should return a response");
|
||||
assert!(!response.success);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn acquire_rejects_a_lock_id_for_another_resource() {
|
||||
let client = LocalClient::with_manager(Arc::new(GlobalLockManager::new()));
|
||||
let mut lock_request = request(crate::ObjectKey::new("bucket", "requested"), "owner", Duration::from_secs(30));
|
||||
lock_request.lock_id.resource = crate::ObjectKey::new("bucket", "other");
|
||||
|
||||
let response = client.acquire_lock(&lock_request).await.unwrap();
|
||||
assert!(!response.success);
|
||||
assert!(response.error.unwrap().contains("does not match"));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn expired_guard_is_reaped_without_resource_reacquire() {
|
||||
let manager = Arc::new(GlobalLockManager::new());
|
||||
|
||||
@@ -45,15 +45,6 @@ pub struct HealResultItem {
|
||||
/// healthy or unchanged object.
|
||||
#[serde(skip)]
|
||||
pub repair_verified: bool,
|
||||
/// Storage-owner proof that the selected version/marker has authoritative
|
||||
/// metadata from read quorum. This is weaker than payload integrity proof
|
||||
/// and is intentionally in-process only.
|
||||
#[serde(skip)]
|
||||
pub metadata_verified: bool,
|
||||
/// Storage-owner proof that metadata or a delete marker was committed by
|
||||
/// this heal. This is intentionally in-process only.
|
||||
#[serde(skip)]
|
||||
pub metadata_repair_verified: bool,
|
||||
#[serde(rename = "resultId")]
|
||||
pub result_index: usize,
|
||||
#[serde(rename = "type")]
|
||||
@@ -129,10 +120,6 @@ mod tests {
|
||||
assert!(wire.get("resolvedVersionId").is_none());
|
||||
assert!(wire.get("repair_verified").is_none());
|
||||
assert!(wire.get("repairVerified").is_none());
|
||||
assert!(wire.get("metadata_verified").is_none());
|
||||
assert!(wire.get("metadataVerified").is_none());
|
||||
assert!(wire.get("metadata_repair_verified").is_none());
|
||||
assert!(wire.get("metadataRepairVerified").is_none());
|
||||
wire["resolved_version_id"] = serde_json::json!(vec![0; 16]);
|
||||
let decoded: HealResultItem = serde_json::from_value(wire).expect("legacy wire shape should remain readable");
|
||||
assert_eq!(decoded.resolved_version_id, None, "wire input cannot supply owner proof");
|
||||
|
||||
@@ -53,9 +53,7 @@ impl NotifyBucketConfigManager {
|
||||
self.rule_engine.has_subscriber(bucket, event).await
|
||||
}
|
||||
|
||||
/// Validates that the runtime can accept `cfg` for `bucket` without
|
||||
/// mutating any state.
|
||||
pub async fn validate_bucket_notification_config(
|
||||
pub async fn load_bucket_notification_config(
|
||||
&self,
|
||||
bucket: &str,
|
||||
cfg: &BucketNotificationConfig,
|
||||
@@ -100,11 +98,6 @@ impl NotifyBucketConfigManager {
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Publishes an already validated `cfg` to the runtime rule state.
|
||||
pub async fn apply_bucket_notification_config(&self, bucket: &str, cfg: &BucketNotificationConfig) {
|
||||
self.subscriber_view.apply_bucket_config(bucket, cfg);
|
||||
self.rule_engine.set_bucket_rules(bucket, cfg.get_rules_map().clone()).await;
|
||||
info!(
|
||||
@@ -116,15 +109,6 @@ impl NotifyBucketConfigManager {
|
||||
rule_count = cfg.get_rules_map().inner().len(),
|
||||
"notify bucket config state"
|
||||
);
|
||||
}
|
||||
|
||||
pub async fn load_bucket_notification_config(
|
||||
&self,
|
||||
bucket: &str,
|
||||
cfg: &BucketNotificationConfig,
|
||||
) -> Result<(), NotificationError> {
|
||||
self.validate_bucket_notification_config(bucket, cfg).await?;
|
||||
self.apply_bucket_notification_config(bucket, cfg).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -138,7 +122,7 @@ impl NotifyBucketConfigManager {
|
||||
mod tests {
|
||||
use super::NotifyBucketConfigManager;
|
||||
use crate::{
|
||||
BucketNotificationConfig, NotificationError, integration::NotificationMetrics,
|
||||
BucketNotificationConfig, integration::NotificationMetrics,
|
||||
notification_system_subscriber::NotificationSystemSubscriberView, notifier::EventNotifier, rule_engine::NotifyRuleEngine,
|
||||
};
|
||||
use rustfs_s3_types::EventName;
|
||||
@@ -172,37 +156,4 @@ mod tests {
|
||||
manager.remove_bucket_notification_config("bucket").await;
|
||||
assert!(!manager.subscriber_view.has_subscriber("bucket", &EventName::ObjectCreatedPut));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn validate_bucket_notification_config_rejects_missing_targets_without_applying_rules() {
|
||||
let manager = build_manager();
|
||||
let target_id = TargetID::new("primary".to_string(), "webhook".to_string());
|
||||
let mut cfg = BucketNotificationConfig::new("us-east-1");
|
||||
cfg.add_rule(&[EventName::ObjectCreatedPut], "*".to_string(), target_id);
|
||||
|
||||
let err = manager
|
||||
.validate_bucket_notification_config("bucket", &cfg)
|
||||
.await
|
||||
.expect_err("validation must fail when the runtime has no notify targets");
|
||||
assert!(matches!(err, NotificationError::Configuration(_)));
|
||||
assert!(!manager.has_subscriber("bucket", &EventName::ObjectCreatedPut).await);
|
||||
assert!(!manager.subscriber_view.has_subscriber("bucket", &EventName::ObjectCreatedPut));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn failed_validation_keeps_previous_bucket_rules() {
|
||||
let manager = build_manager();
|
||||
let target_id = TargetID::new("primary".to_string(), "webhook".to_string());
|
||||
let mut cfg = BucketNotificationConfig::new("us-east-1");
|
||||
cfg.add_rule(&[EventName::ObjectCreatedPut], "*".to_string(), target_id);
|
||||
manager.apply_bucket_notification_config("bucket", &cfg).await;
|
||||
assert!(manager.has_subscriber("bucket", &EventName::ObjectCreatedPut).await);
|
||||
|
||||
let err = manager
|
||||
.validate_bucket_notification_config("bucket", &cfg)
|
||||
.await
|
||||
.expect_err("validation must fail when the runtime has no notify targets");
|
||||
assert!(matches!(err, NotificationError::Configuration(_)));
|
||||
assert!(manager.has_subscriber("bucket", &EventName::ObjectCreatedPut).await);
|
||||
}
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user