mirror of
https://github.com/rustfs/rustfs.git
synced 2026-10-29 23:37:04 +00:00
Compare commits
282 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| d47f54bfb2 | |||
| 9cc9c48677 | |||
| 40b2aadb93 | |||
| 3d3debf938 | |||
| a708820cda | |||
| 5fdc16fec7 | |||
| a9596debb9 | |||
| 5249a90540 | |||
| 8c5ce4c9bc | |||
| 08a6f3ad42 | |||
| 2bdec2fdcd | |||
| c5a327df45 | |||
| 3774b75b93 | |||
| 3b7d11037d | |||
| e1e3cf4620 | |||
| 3999544ece | |||
| 6a2dc68130 | |||
| 230f31beff | |||
| 646c7c087f | |||
| 756968d735 | |||
| 9e84ede33f | |||
| 343cd86551 | |||
| 2822cb408a | |||
| 63fec31f72 | |||
| 14a464a313 | |||
| a7deb27bcf | |||
| 3dc4d9e40f | |||
| 1c06aa29a7 | |||
| b1b79d3c54 | |||
| 002b000055 | |||
| 12261efd1d | |||
| e97dff1201 | |||
| 1c47ca22eb | |||
| c9270c48a2 | |||
| 18547b1bb8 | |||
| 1eb14a8874 | |||
| 3e69654f91 | |||
| de60f6ad6e | |||
| 397ee852da | |||
| 6fc748d6de | |||
| de0a7f1090 | |||
| 7192eecbfb | |||
| 201e314dd9 | |||
| 28db20f7c4 | |||
| c020372948 | |||
| e40ed3c9e3 | |||
| 9fbe8079af | |||
| 89fd23fcae | |||
| 8c3b5afbea | |||
| 31d64c8ef2 | |||
| 203a9e25ed | |||
| 34d0b1b5bb | |||
| 7abb0f5f1d | |||
| 55f7514f90 | |||
| 703b1a5c9c | |||
| b61dc6519f | |||
| 95e90dbbc9 | |||
| e4031940e2 | |||
| 73552e40bc | |||
| cb3efc4a98 | |||
| 5dd92bd783 | |||
| 4e16705873 | |||
| 8982b4a3d2 | |||
| 568e8cbe0c | |||
| ed7baf534b | |||
| 0900d78877 | |||
| 8f6d06db21 | |||
| 3e156ee61e | |||
| 3ca3e26cec | |||
| 13093c5afc | |||
| 41770983d6 | |||
| 244e7dfb99 | |||
| 17ddecb075 | |||
| d0c7aec0b4 | |||
| 39ccd3abb0 | |||
| f21b06dfd2 | |||
| 71c002d467 | |||
| 9b835bcca5 | |||
| 7a7b080265 | |||
| cfe1d4d456 | |||
| ae8d079658 | |||
| 2cfefcfd65 | |||
| 7776004977 | |||
| 19c7ed68be | |||
| 455c36a3fd | |||
| 684900fbb9 | |||
| 023a674cd4 | |||
| adfde0110b | |||
| bcf5184a85 | |||
| f90d4071a5 | |||
| e3ecbf5db9 | |||
| 59f1c68759 | |||
| 4f0e050b39 | |||
| 741cd0ec3b | |||
| 824231ff1a | |||
| 66026d7d7b | |||
| 82a4083981 | |||
| 9f5ff23fd8 | |||
| 5fc92cdd27 | |||
| 7bd09a00b0 | |||
| 729cff5f32 | |||
| c478a392e7 | |||
| c8ccc1e198 | |||
| 74ba5c205c | |||
| 9fcd54734b | |||
| 6eb200022e | |||
| 92ce782999 | |||
| bf425ca32c | |||
| 0ae38aabbc | |||
| 77ba4f1b2e | |||
| 7b6e45d372 | |||
| 09c85a5f29 | |||
| 00aeb12914 | |||
| ffb18979f8 | |||
| 97c7b451d2 | |||
| 9ba95cac37 | |||
| cec796328c | |||
| 1aa4fa145f | |||
| d286f3d06c | |||
| 3391528025 | |||
| 0d1490da3c | |||
| 6421a7ff60 | |||
| 27141f6312 | |||
| 940e221988 | |||
| 8b66e9b62a | |||
| 6a879be1fb | |||
| 0b40d47a8a | |||
| e292637ee0 | |||
| 8b6b1e53a3 | |||
| 358bf6832d | |||
| 70fb8bf504 | |||
| 110f630a5c | |||
| 54bd3c8e24 | |||
| 1643cb29f1 | |||
| bf8d5a32b6 | |||
| 230eeb5fb5 | |||
| 2c6f5f22c0 | |||
| d7b6a8c10d | |||
| e8ffd72575 | |||
| 0b03d535c3 | |||
| 3afb389247 | |||
| 86b6569071 | |||
| 0940fbe1b2 | |||
| 9e7c5dc932 | |||
| 45806bf295 | |||
| 42f00d8532 | |||
| 3e46d61a91 | |||
| 88183daf10 | |||
| 8be1e9b2c1 | |||
| 9b4b366209 | |||
| 43436ad5a7 | |||
| 0c7b6188b2 | |||
| 3ffc3704af | |||
| de3ac27a6a | |||
| 2d0e82e9f7 | |||
| c92fabea3a | |||
| b3b0d892bb | |||
| d9ae1654b6 | |||
| 15949180c9 | |||
| f65305e97f | |||
| 28ac3ab4e7 | |||
| cf41b1d3cf | |||
| ca876930b3 | |||
| c05b3ee01e | |||
| 1076e2fb4f | |||
| 51e6733a9c | |||
| ed2b2cdd19 | |||
| 084477e079 | |||
| bb1b5dea16 | |||
| 1c87413788 | |||
| e1fee0569a | |||
| a6e5bdcbd2 | |||
| 923f51cf12 | |||
| e16274ab14 | |||
| 5eecd657e7 | |||
| 929d906131 | |||
| 6b802a91eb | |||
| 14b3cffe8f | |||
| c62384e58d | |||
| 3f30f6c139 | |||
| 584a52ce3d | |||
| 76e025a50c | |||
| b40eb193df | |||
| 023c987397 | |||
| a4b265bf77 | |||
| c176220a34 | |||
| 82ac2dff05 | |||
| e24eae9eaa | |||
| 8617f2701b | |||
| 8d5592f83e | |||
| b5c9229e77 | |||
| a974e50b1d | |||
| 081910e825 | |||
| 7f7e4fe40b | |||
| 684e6f313a | |||
| b3a02f305e | |||
| 0ea9c19569 | |||
| ae662da256 | |||
| b5d33a1f4e | |||
| d288496752 | |||
| 40baedba4f | |||
| 0c3fd48c22 | |||
| 3e8b0e7c83 | |||
| dd2af9ff88 | |||
| 5db0e14d04 | |||
| 9429225cf4 | |||
| 857584b3c2 | |||
| cfda9b451c | |||
| 354e49de4c | |||
| 8e987ce0a6 | |||
| 929f836e40 | |||
| c7dec044eb | |||
| 14a59f7770 | |||
| 084e9c4b82 | |||
| 921cc7ad94 | |||
| bfa8df00e7 | |||
| 33a8469e27 | |||
| 128080ceb9 | |||
| ae9fe62fb1 | |||
| 929a9f27c4 | |||
| 15376a7fa6 | |||
| 5d789006dd | |||
| f4bd53a63c | |||
| 21fee7f7ff | |||
| 23ed00fc85 | |||
| f49ffa2fec | |||
| ee5f287324 | |||
| 158c613d0e | |||
| a950f914ca | |||
| 1de9b40a30 | |||
| 831468b2e9 | |||
| 600d037b25 | |||
| 3fa2b334be | |||
| 1b549d5907 | |||
| 11c4ce96eb | |||
| 4508a0985d | |||
| ed1b9f25d6 | |||
| 274c2bf402 | |||
| 9aebcefa9c | |||
| 1748814bbf | |||
| ee5f76c180 | |||
| 4d68c32b75 | |||
| 8c6689ff13 | |||
| 595f9f662d | |||
| 4a2b15cb82 | |||
| 2ba7f95547 | |||
| 7b3dad6bae | |||
| c0754f5b1c | |||
| 5dc0e3b402 | |||
| cc5487e7de | |||
| 0b05b6c6ff | |||
| c507da8f75 | |||
| 550dabeffd | |||
| 4d7f0344d3 | |||
| 084338add6 | |||
| 8d339da706 | |||
| 817ad0a682 | |||
| 80321e5bb4 | |||
| e7475cfa4d | |||
| 8c15025a5a | |||
| 6f2ec66263 | |||
| 62542ddc57 | |||
| a8bb53218d | |||
| 5355d9f8f8 | |||
| 50d7a049ee | |||
| 3149c87cf2 | |||
| b7fa6a4615 | |||
| 6fe83f87a4 | |||
| df6981d88e | |||
| 590adad5ae | |||
| be5d14c985 | |||
| 703086d71d | |||
| a159f312f0 | |||
| 2f6f095298 | |||
| efd8ef005f | |||
| 60fa33773a | |||
| ee752b0b03 | |||
| 99c1f4418b | |||
| 7ce0ac72cf | |||
| 944e26d432 | |||
| f0b0a99260 | |||
| 33ddc10ffd |
@@ -101,7 +101,6 @@ Update this file only when an advisory adds or changes a reusable lesson, affect
|
||||
### SSE and on-disk storage invariants
|
||||
|
||||
- `GHSA-xrrf-67jm-3c2r`: SSE metadata reported encryption while reader composition bypassed `EncryptReader` and stored plaintext. Lesson: test actual bytes on disk and wrapper order, not only API metadata.
|
||||
- `GHSA-wqmc-vjgv-jrpw`: SSE-C persisted and listed the plaintext MD5 as the ETag, allowing list-only users to confirm or brute-force low-entropy content and correlate equal plaintext across keys. Lesson: encrypted-object ETags must not expose deterministic plaintext fingerprints; protect them with per-object key material and keep listing, conditional, copy, and multipart semantics consistent.
|
||||
|
||||
### Object Lock and retention invariants
|
||||
|
||||
@@ -127,7 +126,7 @@ rg -n "normalize_extract_entry_key|Snowball|auto-extract|PathBuf::join|canonical
|
||||
rg -n "DEFAULT_SECRET|DEFAULT_ACCESS|TEST_PRIVATE_KEY|rustfs rpc|RUSTFS_RPC_SECRET" rustfs crates
|
||||
rg -n "TONIC_RPC_PREFIX|verify_rpc_signature|check_auth|NodeServiceServer|x-rustfs-signature" rustfs crates
|
||||
rg -n "debug!|trace!|info!|error!|\\?resp|\\?merged_config|session_token|secret_key" rustfs crates
|
||||
rg -n "HashReader|EncryptReader|try_resolve_etag|ETag|SSE|server-side encryption|Access-Control-Allow-Credentials|Origin" rustfs crates
|
||||
rg -n "HashReader|EncryptReader|SSE|server-side encryption|Access-Control-Allow-Credentials|Origin" rustfs crates
|
||||
rg -n "ObjectLock|object_lock|retention|COMPLIANCE|GOVERNANCE|delete_prefix|lifecycle|scanner" rustfs crates
|
||||
rg -n "deny_unknown_fields|serde.default|as u32|as usize|as i32" rustfs crates
|
||||
```
|
||||
@@ -146,5 +145,5 @@ rg -n "deny_unknown_fields|serde.default|as u32|as usize|as i32" rustfs crates
|
||||
- IAM export fixes: assert exported archives omit plaintext user and service-account secrets unless the format deliberately encrypts or seals them.
|
||||
- RPC auth fixes: include captured metadata replay across two concrete methods, stale timestamps, wrong path, wrong method surrogate, wrong secret, and valid same-method calls.
|
||||
- Browser/CORS fixes: assert no credentials on reflected/default origins, correct behavior for explicit allowlists, and no same-origin script execution for previewed object content.
|
||||
- SSE fixes: inspect stored bytes and verify API metadata, read-back behavior, and on-disk ciphertext together; for ETags, upload equal plaintext under different keys and assert list-only callers cannot derive or correlate the plaintext digest, including multipart objects.
|
||||
- SSE fixes: inspect stored bytes and verify API metadata, read-back behavior, and on-disk ciphertext together.
|
||||
- Object Lock fixes: include unreadable metadata, fabricated metadata defaults, unparsable config, confirmed absent config, COMPLIANCE/GOVERNANCE retention, lifecycle expiry, scanner sweeps, and force-delete paths.
|
||||
|
||||
@@ -29,7 +29,6 @@ script-tests: ## Run shell script tests
|
||||
./scripts/test_entrypoint_credentials.sh
|
||||
./scripts/test_internode_grpc_ab_bench.sh
|
||||
./scripts/test_object_batch_bench_enhanced.sh
|
||||
./scripts/test_package_service_scripts.sh
|
||||
./scripts/test_hotpath_warp_ab_gate.sh
|
||||
./scripts/test_hotpath_warp_abba.sh
|
||||
./scripts/test_scanner_validation_harness.sh
|
||||
|
||||
@@ -132,12 +132,6 @@ test-group = 'ecstore-serial-flaky'
|
||||
filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnostic_capability_e2e))'
|
||||
test-group = 'embedded-test-ports'
|
||||
|
||||
# The real object probe includes bucket creation and cleanup in its short
|
||||
# measurement budget. Keep competing storage fixtures outside that budget.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs) & binary(connect_perf_object) & test(=real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put)'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# Serialize the durable manual-transition checkpoint test across nextest's
|
||||
# process boundary; it mutates bucket lifecycle metadata and is not quarantined.
|
||||
[[profile.default.overrides]]
|
||||
@@ -333,10 +327,6 @@ test-group = 'ecstore-serial-flaky'
|
||||
filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnostic_capability_e2e))'
|
||||
test-group = 'embedded-test-ports'
|
||||
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs) & binary(connect_perf_object) & test(=real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put)'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# Serialize the durable manual-transition checkpoint test under the ci profile
|
||||
# too. No retries: failures stay visible.
|
||||
[[profile.ci.overrides]]
|
||||
|
||||
@@ -24,11 +24,8 @@ on:
|
||||
- '.github/actions/**'
|
||||
- '.github/workflows/**'
|
||||
- 'scripts/release/create_or_update_release.sh'
|
||||
- 'scripts/release/package_service_scripts.sh'
|
||||
- 'scripts/release/package_versions.sh'
|
||||
- 'scripts/test_package_service_scripts.sh'
|
||||
- 'scripts/test_package_versions.sh'
|
||||
- 'scripts/test_docker_workflow.py'
|
||||
- 'scripts/security/check_performance_ab_workflow.sh'
|
||||
- 'scripts/security/check_preview_release_workflow.sh'
|
||||
- 'scripts/security/check_tier_artifact_workflow.sh'
|
||||
@@ -43,11 +40,8 @@ on:
|
||||
- '.github/actions/**'
|
||||
- '.github/workflows/**'
|
||||
- 'scripts/release/create_or_update_release.sh'
|
||||
- 'scripts/release/package_service_scripts.sh'
|
||||
- 'scripts/release/package_versions.sh'
|
||||
- 'scripts/test_package_service_scripts.sh'
|
||||
- 'scripts/test_package_versions.sh'
|
||||
- 'scripts/test_docker_workflow.py'
|
||||
- 'scripts/security/check_performance_ab_workflow.sh'
|
||||
- 'scripts/security/check_preview_release_workflow.sh'
|
||||
- 'scripts/security/check_tier_artifact_workflow.sh'
|
||||
@@ -164,9 +158,6 @@ jobs:
|
||||
- name: Check package version contract
|
||||
run: ./scripts/test_package_versions.sh
|
||||
|
||||
- name: Check package service lifecycle contract
|
||||
run: ./scripts/test_package_service_scripts.sh
|
||||
|
||||
dependency-review:
|
||||
name: Dependency Review
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -818,7 +818,9 @@ jobs:
|
||||
|
||||
echo ""
|
||||
echo "🐳 Docker Images:"
|
||||
if [[ "$INPUT_BUILD_DOCKER" == "false" ]]; then
|
||||
if [[ "$BUILD_TYPE" == "preview" ]]; then
|
||||
echo "⏭️ Preview tags do not publish Docker images"
|
||||
elif [[ "$INPUT_BUILD_DOCKER" == "false" ]]; then
|
||||
echo "⏭️ Docker image build was skipped (binary only build)"
|
||||
elif [[ "$BUILD_STATUS" == "success" ]]; then
|
||||
echo "🔄 Docker images will be built and pushed automatically via workflow_run event"
|
||||
|
||||
@@ -16,7 +16,7 @@ name: Continuous Integration
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main ]
|
||||
branches: [ main, release ]
|
||||
paths-ignore:
|
||||
- "**.md"
|
||||
- "docs/**"
|
||||
@@ -36,7 +36,7 @@ on:
|
||||
- "flake.lock"
|
||||
pull_request:
|
||||
types: [ opened, synchronize, reopened, closed ]
|
||||
branches: [ main ]
|
||||
branches: [ main, release ]
|
||||
merge_group:
|
||||
types: [ checks_requested ]
|
||||
schedule:
|
||||
@@ -867,13 +867,14 @@ jobs:
|
||||
# Merge gate only (backlog#1149 ci-5): the never-automated user-visible
|
||||
# suites — KMS, object_lock, multipart_auth, quota, checksum, encryption,
|
||||
# security-boundary, ... — via the e2e-full nextest profile. Too heavy for
|
||||
# every PR, so it is gated to main pushes, the merge queue, and manual
|
||||
# every PR, so it is gated to main/release pushes, the merge queue, and manual
|
||||
# dispatch. protocols / the 7 cluster suites / replication / #[ignore] are
|
||||
# owned by other lanes (see .config/nextest.toml profile.e2e-full).
|
||||
if: >-
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
github.event_name == 'merge_group' ||
|
||||
(github.event_name == 'push' && github.ref == 'refs/heads/main')
|
||||
(github.event_name == 'push' &&
|
||||
(github.ref == 'refs/heads/main' || github.ref == 'refs/heads/release'))
|
||||
needs: [ build-rustfs-debug-binary ]
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 55
|
||||
|
||||
@@ -47,7 +47,7 @@ on:
|
||||
default: true
|
||||
type: boolean
|
||||
version:
|
||||
description: "Version to build (latest, v1.0.0, v1.0.0-alpha1, or v1.0.1-preview.1)"
|
||||
description: "Version to build (latest for stable release, or specific version like v1.0.0, v1.0.0-alpha1)"
|
||||
required: false
|
||||
default: "latest"
|
||||
type: string
|
||||
@@ -82,7 +82,8 @@ jobs:
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
(github.event.workflow_run.conclusion == 'success' &&
|
||||
github.event.workflow_run.event == 'push' &&
|
||||
github.event.workflow_run.head_branch != 'main')
|
||||
github.event.workflow_run.head_branch != 'main' &&
|
||||
!contains(github.event.workflow_run.head_branch, '-preview'))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
outputs:
|
||||
@@ -196,14 +197,7 @@ jobs:
|
||||
version="${version#v}"
|
||||
fi
|
||||
|
||||
if [[ "$version" =~ -preview\.[0-9]+$ ]]; then
|
||||
build_type="preview"
|
||||
is_prerelease=true
|
||||
echo "🔍 Building Docker image for preview: $version (version tags only)"
|
||||
elif [[ "$version" == *"-preview"* ]]; then
|
||||
echo "❌ Invalid preview tag: $version (expected suffix: -preview.<number>)" >&2
|
||||
exit 1
|
||||
elif [[ "$version" == *"alpha"* ]] || [[ "$version" == *"beta"* ]] || [[ "$version" == *"rc"* ]]; then
|
||||
if [[ "$version" == *"alpha"* ]] || [[ "$version" == *"beta"* ]] || [[ "$version" == *"rc"* ]]; then
|
||||
build_type="prerelease"
|
||||
is_prerelease=true
|
||||
# Pre-GA policy: prereleases update latest until the first stable tag exists.
|
||||
@@ -255,14 +249,11 @@ jobs:
|
||||
echo "🚀 Building with latest stable release version"
|
||||
;;
|
||||
*-preview*)
|
||||
if [[ ! "$input_version" =~ ^v?[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?-preview\.[0-9]+$ ]]; then
|
||||
echo "❌ Invalid preview version: $input_version" >&2
|
||||
exit 1
|
||||
fi
|
||||
build_type="preview"
|
||||
is_prerelease=true
|
||||
version="${input_version#v}"
|
||||
echo "🔍 Building with preview version: $version (version tags only)"
|
||||
should_build=false
|
||||
should_push=false
|
||||
echo "⏭️ Preview tags do not publish Docker images"
|
||||
;;
|
||||
# Prerelease versions (must match first, more specific)
|
||||
v*alpha*|v*beta*|v*rc*|*alpha*|*beta*|*rc*)
|
||||
@@ -551,54 +542,6 @@ jobs:
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
|
||||
# Docker Hub's overview is a separate `full_description` field that
|
||||
# `docker push` never touches, so it drifted from README.md and kept
|
||||
# linking to pages that no longer exist (#7995). Republish it from the
|
||||
# commit the images were built from, only when images were pushed.
|
||||
sync-dockerhub-description:
|
||||
name: Sync Docker Hub Description
|
||||
needs: [ build-check, build-docker ]
|
||||
if: needs.build-check.outputs.should_build == 'true' && needs.build-check.outputs.should_push == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ needs.build-check.outputs.source_ref }}
|
||||
sparse-checkout: README.md
|
||||
sparse-checkout-cone-mode: false
|
||||
|
||||
- name: Sync Docker Hub description
|
||||
uses: peter-evans/dockerhub-description@1b9a80c056b620d92cedb9d9b5a223409c68ddfa # v5.0.0
|
||||
with:
|
||||
username: ${{ env.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
repository: ${{ env.REGISTRY_DOCKERHUB }}
|
||||
readme-filepath: ./README.md
|
||||
# README.md links to docs/ and CONTRIBUTING.md by relative path;
|
||||
# rewrite those to github.com URLs so they resolve on Docker Hub.
|
||||
enable-url-completion: true
|
||||
|
||||
# Docker Hub caps `full_description` at 25,000 bytes and the action
|
||||
# truncates to fit with only a warning, which would publish a README
|
||||
# cut off mid-sentence. README.md is already near the cap, so read the
|
||||
# published overview back and fail if it hit the limit.
|
||||
- name: Check published description was not truncated
|
||||
env:
|
||||
REPOSITORY: ${{ env.REGISTRY_DOCKERHUB }}
|
||||
run: |
|
||||
limit=25000
|
||||
published=$(curl -sSf "https://hub.docker.com/v2/repositories/${REPOSITORY}/" \
|
||||
| jq -j '.full_description' | wc -c | tr -d ' ')
|
||||
echo "Published overview: ${published} bytes (Docker Hub limit: ${limit})"
|
||||
# Truncation lands within one UTF-8 character of the cap.
|
||||
if [ "$published" -ge $((limit - 4)) ]; then
|
||||
echo "::error file=README.md::Docker Hub overview was truncated at ${limit} bytes; trim README.md before the next release."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Docker build summary
|
||||
docker-summary:
|
||||
name: Docker Build Summary
|
||||
@@ -620,10 +563,6 @@ jobs:
|
||||
echo ""
|
||||
|
||||
case "$BUILD_TYPE" in
|
||||
"preview")
|
||||
echo "🔍 Preview Docker image has been built with ${VERSION} tags"
|
||||
echo "⏭️ Preview images do not update latest or prerelease channels"
|
||||
;;
|
||||
"release")
|
||||
echo "🚀 Release Docker image has been built with ${VERSION} tags"
|
||||
echo "✅ This image is ready for production use"
|
||||
|
||||
@@ -117,15 +117,10 @@ jobs:
|
||||
|
||||
cat > "${PKG_DIR}/etc/default/rustfs" << 'ENVEOF'
|
||||
# RustFS Environment Configuration
|
||||
# See https://docs.rustfs.com for more information
|
||||
# RUSTFS_ACCESS_KEY=<changeme>
|
||||
# RUSTFS_SECRET_KEY=<changeme>
|
||||
# See https://rustfs.com/docs/ for more information
|
||||
# RUSTFS_VOLUMES=""
|
||||
# RUSTFS_ADDRESS=":9000"
|
||||
# RUSTFS_CONSOLE_ADDRESS=":9001"
|
||||
# RUSTFS_CONSOLE_ENABLE=true
|
||||
# RUSTFS_OBS_LOGGER_LEVEL=error
|
||||
# RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/"
|
||||
# RUSTFS_ROOT_USER=""
|
||||
# RUSTFS_ROOT_PASSWORD=""
|
||||
ENVEOF
|
||||
|
||||
# dpkg versions must start with a digit and cannot contain hyphens;
|
||||
@@ -151,13 +146,37 @@ jobs:
|
||||
/etc/default/rustfs
|
||||
CONFFILES
|
||||
|
||||
./scripts/release/package_service_scripts.sh after-install > "${PKG_DIR}/DEBIAN/postinst"
|
||||
cat > "${PKG_DIR}/DEBIAN/postinst" << 'POSTINST'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if ! getent passwd rustfs > /dev/null 2>&1; then
|
||||
useradd -r -s /bin/false -d /opt/rustfs rustfs
|
||||
fi
|
||||
mkdir -p /opt/rustfs /data/rustfs /var/log/rustfs
|
||||
chown rustfs:rustfs /opt/rustfs /data/rustfs /var/log/rustfs
|
||||
if [ -d /run/systemd/system ]; then
|
||||
systemctl daemon-reload
|
||||
fi
|
||||
echo "RustFS installed. Configure /etc/default/rustfs then: systemctl start rustfs"
|
||||
POSTINST
|
||||
chmod 755 "${PKG_DIR}/DEBIAN/postinst"
|
||||
|
||||
./scripts/release/package_service_scripts.sh before-remove > "${PKG_DIR}/DEBIAN/prerm"
|
||||
cat > "${PKG_DIR}/DEBIAN/prerm" << 'PRERM'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if [ -d /run/systemd/system ] && systemctl is-active --quiet rustfs; then
|
||||
systemctl stop rustfs
|
||||
fi
|
||||
PRERM
|
||||
chmod 755 "${PKG_DIR}/DEBIAN/prerm"
|
||||
|
||||
./scripts/release/package_service_scripts.sh after-remove > "${PKG_DIR}/DEBIAN/postrm"
|
||||
cat > "${PKG_DIR}/DEBIAN/postrm" << 'POSTRM'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if [ -d /run/systemd/system ]; then
|
||||
systemctl daemon-reload
|
||||
fi
|
||||
POSTRM
|
||||
chmod 755 "${PKG_DIR}/DEBIAN/postrm"
|
||||
|
||||
cp LICENSE "${PKG_DIR}/usr/share/doc/rustfs/"
|
||||
@@ -195,15 +214,10 @@ jobs:
|
||||
mkdir -p ./tmp-pkg/etc/default
|
||||
cat > ./tmp-pkg/etc/default/rustfs << 'ENVEOF'
|
||||
# RustFS Environment Configuration
|
||||
# See https://docs.rustfs.com for more information
|
||||
# RUSTFS_ACCESS_KEY=<changeme>
|
||||
# RUSTFS_SECRET_KEY=<changeme>
|
||||
# See https://rustfs.com/docs/ for more information
|
||||
# RUSTFS_VOLUMES=""
|
||||
# RUSTFS_ADDRESS=":9000"
|
||||
# RUSTFS_CONSOLE_ADDRESS=":9001"
|
||||
# RUSTFS_CONSOLE_ENABLE=true
|
||||
# RUSTFS_OBS_LOGGER_LEVEL=error
|
||||
# RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/"
|
||||
# RUSTFS_ROOT_USER=""
|
||||
# RUSTFS_ROOT_PASSWORD=""
|
||||
ENVEOF
|
||||
|
||||
fpm -s dir -t rpm \
|
||||
@@ -217,13 +231,35 @@ jobs:
|
||||
--description "High-performance distributed object storage" \
|
||||
--url "https://rustfs.com" \
|
||||
--license "Apache-2.0" \
|
||||
--before-install <(./scripts/release/package_service_scripts.sh rpm-before-install) \
|
||||
--before-upgrade <(./scripts/release/package_service_scripts.sh rpm-before-upgrade) \
|
||||
--after-install <(./scripts/release/package_service_scripts.sh after-install) \
|
||||
--after-upgrade <(./scripts/release/package_service_scripts.sh after-install) \
|
||||
--before-remove <(./scripts/release/package_service_scripts.sh before-remove) \
|
||||
--after-remove <(./scripts/release/package_service_scripts.sh after-remove) \
|
||||
--rpm-posttrans <(./scripts/release/package_service_scripts.sh rpm-posttrans) \
|
||||
--after-install <(cat << 'POSTINST'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if ! getent passwd rustfs > /dev/null 2>&1; then
|
||||
useradd -r -s /bin/false -d /opt/rustfs rustfs
|
||||
fi
|
||||
mkdir -p /opt/rustfs /data/rustfs /var/log/rustfs
|
||||
chown rustfs:rustfs /opt/rustfs /data/rustfs /var/log/rustfs
|
||||
if [ -d /run/systemd/system ]; then
|
||||
systemctl daemon-reload
|
||||
fi
|
||||
POSTINST
|
||||
) \
|
||||
--before-remove <(cat << 'PRERM'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if [ -d /run/systemd/system ] && systemctl is-active --quiet rustfs; then
|
||||
systemctl stop rustfs
|
||||
fi
|
||||
PRERM
|
||||
) \
|
||||
--after-remove <(cat << 'POSTRM'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if [ -d /run/systemd/system ]; then
|
||||
systemctl daemon-reload
|
||||
fi
|
||||
POSTRM
|
||||
) \
|
||||
--config-files /etc/default/rustfs \
|
||||
"rustfs-nightly-${DEB_DATE}/usr/bin/rustfs=/usr/bin/rustfs" \
|
||||
./tmp-pkg/etc/default/rustfs=/etc/default/rustfs \
|
||||
|
||||
@@ -328,15 +328,10 @@ jobs:
|
||||
|
||||
cat > "${PKG_DIR}/etc/default/rustfs" << 'ENVEOF'
|
||||
# RustFS Environment Configuration
|
||||
# See https://docs.rustfs.com for more information
|
||||
# RUSTFS_ACCESS_KEY=<changeme>
|
||||
# RUSTFS_SECRET_KEY=<changeme>
|
||||
# See https://rustfs.com/docs/ for more information
|
||||
# RUSTFS_VOLUMES=""
|
||||
# RUSTFS_ADDRESS=":9000"
|
||||
# RUSTFS_CONSOLE_ADDRESS=":9001"
|
||||
# RUSTFS_CONSOLE_ENABLE=true
|
||||
# RUSTFS_OBS_LOGGER_LEVEL=error
|
||||
# RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/"
|
||||
# RUSTFS_ROOT_USER=""
|
||||
# RUSTFS_ROOT_PASSWORD=""
|
||||
ENVEOF
|
||||
|
||||
cat > "${PKG_DIR}/DEBIAN/control" << EOF
|
||||
@@ -359,13 +354,37 @@ jobs:
|
||||
/etc/default/rustfs
|
||||
CONFFILES
|
||||
|
||||
./scripts/release/package_service_scripts.sh after-install > "${PKG_DIR}/DEBIAN/postinst"
|
||||
cat > "${PKG_DIR}/DEBIAN/postinst" << 'POSTINST'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if ! getent passwd rustfs > /dev/null 2>&1; then
|
||||
useradd -r -s /bin/false -d /opt/rustfs rustfs
|
||||
fi
|
||||
mkdir -p /opt/rustfs /data/rustfs /var/log/rustfs
|
||||
chown rustfs:rustfs /opt/rustfs /data/rustfs /var/log/rustfs
|
||||
if [ -d /run/systemd/system ]; then
|
||||
systemctl daemon-reload
|
||||
fi
|
||||
echo "RustFS installed. Configure /etc/default/rustfs then: systemctl start rustfs"
|
||||
POSTINST
|
||||
chmod 755 "${PKG_DIR}/DEBIAN/postinst"
|
||||
|
||||
./scripts/release/package_service_scripts.sh before-remove > "${PKG_DIR}/DEBIAN/prerm"
|
||||
cat > "${PKG_DIR}/DEBIAN/prerm" << 'PRERM'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if [ -d /run/systemd/system ] && systemctl is-active --quiet rustfs; then
|
||||
systemctl stop rustfs
|
||||
fi
|
||||
PRERM
|
||||
chmod 755 "${PKG_DIR}/DEBIAN/prerm"
|
||||
|
||||
./scripts/release/package_service_scripts.sh after-remove > "${PKG_DIR}/DEBIAN/postrm"
|
||||
cat > "${PKG_DIR}/DEBIAN/postrm" << 'POSTRM'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if [ -d /run/systemd/system ]; then
|
||||
systemctl daemon-reload
|
||||
fi
|
||||
POSTRM
|
||||
chmod 755 "${PKG_DIR}/DEBIAN/postrm"
|
||||
|
||||
cp LICENSE "${PKG_DIR}/usr/share/doc/rustfs/"
|
||||
@@ -403,15 +422,10 @@ jobs:
|
||||
mkdir -p ./tmp-pkg/etc/default
|
||||
cat > ./tmp-pkg/etc/default/rustfs << 'ENVEOF'
|
||||
# RustFS Environment Configuration
|
||||
# See https://docs.rustfs.com for more information
|
||||
# RUSTFS_ACCESS_KEY=<changeme>
|
||||
# RUSTFS_SECRET_KEY=<changeme>
|
||||
# See https://rustfs.com/docs/ for more information
|
||||
# RUSTFS_VOLUMES=""
|
||||
# RUSTFS_ADDRESS=":9000"
|
||||
# RUSTFS_CONSOLE_ADDRESS=":9001"
|
||||
# RUSTFS_CONSOLE_ENABLE=true
|
||||
# RUSTFS_OBS_LOGGER_LEVEL=error
|
||||
# RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/"
|
||||
# RUSTFS_ROOT_USER=""
|
||||
# RUSTFS_ROOT_PASSWORD=""
|
||||
ENVEOF
|
||||
|
||||
fpm -s dir -t rpm \
|
||||
@@ -425,13 +439,35 @@ jobs:
|
||||
--description "High-performance distributed object storage" \
|
||||
--url "https://rustfs.com" \
|
||||
--license "Apache-2.0" \
|
||||
--before-install <(./scripts/release/package_service_scripts.sh rpm-before-install) \
|
||||
--before-upgrade <(./scripts/release/package_service_scripts.sh rpm-before-upgrade) \
|
||||
--after-install <(./scripts/release/package_service_scripts.sh after-install) \
|
||||
--after-upgrade <(./scripts/release/package_service_scripts.sh after-install) \
|
||||
--before-remove <(./scripts/release/package_service_scripts.sh before-remove) \
|
||||
--after-remove <(./scripts/release/package_service_scripts.sh after-remove) \
|
||||
--rpm-posttrans <(./scripts/release/package_service_scripts.sh rpm-posttrans) \
|
||||
--after-install <(cat <<'POSTINST'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if ! getent passwd rustfs > /dev/null 2>&1; then
|
||||
useradd -r -s /bin/false -d /opt/rustfs rustfs
|
||||
fi
|
||||
mkdir -p /opt/rustfs /data/rustfs /var/log/rustfs
|
||||
chown rustfs:rustfs /opt/rustfs /data/rustfs /var/log/rustfs
|
||||
if [ -d /run/systemd/system ]; then
|
||||
systemctl daemon-reload
|
||||
fi
|
||||
POSTINST
|
||||
) \
|
||||
--before-remove <(cat <<'PRERM'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if [ -d /run/systemd/system ] && systemctl is-active --quiet rustfs; then
|
||||
systemctl stop rustfs
|
||||
fi
|
||||
PRERM
|
||||
) \
|
||||
--after-remove <(cat <<'POSTRM'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if [ -d /run/systemd/system ]; then
|
||||
systemctl daemon-reload
|
||||
fi
|
||||
POSTRM
|
||||
) \
|
||||
--config-files /etc/default/rustfs \
|
||||
./bin/rustfs=/usr/bin/rustfs \
|
||||
./tmp-pkg/etc/default/rustfs=/etc/default/rustfs \
|
||||
|
||||
@@ -1,221 +0,0 @@
|
||||
# RustFS Fault-Tolerance (degradation) Matrix Sweep
|
||||
#
|
||||
# Exhaustive topology x EC outage sweep, complementary to the scenario suite
|
||||
# (rustfs-fault-tolerance-test.yml). For each topology (nodes x drives per
|
||||
# node) and each EC level P in 1..N*drives/2 the runner installs that exact
|
||||
# cluster shape and drives progressive node outages plus recovery:
|
||||
#
|
||||
# M-<topo>EC<P>-full full-capacity reads/writes must succeed
|
||||
# M-<topo>EC<P>-n<k> with k nodes online: reads succeed iff the online
|
||||
# drives cover the read quorum (N-P), writes iff they
|
||||
# cover the write quorum (N-P+1); probes below quorum
|
||||
# assert the expected outage
|
||||
# M-<topo>EC<P>-rec after restoring every node, reads/writes succeed
|
||||
#
|
||||
# 10 topologies x EC levels = 38 combinations, 320 cases. The full sweep runs
|
||||
# 3-5 hours and is dispatched on demand (e.g. after a nightly chain goes
|
||||
# green), not nightly. Case ids and expectations come from product source
|
||||
# (default_parity_count, erasure set sizing) via rustfs/auto-testing.
|
||||
|
||||
name: RustFS Fault-Tolerance Matrix
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
package_url:
|
||||
description: 'Direct .deb URL. Required unless the nightly default is wanted.'
|
||||
required: false
|
||||
type: string
|
||||
auto_testing_ref:
|
||||
description: 'auto-testing branch, tag, or SHA (defaults to main)'
|
||||
required: false
|
||||
default: main
|
||||
type: string
|
||||
matrix_topo:
|
||||
description: 'Topology filter like 4x2; empty sweeps all ten topologies'
|
||||
required: false
|
||||
default: ''
|
||||
type: string
|
||||
matrix_ec:
|
||||
description: 'EC/parity filter; empty sweeps every EC level'
|
||||
required: false
|
||||
default: ''
|
||||
type: string
|
||||
cleanup_before:
|
||||
description: 'Reset the nodes before the test (DESTROYS existing data/config)'
|
||||
type: boolean
|
||||
default: true
|
||||
cleanup_after:
|
||||
description: 'Reset the nodes after the test (DESTROYS test data/config)'
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# The suite stops services and revokes drive access on the shared fleet; only one
|
||||
# functional suite may touch the environment at a time.
|
||||
concurrency:
|
||||
group: rustfs-shared-functional-tests-v2
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
env:
|
||||
RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
|
||||
RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
|
||||
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
|
||||
RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
|
||||
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
|
||||
|
||||
jobs:
|
||||
fault-tolerance-matrix:
|
||||
runs-on: smoke-testing
|
||||
# The full 38-combination sweep takes 3-5h; 360 is the runner job ceiling.
|
||||
timeout-minutes: 360
|
||||
steps:
|
||||
- name: Initialize functional evidence
|
||||
id: evidence
|
||||
run: |
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-fault-tolerance-matrix-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch" "${FUNCTIONAL_ARTIFACTS_DIR}/evidence"
|
||||
{
|
||||
printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'RESULTS_FILE=%s/results.json\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'EVIDENCE_DIR=%s/evidence\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Checkout auto-testing scripts
|
||||
timeout-minutes: 5
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
repository: rustfs/auto-testing
|
||||
ref: ${{ inputs.auto_testing_ref }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
path: auto-testing
|
||||
persist-credentials: false
|
||||
|
||||
- name: Record auto-testing revision
|
||||
run: |
|
||||
set -euo pipefail
|
||||
AUTO_TESTING_SHA="$(git -C auto-testing rev-parse HEAD)"
|
||||
printf 'AUTO_TESTING_SHA=%s\n' "${AUTO_TESTING_SHA}" >> "${GITHUB_ENV}"
|
||||
echo "auto-testing revision: ${AUTO_TESTING_SHA}"
|
||||
|
||||
- name: Show environment
|
||||
run: |
|
||||
uname -a
|
||||
jq --version
|
||||
aws --version
|
||||
echo "auto-testing: ${AUTO_TESTING_SHA}"
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
if: ${{ inputs.cleanup_before != 'false' }}
|
||||
run: |
|
||||
./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}"
|
||||
|
||||
- name: Run matrix sweep (topology x EC outage combinations)
|
||||
timeout-minutes: 350
|
||||
id: test
|
||||
# No continue-on-error: an UNEXPECTED verdict (a probe that should
|
||||
# fail by quorum math but passes, or vice versa) is the finding this
|
||||
# sweep exists to surface, and the red run is the signal until matrix
|
||||
# runs get backlog integration like the scenario suite has.
|
||||
env:
|
||||
MATRIX_EC: ${{ inputs.matrix_ec }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ARGS=(--matrix -y --package-url "${{ inputs.package_url || env.RUSTFS_NIGHTLY_PACKAGE_URL }}" --log-file "${LOG_FILE}" --results-file "${RESULTS_FILE}")
|
||||
if [ -n "${{ inputs.matrix_topo }}" ]; then
|
||||
# Trailing value form: consumed as `--matrix-topo <topo> [ec]`.
|
||||
ARGS+=(--matrix-topo "${{ inputs.matrix_topo }}")
|
||||
if [ -n "${MATRIX_EC}" ]; then
|
||||
ARGS+=("${MATRIX_EC}")
|
||||
fi
|
||||
fi
|
||||
./auto-testing/rustfs-fault-tolerance-test.sh "${ARGS[@]}"
|
||||
|
||||
- name: Generate report
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
{
|
||||
echo "# RustFS fault-tolerance matrix report"
|
||||
echo ""
|
||||
echo "- Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo "- Package: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
|
||||
echo "- auto-testing: \`${AUTO_TESTING_SHA}\`"
|
||||
echo "- Topology filter: ${{ inputs.matrix_topo || 'all (10 topologies)' }}"
|
||||
echo "- EC filter: ${{ inputs.matrix_ec || 'all levels' }}"
|
||||
echo ""
|
||||
echo "## Per-probe results"
|
||||
echo ""
|
||||
echo '```'
|
||||
grep -E '^FT-(CASE|COMPLETE|SUMMARY|REPORT)' "${LOG_FILE}" || echo "(no FT-CASE lines found)"
|
||||
echo '```'
|
||||
echo ""
|
||||
echo "## Health snapshots"
|
||||
echo ""
|
||||
for f in "${FUNCTIONAL_ARTIFACTS_DIR}"/evidence/*.code; do
|
||||
[ -e "${f}" ] || continue
|
||||
printf '%s -> %s\n' "$(basename "${f}" .code)" "$(cat "${f}")"
|
||||
done
|
||||
} > "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
# An unfiltered sweep must land exactly 320 cases (38 topology x EC
|
||||
# combinations at 2*(nodes+1) cases each); filtered sweeps are held
|
||||
# to internal consistency. Missing/harness-error rows always fail.
|
||||
EXPECTED_TOTAL=""
|
||||
if [ -z "${{ inputs.matrix_topo }}" ] && [ -z "${MATRIX_EC}" ]; then
|
||||
EXPECTED_TOTAL=320
|
||||
fi
|
||||
jq -e --arg testing_sha "${AUTO_TESTING_SHA}" --argjson expected_total "${EXPECTED_TOTAL:-null}" '
|
||||
.schema_version == 1 and
|
||||
.auto_testing_sha == $testing_sha and
|
||||
.complete == true and
|
||||
.counts.expected == ($expected_total // .counts.expected) and
|
||||
.counts.expected == .counts.seen and
|
||||
.counts.harness_errors == 0 and
|
||||
(.expected_cases | length) == .counts.expected and
|
||||
(.expected_cases | unique | length) == .counts.expected and
|
||||
(.seen_cases | length) == .counts.seen and
|
||||
(.missing_cases | length) == 0 and
|
||||
(.harness_errors | length) == 0 and
|
||||
(.cases | length) == .counts.seen and
|
||||
([.cases[].case_id] | unique | length) == .counts.seen and
|
||||
([.cases[].case_id] | sort) == (.expected_cases | sort) and
|
||||
all(.cases[]; (.case_id | startswith("M-")) and (.verdict == "pass" or .verdict == "known-divergence" or .verdict == "UNEXPECTED")) and
|
||||
.counts.unexpected == ([.cases[] | select(.verdict == "UNEXPECTED")] | length) and
|
||||
.counts.known_divergence == ([.cases[] | select(.verdict == "known-divergence")] | length)
|
||||
' "${RESULTS_FILE}" >/dev/null
|
||||
|
||||
- name: Upload test logs & evidence
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: rustfs-fault-tolerance-matrix-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results.json
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/evidence/
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && inputs.cleanup_after != 'false' }}
|
||||
run: |
|
||||
./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}" || true
|
||||
@@ -4,14 +4,13 @@
|
||||
# behavior under drive and node loss against the erasure-coding contract and
|
||||
# snapshots health-endpoint responses at every tier.
|
||||
#
|
||||
# A single-node 4 drives (SNMD): revoke access to 1/2/3 drives, restore
|
||||
# A single-node 4 drives (SNMD): hide 1/2/3 drives, restore
|
||||
# B multi-node 4x1 (one drive per node): stop 1/2/3 nodes, restore
|
||||
# C multi-node 4x4 (16 drives, EC:4): stop 1 node (read-quorum boundary),
|
||||
# stop 2 nodes, restore
|
||||
# C2 multi-node 4x4 with EC:8: 2 nodes down puts 8 drives online -- reads
|
||||
# satisfy the EC read quorum while the lock majority is broken (the
|
||||
# reported divergence window: reads 503 with lock_quorum_unavailable)
|
||||
# D multi-node 4x1: stop 1/2 nodes, then restore
|
||||
#
|
||||
# Expectations come from product source (default_parity_count, erasure set
|
||||
# sizing). By default a "reads refused although the read quorum is met"
|
||||
@@ -33,11 +32,6 @@ on:
|
||||
description: 'Direct .deb URL. Required unless the nightly default is wanted.'
|
||||
required: false
|
||||
type: string
|
||||
auto_testing_ref:
|
||||
description: 'auto-testing branch, tag, or SHA (defaults to main)'
|
||||
required: false
|
||||
default: main
|
||||
type: string
|
||||
strict:
|
||||
description: 'Fail the suite when reads are refused despite a met read quorum'
|
||||
type: boolean
|
||||
@@ -58,7 +52,7 @@ on:
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# The suite stops services and revokes drive access on the shared fleet; only one
|
||||
# The suite stops services and hides drive dirs on the shared fleet; only one
|
||||
# functional suite may touch the environment at a time.
|
||||
concurrency:
|
||||
group: rustfs-shared-functional-tests-v2
|
||||
@@ -79,7 +73,7 @@ env:
|
||||
jobs:
|
||||
fault-tolerance-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: 480
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout repository (for report parser)
|
||||
@@ -92,15 +86,13 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-fault-tolerance-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch" "${FUNCTIONAL_ARTIFACTS_DIR}/evidence"
|
||||
FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-ft-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}/evidence"
|
||||
{
|
||||
printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'RESULTS_FILE=%s/results.json\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'EVIDENCE_DIR=%s/evidence\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
|
||||
# Pin the private suite checkout when the chain supplies a candidate.
|
||||
@@ -112,32 +104,22 @@ jobs:
|
||||
run: python3 scripts/functional_chain_evidence.py consume
|
||||
|
||||
- name: Checkout auto-testing scripts
|
||||
timeout-minutes: 5
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
repository: rustfs/auto-testing
|
||||
ref: ${{ steps.chain.outputs.testing_sha || inputs.auto_testing_ref || 'main' }}
|
||||
ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
path: auto-testing
|
||||
persist-credentials: false
|
||||
|
||||
- name: Record auto-testing revision
|
||||
run: |
|
||||
set -euo pipefail
|
||||
AUTO_TESTING_SHA="$(git -C auto-testing rev-parse HEAD)"
|
||||
printf 'AUTO_TESTING_SHA=%s\n' "${AUTO_TESTING_SHA}" >> "${GITHUB_ENV}"
|
||||
echo "auto-testing revision: ${AUTO_TESTING_SHA}"
|
||||
|
||||
- name: Show environment
|
||||
run: |
|
||||
uname -a
|
||||
jq --version
|
||||
aws --version
|
||||
echo "auto-testing: ${AUTO_TESTING_SHA}"
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
if: ${{ inputs.cleanup_before != 'false' }}
|
||||
run: |
|
||||
./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}"
|
||||
@@ -149,21 +131,19 @@ jobs:
|
||||
CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
|
||||
run: python3 scripts/prepare_functional_package.py prepare
|
||||
|
||||
- name: Run fault-tolerance scenarios (A, B, C, C2, D, E)
|
||||
timeout-minutes: 45
|
||||
- name: Run fault-tolerance scenarios (A, B, C, C2)
|
||||
id: test
|
||||
# Standalone case failures are reported by the backlog manager.
|
||||
# Chain evidence separately requires complete passing cases.
|
||||
continue-on-error: true
|
||||
run: |
|
||||
ARGS=(--all -y --package-url "${{ steps.chain_package.outputs.package_url || inputs.package_url || env.RUSTFS_NIGHTLY_PACKAGE_URL }}" --log-file "${LOG_FILE}" --results-file "${RESULTS_FILE}")
|
||||
ARGS=(--all -y --package-url "${{ steps.chain_package.outputs.package_url || inputs.package_url || env.RUSTFS_NIGHTLY_PACKAGE_URL }}" --log-file "${LOG_FILE}")
|
||||
if [ "${{ inputs.strict }}" = "true" ]; then
|
||||
ARGS+=(--strict)
|
||||
fi
|
||||
./auto-testing/rustfs-fault-tolerance-test.sh "${ARGS[@]}"
|
||||
|
||||
- name: Generate report
|
||||
timeout-minutes: 2
|
||||
id: chain_report
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
run: |
|
||||
@@ -173,13 +153,12 @@ jobs:
|
||||
echo ""
|
||||
echo "- Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo "- Package: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
|
||||
echo "- auto-testing: \`${AUTO_TESTING_SHA}\`"
|
||||
echo "- Strict mode: ${{ inputs.strict || 'false' }}"
|
||||
echo ""
|
||||
echo "## Per-probe results"
|
||||
echo ""
|
||||
echo '```'
|
||||
grep -E '^FT-(CASE|COMPLETE|SUMMARY|REPORT)' "${LOG_FILE}" || echo "(no FT-CASE lines found)"
|
||||
grep -E '^FT-(CASE|SUMMARY|REPORT)' "${LOG_FILE}" || echo "(no FT-CASE lines found)"
|
||||
echo '```'
|
||||
echo ""
|
||||
echo "## Health snapshots"
|
||||
@@ -190,40 +169,28 @@ jobs:
|
||||
done
|
||||
} > "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
printf 'complete=false\n' >> "${GITHUB_OUTPUT}"
|
||||
jq -e --arg testing_sha "${AUTO_TESTING_SHA}" '
|
||||
.schema_version == 1 and
|
||||
.auto_testing_sha == $testing_sha and
|
||||
.selected_scenarios == ["A", "B", "C", "C2", "D", "E"] and
|
||||
.complete == true and
|
||||
.counts.expected == 44 and
|
||||
.counts.seen == 44 and
|
||||
.counts.harness_errors == 0 and
|
||||
(.expected_cases | length) == 44 and
|
||||
(.expected_cases | unique | length) == 44 and
|
||||
(.seen_cases | length) == 44 and
|
||||
(.seen_cases | unique | length) == 44 and
|
||||
(.seen_cases | sort) == (.expected_cases | sort) and
|
||||
(.missing_cases | length) == 0 and
|
||||
(.harness_errors | length) == 0 and
|
||||
(.cases | length) == 44 and
|
||||
([.cases[].case_id] | sort) == (.expected_cases | sort) and
|
||||
([.cases[].case_id] | unique | length) == 44 and
|
||||
all(.cases[]; .verdict == "pass" or .verdict == "known-divergence" or .verdict == "UNEXPECTED") and
|
||||
.counts.unexpected == ([.cases[] | select(.verdict == "UNEXPECTED")] | length) and
|
||||
.counts.known_divergence == ([.cases[] | select(.verdict == "known-divergence")] | length)
|
||||
' "${RESULTS_FILE}" >/dev/null
|
||||
printf 'complete=true\n' >> "${GITHUB_OUTPUT}"
|
||||
# Product gate: failing FT cases keep the run green — the report above and
|
||||
# the backlog issue manager carry the signal. Only harness/environment
|
||||
# breakdowns (no case verdicts at all) turn the workflow red.
|
||||
FT_CASES="$(grep -cE '^FT-CASE:' "${LOG_FILE}" 2>/dev/null || true)"
|
||||
FT_FAILED="$(grep -cE '^FT-CASE: .* verdict=UNEXPECTED' "${LOG_FILE}" 2>/dev/null || true)"
|
||||
FT_PASSED=$(( ${FT_CASES:-0} - ${FT_FAILED:-0} ))
|
||||
HARNESS_OK=0
|
||||
if [ "${{ steps.test.outcome }}" = "success" ]; then
|
||||
HARNESS_OK=1
|
||||
elif [ "${{ steps.test.outcome }}" = "failure" ] && [ "${FT_CASES}" -gt 0 ] && [ "${FT_PASSED}" -ge 1 ]; then
|
||||
HARNESS_OK=1
|
||||
fi
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
|
||||
- name: Manage backlog issues (dedup / label / auto-close)
|
||||
timeout-minutes: 2
|
||||
# Signal-based lifecycle: dedups against open issues by label
|
||||
# (fault-tolerance + FT case), labels new issues (functional-test,
|
||||
# category, case IDs, env), and closes fixed issues after a fully
|
||||
# green run. Never acts on cancelled runs. Logic lives in
|
||||
# auto-testing/scripts/issue_manager.py, which parses the
|
||||
# FT-CASE verdict lines from the suite log.
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' && steps.chain_report.outputs.complete == 'true' }}
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
@@ -261,51 +228,18 @@ jobs:
|
||||
--package-source "${PACKAGE_SOURCE}" \
|
||||
--date "$(date -u +%Y-%m-%d)"
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
REPORT_FILE: ${{ env.REPORT_FILE }}
|
||||
SUITE: fault-tolerance
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
|
||||
exit 0
|
||||
fi
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
||||
B64_FILE="$(mktemp)"
|
||||
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n"))}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
fi
|
||||
echo "dashboard updated: ${REPORT_PATH}"
|
||||
|
||||
- name: Upload test logs & evidence
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: rustfs-fault-tolerance-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results.json
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/evidence/
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && inputs.cleanup_after != 'false' }}
|
||||
run: |
|
||||
./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}" || true
|
||||
|
||||
@@ -46,7 +46,6 @@ jobs:
|
||||
timeout-minutes: 10
|
||||
outputs:
|
||||
manifest: ${{ steps.candidate.outputs.manifest }}
|
||||
performance_ready: ${{ steps.perf_probe.outputs.performance_ready }}
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
@@ -54,10 +53,7 @@ jobs:
|
||||
- name: Resolve published candidate
|
||||
id: candidate
|
||||
env:
|
||||
# The staleness probe in resolve_functional_candidate.py reads the
|
||||
# private rustfs/auto-testing repo; github.token cannot, and a probe
|
||||
# failure silently keeps a stale testing_sha pinned for the chain.
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
BUILD_RUN_ID: ${{ inputs.build_run_id }}
|
||||
BUILD_RUN_ATTEMPT: ${{ inputs.build_run_attempt }}
|
||||
CHAIN_OUTPUT: ${{ runner.temp }}/chain-candidate.json
|
||||
@@ -69,25 +65,10 @@ jobs:
|
||||
path: ${{ runner.temp }}/chain-candidate.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Check shared functional fleet runner before scheduling suites
|
||||
# The eleven functional suites run on the smoke-testing fleet; a hard
|
||||
# requirement. The performance lane runs on pf-testing, which is a
|
||||
# separate fleet - probe it without failing the whole chain so an
|
||||
# offline performance runner skips only the performance lane.
|
||||
- name: Check shared functional runners before scheduling suites
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: python3 scripts/check_functional_runners.py smoke-testing
|
||||
|
||||
- name: Probe performance fleet runner
|
||||
id: perf_probe
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
if python3 scripts/check_functional_runners.py pf-testing; then
|
||||
echo "performance_ready=online" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "performance_ready=offline" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
run: python3 scripts/check_functional_runners.py smoke-testing pf-testing
|
||||
|
||||
upgrade:
|
||||
needs: [prepare]
|
||||
@@ -179,7 +160,7 @@ jobs:
|
||||
|
||||
performance:
|
||||
needs: [prepare, table]
|
||||
if: ${{ always() && needs.prepare.result == 'success' && needs.prepare.outputs.performance_ready == 'online' }}
|
||||
if: ${{ always() && needs.prepare.result == 'success' }}
|
||||
uses: ./.github/workflows/rustfs-performance-test.yml
|
||||
with:
|
||||
chain_manifest: ${{ needs.prepare.outputs.manifest }}
|
||||
@@ -227,7 +208,6 @@ jobs:
|
||||
CHAIN_NEEDS: ${{ toJSON(needs) }}
|
||||
run: >-
|
||||
python3 scripts/functional_chain_evidence.py aggregate
|
||||
--allow-skipped "${{ needs.prepare.outputs.performance_ready == 'online' && '' || 'performance' }}"
|
||||
--directory "${RUNNER_TEMP}/chain-evidence"
|
||||
--output "${RUNNER_TEMP}/chain-complete.json"
|
||||
- name: Upload complete-chain evidence
|
||||
|
||||
@@ -60,7 +60,7 @@ env:
|
||||
jobs:
|
||||
heal-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: 480
|
||||
# Standalone manual run, or one link of the nightly functional chain
|
||||
# (storage -> heal -> pool). Pool expansion no longer re-runs heal.
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
@@ -110,7 +110,6 @@ jobs:
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
if: ${{ inputs.cleanup_before != 'false' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -130,7 +129,6 @@ jobs:
|
||||
done
|
||||
|
||||
- name: Install RustFS package & start cluster
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
ARGS=(--steps "1,2" -y --endpoint "${{ env.RUSTFS_API_ENDPOINT }}")
|
||||
if [ -n "${{ inputs.package_url }}" ]; then
|
||||
@@ -141,7 +139,6 @@ jobs:
|
||||
./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
|
||||
|
||||
- name: Preflight checks
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
ARGS=(--preflight --endpoint "${{ env.RUSTFS_API_ENDPOINT }}")
|
||||
if [ -n "${{ inputs.package_url }}" ]; then
|
||||
@@ -152,7 +149,6 @@ jobs:
|
||||
./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
|
||||
|
||||
- name: Run heal test (write -> outage -> heal -> verify)
|
||||
timeout-minutes: 45
|
||||
id: test
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
@@ -167,7 +163,6 @@ jobs:
|
||||
|
||||
- name: Generate report
|
||||
id: chain_report
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -285,7 +280,6 @@ jobs:
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
@@ -317,7 +311,6 @@ jobs:
|
||||
rm -f "${B64_FILE}"
|
||||
|
||||
- name: Manage backlog issues (dedup / label / auto-close)
|
||||
timeout-minutes: 2
|
||||
# Replaces the old per-run failure filing. One entry point that:
|
||||
# - dedups by signal: failing cases are matched against open backlog
|
||||
# issues by label (category + case ID); covered cases become a
|
||||
@@ -364,7 +357,6 @@ jobs:
|
||||
--date "$(date -u +%Y-%m-%d)"
|
||||
|
||||
- name: Upload test logs
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
@@ -377,7 +369,6 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && inputs.cleanup_after != 'false' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
@@ -55,7 +55,7 @@ env:
|
||||
jobs:
|
||||
kms-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: 420
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout repository (for report parser)
|
||||
@@ -101,7 +101,6 @@ jobs:
|
||||
docker --version || true
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
@@ -130,7 +129,6 @@ jobs:
|
||||
|
||||
- name: Run KMS suite
|
||||
id: test
|
||||
timeout-minutes: 45
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
continue-on-error: true
|
||||
@@ -164,7 +162,6 @@ jobs:
|
||||
|
||||
- name: Generate report
|
||||
id: chain_report
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -230,7 +227,6 @@ jobs:
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
@@ -262,7 +258,6 @@ jobs:
|
||||
rm -f "${B64_FILE}"
|
||||
|
||||
- name: Manage backlog issues (dedup / label / auto-close)
|
||||
timeout-minutes: 2
|
||||
# Replaces the old per-run failure filing. One entry point that:
|
||||
# - dedups by signal: failing cases are matched against open backlog
|
||||
# issues by label (category + case ID); covered cases become a
|
||||
@@ -313,7 +308,6 @@ jobs:
|
||||
--date "$(date -u +%Y-%m-%d)"
|
||||
|
||||
- name: Upload report and logs
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
@@ -325,7 +319,6 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
timeout-minutes: 5
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
@@ -61,13 +61,10 @@ on:
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# Workflow-level so the lock is held through cleanup as well.
|
||||
# The default performance nodes overlap the other suites' remote VMs, even
|
||||
# though the runner differs. Hold the shared lock through cleanup as well.
|
||||
concurrency:
|
||||
# Own group: the performance suite runs on the pf-testing fleet - a fully
|
||||
# separate environment from the smoke-testing fleet the functional suites
|
||||
# use. No shared node, no shared runner: it must never queue behind them
|
||||
# (or block them).
|
||||
group: rustfs-performance-suite
|
||||
group: rustfs-shared-functional-tests-v2
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
|
||||
@@ -82,7 +82,7 @@ jobs:
|
||||
pool-expansion-test:
|
||||
name: Pool expansion / decommission test
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: 360
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
env:
|
||||
RUSTFS_POOL_ADMIN_ENDPOINT: ${{ secrets.RUSTFS_POOL_ADMIN_ENDPOINT || vars.RUSTFS_POOL_ADMIN_ENDPOINT || 'http://rustfs-node1:9000' }}
|
||||
@@ -140,7 +140,6 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
if: ${{ inputs.cleanup_before != 'false' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -160,7 +159,6 @@ jobs:
|
||||
done
|
||||
|
||||
- name: Install RustFS package & start cluster
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
ARGS=(--steps "1,2,3" -y \
|
||||
--admin-endpoint "${RUSTFS_POOL_ADMIN_ENDPOINT}" \
|
||||
@@ -177,7 +175,6 @@ jobs:
|
||||
./auto-testing/rustfs_pool_expand.sh "${ARGS[@]}"
|
||||
|
||||
- name: Preflight checks
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
ARGS=(--preflight \
|
||||
--admin-endpoint "${RUSTFS_POOL_ADMIN_ENDPOINT}" \
|
||||
@@ -194,7 +191,6 @@ jobs:
|
||||
./auto-testing/rustfs_pool_expand.sh "${ARGS[@]}"
|
||||
|
||||
- name: Reset dedicated pool proxy
|
||||
timeout-minutes: 2
|
||||
run: |
|
||||
set -euo pipefail
|
||||
RUSTFS_POOL_NGINX_CONFIG_PATH=/etc/nginx/conf.d/rustfs-pool-test.conf \
|
||||
@@ -204,7 +200,6 @@ jobs:
|
||||
./auto-testing/rustfs_pool_nginx_stage.sh cleanup
|
||||
|
||||
- name: Capture pool test baseline
|
||||
timeout-minutes: 2
|
||||
run: |
|
||||
set -uo pipefail
|
||||
BASELINE_FILE="${POOL_ARTIFACT_DIR}/pool-baseline.log"
|
||||
@@ -251,7 +246,6 @@ jobs:
|
||||
|
||||
- name: Run pool expansion & decommission test
|
||||
id: pool_test
|
||||
timeout-minutes: 45
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
continue-on-error: true
|
||||
@@ -291,7 +285,6 @@ jobs:
|
||||
./auto-testing/rustfs_pool_expand.sh "${ARGS[@]}"
|
||||
|
||||
- name: Collect pool test diagnostics
|
||||
timeout-minutes: 2
|
||||
if: always()
|
||||
run: |
|
||||
set -uo pipefail
|
||||
@@ -393,7 +386,6 @@ jobs:
|
||||
|
||||
- name: Generate report
|
||||
id: chain_report
|
||||
timeout-minutes: 2
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -481,7 +473,6 @@ jobs:
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Validate pool diagnostic completeness
|
||||
timeout-minutes: 2
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -571,7 +562,6 @@ jobs:
|
||||
[ "${failed}" -eq 0 ] || exit 1
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
timeout-minutes: 2
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
env:
|
||||
@@ -604,7 +594,6 @@ jobs:
|
||||
rm -f "${B64_FILE}"
|
||||
|
||||
- name: Manage backlog issues (dedup / label / auto-close)
|
||||
timeout-minutes: 2
|
||||
# Replaces the old per-run failure filing. One entry point that:
|
||||
# - dedups by signal: failing cases are matched against open backlog
|
||||
# issues by label (category + case ID); covered cases become a
|
||||
@@ -654,7 +643,6 @@ jobs:
|
||||
--date "$(date -u +%Y-%m-%d)"
|
||||
|
||||
- name: Upload test logs
|
||||
timeout-minutes: 2
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
@@ -663,7 +651,6 @@ jobs:
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Restore dedicated pool proxy
|
||||
timeout-minutes: 2
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -674,7 +661,6 @@ jobs:
|
||||
./auto-testing/rustfs_pool_nginx_stage.sh cleanup
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && steps.topology.outcome == 'success' && inputs.cleanup_after != 'false' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
@@ -67,7 +67,7 @@ env:
|
||||
jobs:
|
||||
replication-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: 360
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout repository (for report parser)
|
||||
@@ -114,7 +114,6 @@ jobs:
|
||||
df -h /data | tail -1 || true
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
@@ -134,7 +133,6 @@ jobs:
|
||||
|
||||
- name: Run replication suite
|
||||
id: test
|
||||
timeout-minutes: 45
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
continue-on-error: true
|
||||
@@ -161,7 +159,6 @@ jobs:
|
||||
|
||||
- name: Generate report
|
||||
id: chain_report
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -238,7 +235,6 @@ jobs:
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
@@ -270,7 +266,6 @@ jobs:
|
||||
rm -f "${B64_FILE}"
|
||||
|
||||
- name: Manage backlog issues (dedup / label / auto-close)
|
||||
timeout-minutes: 2
|
||||
# Replaces the old per-run failure filing. One entry point that:
|
||||
# - dedups by signal: failing cases are matched against open backlog
|
||||
# issues by label (category + case ID); covered cases become a
|
||||
@@ -321,7 +316,6 @@ jobs:
|
||||
--date "$(date -u +%Y-%m-%d)"
|
||||
|
||||
- name: Upload report and logs
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
@@ -333,7 +327,6 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
timeout-minutes: 5
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
@@ -43,7 +43,7 @@ env:
|
||||
jobs:
|
||||
s3-compat-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: 360
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout repository (for report parser)
|
||||
@@ -89,7 +89,6 @@ jobs:
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
@@ -109,7 +108,6 @@ jobs:
|
||||
|
||||
- name: Run S3 compatibility suite
|
||||
id: test
|
||||
timeout-minutes: 45
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
continue-on-error: true
|
||||
@@ -130,7 +128,6 @@ jobs:
|
||||
|
||||
- name: Generate report
|
||||
id: chain_report
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -207,7 +204,6 @@ jobs:
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
@@ -239,7 +235,6 @@ jobs:
|
||||
rm -f "${B64_FILE}"
|
||||
|
||||
- name: Manage backlog issues (dedup / label / auto-close)
|
||||
timeout-minutes: 2
|
||||
# Replaces the old per-run failure filing. One entry point that:
|
||||
# - dedups by signal: failing cases are matched against open backlog
|
||||
# issues by label (category + case ID); covered cases become a
|
||||
@@ -290,7 +285,6 @@ jobs:
|
||||
--date "$(date -u +%Y-%m-%d)"
|
||||
|
||||
- name: Upload report and logs
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
@@ -302,7 +296,6 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
timeout-minutes: 5
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
@@ -80,7 +80,7 @@ env:
|
||||
jobs:
|
||||
security-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: 360
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
# Checkout the repository into its own subdirectory. Checking out at
|
||||
@@ -127,7 +127,6 @@ jobs:
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
if: ${{ inputs.cleanup_before != 'false' || github.event_name != 'workflow_dispatch' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -148,7 +147,6 @@ jobs:
|
||||
|
||||
- name: Run security suite
|
||||
id: test
|
||||
timeout-minutes: 45
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
continue-on-error: true
|
||||
@@ -182,7 +180,6 @@ jobs:
|
||||
|
||||
- name: Generate report
|
||||
id: report
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
env:
|
||||
TEST_OUTCOME: ${{ steps.test.outcome }}
|
||||
@@ -240,7 +237,6 @@ jobs:
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
@@ -273,7 +269,6 @@ jobs:
|
||||
rm -f "${B64_FILE}"
|
||||
|
||||
- name: Manage backlog issues (dedup / label / auto-close)
|
||||
timeout-minutes: 2
|
||||
# Replaces the old per-run failure filing. One entry point that:
|
||||
# - dedups by signal: failing cases are matched against open backlog
|
||||
# issues by label (category + case ID); covered cases become a
|
||||
@@ -323,7 +318,6 @@ jobs:
|
||||
--date "$(date -u +%Y-%m-%d)"
|
||||
|
||||
- name: Upload report and logs
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
@@ -336,7 +330,6 @@ jobs:
|
||||
retention-days: 3
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
@@ -52,7 +52,7 @@ env:
|
||||
jobs:
|
||||
storage-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: 360
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout repository (for report parser)
|
||||
@@ -98,7 +98,6 @@ jobs:
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
@@ -118,7 +117,6 @@ jobs:
|
||||
|
||||
- name: Run storage engine suite
|
||||
id: test
|
||||
timeout-minutes: 45
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
continue-on-error: true
|
||||
@@ -145,7 +143,6 @@ jobs:
|
||||
|
||||
- name: Generate report
|
||||
id: chain_report
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -222,7 +219,6 @@ jobs:
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
@@ -254,7 +250,6 @@ jobs:
|
||||
rm -f "${B64_FILE}"
|
||||
|
||||
- name: Manage backlog issues (dedup / label / auto-close)
|
||||
timeout-minutes: 2
|
||||
# Replaces the old per-run failure filing. One entry point that:
|
||||
# - dedups by signal: failing cases are matched against open backlog
|
||||
# issues by label (category + case ID); covered cases become a
|
||||
@@ -305,7 +300,6 @@ jobs:
|
||||
--date "$(date -u +%Y-%m-%d)"
|
||||
|
||||
- name: Upload report and logs
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
@@ -317,7 +311,6 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
timeout-minutes: 5
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
@@ -43,7 +43,7 @@ jobs:
|
||||
table-test:
|
||||
name: table-test
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: 90
|
||||
steps:
|
||||
- name: Checkout repository (for report parser)
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
@@ -72,7 +72,6 @@ jobs:
|
||||
run: python3 scripts/functional_chain_evidence.py consume
|
||||
|
||||
- name: Checkout auto-testing scripts
|
||||
timeout-minutes: 5
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
repository: rustfs/auto-testing
|
||||
@@ -90,7 +89,6 @@ jobs:
|
||||
python3 --version
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
continue-on-error: true
|
||||
run: |
|
||||
set -uo pipefail
|
||||
@@ -116,7 +114,6 @@ jobs:
|
||||
|
||||
- name: Run table suite
|
||||
id: test
|
||||
timeout-minutes: 45
|
||||
# Standalone case failures are reported by the backlog manager.
|
||||
# Chain evidence separately requires complete passing cases.
|
||||
continue-on-error: true
|
||||
@@ -140,7 +137,6 @@ jobs:
|
||||
./auto-testing/rustfs-table-test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
|
||||
|
||||
- name: Generate report
|
||||
timeout-minutes: 2
|
||||
id: chain_report
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
run: |
|
||||
@@ -207,7 +203,6 @@ jobs:
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
@@ -237,7 +232,6 @@ jobs:
|
||||
echo "dashboard updated: ${REPORT_PATH}"
|
||||
|
||||
- name: Manage backlog issues (dedup / label / auto-close)
|
||||
timeout-minutes: 2
|
||||
# Replaces the old per-run failure filing. One entry point that:
|
||||
# - dedups by signal: failing cases are matched against open backlog
|
||||
# issues by label (category + case ID); covered cases become a
|
||||
@@ -279,7 +273,6 @@ jobs:
|
||||
--date "$(date -u +%Y-%m-%d)"
|
||||
|
||||
- name: Upload report and logs
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
@@ -292,7 +285,6 @@ jobs:
|
||||
retention-days: 30
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
timeout-minutes: 5
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
run: |
|
||||
|
||||
@@ -67,7 +67,7 @@ env:
|
||||
jobs:
|
||||
tier-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: 420
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout chain tooling
|
||||
@@ -104,7 +104,6 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Prepare pinned RustFS CLI
|
||||
timeout-minutes: 5
|
||||
id: rc
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -217,7 +216,6 @@ jobs:
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sudo docker rm -f rustfs-test-mqtt >/dev/null 2>&1 || true
|
||||
@@ -238,7 +236,6 @@ jobs:
|
||||
done
|
||||
|
||||
- name: Ensure MQTT broker + clients
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if ! command -v mosquitto_sub >/dev/null 2>&1; then
|
||||
@@ -268,7 +265,6 @@ jobs:
|
||||
|
||||
- name: Run tier suite
|
||||
id: test
|
||||
timeout-minutes: 45
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
continue-on-error: true
|
||||
@@ -313,7 +309,6 @@ jobs:
|
||||
|
||||
- name: Generate report
|
||||
id: chain_report
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
env:
|
||||
PACKAGE_URL_INPUT: ${{ inputs.package_url }}
|
||||
@@ -394,7 +389,6 @@ jobs:
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
@@ -427,7 +421,6 @@ jobs:
|
||||
rm -f "${B64_FILE}"
|
||||
|
||||
- name: Verify required tier evidence
|
||||
timeout-minutes: 2
|
||||
id: evidence_verify
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
run: |
|
||||
@@ -458,7 +451,6 @@ jobs:
|
||||
[ "${failed}" -eq 0 ]
|
||||
|
||||
- name: Upload report and logs
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
@@ -467,7 +459,6 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
timeout-minutes: 5
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -489,7 +480,6 @@ jobs:
|
||||
done
|
||||
|
||||
- name: Cleanup pinned RustFS CLI
|
||||
timeout-minutes: 2
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -508,103 +498,23 @@ jobs:
|
||||
GATE_RC_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier-gate.rc
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Product gate: failing cases keep the run green (the structured gate
|
||||
# rc and the report carry the per-case signal, and the backlog issue
|
||||
# manager tracks failures). Only harness/environment breakdowns —
|
||||
# evidence initialization or a missing structured gate result — turn
|
||||
# the workflow red.
|
||||
failed=0
|
||||
if [ "${EVIDENCE_OUTCOME}" != "success" ]; then
|
||||
echo "tier evidence directory initialization is ${EVIDENCE_OUTCOME}, expected success" >&2
|
||||
failed=1
|
||||
fi
|
||||
if [ "${TEST_OUTCOME}" != "success" ]; then
|
||||
echo "tier suite step outcome is ${TEST_OUTCOME}, expected success" >&2
|
||||
failed=1
|
||||
fi
|
||||
if [ "${EVIDENCE_OUTCOME}" != "success" ]; then
|
||||
echo "structured gate result is unavailable because evidence initialization failed" >&2
|
||||
elif [ ! -s "${GATE_RC_FILE}" ]; then
|
||||
echo "structured gate result is missing" >&2
|
||||
failed=1
|
||||
else
|
||||
GATE_RC="$(tr -d '[:space:]' < "${GATE_RC_FILE}")"
|
||||
if ! [[ "${GATE_RC}" =~ ^[0-9]+$ ]] || [ "${GATE_RC}" -ne 0 ]; then
|
||||
echo "structured 56-case gate failed with exit ${GATE_RC:-invalid}" >&2
|
||||
failed=1
|
||||
fi
|
||||
fi
|
||||
[ "${failed}" -eq 0 ]
|
||||
|
||||
- name: File failure issue in rustfs/backlog
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && (steps.evidence.outcome != 'success' || steps.test.outcome == 'cancelled' || steps.evidence_verify.outcome == 'failure' || steps.evidence_verify.outcome == 'cancelled' || (steps.gate.outcome == 'failure' && steps.test.outcome != 'failure') || steps.gate.outcome == 'cancelled') }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
SUITE: 'tier'
|
||||
SUITE_LABEL: 'Tier'
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
EVIDENCE_DIR: ${{ env.TIER_ARTIFACTS_DIR }}
|
||||
EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
|
||||
VERIFY_OUTCOME: ${{ steps.evidence_verify.outcome }}
|
||||
GATE_OUTCOME: ${{ steps.gate.outcome }}
|
||||
REPORT_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier-report.md
|
||||
LOG_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier.log
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
|
||||
exit 0
|
||||
fi
|
||||
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID}, attempt ${GITHUB_RUN_ATTEMPT})"
|
||||
EXISTING="$(gh issue list -R rustfs/backlog --state all \
|
||||
--search "in:title \"run ${GITHUB_RUN_ID}\"" \
|
||||
--json number,title --jq ".[] | select(.title == \"${TITLE}\") | .number")"
|
||||
if [ -n "${EXISTING}" ]; then
|
||||
echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
|
||||
exit 0
|
||||
fi
|
||||
redact() {
|
||||
sed -E \
|
||||
-e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
|
||||
-e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
|
||||
-e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
|
||||
-e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
|
||||
}
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The **${SUITE_LABEL}** functional suite failed."
|
||||
echo ""
|
||||
echo "- Suite: \`${SUITE}\`"
|
||||
echo "- Run: ${RUN_URL}"
|
||||
echo "- Trigger: ${GITHUB_EVENT_NAME}"
|
||||
echo "- Date: $(date -u +%Y-%m-%d)"
|
||||
echo "- Evidence initialization: ${EVIDENCE_OUTCOME}"
|
||||
echo "- Evidence verification: ${VERIFY_OUTCOME}"
|
||||
echo "- Final gate: ${GATE_OUTCOME}"
|
||||
echo ""
|
||||
echo "## Report (errors and symptoms)"
|
||||
echo ""
|
||||
if [ "${EVIDENCE_OUTCOME}" != "success" ]; then
|
||||
echo "(the run evidence directory was rejected; its contents were not read)"
|
||||
elif [ ! -d "${EVIDENCE_DIR}" ] || [ -L "${EVIDENCE_DIR}" ]; then
|
||||
echo "(the run evidence directory is missing or unsafe; its contents were not read)"
|
||||
elif [ -s "${REPORT_FILE}" ]; then
|
||||
redact < "${REPORT_FILE}"
|
||||
elif [ -s "${LOG_FILE:-}" ]; then
|
||||
echo "(report file missing; log tail below)"
|
||||
echo ""
|
||||
tail -n 200 "${LOG_FILE}" | redact
|
||||
else
|
||||
echo "(no report or log file was produced)"
|
||||
fi
|
||||
} | head -c 55000 > "${BODY_FILE}"
|
||||
gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
|
||||
if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test; then
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
|
||||
fi
|
||||
echo "filed backlog issue for suite ${SUITE}"
|
||||
|
||||
- name: Manage backlog issues (dedup / label / auto-close)
|
||||
timeout-minutes: 2
|
||||
# Case-level issue management complements the infrastructure failure report:
|
||||
# Replaces the old per-run failure filing. One entry point that:
|
||||
# - dedups by signal: failing cases are matched against open backlog
|
||||
# issues by label (category + case ID); covered cases become a
|
||||
# comment on the existing issue, only uncovered cases file a new one
|
||||
@@ -615,9 +525,6 @@ jobs:
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
TEST_OUTCOME: ${{ steps.test.outcome }}
|
||||
VERIFY_OUTCOME: ${{ steps.evidence_verify.outcome }}
|
||||
GATE_OUTCOME: ${{ steps.gate.outcome }}
|
||||
SUITE: 'tier'
|
||||
SUITE_LABEL: 'Tier'
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
@@ -631,14 +538,6 @@ jobs:
|
||||
echo "issue_manager.py not found in auto-testing checkout; skipping"
|
||||
exit 0
|
||||
fi
|
||||
if [ ! -d "${TIER_ARTIFACTS_DIR}" ] || [ -L "${TIER_ARTIFACTS_DIR}" ]; then
|
||||
echo "tier evidence directory is missing or unsafe; skipping backlog issue management" >&2
|
||||
exit 1
|
||||
fi
|
||||
OUTCOME="${TEST_OUTCOME}"
|
||||
if [ "${OUTCOME}" != "cancelled" ] && { [ "${VERIFY_OUTCOME}" != "success" ] || [ "${GATE_OUTCOME}" != "success" ]; }; then
|
||||
OUTCOME=failure
|
||||
fi
|
||||
PACKAGE_URL='${{ inputs.package_url }}'
|
||||
RUSTFS_VERSION='${{ inputs.rustfs_version }}'
|
||||
PACKAGE_SOURCE=""
|
||||
@@ -652,7 +551,7 @@ jobs:
|
||||
python3 auto-testing/scripts/issue_manager.py handle \
|
||||
--repo rustfs/backlog \
|
||||
--suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
|
||||
--outcome "${OUTCOME}" \
|
||||
--outcome "${{ steps.test.outcome }}" \
|
||||
--report "${TIER_ARTIFACTS_DIR}/rustfs-tier-cases.md" \
|
||||
--report-file "${TIER_ARTIFACTS_DIR}/rustfs-tier-report.md" \
|
||||
--log "${TIER_ARTIFACTS_DIR}/rustfs-tier.log" \
|
||||
|
||||
@@ -85,7 +85,7 @@ env:
|
||||
jobs:
|
||||
upgrade-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: 420
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout repository (for report parser)
|
||||
@@ -133,7 +133,6 @@ jobs:
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
if: ${{ inputs.cleanup_before != 'false' || github.event_name != 'workflow_dispatch' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -153,7 +152,6 @@ jobs:
|
||||
done
|
||||
|
||||
- name: Ensure docker (Vault container)
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
sudo apt-get update
|
||||
@@ -164,7 +162,6 @@ jobs:
|
||||
|
||||
- name: Run upgrade compatibility suite
|
||||
id: test
|
||||
timeout-minutes: 45
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
continue-on-error: true
|
||||
@@ -227,7 +224,6 @@ jobs:
|
||||
|
||||
- name: Generate report
|
||||
id: chain_report
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -308,7 +304,6 @@ jobs:
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
@@ -340,7 +335,6 @@ jobs:
|
||||
rm -f "${B64_FILE}"
|
||||
|
||||
- name: Manage backlog issues (dedup / label / auto-close)
|
||||
timeout-minutes: 2
|
||||
# Replaces the old per-run failure filing. One entry point that:
|
||||
# - dedups by signal: failing cases are matched against open backlog
|
||||
# issues by label (category + case ID); covered cases become a
|
||||
@@ -398,7 +392,6 @@ jobs:
|
||||
--date "$(date -u +%Y-%m-%d)"
|
||||
|
||||
- name: Upload report and logs
|
||||
timeout-minutes: 2
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
@@ -412,7 +405,6 @@ jobs:
|
||||
retention-days: 3
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
@@ -16,7 +16,7 @@ name: Windows Filesystem Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main ]
|
||||
branches: [ main, release ]
|
||||
paths:
|
||||
- "crates/ecstore/src/disk/**"
|
||||
- "crates/ecstore/src/store/init_format.rs"
|
||||
@@ -26,7 +26,7 @@ on:
|
||||
- ".github/actions/setup/**"
|
||||
- ".github/workflows/windows-filesystem.yml"
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
branches: [ main, release ]
|
||||
paths:
|
||||
- "crates/ecstore/src/disk/**"
|
||||
- "crates/ecstore/src/store/init_format.rs"
|
||||
|
||||
Generated
+204
-279
File diff suppressed because it is too large
Load Diff
+67
-68
@@ -73,7 +73,7 @@ edition = "2024"
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/rustfs/rustfs"
|
||||
rust-version = "1.98.1"
|
||||
version = "1.0.1"
|
||||
version = "1.0.0"
|
||||
homepage = "https://rustfs.com"
|
||||
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
||||
keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
|
||||
@@ -90,62 +90,62 @@ redundant_clone = "warn"
|
||||
|
||||
[workspace.dependencies]
|
||||
# RustFS Internal Crates
|
||||
rustfs = { path = "./rustfs", version = "1.0.1" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.1" }
|
||||
rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.1" }
|
||||
rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.1" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.1" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.1" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.1" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.1" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.1" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.1" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.1" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.1" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.1" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.1" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.1" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.1" }
|
||||
rustfs-license = { path = "crates/license", version = "1.0.1" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.1" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.1" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.1" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.1" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.1" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.1" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.1" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.1" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.1", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.1" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.1" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.1" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.1" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.1" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.1" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.1" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.1" }
|
||||
rustfs-s3-client = { path = "crates/s3-client", version = "1.0.1" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.1" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.1" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.1" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.1" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.1" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.1" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.1" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.1" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.1" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.1" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.1" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.1" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.1" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.1" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.1" }
|
||||
rustfs = { path = "./rustfs", version = "1.0.0" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0" }
|
||||
rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.0" }
|
||||
rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.0" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0" }
|
||||
rustfs-license = { path = "crates/license", version = "1.0.0" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0" }
|
||||
rustfs-s3-client = { path = "crates/s3-client", version = "1.0.0" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0" }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
async_zip = { default-features = false, version = "0.0.19" }
|
||||
mysql_async = { default-features = false, version = "0.37.1" }
|
||||
async-compression = { version = "0.4.48" }
|
||||
async-compression = { version = "0.4.47" }
|
||||
async-recursion = "1.1.1"
|
||||
async-trait = "0.1.92"
|
||||
async-nats = { version = "0.50.0", default-features = false }
|
||||
@@ -163,7 +163,6 @@ hyper-util = { version = "0.1.20" }
|
||||
http = "1.5.0"
|
||||
http-body = "1.1.0"
|
||||
http-body-util = "0.1.5"
|
||||
memchr = "2.8.3"
|
||||
minlz = "1.2.3"
|
||||
reqwest = "0.13.5"
|
||||
rustfs-kafka-async = { version = "1.3.1" }
|
||||
@@ -207,7 +206,7 @@ blake2 = "0.11.0"
|
||||
chacha20poly1305 = { version = "0.11.0" }
|
||||
crc-fast = "1.10.0"
|
||||
hmac = { version = "0.13.0" }
|
||||
jsonwebtoken = { version = "11.1.0" }
|
||||
jsonwebtoken = { version = "11.0.0" }
|
||||
openidconnect = { default-features = false, version = "4.0" }
|
||||
pbkdf2 = "0.13.0"
|
||||
p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
|
||||
@@ -244,13 +243,13 @@ atoi = "3.1.0"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.12.0" }
|
||||
aws-credential-types = { version = "1.3.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.120.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.148.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.116.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.118.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.146.1" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.114.0" }
|
||||
aws-smithy-async = { version = "1.3.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.4.2" }
|
||||
aws-smithy-runtime-api = { version = "1.16.2" }
|
||||
aws-smithy-types = { version = "1.6.4" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.4.0" }
|
||||
aws-smithy-runtime-api = { version = "1.16.0" }
|
||||
aws-smithy-types = { version = "1.6.3" }
|
||||
base64-simd = "0.8.0"
|
||||
brotli = "9.0.0"
|
||||
clap = { version = "4.6.7" }
|
||||
@@ -298,23 +297,23 @@ percent-encoding = "2.3.2"
|
||||
# Server-side QR rendering for TOTP enrollment, so neither the console nor the
|
||||
# CLI needs its own QR encoder. No default features: the image/render backends
|
||||
# pull in an image stack this only needs SVG and text output from.
|
||||
qrcode-rs = { version = "2.1.1", default-features = false, features = ["std", "svg"] }
|
||||
qrcode-rs = { version = "2.1.0", default-features = false, features = ["std", "svg"] }
|
||||
pin-project-lite = "0.2.17"
|
||||
pretty_assertions = "1.4.1"
|
||||
rand = { version = "0.10.2" }
|
||||
ratelimit = "2.0.0"
|
||||
rayon = "1.12.0"
|
||||
rustfs-erasure-codec = { version = "9.0.0" }
|
||||
rustfs-erasure-codec = { version = "8.0.2" }
|
||||
reed-solomon-simd = "3.1.0"
|
||||
regex = { version = "1.13.1" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.34.0" }
|
||||
redis = { version = "1.7.0" }
|
||||
rustify = { version = "0.7", default-features = false }
|
||||
rustix = { version = "1.1.5" }
|
||||
rustix = { version = "1.1.4" }
|
||||
rust-embed = { version = "8.12.0" }
|
||||
rustc-hash = { version = "2.1.3" }
|
||||
s3s = { git = "https://github.com/s3s-project/s3s.git", rev = "50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6", features = ["minio"] }
|
||||
s3s-sigv4 = { git = "https://github.com/s3s-project/s3s.git", rev = "50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6" }
|
||||
s3s = { version = "0.16.0", features = ["minio"] }
|
||||
s3s-sigv4 = { version = "0.16.0" }
|
||||
serial_test = "4.0.1"
|
||||
shadow-rs = { default-features = false, version = "2.0.0" }
|
||||
siphasher = "1.0.3"
|
||||
@@ -351,7 +350,7 @@ zstd = "0.14.0"
|
||||
# Observability and Metrics
|
||||
metrics = "0.24.6"
|
||||
metrics-util = "0.20"
|
||||
dial9 = { version = "0.5.1", default-features = false }
|
||||
dial9-tokio-telemetry = "0.5.0"
|
||||
opentelemetry = { version = "0.32.0" }
|
||||
opentelemetry-appender-tracing = { version = "0.32.0" }
|
||||
opentelemetry-otlp = { version = "0.32.0" }
|
||||
@@ -373,9 +372,9 @@ russh-sftp = "3.0.0"
|
||||
dav-server = "0.11.0"
|
||||
|
||||
# Performance Analysis and Memory Profiling
|
||||
rustfs-mimalloc = { version = "0.5.4" }
|
||||
rustfs-mimalloc = { version = "0.5.3" }
|
||||
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
|
||||
hotpath = { version = "0.26.0", default-features = false }
|
||||
hotpath = { version = "0.25.1", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.48" }
|
||||
|
||||
|
||||
@@ -141,7 +141,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# Using specific version
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.1
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0
|
||||
```
|
||||
|
||||
If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
|
||||
|
||||
+1
-1
@@ -138,7 +138,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# 使用指定版本运行
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.1
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0
|
||||
```
|
||||
|
||||
如果您通过绑定挂载启用 TLS 证书目录,也请用同样方式准备该目录:
|
||||
|
||||
@@ -663,15 +663,6 @@ pub const ENV_OBJECT_LOCK_RPC_DETACHED_LIMIT: &str = "RUSTFS_OBJECT_LOCK_RPC_DET
|
||||
/// Default per-peer budget of detached (timed-out but still running) lock RPCs: 256.
|
||||
pub const DEFAULT_OBJECT_LOCK_RPC_DETACHED_LIMIT: usize = 256;
|
||||
|
||||
/// Environment variable for the maximum number of in-flight lock acquisition
|
||||
/// RPCs admitted to one peer. Requests beyond this bound fail fast as
|
||||
/// retryable contention so a slow endpoint cannot accumulate an unbounded
|
||||
/// queue while its channel remains healthy.
|
||||
pub const ENV_OBJECT_LOCK_RPC_REQUEST_LIMIT: &str = "RUSTFS_OBJECT_LOCK_RPC_REQUEST_LIMIT";
|
||||
|
||||
/// Default per-peer in-flight lock acquisition RPC admission limit.
|
||||
pub const DEFAULT_OBJECT_LOCK_RPC_REQUEST_LIMIT: usize = 128;
|
||||
|
||||
/// Environment variable to enable object namespace lock diagnostics.
|
||||
///
|
||||
/// When enabled, RustFS emits slow lock acquisition and long lock hold
|
||||
|
||||
@@ -39,7 +39,7 @@ pub const ENV_RUNTIME_DIAL9_S3_BUCKET: &str = "RUSTFS_RUNTIME_DIAL9_S3_BUCKET";
|
||||
/// Accepted but not honoured; see [`ENV_RUNTIME_DIAL9_S3_BUCKET`].
|
||||
pub const ENV_RUNTIME_DIAL9_S3_PREFIX: &str = "RUSTFS_RUNTIME_DIAL9_S3_PREFIX";
|
||||
// Note: there are deliberately no task-dump knobs. dial9 only captures task
|
||||
// dumps for futures spawned through `dial9::spawn`, and RustFS
|
||||
// dumps for futures spawned through `dial9_tokio_telemetry::spawn`, and RustFS
|
||||
// spawns with `tokio::spawn` throughout, so the switch could never do anything.
|
||||
// Measured: 0 dumps via tokio::spawn vs 14709 via dial9::spawn on an identical
|
||||
// workload. See rustfs/backlog#1157 (D9-16).
|
||||
|
||||
@@ -30,9 +30,6 @@ pub const ENV_OBS_ENDPOINT_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_TIMEOUT_M
|
||||
pub const ENV_OBS_ENDPOINT_TRACES_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_TRACES_TIMEOUT_MILLIS";
|
||||
pub const ENV_OBS_ENDPOINT_METRICS_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_METRICS_TIMEOUT_MILLIS";
|
||||
pub const ENV_OBS_ENDPOINT_LOGS_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_LOGS_TIMEOUT_MILLIS";
|
||||
/// Absolute path to a PEM bundle whose certificates are added to the system
|
||||
/// trust roots used by OTLP/HTTP exporters.
|
||||
pub const ENV_OBS_TLS_CA_FILE: &str = "RUSTFS_OBS_TLS_CA_FILE";
|
||||
pub const ENV_OBS_PROFILING_ENDPOINT: &str = "RUSTFS_OBS_PROFILING_ENDPOINT";
|
||||
pub const ENV_OBS_USE_STDOUT: &str = "RUSTFS_OBS_USE_STDOUT";
|
||||
pub const ENV_OBS_SAMPLE_RATIO: &str = "RUSTFS_OBS_SAMPLE_RATIO";
|
||||
@@ -58,13 +55,6 @@ pub const ENV_OBS_LOG_FILENAME: &str = "RUSTFS_OBS_LOG_FILENAME";
|
||||
pub const ENV_OBS_LOG_ROTATION_TIME: &str = "RUSTFS_OBS_LOG_ROTATION_TIME";
|
||||
pub const ENV_OBS_LOG_KEEP_FILES: &str = "RUSTFS_OBS_LOG_KEEP_FILES";
|
||||
|
||||
/// Span lifecycle events emitted by the JSON log layers (`none`|`close`|`full`).
|
||||
///
|
||||
/// Defaults to `none`: a record per span lifecycle multiplies log volume on
|
||||
/// object hot paths, and emitting the ancestor span list on every record
|
||||
/// produced multi-megabyte lines that journald truncated (backlog#2642).
|
||||
pub const ENV_OBS_LOG_SPAN_EVENTS: &str = "RUSTFS_OBS_LOG_SPAN_EVENTS";
|
||||
|
||||
/// Log cleanup related configurations
|
||||
pub const ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES: &str = "RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES";
|
||||
pub const ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES: &str = "RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES";
|
||||
@@ -109,8 +99,6 @@ pub const DEFAULT_OBS_LOG_DRY_RUN: bool = false;
|
||||
pub const DEFAULT_OBS_LOG_MATCH_MODE_PREFIX: &str = "prefix";
|
||||
pub const DEFAULT_OBS_LOG_MATCH_MODE: &str = "suffix";
|
||||
|
||||
pub const DEFAULT_OBS_LOG_SPAN_EVENTS: &str = "none";
|
||||
|
||||
/// Default values for observability configuration
|
||||
// ### Supported Environment Values
|
||||
// - `production` - Secure file-only logging
|
||||
@@ -153,7 +141,6 @@ mod tests {
|
||||
assert_eq!(ENV_OBS_LOG_FILENAME, "RUSTFS_OBS_LOG_FILENAME");
|
||||
assert_eq!(ENV_OBS_LOG_ROTATION_TIME, "RUSTFS_OBS_LOG_ROTATION_TIME");
|
||||
assert_eq!(ENV_OBS_LOG_KEEP_FILES, "RUSTFS_OBS_LOG_KEEP_FILES");
|
||||
assert_eq!(ENV_OBS_LOG_SPAN_EVENTS, "RUSTFS_OBS_LOG_SPAN_EVENTS");
|
||||
assert_eq!(ENV_OBS_TRACES_EXPORT_ENABLED, "RUSTFS_OBS_TRACES_EXPORT_ENABLED");
|
||||
assert_eq!(ENV_OBS_METRICS_EXPORT_ENABLED, "RUSTFS_OBS_METRICS_EXPORT_ENABLED");
|
||||
assert_eq!(ENV_OBS_LOGS_EXPORT_ENABLED, "RUSTFS_OBS_LOGS_EXPORT_ENABLED");
|
||||
@@ -191,7 +178,6 @@ mod tests {
|
||||
assert_eq!(DEFAULT_OBS_LOG_COMPRESSION_ALGORITHM_GZIP, "gzip");
|
||||
assert_eq!(DEFAULT_OBS_LOG_COMPRESSION_ALGORITHM_ZSTD, "zstd");
|
||||
assert_eq!(DEFAULT_OBS_LOG_MATCH_MODE_PREFIX, "prefix");
|
||||
assert_eq!(DEFAULT_OBS_LOG_SPAN_EVENTS, "none");
|
||||
assert_eq!(DEFAULT_OBS_LOG_MATCH_MODE, "suffix");
|
||||
assert_eq!(DEFAULT_OBS_LOG_COMPRESSION_ALGORITHM, "zstd");
|
||||
}
|
||||
|
||||
@@ -152,29 +152,9 @@ async fn test_compression_roundtrip() -> Result<(), Box<dyn std::error::Error +
|
||||
let content_length = head_response.content_length().unwrap_or(0);
|
||||
assert_eq!(content_length as usize, original_size, "Content-Length should be original size");
|
||||
|
||||
// A 5 KiB compressed object sits inside the inline budget, so its shard is
|
||||
// embedded in xl.meta rather than written as a part file; the stored size
|
||||
// recorded in the metadata is the compressed length.
|
||||
let part_files = find_part_files(&env.temp_dir, COMPRESSION_TEST_BUCKET, object_key)?;
|
||||
assert!(
|
||||
part_files.is_empty(),
|
||||
"small compressed object must be stored inline, found part files {part_files:?}"
|
||||
);
|
||||
let xl_meta_path = PathBuf::from(&env.temp_dir)
|
||||
.join(COMPRESSION_TEST_BUCKET)
|
||||
.join(object_key)
|
||||
.join("xl.meta");
|
||||
let file_info = rustfs_filemeta::FileMeta::load(&fs::read(&xl_meta_path)?)?.into_fileinfo(
|
||||
COMPRESSION_TEST_BUCKET,
|
||||
object_key,
|
||||
"",
|
||||
true,
|
||||
false,
|
||||
true,
|
||||
)?;
|
||||
assert!(file_info.inline_data(), "xl.meta must carry the inline marker for the compressed object");
|
||||
assert!(file_info.is_compressed(), "xl.meta must carry the compression marker");
|
||||
let total_physical_size = u64::try_from(file_info.size)?;
|
||||
assert!(!part_files.is_empty(), "expected on-disk part files for the compressed object");
|
||||
let total_physical_size = part_files_total_size(&part_files)?;
|
||||
|
||||
assert!(
|
||||
total_physical_size < original_size as u64,
|
||||
|
||||
@@ -184,7 +184,7 @@ async fn four_node_health_inventory_metrics_and_audit_delivery_are_consistent()
|
||||
let (status, metrics_body) = admin_request(
|
||||
&node.url,
|
||||
Method::GET,
|
||||
"/rustfs/admin/v3/realtime?n=1&by-host=true&by-disk=true",
|
||||
"/rustfs/admin/v3/metrics?n=1&by-host=true&by-disk=true",
|
||||
None,
|
||||
&dist.cluster.access_key,
|
||||
&dist.cluster.secret_key,
|
||||
@@ -257,7 +257,7 @@ async fn node_admin_body(dist: &DistCluster, node: usize, path: &str) -> TestRes
|
||||
}
|
||||
|
||||
async fn http_put_counts(dist: &DistCluster, node: usize) -> TestResult<[u64; 2]> {
|
||||
let body = node_admin_body(dist, node, "/rustfs/admin/v3/realtime?types=512&by-host=true&n=1").await?;
|
||||
let body = node_admin_body(dist, node, "/rustfs/admin/v3/metrics?types=512&by-host=true&n=1").await?;
|
||||
let sample: RealtimeMetrics = serde_json::from_str(body.lines().next().ok_or("empty HTTP metrics stream")?)?;
|
||||
assert!(sample.errors.is_empty(), "HTTP metrics returned errors: {:?}", sample.errors);
|
||||
let http = sample.aggregated.http.ok_or("HTTP metrics missing at WARN log level")?;
|
||||
@@ -400,36 +400,17 @@ async fn verify_write_observations_during_peer_failure(dist: &DistCluster, bucke
|
||||
let observations = storage["info"]["observations"]
|
||||
.as_array()
|
||||
.ok_or("recovery omitted observations")?;
|
||||
let disks = storage["info"]["disks"].as_array().ok_or("recovery omitted disks")?;
|
||||
Ok(observations.len() == 4
|
||||
&& observations
|
||||
.iter()
|
||||
.all(|item| item["status"] == "succeeded" && item["cached"] == false)
|
||||
&& disks.len() == 16
|
||||
&& disks.iter().all(|disk| {
|
||||
disk["state"].as_str().is_some_and(|state| state.eq_ignore_ascii_case("ok"))
|
||||
&& disk["runtimeState"]
|
||||
.as_str()
|
||||
.is_some_and(|state| state.eq_ignore_ascii_case("online"))
|
||||
}))
|
||||
.all(|item| item["status"] == "succeeded" && item["cached"] == false))
|
||||
},
|
||||
"peer probes and remote disks recover to fresh successful observations",
|
||||
"peer probes recover to fresh successful observations",
|
||||
)
|
||||
.await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
let recovered_before = http_put_counts(dist, 0).await?;
|
||||
wait_until(
|
||||
Duration::from_secs(90),
|
||||
|| async { Ok(observed_put(dist, 0, bucket, "recovered").await?.is_success()) },
|
||||
"recovered PUT after remote disk and lock convergence",
|
||||
)
|
||||
.await?;
|
||||
let recovered_after = http_put_counts(dist, 0).await?;
|
||||
assert_eq!(recovered_after[0], recovered_before[0] + 1);
|
||||
assert!(
|
||||
recovered_after[1] >= baseline[0][1] + 2,
|
||||
"recovery retries must retain the two intentional sub-quorum failures: before={recovered_before:?}, after={recovered_after:?}"
|
||||
);
|
||||
assert!(observed_put(dist, 0, bucket, "recovered").await?.is_success());
|
||||
assert_eq!(http_put_counts(dist, 0).await?, [baseline[0][0] + 1, baseline[0][1] + 2]);
|
||||
for node in 0..dist.cluster.nodes.len() {
|
||||
assert_object_bytes(&dist.client(node)?, bucket, "healthy-0", b"write-observation").await?;
|
||||
assert_object_bytes(&dist.client(node)?, bucket, "recovered", b"write-observation").await?;
|
||||
|
||||
@@ -1794,8 +1794,6 @@ async fn four_node_inline_fallback_controls() -> TestResult {
|
||||
configure_reader_metric_cluster(&mut cluster, &collector);
|
||||
let sse_master_key = base64_simd::STANDARD.encode_to_string([0x42u8; 32]);
|
||||
cluster.set_env("RUSTFS_SSE_S3_MASTER_KEY", &sse_master_key);
|
||||
// Inspect every disk only after the PUT rename fanout has drained.
|
||||
cluster.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
|
||||
cluster.start().await?;
|
||||
|
||||
let bucket = "inline-fallback-controls";
|
||||
@@ -1847,10 +1845,6 @@ async fn four_node_inline_fallback_controls() -> TestResult {
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
// 16 KiB of plaintext is 8 KiB per data shard on EC 2+2, inside the inline
|
||||
// budget: an SSE-S3 PUT is admitted inline by its plaintext size even though
|
||||
// the ciphertext length is unknown up front (backlog#2393 STOR-115).
|
||||
assert_storage_layout(&cluster, bucket, encrypted_key, None, true)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1863,8 +1857,6 @@ async fn four_node_compressed_inline_fallback() -> TestResult {
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
|
||||
configure_reader_metric_cluster(&mut cluster, &collector);
|
||||
cluster.set_env("RUSTFS_COMPRESSION_ENABLED", "true");
|
||||
// Inspect every disk only after the PUT rename fanout has drained.
|
||||
cluster.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
|
||||
cluster.start().await?;
|
||||
|
||||
let bucket = "inline-compressed-fallback";
|
||||
@@ -1885,33 +1877,6 @@ async fn four_node_compressed_inline_fallback() -> TestResult {
|
||||
ReaderPathExpectation::for_class(ReaderObject::new(bucket, key, &body, put.e_tag(), None), LEGACY_DUPLEX, COMPRESSED),
|
||||
)
|
||||
.await?;
|
||||
// 64 KiB of plaintext is 32 KiB per data shard on EC 2+2, inside the inline
|
||||
// budget: a compressed PUT is admitted inline by its plaintext size even
|
||||
// though its stored size is unknown up front (backlog#2393 STOR-115).
|
||||
assert_storage_layout(&cluster, bucket, key, None, true)?;
|
||||
|
||||
// A server-side copy onto another compressible key re-compresses the
|
||||
// decompressed source stream and lands inline the same way (STOR-125).
|
||||
let copy_key = "compressed/copy.txt";
|
||||
let copy = client
|
||||
.copy_object()
|
||||
.bucket(bucket)
|
||||
.key(copy_key)
|
||||
.copy_source(format!("{bucket}/{key}"))
|
||||
.send()
|
||||
.await?;
|
||||
let copy_etag = copy.copy_object_result().and_then(|result| result.e_tag()).map(str::to_owned);
|
||||
assert_reader_path(
|
||||
&collector,
|
||||
&client,
|
||||
ReaderPathExpectation::for_class(
|
||||
ReaderObject::new(bucket, copy_key, &body, copy_etag.as_deref(), None),
|
||||
LEGACY_DUPLEX,
|
||||
COMPRESSED,
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
assert_storage_layout(&cluster, bucket, copy_key, None, true)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -391,9 +391,8 @@ pub mod data_usage {
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::data_movement::scanner_backlog::test_util::NativeScannerPauseBacklogWriteFault;
|
||||
pub use crate::data_movement::scanner_backlog::{
|
||||
MAX_SCANNER_PAUSE_BACKLOG_BYTES, ScannerPauseBacklogRetirementError, ScannerPauseBacklogRetirementPlan,
|
||||
ScannerPauseBacklogRetirementPlanner, ScannerPauseBacklogRetirementReplica,
|
||||
register_scanner_pause_backlog_retirement_planner,
|
||||
MAX_SCANNER_PAUSE_BACKLOG_BYTES, ScannerPauseBacklogRetirementPlan, ScannerPauseBacklogRetirementPlanner,
|
||||
ScannerPauseBacklogRetirementReplica, register_scanner_pause_backlog_retirement_planner,
|
||||
};
|
||||
pub use crate::data_usage::{
|
||||
DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
|
||||
@@ -504,13 +503,6 @@ pub mod notification {
|
||||
};
|
||||
}
|
||||
|
||||
pub mod integrity {
|
||||
pub use crate::services::integrity::{
|
||||
IntegrityError, InventoryItem, InventoryPage, ItemRequest, ItemResult, ItemState, Job, JobMode, JobRequest, JobState,
|
||||
Protection, Readiness, control_job, create_job, get_job, inventory, readiness, resume_job,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod object {
|
||||
pub use crate::object_api::{
|
||||
BLOCK_SIZE_V2, ERASURE_ALGORITHM, EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook,
|
||||
|
||||
@@ -515,26 +515,20 @@ impl ExpiryStats {
|
||||
Self::add_nonnegative(&self.missed_tier_journal_tasks, 1);
|
||||
}
|
||||
|
||||
// The pending and active gauges are balanced by design: every increment
|
||||
// has exactly one matching decrement. They must not saturate at zero on
|
||||
// update, because a worker can dequeue (and decrement) before the
|
||||
// enqueuing side has recorded its increment. Clamping that transient -1
|
||||
// to 0 turns the later +1 into a phantom task that never drains
|
||||
// (rustfs#7921). Readers clamp negative snapshots instead.
|
||||
fn increment_pending_tasks(&self) {
|
||||
self.pending_tasks.fetch_add(1, Ordering::AcqRel);
|
||||
Self::add_nonnegative(&self.pending_tasks, 1);
|
||||
}
|
||||
|
||||
fn decrement_pending_tasks(&self) {
|
||||
self.pending_tasks.fetch_sub(1, Ordering::AcqRel);
|
||||
Self::add_nonnegative(&self.pending_tasks, -1);
|
||||
}
|
||||
|
||||
fn increment_active_tasks(&self) {
|
||||
self.active_tasks.fetch_add(1, Ordering::AcqRel);
|
||||
Self::add_nonnegative(&self.active_tasks, 1);
|
||||
}
|
||||
|
||||
fn decrement_active_tasks(&self) {
|
||||
self.active_tasks.fetch_sub(1, Ordering::AcqRel);
|
||||
Self::add_nonnegative(&self.active_tasks, -1);
|
||||
}
|
||||
|
||||
fn increment_workers(&self) {
|
||||
@@ -1080,13 +1074,9 @@ impl ExpiryState {
|
||||
}
|
||||
|
||||
fn send_expiry_task(&self, wrkr: Sender<Option<ExpiryOpType>>, task: ExpiryOpType) -> bool {
|
||||
// Account for the task before a worker can observe it. The worker
|
||||
// decrements on dequeue, so incrementing after `try_send` would let a
|
||||
// fast dequeue run the gauge through zero first.
|
||||
self.stats.increment_pending_tasks();
|
||||
let queued = wrkr.try_send(Some(task)).is_ok();
|
||||
if !queued {
|
||||
self.stats.decrement_pending_tasks();
|
||||
if queued {
|
||||
self.stats.increment_pending_tasks();
|
||||
}
|
||||
queued
|
||||
}
|
||||
@@ -1454,14 +1444,11 @@ async fn enqueue_recovered_free_version_with_state(state: &Arc<RwLock<ExpiryStat
|
||||
return false;
|
||||
};
|
||||
|
||||
// Same ordering rule as `ExpiryState::send_expiry_task`: count first, so
|
||||
// a worker that dequeues immediately cannot decrement before this
|
||||
// increment lands.
|
||||
stats.increment_pending_tasks();
|
||||
let queued = wrkr.try_send(Some(Box::new(task))).is_ok();
|
||||
if !queued {
|
||||
stats.decrement_pending_tasks();
|
||||
stats.increment_missed_freevers_tasks();
|
||||
} else {
|
||||
stats.increment_pending_tasks();
|
||||
}
|
||||
stats.record_scanner_expiry_state();
|
||||
queued
|
||||
@@ -3770,8 +3757,11 @@ pub async fn enqueue_transition_immediate(oi: &ObjectInfo, src: LcEventSrc) {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn enqueue_immediate_expiry(api: Arc<ECStore>, oi: &ObjectInfo, src: LcEventSrc, opts: &ObjectOptions) {
|
||||
let configs = match metadata_boundary::get_expiry_configs_for_options(&api, &oi.bucket, opts).await {
|
||||
pub async fn enqueue_immediate_expiry(oi: &ObjectInfo, src: LcEventSrc) {
|
||||
let Some(api) = runtime_sources::object_store_handle() else {
|
||||
return;
|
||||
};
|
||||
let configs = match metadata_boundary::get_expiry_configs(&api, &oi.bucket).await {
|
||||
Ok(configs) => configs,
|
||||
Err(err) => {
|
||||
observe_lifecycle_observability_event(EVENT_LIFECYCLE_EVALUATION_FAILED, "failed", Some("metadata_unavailable"));
|
||||
@@ -7582,65 +7572,6 @@ mod tests {
|
||||
assert!(recovery_notify.notified().now_or_never().is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn expiry_pending_gauge_survives_dequeue_landing_before_enqueue_accounting() {
|
||||
// A worker may dequeue and decrement before the enqueuing side records
|
||||
// its increment. The gauge must return to zero afterwards instead of
|
||||
// clamping the transient -1 away and reporting a phantom pending task
|
||||
// that no idle check can ever drain (rustfs#7921).
|
||||
let state = ExpiryState::new();
|
||||
let stats = Arc::clone(&state.read().await.stats);
|
||||
|
||||
stats.decrement_pending_tasks();
|
||||
stats.increment_pending_tasks();
|
||||
assert_eq!(stats.pending_tasks(), 0);
|
||||
assert_eq!(state.read().await.pending_tasks(), 0);
|
||||
|
||||
stats.decrement_active_tasks();
|
||||
stats.increment_active_tasks();
|
||||
assert_eq!(stats.active_tasks(), 0);
|
||||
assert_eq!(state.read().await.active_tasks(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn free_version_enqueue_rolls_back_pending_when_queue_full() {
|
||||
// Single-threaded, so this cannot observe the count-before-publish
|
||||
// ordering itself; it pins the rollback that ordering requires: a
|
||||
// rejected send must not leave its speculative increment behind.
|
||||
let state = ExpiryState::new_with_unconsumed_worker_channel(1);
|
||||
let oi = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
transitioned_object: TransitionedObject {
|
||||
name: "remote/object".to_string(),
|
||||
version_id: "remote-version".to_string(),
|
||||
tier: "WARM".to_string(),
|
||||
free_version: true,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(state.read().await.enqueue_free_version(oi.clone()));
|
||||
assert_eq!(state.read().await.stats.pending_tasks(), 1);
|
||||
|
||||
// The single-slot queue is full for both enqueue paths.
|
||||
assert!(!enqueue_recovered_free_version_with_state(&state, oi.clone()).await);
|
||||
assert_eq!(state.read().await.stats.pending_tasks(), 1);
|
||||
assert_eq!(state.read().await.stats.missed_free_vers_tasks(), 1);
|
||||
|
||||
assert!(!state.read().await.enqueue_free_version(oi));
|
||||
assert_eq!(state.read().await.stats.pending_tasks(), 1);
|
||||
assert_eq!(state.read().await.stats.missed_free_vers_tasks(), 2);
|
||||
|
||||
// Draining the one real task returns the gauge to zero.
|
||||
let receiver = state.read().await.tasks_rx[0].clone();
|
||||
let task = receiver.lock().await.recv().await.expect("queued task");
|
||||
assert!(task.is_some());
|
||||
state.read().await.stats.decrement_pending_tasks();
|
||||
assert_eq!(state.read().await.pending_tasks(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn enqueue_recovered_free_version_reports_false_without_worker_channel() {
|
||||
let state = ExpiryState::new();
|
||||
@@ -12772,8 +12703,7 @@ mod tests {
|
||||
.push((event, state, reason));
|
||||
});
|
||||
|
||||
super::enqueue_immediate_expiry(Arc::clone(&ecstore), &object_info, LcEventSrc::S3PutObject, &ObjectOptions::default())
|
||||
.await;
|
||||
super::enqueue_immediate_expiry(&object_info, LcEventSrc::S3PutObject).await;
|
||||
|
||||
assert!(
|
||||
observed.lock().expect("observed events should not poison").contains(&(
|
||||
|
||||
@@ -55,15 +55,7 @@ pub(crate) async fn lifecycle_expiry_allowed(
|
||||
}
|
||||
|
||||
pub(crate) async fn get_expiry_configs(api: &crate::store::ECStore, bucket: &str) -> Result<LifecycleExpiryConfigs> {
|
||||
get_expiry_configs_for_options(api, bucket, &crate::object_api::ObjectOptions::default()).await
|
||||
}
|
||||
|
||||
pub(crate) async fn get_expiry_configs_for_options(
|
||||
api: &crate::store::ECStore,
|
||||
bucket: &str,
|
||||
opts: &crate::object_api::ObjectOptions,
|
||||
) -> Result<LifecycleExpiryConfigs> {
|
||||
let bucket_incarnation_id = metadata_sys::get_bucket_incarnation_id_for_options_in(&api.ctx, bucket, opts).await?;
|
||||
let bucket_incarnation_id = api.bucket_incarnation_id_from_disk(bucket).await?;
|
||||
let metadata = get_authoritative_metadata(api, bucket, bucket_incarnation_id).await?;
|
||||
let table_bucket_enabled = metadata.table_bucket_enabled();
|
||||
|
||||
|
||||
@@ -35,7 +35,6 @@ use crate::bucket::lifecycle::tier_sweeper::{
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::{Error, Result as EcstoreResult};
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions};
|
||||
use crate::services::notification_sys::acquire_remote_version_state_writer_fleet_proof;
|
||||
use crate::services::tier::{tier::TierConfigMgr, warm_backend::TransitionCandidateProbe};
|
||||
use crate::storage_api_contracts::{
|
||||
list::ListOperations as _,
|
||||
@@ -1238,12 +1237,6 @@ pub async fn delete_transition_candidate_for_operator(
|
||||
lease
|
||||
.validate_remote_version_id(remote_version_id)
|
||||
.map_err(TransitionOperatorError::Remote)?;
|
||||
if remote_version_requires_fleet_proof(remote_version_id) && acquire_remote_version_state_writer_fleet_proof().is_none() {
|
||||
return Err(TransitionOperatorError::Remote(std::io::Error::new(
|
||||
std::io::ErrorKind::Unsupported,
|
||||
"opaque remote tier version cleanup requires the operator-attested live fleet capability gate",
|
||||
)));
|
||||
}
|
||||
let before_delete_probe = lease
|
||||
.probe_transition_candidate_for(&transaction.remote_object, transaction.transaction_id)
|
||||
.await
|
||||
@@ -1852,11 +1845,6 @@ async fn recover_unknown_upload_outcome(
|
||||
))
|
||||
}
|
||||
TransitionCandidateProbe::VersionedPresent(version_id) => {
|
||||
if remote_version_requires_fleet_proof(&version_id) && acquire_remote_version_state_writer_fleet_proof().is_none() {
|
||||
return Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(
|
||||
IlmRecoveryErrorCode::RemoteVersionUnknown,
|
||||
));
|
||||
}
|
||||
cleanup_recovered_unknown_upload_candidate(api, transaction, TransitionRemoteVersion::versioned(version_id)).await
|
||||
}
|
||||
TransitionCandidateProbe::Ambiguous => Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(
|
||||
@@ -1952,14 +1940,6 @@ fn transition_source_lookup_options(transaction: &TransitionTransaction) -> Obje
|
||||
async fn delete_transition_remote_candidate(api: Arc<ECStore>, transaction: &TransitionTransaction) -> EcstoreResult<()> {
|
||||
let version_id = transaction.remote_version.tier_delete_version_id().unwrap_or_default();
|
||||
let version_id_exact = transaction.remote_version.kind == TransitionRemoteVersionKind::Versioned;
|
||||
if version_id_exact
|
||||
&& remote_version_requires_fleet_proof(version_id)
|
||||
&& acquire_remote_version_state_writer_fleet_proof().is_none()
|
||||
{
|
||||
return Err(Error::other(
|
||||
"opaque remote tier version cleanup requires the operator-attested live fleet capability gate",
|
||||
));
|
||||
}
|
||||
delete_object_from_remote_tier_idempotent_with_manager_and_identity(
|
||||
&transaction.remote_object,
|
||||
version_id,
|
||||
@@ -1973,10 +1953,6 @@ async fn delete_transition_remote_candidate(api: Arc<ECStore>, transaction: &Tra
|
||||
.map_err(Error::other)
|
||||
}
|
||||
|
||||
fn remote_version_requires_fleet_proof(version_id: &str) -> bool {
|
||||
!version_id.is_empty() && Uuid::parse_str(version_id).is_err()
|
||||
}
|
||||
|
||||
pub async fn recover_transition_transaction_records(
|
||||
api: Arc<ECStore>,
|
||||
limit: usize,
|
||||
@@ -2463,14 +2439,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn opaque_remote_versions_require_the_fleet_proof_boundary() {
|
||||
assert!(!remote_version_requires_fleet_proof(""));
|
||||
assert!(!remote_version_requires_fleet_proof(&Uuid::new_v4().to_string()));
|
||||
assert!(remote_version_requires_fleet_proof("null"));
|
||||
assert!(remote_version_requires_fleet_proof("b2-opaque-version"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_distinguishes_unknown_unversioned_and_versioned() {
|
||||
assert_eq!(TransitionRemoteVersion::known_from_put_response("").tier_delete_version_id(), None);
|
||||
|
||||
@@ -1464,13 +1464,9 @@ pub(crate) async fn load_bucket_metadata_parse_with_presence(
|
||||
"bucket incarnation sidecar is missing for new-format metadata: {bucket}"
|
||||
)));
|
||||
}
|
||||
} else if incarnation.is_some() {
|
||||
return Err(Error::other("bucket incarnation sidecar exists without bucket metadata"));
|
||||
}
|
||||
// A sidecar without `.metadata.bin` is the window between the two legacy
|
||||
// migration writes (rustfs/rustfs#8003): the sidecar lands first, and a
|
||||
// crash or lost namespace lease before the metadata write leaves the
|
||||
// bucket in this state on every node. Report it as not persisted so the
|
||||
// migration runs again; the writers adopt the stored incarnation instead
|
||||
// of minting a new one, so the sidecar keeps its authority.
|
||||
|
||||
bm.default_timestamps();
|
||||
|
||||
|
||||
@@ -24,7 +24,6 @@ use crate::bucket::metadata::{load_bucket_metadata_parse, load_bucket_metadata_p
|
||||
use crate::bucket::utils::is_meta_bucketname;
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::{Error, Result, is_err_bucket_not_found, is_err_strict_volume_not_found};
|
||||
use crate::object_api::ObjectOptions;
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use crate::storage_api_contracts::heal::HealOperations as _;
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
@@ -491,18 +490,6 @@ pub(crate) async fn get_bucket_incarnation_id_in(ctx: &crate::runtime::instance:
|
||||
sys.get_bucket_incarnation_id_from_disk(bucket).await
|
||||
}
|
||||
|
||||
/// Validate against disk while retaining an already-held Object Lock fence.
|
||||
pub(crate) async fn get_bucket_incarnation_id_for_options_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<Uuid> {
|
||||
let sys = bucket_metadata_sys_of(ctx)?;
|
||||
let sys = sys.read().await.clone();
|
||||
let guard = acquire_bucket_metadata_transaction_read_lock_for_options_in(ctx, bucket, opts).await?;
|
||||
sys.get_bucket_incarnation_id_under_transaction_lock(bucket, &guard).await
|
||||
}
|
||||
|
||||
pub(crate) async fn get_cached_bucket_incarnation_id_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
@@ -1068,24 +1055,6 @@ pub(crate) async fn acquire_bucket_metadata_transaction_read_lock_in(
|
||||
Ok(lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?)
|
||||
}
|
||||
|
||||
/// Readers already holding an Object Lock snapshot must share its guard:
|
||||
/// a fresh read acquisition can queue behind a writer waiting for that snapshot.
|
||||
pub(crate) async fn acquire_bucket_metadata_transaction_read_lock_for_options_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<Arc<rustfs_lock::NamespaceLockGuard>> {
|
||||
if let Some(snapshot) = opts.object_lock_config_snapshot.as_ref() {
|
||||
let store = object_store_in(ctx).await?;
|
||||
return snapshot
|
||||
.metadata_transaction_guard_for(store.id, bucket, opts.expected_bucket_incarnation_id)
|
||||
.ok_or_else(|| {
|
||||
Error::other("Object Lock snapshot does not hold a valid metadata transaction fence for this bucket")
|
||||
});
|
||||
}
|
||||
Ok(Arc::new(acquire_bucket_metadata_transaction_read_lock_in(ctx, bucket).await?))
|
||||
}
|
||||
|
||||
async fn acquire_transaction_lock_with_sys(
|
||||
sys: &Arc<RwLock<BucketMetadataSys>>,
|
||||
bucket: &str,
|
||||
@@ -1291,28 +1260,6 @@ pub(crate) async fn get_object_lock_config_and_incarnation_from_disk_in(
|
||||
}
|
||||
}
|
||||
|
||||
/// Inspect all migration-relevant settings while the caller holds an Object Lock
|
||||
/// snapshot's lifecycle and metadata transaction guards. Cached settings are not
|
||||
/// sufficient to authorize a direct storage writer that cannot apply S3 defaults.
|
||||
pub(crate) async fn integrity_migration_metadata_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
) -> Result<Arc<BucketMetadata>> {
|
||||
let sys = bucket_metadata_sys_of(ctx)?.read().await.clone();
|
||||
match sys
|
||||
.read_authoritative_metadata_from_disk_under_transaction_lock(bucket)
|
||||
.await?
|
||||
{
|
||||
BucketMetadataAuthority::Authoritative(metadata)
|
||||
if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
|
||||
{
|
||||
Ok(metadata)
|
||||
}
|
||||
BucketMetadataAuthority::MissingBucket => Err(Error::BucketNotFound(bucket.to_string())),
|
||||
_ => Err(Error::other("migration requires authoritative bucket metadata")),
|
||||
}
|
||||
}
|
||||
|
||||
/// Re-read the quota configuration and bucket incarnation from the same
|
||||
/// authoritative metadata blob while the caller holds the bucket metadata
|
||||
/// transaction read lock.
|
||||
@@ -2410,17 +2357,8 @@ impl BucketMetadataSys {
|
||||
let _transaction_guard = transaction_lock
|
||||
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await?;
|
||||
self.get_bucket_incarnation_id_under_transaction_lock(bucket, &_transaction_guard)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get_bucket_incarnation_id_under_transaction_lock(
|
||||
&self,
|
||||
bucket: &str,
|
||||
transaction_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
) -> Result<Uuid> {
|
||||
let incarnation_id = load_bucket_incarnation(self.object_store(), bucket).await?;
|
||||
if transaction_guard.is_lock_lost() {
|
||||
if _transaction_guard.is_lock_lost() {
|
||||
return Err(Error::other(format!("bucket incarnation metadata transaction lock was lost: {bucket}")));
|
||||
}
|
||||
match incarnation_id {
|
||||
@@ -2526,16 +2464,6 @@ impl BucketMetadataSys {
|
||||
if !persisted {
|
||||
metadata = BucketMetadata::new(bucket);
|
||||
metadata.created = bucket_info.created.unwrap_or(OffsetDateTime::UNIX_EPOCH);
|
||||
// An interrupted migration may already have published this
|
||||
// bucket's incarnation; re-read it under the transaction lock so
|
||||
// the retry never replaces an identity other nodes fenced on. A
|
||||
// retired incarnation is residue of a deleted bucket and must not
|
||||
// come back, or heal would reclaim the bucket's new objects.
|
||||
if let Some(stored) = load_bucket_incarnation(self.object_store(), bucket).await?
|
||||
&& !crate::bucket::retirement::is_retired(self.object_store(), bucket, stored).await?
|
||||
{
|
||||
metadata.bucket_incarnation_id = stored;
|
||||
}
|
||||
} else if metadata.bucket_incarnation_id.is_nil() {
|
||||
metadata.bucket_incarnation_id = Uuid::new_v4();
|
||||
}
|
||||
@@ -3588,98 +3516,6 @@ mod tests {
|
||||
assert!(err.to_string().contains("sidecar is missing"));
|
||||
}
|
||||
|
||||
/// rustfs/rustfs#8003: the legacy migration writes the incarnation sidecar
|
||||
/// before `.metadata.bin`. A crash or lost namespace lease between the two
|
||||
/// left every pre-existing bucket answering 500 on every request, with no
|
||||
/// path back. The sidecar-only state must load as a legacy bucket, and the
|
||||
/// retried migration must keep the stored incarnation.
|
||||
#[tokio::test]
|
||||
async fn issue_8003_sidecar_without_metadata_loads_as_legacy_and_migrates_with_its_incarnation() {
|
||||
let (dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
let sys = BucketMetadataSys::new(ecstore.clone());
|
||||
let bucket = "issue-8003-sidecar-only";
|
||||
for dir in &dirs {
|
||||
std::fs::create_dir_all(dir.path().join(bucket)).unwrap();
|
||||
}
|
||||
let stored = Uuid::new_v4();
|
||||
save_bucket_incarnation(ecstore.clone(), bucket, stored)
|
||||
.await
|
||||
.expect("simulate the interrupted migration's first write");
|
||||
|
||||
let (fabricated, persisted) = load_bucket_metadata_parse_with_presence(ecstore.clone(), bucket, true)
|
||||
.await
|
||||
.expect("a sidecar without metadata must read as a legacy bucket, not fail closed");
|
||||
assert!(!persisted);
|
||||
assert!(!fabricated.bucket_incarnation_sidecar);
|
||||
|
||||
let (_, fabricated_read) = sys
|
||||
.get_config(bucket)
|
||||
.await
|
||||
.expect("request-path metadata reads must not fail closed on the sidecar-only state");
|
||||
assert!(fabricated_read);
|
||||
|
||||
let migrated = sys
|
||||
.get_authoritative_metadata(bucket)
|
||||
.await
|
||||
.expect("the retried legacy migration must complete");
|
||||
assert!(migrated.bucket_incarnation_sidecar);
|
||||
assert_eq!(
|
||||
migrated.bucket_incarnation_id, stored,
|
||||
"the retry must adopt the published incarnation instead of minting a new one"
|
||||
);
|
||||
assert_eq!(sys.get_bucket_incarnation_id(bucket).await.unwrap(), stored);
|
||||
|
||||
let on_disk = sys.get_config_from_disk(bucket).await.expect("metadata is now persisted");
|
||||
assert!(on_disk.bucket_incarnation_sidecar);
|
||||
assert_eq!(on_disk.bucket_incarnation_id, stored);
|
||||
assert_eq!(load_bucket_incarnation(ecstore, bucket).await.unwrap(), Some(stored));
|
||||
}
|
||||
|
||||
/// A sidecar left behind by a deleted bucket names a retired incarnation.
|
||||
/// The migration must mint a new identity for a same-name volume instead
|
||||
/// of re-publishing the retired one, or heal would treat the bucket's new
|
||||
/// objects as reclaimable residue.
|
||||
#[tokio::test]
|
||||
async fn issue_8003_sidecar_only_migration_does_not_adopt_a_retired_incarnation() {
|
||||
let (dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
let sys = BucketMetadataSys::new(ecstore.clone());
|
||||
let bucket = "issue-8003-retired-sidecar";
|
||||
for dir in &dirs {
|
||||
std::fs::create_dir_all(dir.path().join(bucket)).unwrap();
|
||||
}
|
||||
let retired = Uuid::new_v4();
|
||||
save_bucket_incarnation(ecstore.clone(), bucket, retired)
|
||||
.await
|
||||
.expect("residual sidecar");
|
||||
crate::bucket::retirement::commit_retirement(
|
||||
ecstore.clone(),
|
||||
bucket,
|
||||
retired,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("retirement record");
|
||||
|
||||
let migrated = sys
|
||||
.get_authoritative_metadata(bucket)
|
||||
.await
|
||||
.expect("the migration must still complete for the same-name volume");
|
||||
assert!(migrated.bucket_incarnation_sidecar);
|
||||
assert_ne!(
|
||||
migrated.bucket_incarnation_id, retired,
|
||||
"a retired incarnation must never be re-published"
|
||||
);
|
||||
assert!(!migrated.bucket_incarnation_id.is_nil());
|
||||
assert_eq!(
|
||||
load_bucket_incarnation(ecstore, bucket).await.unwrap(),
|
||||
Some(migrated.bucket_incarnation_id),
|
||||
"the sidecar must be rewritten to the new incarnation"
|
||||
);
|
||||
}
|
||||
|
||||
/// Concurrent cache misses for one bucket must collapse into a single disk
|
||||
/// load.
|
||||
///
|
||||
|
||||
@@ -38,7 +38,6 @@ const MAX_ORPHANS_REAPED_PER_WRITE: usize = 64;
|
||||
const MAX_ORPHAN_PROBES_PER_WRITE: usize = 128;
|
||||
const ORPHAN_PROBE_CONCURRENCY: usize = 32;
|
||||
const EVENT_QUOTA_LEDGER_SETTLEMENT: &str = "quota_ledger_settlement";
|
||||
const EVENT_QUOTA_ADMISSION: &str = "quota_admission";
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_QUOTA: &str = "quota";
|
||||
|
||||
@@ -47,8 +46,6 @@ static FAIL_NEXT_LEDGER_SAVE: std::sync::atomic::AtomicBool = std::sync::atomic:
|
||||
|
||||
// Lock order: caller-held destination object/upload, bucket metadata
|
||||
// transaction (read), operation reservation, then quota ledger.
|
||||
// When Object Lock already holds the metadata transaction read lock, reuse
|
||||
// that guard: reacquiring it behind a waiting metadata writer would deadlock.
|
||||
|
||||
#[cfg(not(any(test, feature = "test-util")))]
|
||||
const ORPHAN_MIN_AGE_SECONDS: i64 = 30;
|
||||
@@ -213,7 +210,7 @@ pub(crate) struct QuotaContext {
|
||||
capability_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
|
||||
snapshot_admission: Option<QuotaAdmission>,
|
||||
legacy_data_movement: bool,
|
||||
metadata_guard: Option<Arc<NamespaceLockGuard>>,
|
||||
metadata_guard: Option<NamespaceLockGuard>,
|
||||
pool_index: Option<usize>,
|
||||
set_index: Option<usize>,
|
||||
}
|
||||
@@ -298,7 +295,7 @@ impl QuotaContext {
|
||||
limit: quota_limit,
|
||||
});
|
||||
}
|
||||
if operation_guard.is_lock_lost() || metadata_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
|
||||
if operation_guard.is_lock_lost() || metadata_guard.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost) {
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "quota_reservation",
|
||||
bucket: ledger_data.bucket.clone(),
|
||||
@@ -335,7 +332,7 @@ struct LedgerReservationData {
|
||||
pub(crate) struct QuotaReservation {
|
||||
ledger: Option<LedgerReservationData>,
|
||||
operation_guard: Option<NamespaceLockGuard>,
|
||||
metadata_guard: Option<Arc<NamespaceLockGuard>>,
|
||||
metadata_guard: Option<NamespaceLockGuard>,
|
||||
capability_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
|
||||
state: ReservationState,
|
||||
}
|
||||
@@ -349,7 +346,7 @@ enum ReservationState {
|
||||
}
|
||||
|
||||
impl QuotaReservation {
|
||||
fn unlimited(metadata_guard: Option<Arc<NamespaceLockGuard>>) -> Self {
|
||||
fn unlimited(metadata_guard: Option<NamespaceLockGuard>) -> Self {
|
||||
Self {
|
||||
ledger: None,
|
||||
operation_guard: None,
|
||||
@@ -361,7 +358,7 @@ impl QuotaReservation {
|
||||
|
||||
pub(crate) fn is_lock_lost(&self) -> bool {
|
||||
self.operation_guard.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost)
|
||||
|| self.metadata_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| self.metadata_guard.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost)
|
||||
}
|
||||
|
||||
pub(crate) fn capability_proof_matches(&self) -> bool {
|
||||
@@ -441,12 +438,11 @@ pub(crate) async fn begin(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
snapshot_admission: Option<QuotaAdmission>,
|
||||
data_movement: bool,
|
||||
pool_index: usize,
|
||||
set_index: usize,
|
||||
) -> Result<QuotaContext> {
|
||||
let snapshot_admission = opts.quota_admission;
|
||||
let data_movement = opts.data_movement;
|
||||
if crate::bucket::utils::is_meta_bucketname(bucket) {
|
||||
return Ok(QuotaContext {
|
||||
store: None,
|
||||
@@ -501,7 +497,7 @@ pub(crate) async fn begin(
|
||||
});
|
||||
}
|
||||
|
||||
let metadata_guard = metadata_sys::acquire_bucket_metadata_transaction_read_lock_for_options_in(ctx, bucket, opts).await?;
|
||||
let metadata_guard = metadata_sys::acquire_bucket_metadata_transaction_read_lock_in(ctx, bucket).await?;
|
||||
let (quota, bucket_incarnation, quota_revision) =
|
||||
metadata_sys::get_quota_config_and_incarnation_from_disk_in(ctx, bucket).await?;
|
||||
if metadata_guard.is_lock_lost() {
|
||||
@@ -517,7 +513,6 @@ pub(crate) async fn begin(
|
||||
.as_ref()
|
||||
.is_some_and(|quota| quota.has_unsupported_reservation_protocol())
|
||||
{
|
||||
log_admission_rejected(bucket, object, "unsupported_reservation_protocol");
|
||||
return Err(StorageError::PartMissingOrCorrupt);
|
||||
}
|
||||
let durable_quota = quota.as_ref().filter(|quota| quota.uses_durable_reservations());
|
||||
@@ -534,16 +529,7 @@ pub(crate) async fn begin(
|
||||
Some(quota) => match (quota.quota, snapshot_admission) {
|
||||
(Some(limit), Some(admission)) if admission.quota_limit() == limit => Some(admission),
|
||||
(Some(_), None) if data_movement => None,
|
||||
(Some(_), _) => {
|
||||
// A snapshot-protocol quota requires the request handler's
|
||||
// admission on every write. Missing or mismatched admission
|
||||
// means a caller rebuilt `ObjectOptions` without carrying it
|
||||
// over (rustfs/rustfs#7674 lost it on CopyObject); fail closed
|
||||
// but leave a diagnosable trace, because the storage error is
|
||||
// the generic `PartMissingOrCorrupt`.
|
||||
log_admission_rejected(bucket, object, "snapshot_quota_admission_missing");
|
||||
return Err(StorageError::PartMissingOrCorrupt);
|
||||
}
|
||||
(Some(_), _) => return Err(StorageError::PartMissingOrCorrupt),
|
||||
(None, _) => None,
|
||||
},
|
||||
None => None,
|
||||
@@ -918,18 +904,6 @@ pub fn fail_next_quota_ledger_save_for_test() {
|
||||
FAIL_NEXT_LEDGER_SAVE.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
}
|
||||
|
||||
fn log_admission_rejected(bucket: &str, object: &str, state: &'static str) {
|
||||
warn!(
|
||||
event = EVENT_QUOTA_ADMISSION,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_QUOTA,
|
||||
state,
|
||||
bucket = %bucket,
|
||||
object = %object,
|
||||
"quota admission rejected the write before commit"
|
||||
);
|
||||
}
|
||||
|
||||
fn log_deferred_settlement(data: &LedgerReservationData, state: &'static str, err: &StorageError) {
|
||||
warn!(
|
||||
event = EVENT_QUOTA_LEDGER_SETTLEMENT,
|
||||
|
||||
@@ -199,38 +199,6 @@ fn has_raw_status(err: &SdkError<HeadObjectError>, status: u16) -> bool {
|
||||
err.raw_response().is_some_and(|r| r.status().as_u16() == status)
|
||||
}
|
||||
|
||||
fn raw_header_value(err: &SdkError<HeadObjectError>, name: &str) -> Option<String> {
|
||||
err.raw_response()
|
||||
.and_then(|response| response.headers().get(name))
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum DeleteMarkerHeadOutcome {
|
||||
Missing,
|
||||
Exists,
|
||||
Ambiguous,
|
||||
}
|
||||
|
||||
fn classify_delete_marker_head_error(err: &SdkError<HeadObjectError>) -> DeleteMarkerHeadOutcome {
|
||||
let (is_not_found, code) = err
|
||||
.as_service_error()
|
||||
.map(|service_err| (service_err.is_not_found(), service_err.code()))
|
||||
.unwrap_or((false, None));
|
||||
|
||||
// Prefer the raw HTTP status when present. HEAD responses commonly have no
|
||||
// body, and a contradictory XML code must not turn a 403/5xx into success.
|
||||
match err.raw_response().map(|response| response.status().as_u16()) {
|
||||
Some(404) => DeleteMarkerHeadOutcome::Missing,
|
||||
Some(405) => DeleteMarkerHeadOutcome::Exists,
|
||||
Some(_) => DeleteMarkerHeadOutcome::Ambiguous,
|
||||
None if is_not_found => DeleteMarkerHeadOutcome::Missing,
|
||||
None if !is_retryable_delete_replication_head_error(false, code) => DeleteMarkerHeadOutcome::Exists,
|
||||
None => DeleteMarkerHeadOutcome::Ambiguous,
|
||||
}
|
||||
}
|
||||
|
||||
fn metadata_requires_existing_target(op_type: ReplicationType, object_info: &ObjectInfo) -> bool {
|
||||
op_type == ReplicationType::Metadata
|
||||
&& object_info
|
||||
@@ -1841,14 +1809,32 @@ async fn verify_resync_head_result(
|
||||
(roi.size, None)
|
||||
}
|
||||
Err(err) if roi.delete_marker => {
|
||||
// A marker creation is verified by either absence (404) or
|
||||
// presence (405). A marker-version purge has the opposite goal:
|
||||
// only absence proves completion. All other outcomes remain
|
||||
// failed (backlog#862 / #799 B13).
|
||||
let retryable = if roi.version_purge_status.is_empty() {
|
||||
matches!(classify_delete_marker_head_error(&err), DeleteMarkerHeadOutcome::Ambiguous)
|
||||
} else {
|
||||
!matches!(classify_delete_marker_head_error(&err), DeleteMarkerHeadOutcome::Missing)
|
||||
// Verifying a replicated delete marker: only a
|
||||
// definitive 404/NoSuchKey or 405/MethodNotAllowed
|
||||
// confirms the marker propagated. Any other
|
||||
// (retryable/ambiguous) HEAD error leaves the outcome
|
||||
// unverified, so it must count as failed — not as a
|
||||
// blanket success (backlog#862 / #799 B13).
|
||||
// A HEAD carries no body, so the SDK only synthesizes an error
|
||||
// code for 404; a target answering the delete marker's version
|
||||
// with 405 (RustFS, MinIO, AWS) leaves `code()` empty and the
|
||||
// raw status is the only evidence (rustfs/backlog#2479).
|
||||
// A marker *version purge* (`version_purge_status` set) asks
|
||||
// for the opposite outcome: the marker must be gone from the
|
||||
// target, so only absence verifies it and a 405 proves the
|
||||
// marker is still there.
|
||||
let retryable = {
|
||||
let (is_not_found, code) = err
|
||||
.as_service_error()
|
||||
.map(|se| (se.is_not_found(), se.code()))
|
||||
.unwrap_or((false, None));
|
||||
let not_found = is_not_found || has_raw_status(&err, 404);
|
||||
if roi.version_purge_status.is_empty() {
|
||||
let code = code.or_else(|| has_raw_status(&err, 405).then_some("405"));
|
||||
is_retryable_delete_replication_head_error(not_found, code)
|
||||
} else {
|
||||
!not_found
|
||||
}
|
||||
};
|
||||
if retryable {
|
||||
st.failed_count += 1;
|
||||
@@ -3567,7 +3553,7 @@ async fn replicate_delete_to_target(
|
||||
}
|
||||
}
|
||||
|
||||
if !is_version_purge && dobj.delete_object.delete_marker && dobj.delete_object.delete_marker_version_id.is_some() {
|
||||
if dobj.delete_object.delete_marker && dobj.delete_object.delete_marker_version_id.is_some() {
|
||||
match head_object_for_worker(
|
||||
tgt_client.as_ref(),
|
||||
&tgt_client.bucket,
|
||||
@@ -3576,48 +3562,20 @@ async fn replicate_delete_to_target(
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(head) if head.delete_marker == Some(true) => {
|
||||
// Some targets expose an existing marker as a successful HEAD
|
||||
// with the marker bit instead of the usual 405. Treat that as
|
||||
// positive convergence evidence and retain the target version
|
||||
// id for any later marker purge.
|
||||
rinfo.target_delete_marker_version_id = head.version_id.filter(|version_id| !version_id.is_empty());
|
||||
rinfo.replication_status = ReplicationStatusType::Completed;
|
||||
return rinfo;
|
||||
}
|
||||
Ok(_) => {
|
||||
// A successful HEAD without the delete-marker bit proves that
|
||||
// the addressed version is a regular object (or that the
|
||||
// target omitted the marker contract). Creating a marker in
|
||||
// that state would hide a live version, so fail closed.
|
||||
rinfo.replication_status = ReplicationStatusType::Failed;
|
||||
rinfo.error = Some("target version is not a delete marker".to_string());
|
||||
return rinfo;
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
match classify_delete_marker_head_error(e.as_ref()) {
|
||||
DeleteMarkerHeadOutcome::Exists => {
|
||||
// An explicit HEAD of a delete-marker version is expected
|
||||
// to answer 405 (often without an SDK error code). That
|
||||
// is positive evidence the marker already exists, so the
|
||||
// creation is complete and must not issue another
|
||||
// versionless DELETE that would mint a second marker.
|
||||
if let Some(version_id) = raw_header_value(e.as_ref(), "x-amz-version-id") {
|
||||
rinfo.target_delete_marker_version_id = Some(version_id);
|
||||
}
|
||||
rinfo.replication_status = ReplicationStatusType::Completed;
|
||||
return rinfo;
|
||||
}
|
||||
DeleteMarkerHeadOutcome::Missing => {}
|
||||
DeleteMarkerHeadOutcome::Ambiguous => {
|
||||
// Only a definitive not-found means the marker is absent
|
||||
// and may be created. Keep transport and ambiguous
|
||||
// failures fail-closed instead of creating blindly.
|
||||
rinfo.replication_status = ReplicationStatusType::Failed;
|
||||
rinfo.error = Some(e.to_string());
|
||||
mark_replication_target_offline_if_needed(&tgt_client, &e).await;
|
||||
return rinfo;
|
||||
}
|
||||
let non_retryable = matches!(
|
||||
e.as_ref(),
|
||||
SdkError::ServiceError(service_err)
|
||||
if is_retryable_delete_replication_head_error(
|
||||
service_err.err().is_not_found(),
|
||||
service_err.err().code(),
|
||||
)
|
||||
);
|
||||
if non_retryable {
|
||||
rinfo.replication_status = ReplicationStatusType::Failed;
|
||||
rinfo.error = Some(e.to_string());
|
||||
return rinfo;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -6571,110 +6529,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn marker_creation_dobj(target_arn: &str) -> DeletedObjectReplicationInfo {
|
||||
let mut state = ReplicationState::default();
|
||||
state.targets.insert(target_arn.to_string(), ReplicationStatusType::Pending);
|
||||
DeletedObjectReplicationInfo {
|
||||
bucket: "source".to_string(),
|
||||
target_arn: target_arn.to_string(),
|
||||
op_type: ReplicationType::Heal,
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: "gone.txt".to_string(),
|
||||
delete_marker: true,
|
||||
delete_marker_version_id: Some(Uuid::new_v4()),
|
||||
replication_state: Some(state),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// A target that already has the marker answers a versioned HEAD with a
|
||||
/// bodiless 405. The live worker must settle the marker as completed and
|
||||
/// avoid minting another marker with a versionless DELETE.
|
||||
#[tokio::test]
|
||||
async fn live_delete_marker_worker_accepts_bodiless_405_without_delete() {
|
||||
let (endpoint, server) = spawn_scripted_target_server(1, |_| {
|
||||
"HTTP/1.1 405 Method Not Allowed\r\nX-Amz-Version-Id: target-marker\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
|
||||
.to_string()
|
||||
});
|
||||
let target = test_target_client(endpoint);
|
||||
register_test_target(&target).await;
|
||||
let dobj = marker_creation_dobj(&target.arn);
|
||||
|
||||
let rinfo = replicate_delete_to_target(&dobj, target.clone(), None).await;
|
||||
|
||||
assert_eq!(rinfo.replication_status, ReplicationStatusType::Completed, "{rinfo:?}");
|
||||
assert!(rinfo.error.is_none(), "{rinfo:?}");
|
||||
assert_eq!(rinfo.target_delete_marker_version_id.as_deref(), Some("target-marker"));
|
||||
let seen = server.join().expect("test HTTP server should finish");
|
||||
assert_eq!(seen.len(), 1, "an existing marker must not be re-created: {seen:?}");
|
||||
assert!(seen[0].starts_with("HEAD "), "{seen:?}");
|
||||
}
|
||||
|
||||
/// A successful HEAD that does not identify a delete marker is not
|
||||
/// convergence evidence: creating a marker would hide a live version.
|
||||
#[tokio::test]
|
||||
async fn live_delete_marker_worker_rejects_successful_regular_head_without_delete() {
|
||||
let (endpoint, server) = spawn_scripted_target_server(1, |_| empty_response("200 OK"));
|
||||
let target = test_target_client(endpoint);
|
||||
register_test_target(&target).await;
|
||||
let dobj = marker_creation_dobj(&target.arn);
|
||||
|
||||
let rinfo = replicate_delete_to_target(&dobj, target.clone(), None).await;
|
||||
|
||||
assert_eq!(rinfo.replication_status, ReplicationStatusType::Failed, "{rinfo:?}");
|
||||
assert_eq!(rinfo.error.as_deref(), Some("target version is not a delete marker"));
|
||||
let seen = server.join().expect("test HTTP server should finish");
|
||||
assert_eq!(seen.len(), 1, "a regular object must not be hidden by a new marker: {seen:?}");
|
||||
assert!(seen[0].starts_with("HEAD "), "{seen:?}");
|
||||
}
|
||||
|
||||
/// A definitive 404 means the marker is absent, so exactly one marker
|
||||
/// creation DELETE is still required.
|
||||
#[tokio::test]
|
||||
async fn live_delete_marker_worker_creates_marker_after_404() {
|
||||
let (endpoint, server) = spawn_scripted_target_server(2, |line| {
|
||||
if line.starts_with("HEAD ") {
|
||||
empty_response("404 Not Found")
|
||||
} else if line.starts_with("DELETE ") {
|
||||
empty_response("204 No Content")
|
||||
} else {
|
||||
empty_response("500 Unexpected")
|
||||
}
|
||||
});
|
||||
let target = test_target_client(endpoint);
|
||||
register_test_target(&target).await;
|
||||
let dobj = marker_creation_dobj(&target.arn);
|
||||
|
||||
let rinfo = replicate_delete_to_target(&dobj, target.clone(), None).await;
|
||||
|
||||
assert_eq!(rinfo.replication_status, ReplicationStatusType::Completed, "{rinfo:?}");
|
||||
let seen = server.join().expect("test HTTP server should finish");
|
||||
assert_eq!(seen.len(), 2, "a missing marker should be created once: {seen:?}");
|
||||
assert!(seen[0].starts_with("HEAD "), "{seen:?}");
|
||||
assert!(seen[1].starts_with("DELETE "), "{seen:?}");
|
||||
assert!(!seen[1].contains("versionId="), "marker creation must remain versionless: {seen:?}");
|
||||
}
|
||||
|
||||
/// Ambiguous service errors must remain failed and must not fall through
|
||||
/// to a blind marker-creation DELETE.
|
||||
#[tokio::test]
|
||||
async fn live_delete_marker_worker_keeps_ambiguous_head_failure_closed() {
|
||||
let (endpoint, server) = spawn_scripted_target_server(1, |_| empty_response("503 Service Unavailable"));
|
||||
let target = test_target_client(endpoint);
|
||||
register_test_target(&target).await;
|
||||
let dobj = marker_creation_dobj(&target.arn);
|
||||
|
||||
let rinfo = replicate_delete_to_target(&dobj, target.clone(), None).await;
|
||||
|
||||
assert_eq!(rinfo.replication_status, ReplicationStatusType::Failed, "{rinfo:?}");
|
||||
assert!(rinfo.error.is_some(), "{rinfo:?}");
|
||||
let seen = server.join().expect("test HTTP server should finish");
|
||||
assert_eq!(seen.len(), 1, "an ambiguous HEAD must not trigger DELETE: {seen:?}");
|
||||
assert!(seen[0].starts_with("HEAD "), "{seen:?}");
|
||||
}
|
||||
|
||||
/// A target answers `HEAD ?versionId=<delete marker>` with a bodiless
|
||||
/// 405, so the SDK reports no error code. That is the marker having
|
||||
/// propagated, not a failure (rustfs/backlog#2479).
|
||||
|
||||
@@ -23,7 +23,7 @@ use std::{
|
||||
future::Future,
|
||||
io::ErrorKind,
|
||||
pin::Pin,
|
||||
sync::{Arc, OnceLock, atomic::AtomicBool},
|
||||
sync::{Arc, OnceLock},
|
||||
task::{Context, Poll},
|
||||
time::Duration,
|
||||
};
|
||||
@@ -234,8 +234,6 @@ pub struct ListPathRawOptions {
|
||||
pub skip_walkdir_total_timeout: bool,
|
||||
pub walkdir_timeout: Option<Duration>,
|
||||
pub walkdir_stall_timeout: Option<Duration>,
|
||||
/// Shared terminal state for bounded producer walks.
|
||||
pub producer_limit_reached: Option<Arc<AtomicBool>>,
|
||||
pub agreed: Option<AgreedFn>,
|
||||
pub partial: Option<PartialFn>,
|
||||
pub finished: Option<FinishedFn>,
|
||||
@@ -269,7 +267,6 @@ impl Clone for ListPathRawOptions {
|
||||
skip_walkdir_total_timeout: self.skip_walkdir_total_timeout,
|
||||
walkdir_timeout: self.walkdir_timeout,
|
||||
walkdir_stall_timeout: self.walkdir_stall_timeout,
|
||||
producer_limit_reached: self.producer_limit_reached.clone(),
|
||||
#[cfg(test)]
|
||||
test_reader_behaviors: self.test_reader_behaviors.clone(),
|
||||
#[cfg(test)]
|
||||
@@ -295,7 +292,6 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
|
||||
skip_total_timeout: opts.skip_walkdir_total_timeout,
|
||||
timeout_ms: opts.walkdir_timeout.map(duration_millis),
|
||||
stall_timeout_ms: opts.walkdir_stall_timeout.map(duration_millis),
|
||||
producer_limit_reached: opts.producer_limit_reached.clone(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1170,11 +1170,6 @@ impl RemoteDisk {
|
||||
self.health.force_runtime_state_for_test(state);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn force_offline_for_test(&self) {
|
||||
self.health.force_offline_for_test();
|
||||
}
|
||||
|
||||
/// Same as [`DiskHealthTracker::reset_for_store_init_retry`]: undo a transient faulty mark before another format load attempt.
|
||||
pub fn reset_health_for_store_init_retry(&self) {
|
||||
self.health.reset_for_store_init_retry(&self.endpoint);
|
||||
@@ -2141,7 +2136,7 @@ impl RemoteDisk {
|
||||
let file_info_bin = encode_file_info_msgpack(fi)?;
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(RenameDataRequest {
|
||||
bucket_incarnation_id: crate::store::bucket_heal_scope_for_object(dst_volume, dst_path)
|
||||
bucket_incarnation_id: crate::store::bucket_heal_scope(dst_volume)
|
||||
.map(|scope| scope.incarnation.as_bytes().to_vec().into())
|
||||
.unwrap_or_default(),
|
||||
disk: self.endpoint.to_string(),
|
||||
@@ -4308,13 +4303,6 @@ mod tests {
|
||||
object_read_all_disks: Arc<StdMutex<Vec<String>>>,
|
||||
format_data: Bytes,
|
||||
read_all_data: Bytes,
|
||||
next_format_read_gate: Arc<StdMutex<Option<Arc<TestFormatReadGate>>>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct TestFormatReadGate {
|
||||
entered: tokio::sync::Notify,
|
||||
release: CancellationToken,
|
||||
}
|
||||
|
||||
impl AuthenticatedReadPeer {
|
||||
@@ -4326,16 +4314,9 @@ mod tests {
|
||||
object_read_all_disks: Arc::default(),
|
||||
format_data,
|
||||
read_all_data,
|
||||
next_format_read_gate: Arc::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn pause_next_format_read(&self) -> Arc<TestFormatReadGate> {
|
||||
let gate = Arc::new(TestFormatReadGate::default());
|
||||
*self.next_format_read_gate.lock().expect("format gate lock poisoned") = Some(Arc::clone(&gate));
|
||||
gate
|
||||
}
|
||||
|
||||
fn disk_info_calls(&self) -> u32 {
|
||||
self.disk_info_calls.load(Ordering::Acquire)
|
||||
}
|
||||
@@ -4443,11 +4424,6 @@ mod tests {
|
||||
let disk = request.disk;
|
||||
peer.read_all_calls.fetch_add(1, Ordering::AcqRel);
|
||||
let data = if is_format_read {
|
||||
let gate = peer.next_format_read_gate.lock().expect("format gate lock poisoned").take();
|
||||
if let Some(gate) = gate {
|
||||
gate.entered.notify_one();
|
||||
gate.release.cancelled().await;
|
||||
}
|
||||
peer.format_data.clone()
|
||||
} else {
|
||||
peer.object_read_all_disks
|
||||
@@ -7150,319 +7126,6 @@ mod tests {
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
async fn missing_read_set(peer: &TestGrpcPeer, format: crate::layout::format::FormatV3) -> Arc<crate::set_disk::SetDisks> {
|
||||
let count = format.erasure.sets[0].len();
|
||||
let endpoints = (0..count)
|
||||
.map(|index| Endpoint {
|
||||
url: url::Url::parse(&format!("{}/data/rustfs{index}", peer.addr)).expect("endpoint should parse"),
|
||||
is_local: false,
|
||||
pool_idx: 0,
|
||||
set_idx: 0,
|
||||
disk_idx: i32::try_from(index).expect("fixture index fits"),
|
||||
})
|
||||
.collect();
|
||||
crate::set_disk::SetDisks::new(
|
||||
"missing-read-test".to_string(),
|
||||
Arc::new(tokio::sync::RwLock::new(vec![None; count])),
|
||||
count,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
endpoints,
|
||||
format,
|
||||
Vec::new(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(remote_disk_recovery_probe)]
|
||||
async fn missing_read_disks_reconnect_before_periodic_monitor_and_coalesce() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let mut format = crate::layout::format::FormatV3::new(1, 1);
|
||||
let disk_id = format.erasure.sets[0][0];
|
||||
format.erasure.this = disk_id;
|
||||
let Some(peer) = TestGrpcPeer::spawn(
|
||||
Bytes::from(format.to_json().expect("format should serialize")),
|
||||
Bytes::from_static(b"startup-recovered-data"),
|
||||
)
|
||||
.await
|
||||
else {
|
||||
panic!("authenticated regression peer is required");
|
||||
};
|
||||
let set = missing_read_set(&peer, format).await;
|
||||
let snapshots = futures::future::join_all((0..8).map(|_| set.get_disks_for_data_read())).await;
|
||||
let disk = snapshots[0][0]
|
||||
.as_ref()
|
||||
.expect("a ready peer must be usable before the periodic reconnect");
|
||||
for snapshot in &snapshots {
|
||||
assert!(Arc::ptr_eq(
|
||||
disk,
|
||||
snapshot[0]
|
||||
.as_ref()
|
||||
.expect("every waiting read should observe the same handle")
|
||||
));
|
||||
}
|
||||
assert_eq!(peer.peer.read_all_calls(), 1, "concurrent requests must share the format probe");
|
||||
assert_eq!(disk.get_disk_id().await.expect("disk identity should be available"), Some(disk_id));
|
||||
let raw = disk
|
||||
.read_all("bucket", "object")
|
||||
.await
|
||||
.expect("reconnected handle should read data");
|
||||
assert_eq!(raw, Bytes::from_static(b"startup-recovered-data"));
|
||||
assert_eq!(peer.peer.object_read_all_disks(), vec![disk_id.to_string()]);
|
||||
let healthy = set.get_disks_for_data_read().await;
|
||||
assert!(Arc::ptr_eq(disk, healthy[0].as_ref().expect("healthy handle must remain registered")));
|
||||
assert_eq!(peer.peer.read_all_calls(), 2, "healthy reads must not re-probe the format");
|
||||
disk.close().await.expect("reconnected disk should close");
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(remote_disk_recovery_probe)]
|
||||
async fn missing_read_disks_reject_foreign_format_and_back_off() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let reference = crate::layout::format::FormatV3::new(1, 1);
|
||||
let mut foreign = reference.clone();
|
||||
foreign.id = Uuid::new_v4();
|
||||
foreign.erasure.this = reference.erasure.sets[0][0];
|
||||
let Some(peer) = TestGrpcPeer::spawn(
|
||||
Bytes::from(foreign.to_json().expect("foreign format should serialize")),
|
||||
Bytes::from_static(b"foreign-data"),
|
||||
)
|
||||
.await
|
||||
else {
|
||||
panic!("authenticated regression peer is required");
|
||||
};
|
||||
let set = missing_read_set(&peer, reference).await;
|
||||
assert!(
|
||||
set.get_disks_for_data_read().await[0].is_none(),
|
||||
"a matching disk ID cannot authorize a foreign deployment"
|
||||
);
|
||||
assert_eq!(peer.peer.read_all_calls(), 1, "the foreign format must actually be inspected");
|
||||
assert!(set.get_disks_for_data_read().await[0].is_none());
|
||||
assert_eq!(peer.peer.read_all_calls(), 1, "failed probes must not repeat on every degraded GET");
|
||||
assert!(peer.peer.object_read_all_disks().is_empty(), "foreign object data must never be read");
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(remote_disk_recovery_probe)]
|
||||
async fn missing_read_disks_reject_wrong_slot_and_invalid_formats() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let mut reference = crate::layout::format::FormatV3::new(1, 2);
|
||||
reference.erasure.this = reference.erasure.sets[0][0];
|
||||
let mut nil = reference.clone();
|
||||
nil.erasure.this = Uuid::nil();
|
||||
let mut unknown = reference.clone();
|
||||
unknown.erasure.this = Uuid::new_v4();
|
||||
let mut future = reference.clone();
|
||||
future.erasure.version = crate::layout::format::FormatErasureVersion::Unknown;
|
||||
for (name, payload, first_accepted) in [
|
||||
("duplicate slot zero", reference.to_json().expect("format should serialize"), true),
|
||||
("nil drive", nil.to_json().expect("nil format should serialize"), false),
|
||||
("unknown drive", unknown.to_json().expect("unknown format should serialize"), false),
|
||||
("future version", future.to_json().expect("future format should serialize"), false),
|
||||
("malformed", "not-json".to_string(), false),
|
||||
] {
|
||||
let peer = TestGrpcPeer::spawn(Bytes::from(payload), Bytes::new())
|
||||
.await
|
||||
.expect("authenticated regression peer is required");
|
||||
let set = missing_read_set(&peer, reference.clone()).await;
|
||||
let disks = set.get_disks_for_data_read().await;
|
||||
assert_eq!(disks[0].is_some(), first_accepted, "{name}: slot zero identity decision");
|
||||
assert!(disks[1].is_none(), "{name}: a peer cannot claim another topology slot");
|
||||
assert_eq!(peer.peer.read_all_calls(), 2, "{name}: both missing slots must be probed");
|
||||
assert!(peer.peer.object_read_all_disks().is_empty(), "{name}: no object payload may be read");
|
||||
for disk in disks.into_iter().flatten() {
|
||||
disk.close().await.expect("accepted disk should close");
|
||||
}
|
||||
peer.stop().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(remote_disk_recovery_probe)]
|
||||
async fn missing_read_disks_do_not_claim_local_or_misconfigured_endpoints() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let mut reference = crate::layout::format::FormatV3::new(1, 1);
|
||||
reference.erasure.this = reference.erasure.sets[0][0];
|
||||
let peer = TestGrpcPeer::spawn(Bytes::from(reference.to_json().expect("format should serialize")), Bytes::new())
|
||||
.await
|
||||
.expect("authenticated regression peer is required");
|
||||
for (local, pool, set_index, disk_index) in [(true, 0, 0, 0), (false, -1, 0, 0), (false, 0, 1, 0), (false, 0, 0, 1)] {
|
||||
let mut set = missing_read_set(&peer, reference.clone()).await;
|
||||
let endpoint = &mut Arc::get_mut(&mut set)
|
||||
.expect("fixture set must be uniquely owned")
|
||||
.set_endpoints[0];
|
||||
endpoint.is_local = local;
|
||||
endpoint.pool_idx = pool;
|
||||
endpoint.set_idx = set_index;
|
||||
endpoint.disk_idx = disk_index;
|
||||
assert!(set.get_disks_for_data_read().await[0].is_none());
|
||||
}
|
||||
assert_eq!(peer.peer.read_all_calls(), 0, "invalid endpoint ownership must fail before I/O");
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(remote_disk_recovery_probe)]
|
||||
async fn missing_read_disks_preserve_concurrently_published_handles() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let mut reference = crate::layout::format::FormatV3::new(1, 1);
|
||||
reference.erasure.this = reference.erasure.sets[0][0];
|
||||
for reader_first in [false, true] {
|
||||
let peer = TestGrpcPeer::spawn(Bytes::from(reference.to_json().expect("format should serialize")), Bytes::new())
|
||||
.await
|
||||
.expect("authenticated regression peer is required");
|
||||
let set = missing_read_set(&peer, reference.clone()).await;
|
||||
let gate = peer.peer.pause_next_format_read();
|
||||
let pending_set = Arc::clone(&set);
|
||||
let pending = tokio::spawn(async move {
|
||||
if reader_first {
|
||||
pending_set.get_disks_for_data_read().await;
|
||||
} else {
|
||||
pending_set.renew_disk(&pending_set.set_endpoints[0]).await;
|
||||
}
|
||||
});
|
||||
time::timeout(Duration::from_secs(2), gate.entered.notified())
|
||||
.await
|
||||
.expect("the first reconnect must reach the format barrier");
|
||||
if reader_first {
|
||||
set.renew_disk(&set.set_endpoints[0]).await;
|
||||
} else {
|
||||
set.get_disks_for_data_read().await;
|
||||
}
|
||||
let published = set.disks.read().await[0]
|
||||
.clone()
|
||||
.expect("the competing reconnect must publish");
|
||||
gate.release.cancel();
|
||||
pending.await.expect("paused reconnect must complete");
|
||||
assert!(
|
||||
Arc::ptr_eq(&published, set.disks.read().await[0].as_ref().expect("published slot must remain")),
|
||||
"neither reconnect may replace a handle published while its format I/O was pending"
|
||||
);
|
||||
published.close().await.expect("published disk should close");
|
||||
peer.stop().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(remote_disk_recovery_probe, internode_metrics)]
|
||||
async fn missing_read_disks_expired_waiter_does_not_start_another_probe() {
|
||||
// Disable cooldown so this independently proves deadline admission,
|
||||
// rather than passing only because the leader has just finished.
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_RETURNING_PROBE_INTERVAL_SECS, Some("0"))], async {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let mut reference = crate::layout::format::FormatV3::new(1, 1);
|
||||
reference.erasure.this = reference.erasure.sets[0][0];
|
||||
let peer = TestGrpcPeer::spawn(Bytes::from(reference.to_json().expect("format should serialize")), Bytes::new())
|
||||
.await
|
||||
.expect("authenticated regression peer is required");
|
||||
let set = missing_read_set(&peer, reference).await;
|
||||
let gate = peer.peer.pause_next_format_read();
|
||||
let mut leader = Box::pin(set.get_disks_for_data_read());
|
||||
time::timeout(Duration::from_secs(2), async {
|
||||
tokio::select! {
|
||||
_ = gate.entered.notified() => {}
|
||||
_ = &mut leader => panic!("leader must remain blocked in its format RPC"),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("leader must reach the format barrier");
|
||||
time::pause();
|
||||
let mut waiter = Box::pin(set.get_disks_for_data_read());
|
||||
assert!(futures::poll!(&mut waiter).is_pending(), "waiter must queue behind the leader");
|
||||
let requests_before =
|
||||
crate::cluster::rpc::runtime_sources::internode_metrics_snapshot_for_test().outgoing_requests_total;
|
||||
|
||||
time::advance(crate::disk::disk_store::get_drive_active_check_timeout()).await;
|
||||
assert!(leader.await[0].is_none(), "expired leader must not publish a disk");
|
||||
assert!(waiter.await[0].is_none(), "expired waiter must not publish a disk");
|
||||
time::resume();
|
||||
|
||||
// Count client dispatches, not server arrivals: a cancelled RPC can still
|
||||
// be buffered in the transport when the expired waiter returns.
|
||||
assert_eq!(
|
||||
crate::cluster::rpc::runtime_sources::internode_metrics_snapshot_for_test().outgoing_requests_total,
|
||||
requests_before,
|
||||
"an expired waiter must not dispatch another format RPC after the leader releases the lock"
|
||||
);
|
||||
gate.release.cancel();
|
||||
peer.stop().await;
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(remote_disk_recovery_probe)]
|
||||
async fn missing_read_disks_timeout_releases_singleflight_and_allows_retry() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let mut reference = crate::layout::format::FormatV3::new(1, 1);
|
||||
reference.erasure.this = reference.erasure.sets[0][0];
|
||||
let peer = TestGrpcPeer::spawn(Bytes::from(reference.to_json().expect("format should serialize")), Bytes::new())
|
||||
.await
|
||||
.expect("authenticated regression peer is required");
|
||||
let set = missing_read_set(&peer, reference).await;
|
||||
let gate = peer.peer.pause_next_format_read();
|
||||
let deadline = crate::disk::disk_store::get_drive_active_check_timeout();
|
||||
let snapshots = time::timeout(
|
||||
deadline + Duration::from_secs(2),
|
||||
futures::future::join_all((0..4).map(|_| set.get_disks_for_data_read())),
|
||||
)
|
||||
.await
|
||||
.expect("all waiting requests must respect the bounded probe deadline");
|
||||
assert!(snapshots.iter().all(|snapshot| snapshot[0].is_none()));
|
||||
assert_eq!(peer.peer.read_all_calls(), 1, "only one format RPC may remain stalled");
|
||||
gate.release.cancel();
|
||||
time::pause();
|
||||
time::advance(crate::disk::health_state::get_drive_returning_probe_interval()).await;
|
||||
time::resume();
|
||||
let disks = set.get_disks_for_data_read().await;
|
||||
let disk = disks[0]
|
||||
.as_ref()
|
||||
.expect("a timed-out probe must not prevent a later read from reconnecting");
|
||||
assert_eq!(peer.peer.read_all_calls(), 2, "retry must validate the format again");
|
||||
disk.close().await.expect("reconnected disk should close");
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(remote_disk_recovery_probe)]
|
||||
async fn missing_read_disks_cancellation_leaves_no_published_handle() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let mut reference = crate::layout::format::FormatV3::new(1, 1);
|
||||
reference.erasure.this = reference.erasure.sets[0][0];
|
||||
let peer = TestGrpcPeer::spawn(Bytes::from(reference.to_json().expect("format should serialize")), Bytes::new())
|
||||
.await
|
||||
.expect("authenticated regression peer is required");
|
||||
let set = missing_read_set(&peer, reference).await;
|
||||
let gate = peer.peer.pause_next_format_read();
|
||||
let pending_set = Arc::clone(&set);
|
||||
let pending = tokio::spawn(async move { pending_set.get_disks_for_data_read().await });
|
||||
time::timeout(Duration::from_secs(2), gate.entered.notified())
|
||||
.await
|
||||
.expect("probe must reach the format barrier");
|
||||
// Cancellation can happen after the start-based cooldown has elapsed.
|
||||
// It must still cool down before another caller probes the same peer.
|
||||
time::pause();
|
||||
time::advance(crate::disk::health_state::get_drive_returning_probe_interval()).await;
|
||||
time::resume();
|
||||
pending.abort();
|
||||
assert!(pending.await.expect_err("probe must be cancelled").is_cancelled());
|
||||
assert!(
|
||||
set.disks.read().await[0].is_none(),
|
||||
"cancelled format I/O must not publish an unvalidated handle"
|
||||
);
|
||||
let waiting = time::timeout(Duration::from_secs(1), set.get_disks_for_data_read())
|
||||
.await
|
||||
.expect("cancelled probe must release the singleflight lock");
|
||||
assert!(waiting[0].is_none(), "retry cooldown must still apply after cancellation");
|
||||
assert_eq!(peer.peer.read_all_calls(), 1);
|
||||
gate.release.cancel();
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_copy_stream_with_buffer_copies_full_payload() {
|
||||
let payload = b"walk-dir-stream".repeat(1024);
|
||||
|
||||
@@ -47,37 +47,10 @@ fn attach_lock_mutation_body_digest<T: rustfs_protos::CanonicalMutationBody>(req
|
||||
|
||||
/// Work to run if an RPC that already timed out for its caller completes later.
|
||||
type LateCompletion<T> = Option<Box<dyn FnOnce(T) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send>>;
|
||||
type LateFailure = LateCompletion<()>;
|
||||
|
||||
struct RpcCleanup<T> {
|
||||
late: LateCompletion<T>,
|
||||
late_failure: LateFailure,
|
||||
admission: Option<LockRequestAdmission>,
|
||||
}
|
||||
|
||||
impl<T> RpcCleanup<T> {
|
||||
fn new(late: LateCompletion<T>, late_failure: LateFailure, admission: Option<LockRequestAdmission>) -> Self {
|
||||
Self {
|
||||
late,
|
||||
late_failure,
|
||||
admission,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The liveness window is this many RPC deadlines: a peer that completed a
|
||||
/// lock RPC within it is slow, not gone, and keeps its channel on a timeout.
|
||||
const LOCK_RPC_LIVENESS_WINDOW_DEADLINES: u32 = 2;
|
||||
const LOCK_RPC_REQUEST_BACKOFF_THRESHOLD: u32 = 3;
|
||||
const LOCK_RPC_REQUEST_BACKOFF_MAX: Duration = Duration::from_secs(30);
|
||||
const LATE_RELEASE_RETRY_DELAYS: [Duration; 3] = [Duration::from_millis(100), Duration::from_millis(500), Duration::from_secs(1)];
|
||||
const LATE_RELEASE_IN_FLIGHT_LIMIT: usize = 32;
|
||||
const TIMEOUT_LOG_STATE_MAX: usize = 1024;
|
||||
const TIMEOUT_LOG_STATE_RETENTION: Duration = Duration::from_secs(600);
|
||||
|
||||
fn is_request_breaker_operation(op: &'static str) -> bool {
|
||||
matches!(op, "lock" | "lock_batch")
|
||||
}
|
||||
|
||||
/// Recent history of the shared lock channel to one peer (issue #7363).
|
||||
///
|
||||
@@ -91,44 +64,15 @@ struct LockPeerChannelHealth {
|
||||
last_success: Option<Instant>,
|
||||
last_eviction: Option<Instant>,
|
||||
consecutive_timeouts: u32,
|
||||
request_consecutive_timeouts: u32,
|
||||
request_backoff_until: Option<Instant>,
|
||||
request_backoff: Duration,
|
||||
request_probe_generation: Option<u64>,
|
||||
next_probe_generation: u64,
|
||||
request_in_flight: usize,
|
||||
late_release_in_flight: usize,
|
||||
/// Timed-out RPCs still running in the background for this peer.
|
||||
detached_rpcs: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default)]
|
||||
struct TimeoutLogHealth {
|
||||
last_logged: Option<Instant>,
|
||||
suppressed: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct LockRequestAdmission {
|
||||
probe_generation: Option<u64>,
|
||||
}
|
||||
|
||||
impl LockRequestAdmission {
|
||||
fn is_probe(self) -> bool {
|
||||
self.probe_generation.is_some()
|
||||
}
|
||||
}
|
||||
|
||||
fn lock_peer_channel_health() -> &'static Mutex<HashMap<String, LockPeerChannelHealth>> {
|
||||
static HEALTH: OnceLock<Mutex<HashMap<String, LockPeerChannelHealth>>> = OnceLock::new();
|
||||
HEALTH.get_or_init(Mutex::default)
|
||||
}
|
||||
|
||||
fn timeout_log_health() -> &'static Mutex<HashMap<(String, &'static str), TimeoutLogHealth>> {
|
||||
static HEALTH: OnceLock<Mutex<HashMap<(String, &'static str), TimeoutLogHealth>>> = OnceLock::new();
|
||||
HEALTH.get_or_init(Mutex::default)
|
||||
}
|
||||
|
||||
fn with_lock_peer_health<R>(addr: &str, update: impl FnOnce(&mut LockPeerChannelHealth) -> R) -> R {
|
||||
let mut peers = lock_peer_channel_health()
|
||||
.lock()
|
||||
@@ -399,163 +343,17 @@ impl RemoteClient {
|
||||
)
|
||||
}
|
||||
|
||||
fn request_limit() -> usize {
|
||||
rustfs_utils::get_env_usize(
|
||||
rustfs_config::ENV_OBJECT_LOCK_RPC_REQUEST_LIMIT,
|
||||
rustfs_config::DEFAULT_OBJECT_LOCK_RPC_REQUEST_LIMIT,
|
||||
)
|
||||
.max(1)
|
||||
}
|
||||
|
||||
fn liveness_window(deadline: Duration) -> Duration {
|
||||
deadline.saturating_mul(LOCK_RPC_LIVENESS_WINDOW_DEADLINES)
|
||||
}
|
||||
|
||||
fn record_rpc_success(&self, op: &'static str, admission: Option<LockRequestAdmission>) {
|
||||
fn record_rpc_success(&self) {
|
||||
with_lock_peer_health(&self.addr, |health| {
|
||||
health.last_success = Some(Instant::now());
|
||||
health.consecutive_timeouts = 0;
|
||||
if is_request_breaker_operation(op) {
|
||||
let is_probe = admission.is_some_and(LockRequestAdmission::is_probe);
|
||||
if health.request_backoff_until.is_none() || is_probe {
|
||||
health.request_consecutive_timeouts = 0;
|
||||
health.request_backoff_until = None;
|
||||
health.request_backoff = Duration::ZERO;
|
||||
}
|
||||
if is_probe && admission.is_some_and(|admission| health.request_probe_generation == admission.probe_generation) {
|
||||
health.request_probe_generation = None;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn admit_late_release(&self) -> bool {
|
||||
with_lock_peer_health(&self.addr, |health| {
|
||||
if health.late_release_in_flight >= LATE_RELEASE_IN_FLIGHT_LIMIT {
|
||||
false
|
||||
} else {
|
||||
health.late_release_in_flight += 1;
|
||||
true
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn release_late_release(&self) {
|
||||
with_lock_peer_health(&self.addr, |health| {
|
||||
health.late_release_in_flight = health.late_release_in_flight.saturating_sub(1);
|
||||
});
|
||||
}
|
||||
|
||||
/// Admit a new lock acquisition unless this peer is in the request-level
|
||||
/// backoff window. A single half-open probe is allowed after the window;
|
||||
/// channel liveness and request admission are deliberately separate.
|
||||
fn admit_lock_request(&self, op: &'static str) -> Option<LockRequestAdmission> {
|
||||
if !is_request_breaker_operation(op) {
|
||||
return Some(LockRequestAdmission { probe_generation: None });
|
||||
}
|
||||
|
||||
let now = Instant::now();
|
||||
let (admission, suppression_reason) = with_lock_peer_health(&self.addr, |health| {
|
||||
let degraded = health.request_backoff_until.is_some() || health.request_consecutive_timeouts > 0;
|
||||
if degraded && health.request_in_flight >= Self::request_limit() {
|
||||
return (None, "in_flight_limit");
|
||||
}
|
||||
let Some(until) = health.request_backoff_until else {
|
||||
health.request_in_flight += 1;
|
||||
return (Some(LockRequestAdmission { probe_generation: None }), "none");
|
||||
};
|
||||
if now < until || health.request_probe_generation.is_some() {
|
||||
return (None, "breaker_open");
|
||||
}
|
||||
health.next_probe_generation = health.next_probe_generation.saturating_add(1);
|
||||
let generation = health.next_probe_generation;
|
||||
health.request_probe_generation = Some(generation);
|
||||
health.request_in_flight += 1;
|
||||
(
|
||||
Some(LockRequestAdmission {
|
||||
probe_generation: Some(generation),
|
||||
}),
|
||||
"none",
|
||||
)
|
||||
});
|
||||
if admission.is_none() {
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_request_suppressed(&self.addr, op, suppression_reason);
|
||||
}
|
||||
admission
|
||||
}
|
||||
|
||||
fn release_lock_request(&self, op: &'static str, admission: Option<LockRequestAdmission>) {
|
||||
if is_request_breaker_operation(op) && admission.is_some() {
|
||||
with_lock_peer_health(&self.addr, |health| {
|
||||
health.request_in_flight = health.request_in_flight.saturating_sub(1);
|
||||
if admission.is_some_and(|admission| health.request_probe_generation == admission.probe_generation) {
|
||||
health.request_probe_generation = None;
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
fn record_request_timeout(&self, op: &'static str, deadline: Duration) {
|
||||
if !is_request_breaker_operation(op) {
|
||||
return;
|
||||
}
|
||||
|
||||
let now = Instant::now();
|
||||
with_lock_peer_health(&self.addr, |health| {
|
||||
health.request_consecutive_timeouts = health.request_consecutive_timeouts.saturating_add(1);
|
||||
if health.request_consecutive_timeouts < LOCK_RPC_REQUEST_BACKOFF_THRESHOLD {
|
||||
return;
|
||||
}
|
||||
|
||||
let initial = deadline / 2;
|
||||
let next = if health.request_backoff.is_zero() {
|
||||
initial.max(Duration::from_millis(1))
|
||||
} else {
|
||||
health.request_backoff.saturating_mul(2)
|
||||
};
|
||||
health.request_backoff = next.min(LOCK_RPC_REQUEST_BACKOFF_MAX);
|
||||
health.request_backoff_until = Some(now + health.request_backoff);
|
||||
});
|
||||
}
|
||||
|
||||
fn timeout_log_decision(&self, op: &'static str, interval: Duration) -> (bool, u64) {
|
||||
let now = Instant::now();
|
||||
let mut peers = timeout_log_health().lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if peers.len() >= TIMEOUT_LOG_STATE_MAX {
|
||||
peers.retain(|_, state| {
|
||||
state
|
||||
.last_logged
|
||||
.is_some_and(|at| now.saturating_duration_since(at) < TIMEOUT_LOG_STATE_RETENTION)
|
||||
});
|
||||
}
|
||||
let key = (self.addr.clone(), op);
|
||||
if peers.len() >= TIMEOUT_LOG_STATE_MAX
|
||||
&& !peers.contains_key(&key)
|
||||
&& let Some(eviction_key) = peers.keys().next().cloned()
|
||||
{
|
||||
peers.remove(&eviction_key);
|
||||
}
|
||||
let state = peers.entry(key).or_default();
|
||||
if state
|
||||
.last_logged
|
||||
.is_some_and(|at| now.saturating_duration_since(at) < interval)
|
||||
{
|
||||
state.suppressed = state.suppressed.saturating_add(1);
|
||||
return (false, state.suppressed);
|
||||
}
|
||||
|
||||
let suppressed = state.suppressed;
|
||||
state.last_logged = Some(now);
|
||||
state.suppressed = 0;
|
||||
(true, suppressed)
|
||||
}
|
||||
|
||||
fn bounded_acquire_request(request: &LockRequest, deadline: Duration) -> LockRequest {
|
||||
let mut bounded = request.clone();
|
||||
bounded.acquire_timeout = bounded.acquire_timeout.min(deadline);
|
||||
bounded
|
||||
}
|
||||
|
||||
/// Apply the per-peer eviction policy after a failed RPC.
|
||||
async fn maybe_evict_connection(
|
||||
&self,
|
||||
@@ -608,8 +406,6 @@ impl RemoteClient {
|
||||
resource_summary: &str,
|
||||
handle: JoinHandle<std::result::Result<T, tonic::Status>>,
|
||||
late: LateCompletion<T>,
|
||||
late_failure: LateFailure,
|
||||
admission: Option<LockRequestAdmission>,
|
||||
) {
|
||||
let limit = Self::detached_rpc_limit();
|
||||
let admitted = with_lock_peer_health(&self.addr, |health| {
|
||||
@@ -622,10 +418,6 @@ impl RemoteClient {
|
||||
});
|
||||
if !admitted {
|
||||
handle.abort();
|
||||
self.release_lock_request(op, admission);
|
||||
if let Some(late_failure) = late_failure {
|
||||
drop(tokio::spawn(late_failure(())));
|
||||
}
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_detached(op, "aborted");
|
||||
debug!(
|
||||
addr = %self.addr,
|
||||
@@ -640,19 +432,12 @@ impl RemoteClient {
|
||||
let addr = self.addr.clone();
|
||||
tokio::spawn(async move {
|
||||
let outcome = handle.await;
|
||||
with_lock_peer_health(&addr, |health| {
|
||||
health.detached_rpcs = health.detached_rpcs.saturating_sub(1);
|
||||
if is_request_breaker_operation(op) && admission.is_some() {
|
||||
health.request_in_flight = health.request_in_flight.saturating_sub(1);
|
||||
if admission.is_some_and(|admission| health.request_probe_generation == admission.probe_generation) {
|
||||
health.request_probe_generation = None;
|
||||
}
|
||||
}
|
||||
});
|
||||
with_lock_peer_health(&addr, |health| health.detached_rpcs = health.detached_rpcs.saturating_sub(1));
|
||||
match outcome {
|
||||
Ok(Ok(response)) => {
|
||||
with_lock_peer_health(&addr, |health| {
|
||||
health.last_success = Some(Instant::now());
|
||||
health.consecutive_timeouts = 0;
|
||||
});
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "success");
|
||||
if let Some(late) = late {
|
||||
@@ -660,9 +445,6 @@ impl RemoteClient {
|
||||
}
|
||||
}
|
||||
Ok(Err(status)) => {
|
||||
if let Some(late_failure) = late_failure {
|
||||
late_failure(()).await;
|
||||
}
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "error");
|
||||
debug!(
|
||||
addr = %addr,
|
||||
@@ -673,9 +455,6 @@ impl RemoteClient {
|
||||
);
|
||||
}
|
||||
Err(join_error) => {
|
||||
if let Some(late_failure) = late_failure {
|
||||
late_failure(()).await;
|
||||
}
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "join_error");
|
||||
debug!(addr = %addr, op, error = %join_error, "Detached remote lock RPC task ended abnormally");
|
||||
}
|
||||
@@ -706,79 +485,24 @@ impl RemoteClient {
|
||||
}))
|
||||
}
|
||||
|
||||
fn late_release_failure_hook(&self, lock_ids: Vec<LockId>) -> LateFailure {
|
||||
let client = self.clone();
|
||||
Some(Box::new(move |_| {
|
||||
Box::pin(async move {
|
||||
client.release_late_acquisitions(lock_ids).await;
|
||||
})
|
||||
}))
|
||||
}
|
||||
|
||||
/// A lock granted after its caller stopped waiting is an orphan until its
|
||||
/// lease expires; hand it back right away, best effort.
|
||||
async fn release_late_acquisitions(&self, mut lock_ids: Vec<LockId>) {
|
||||
if !self.admit_late_release() {
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_late_release("failed");
|
||||
let (log_warning, suppressed_timeouts) = self.timeout_log_decision("late_release", Self::rpc_timeout());
|
||||
if log_warning {
|
||||
warn!(
|
||||
addr = %self.addr,
|
||||
count = lock_ids.len(),
|
||||
outcome = "budget_exhausted",
|
||||
suppressed_timeouts,
|
||||
"Skipped late remote lock release because the cleanup budget is exhausted"
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
let original_count = lock_ids.len();
|
||||
let mut outcome = "failed";
|
||||
for (attempt, delay) in std::iter::once(Duration::ZERO).chain(LATE_RELEASE_RETRY_DELAYS).enumerate() {
|
||||
if !delay.is_zero() {
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
|
||||
match self.release_locks_batch(&lock_ids).await {
|
||||
Ok(released) => {
|
||||
lock_ids = lock_ids
|
||||
.into_iter()
|
||||
.zip(released)
|
||||
.filter_map(|(lock_id, released)| (!released).then_some(lock_id))
|
||||
.collect();
|
||||
if lock_ids.is_empty() {
|
||||
outcome = "released";
|
||||
break;
|
||||
}
|
||||
outcome = if attempt == LATE_RELEASE_RETRY_DELAYS.len() {
|
||||
"partial"
|
||||
} else {
|
||||
"failed"
|
||||
};
|
||||
}
|
||||
Err(_) => {
|
||||
outcome = "failed";
|
||||
}
|
||||
}
|
||||
}
|
||||
self.release_late_release();
|
||||
async fn release_late_acquisitions(&self, lock_ids: Vec<LockId>) {
|
||||
let outcome = match self.release_locks_batch(&lock_ids).await {
|
||||
Ok(released) if released.iter().all(|released| *released) => "released",
|
||||
Ok(_) => "partial",
|
||||
Err(_) => "failed",
|
||||
};
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_late_release(outcome);
|
||||
if outcome == "released" {
|
||||
debug!(addr = %self.addr, count = original_count, "Released remote locks granted after their caller timed out");
|
||||
debug!(addr = %self.addr, count = lock_ids.len(), "Released remote locks granted after their caller timed out");
|
||||
} else {
|
||||
let (log_warning, suppressed_timeouts) = self.timeout_log_decision("late_release", Self::rpc_timeout());
|
||||
if log_warning {
|
||||
warn!(
|
||||
addr = %self.addr,
|
||||
count = original_count,
|
||||
outcome,
|
||||
suppressed_timeouts,
|
||||
"Could not release every remote lock granted after its caller timed out; the server lease will expire it"
|
||||
);
|
||||
} else {
|
||||
debug!(addr = %self.addr, count = original_count, outcome, suppressed_timeouts, "Suppressed repeated late remote lock release warning");
|
||||
}
|
||||
warn!(
|
||||
addr = %self.addr,
|
||||
count = lock_ids.len(),
|
||||
outcome,
|
||||
"Could not release every remote lock granted after its caller timed out; the server lease will expire it"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -788,74 +512,26 @@ impl RemoteClient {
|
||||
resource_summary: &str,
|
||||
deadline: Duration,
|
||||
future: Fut,
|
||||
cleanup: RpcCleanup<T>,
|
||||
late: LateCompletion<T>,
|
||||
) -> std::result::Result<T, LockError>
|
||||
where
|
||||
Fut: Future<Output = std::result::Result<T, tonic::Status>> + Send + 'static,
|
||||
T: Send + 'static,
|
||||
{
|
||||
let RpcCleanup {
|
||||
late,
|
||||
late_failure,
|
||||
admission,
|
||||
} = cleanup;
|
||||
let mut handle = tokio::spawn(future);
|
||||
match timeout(deadline, &mut handle).await {
|
||||
Ok(Ok(Ok(response))) => {
|
||||
self.record_rpc_success(op, admission);
|
||||
self.release_lock_request(op, admission);
|
||||
self.record_rpc_success();
|
||||
Ok(response)
|
||||
}
|
||||
Ok(Ok(Err(err))) => {
|
||||
let reason = err.to_string();
|
||||
if err.code() == tonic::Code::DeadlineExceeded {
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_timeout(&self.addr, op);
|
||||
self.record_request_timeout(op, deadline);
|
||||
let (log_timeout, suppressed_timeouts) = self.timeout_log_decision(op, deadline);
|
||||
if Self::is_scanner_leader_lock(resource_summary) {
|
||||
debug!(
|
||||
addr = %self.addr,
|
||||
op,
|
||||
timeout_ms = deadline.as_millis(),
|
||||
resource_summary,
|
||||
"Remote lock RPC returned deadline exceeded for scanner leader lock"
|
||||
);
|
||||
} else if log_timeout {
|
||||
warn!(
|
||||
addr = %self.addr,
|
||||
op,
|
||||
timeout_ms = deadline.as_millis(),
|
||||
resource_summary,
|
||||
suppressed_timeouts,
|
||||
"Remote lock RPC returned deadline exceeded"
|
||||
);
|
||||
} else {
|
||||
debug!(
|
||||
addr = %self.addr,
|
||||
op,
|
||||
timeout_ms = deadline.as_millis(),
|
||||
resource_summary,
|
||||
suppressed_timeouts,
|
||||
"Suppressed repeated remote lock RPC deadline"
|
||||
);
|
||||
}
|
||||
self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Timeout, deadline)
|
||||
.await;
|
||||
self.release_lock_request(op, admission);
|
||||
if let Some(late_failure) = late_failure {
|
||||
drop(tokio::spawn(late_failure(())));
|
||||
}
|
||||
return Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), deadline));
|
||||
}
|
||||
// Only evict (and re-dial) the cached channel when the failure is a genuine
|
||||
// transport problem. A server-produced application status (auth denied, peer
|
||||
// lock service not ready, invalid args, ...) arrives on a perfectly healthy
|
||||
// channel; evicting it just churns the connection and pushes the peer toward
|
||||
// the offline threshold for no benefit. See issue #4567.
|
||||
let transport_failure = Self::is_transport_failure(&err);
|
||||
if transport_failure {
|
||||
self.record_request_timeout(op, deadline);
|
||||
}
|
||||
if Self::is_scanner_leader_lock(resource_summary) {
|
||||
debug!(
|
||||
addr = %self.addr,
|
||||
@@ -867,31 +543,6 @@ impl RemoteClient {
|
||||
transport_failure,
|
||||
"Remote lock RPC returned tonic error for scanner leader lock"
|
||||
);
|
||||
} else if transport_failure {
|
||||
let (log_failure, suppressed_failures) = self.timeout_log_decision(op, deadline);
|
||||
if log_failure {
|
||||
warn!(
|
||||
addr = %self.addr,
|
||||
op,
|
||||
timeout_ms = deadline.as_millis(),
|
||||
resource_summary,
|
||||
tonic_code = ?err.code(),
|
||||
tonic_message = err.message(),
|
||||
transport_failure,
|
||||
suppressed_failures,
|
||||
"Remote lock RPC returned transport error"
|
||||
);
|
||||
} else {
|
||||
debug!(
|
||||
addr = %self.addr,
|
||||
op,
|
||||
timeout_ms = deadline.as_millis(),
|
||||
resource_summary,
|
||||
tonic_code = ?err.code(),
|
||||
suppressed_failures,
|
||||
"Suppressed repeated remote lock transport error"
|
||||
);
|
||||
}
|
||||
} else {
|
||||
warn!(
|
||||
addr = %self.addr,
|
||||
@@ -908,10 +559,6 @@ impl RemoteClient {
|
||||
self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Transport, deadline)
|
||||
.await;
|
||||
}
|
||||
self.release_lock_request(op, admission);
|
||||
if transport_failure && let Some(late_failure) = late_failure {
|
||||
drop(tokio::spawn(late_failure(())));
|
||||
}
|
||||
Err(LockError::internal(format!("{op} RPC failed: {reason}")))
|
||||
}
|
||||
Ok(Err(join_error)) => {
|
||||
@@ -922,17 +569,11 @@ impl RemoteClient {
|
||||
error = %join_error,
|
||||
"Remote lock RPC task ended abnormally"
|
||||
);
|
||||
self.release_lock_request(op, admission);
|
||||
if let Some(late_failure) = late_failure {
|
||||
drop(tokio::spawn(late_failure(())));
|
||||
}
|
||||
Err(LockError::internal(format!("{op} RPC task failed: {join_error}")))
|
||||
}
|
||||
Err(_) => {
|
||||
let reason = format!("RPC timed out after {deadline:?}");
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_timeout(&self.addr, op);
|
||||
self.record_request_timeout(op, deadline);
|
||||
let (log_timeout, suppressed_timeouts) = self.timeout_log_decision(op, deadline);
|
||||
if Self::is_scanner_leader_lock(resource_summary) {
|
||||
debug!(
|
||||
addr = %self.addr,
|
||||
@@ -941,28 +582,18 @@ impl RemoteClient {
|
||||
resource_summary,
|
||||
"Remote lock RPC timed out for scanner leader lock"
|
||||
);
|
||||
} else if log_timeout {
|
||||
} else {
|
||||
warn!(
|
||||
addr = %self.addr,
|
||||
op,
|
||||
timeout_ms = deadline.as_millis(),
|
||||
resource_summary,
|
||||
suppressed_timeouts,
|
||||
"Remote lock RPC timed out"
|
||||
);
|
||||
} else {
|
||||
debug!(
|
||||
addr = %self.addr,
|
||||
op,
|
||||
timeout_ms = deadline.as_millis(),
|
||||
resource_summary,
|
||||
suppressed_timeouts,
|
||||
"Suppressed repeated remote lock RPC timeout"
|
||||
);
|
||||
}
|
||||
self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Timeout, deadline)
|
||||
.await;
|
||||
self.detach_timed_out_rpc(op, resource_summary, handle, late, late_failure, admission);
|
||||
self.detach_timed_out_rpc(op, resource_summary, handle, late);
|
||||
Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), deadline))
|
||||
}
|
||||
}
|
||||
@@ -990,14 +621,6 @@ impl RemoteClient {
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn rpc_backoff_failure_response(request: &LockRequest) -> LockResponse {
|
||||
LockResponse::failure("Remote lock RPC timed out: peer request breaker is open", request.acquire_timeout)
|
||||
}
|
||||
|
||||
fn rpc_backoff_failure_batch(requests: &[LockRequest]) -> Vec<LockResponse> {
|
||||
requests.iter().map(Self::rpc_backoff_failure_response).collect()
|
||||
}
|
||||
|
||||
fn build_lock_info(request: &LockRequest, lock_info_json: Option<String>) -> LockInfo {
|
||||
if let Some(lock_info_json) = lock_info_json {
|
||||
match serde_json::from_str::<LockInfo>(&lock_info_json) {
|
||||
@@ -1057,45 +680,22 @@ impl RemoteClient {
|
||||
impl LockClient for RemoteClient {
|
||||
async fn acquire_lock(&self, request: &LockRequest) -> Result<LockResponse> {
|
||||
info!("remote acquire_exclusive for {}", request.resource);
|
||||
let admission = match self.admit_lock_request("lock") {
|
||||
Some(admission) => admission,
|
||||
None => return Ok(Self::rpc_backoff_failure_response(request)),
|
||||
};
|
||||
|
||||
let rpc_timeout = Self::rpc_timeout();
|
||||
let bounded_request = Self::bounded_acquire_request(request, rpc_timeout);
|
||||
let mut client = match self.get_client().await {
|
||||
Ok(client) => client,
|
||||
Err(err) => {
|
||||
self.record_request_timeout("lock", rpc_timeout);
|
||||
self.release_lock_request("lock", Some(admission));
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = request.resource.to_string();
|
||||
let args = match serde_json::to_string(&bounded_request) {
|
||||
Ok(args) => args,
|
||||
Err(err) => {
|
||||
self.release_lock_request("lock", Some(admission));
|
||||
return Err(LockError::internal(format!("Failed to serialize request: {err}")));
|
||||
}
|
||||
};
|
||||
let mut req = Request::new(GenerallyLockRequest { args });
|
||||
req.set_timeout(rpc_timeout);
|
||||
if let Err(err) = attach_lock_mutation_body_digest(&mut req) {
|
||||
self.release_lock_request("lock", Some(admission));
|
||||
return Err(err.into());
|
||||
}
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
|
||||
});
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
let late = self.late_release_hook(request.lock_id.clone());
|
||||
let late_failure = self.late_release_failure_hook(vec![request.lock_id.clone()]);
|
||||
|
||||
let resp = match self
|
||||
.execute_rpc(
|
||||
"lock",
|
||||
&resource_summary,
|
||||
rpc_timeout,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.lock(req).await },
|
||||
RpcCleanup::new(late, late_failure, Some(admission)),
|
||||
late,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -1125,54 +725,26 @@ impl LockClient for RemoteClient {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let admission = match self.admit_lock_request("lock_batch") {
|
||||
Some(admission) => admission,
|
||||
None => return Ok(Self::rpc_backoff_failure_batch(requests)),
|
||||
};
|
||||
|
||||
let rpc_timeout = Self::rpc_timeout();
|
||||
let bounded_requests = requests
|
||||
.iter()
|
||||
.map(|request| Self::bounded_acquire_request(request, rpc_timeout))
|
||||
.collect::<Vec<_>>();
|
||||
let mut client = match self.get_client().await {
|
||||
Ok(client) => client,
|
||||
Err(err) => {
|
||||
self.record_request_timeout("lock_batch", rpc_timeout);
|
||||
self.release_lock_request("lock_batch", Some(admission));
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = Self::summarize_resources(requests);
|
||||
let args = match bounded_requests
|
||||
.iter()
|
||||
.map(|request| {
|
||||
serde_json::to_string(request).map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()
|
||||
{
|
||||
Ok(args) => args,
|
||||
Err(err) => {
|
||||
self.release_lock_request("lock_batch", Some(admission));
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
let mut req = Request::new(BatchGenerallyLockRequest { args });
|
||||
req.set_timeout(rpc_timeout);
|
||||
if let Err(err) = attach_lock_mutation_body_digest(&mut req) {
|
||||
self.release_lock_request("lock_batch", Some(admission));
|
||||
return Err(err.into());
|
||||
}
|
||||
let mut req = Request::new(BatchGenerallyLockRequest {
|
||||
args: requests
|
||||
.iter()
|
||||
.map(|request| {
|
||||
serde_json::to_string(request).map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?,
|
||||
});
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
let late = self.late_release_batch_hook(requests.iter().map(|request| request.lock_id.clone()).collect());
|
||||
let late_failure = self.late_release_failure_hook(requests.iter().map(|request| request.lock_id.clone()).collect());
|
||||
|
||||
let resp = match self
|
||||
.execute_rpc(
|
||||
"lock_batch",
|
||||
&resource_summary,
|
||||
rpc_timeout,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.lock_batch(req).await },
|
||||
RpcCleanup::new(late, late_failure, Some(admission)),
|
||||
late,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -1219,7 +791,7 @@ impl LockClient for RemoteClient {
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.un_lock(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
None,
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
@@ -1253,7 +825,7 @@ impl LockClient for RemoteClient {
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.un_lock_batch(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
None,
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
@@ -1281,7 +853,7 @@ impl LockClient for RemoteClient {
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.refresh(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
None,
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
@@ -1307,7 +879,7 @@ impl LockClient for RemoteClient {
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.force_un_lock(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
None,
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
@@ -1322,31 +894,26 @@ impl LockClient for RemoteClient {
|
||||
|
||||
// Since there's no direct status query in the gRPC service,
|
||||
// we attempt a non-blocking lock acquisition to check if the resource is available
|
||||
let probe_lock_id = LockId::new_unique(&lock_id.resource);
|
||||
let rpc_timeout = Self::rpc_timeout();
|
||||
let status_request = Self::create_unlock_request(&probe_lock_id).with_ttl(rpc_timeout);
|
||||
let bounded_status_request = Self::bounded_acquire_request(&status_request, rpc_timeout);
|
||||
let status_request = Self::create_unlock_request(lock_id);
|
||||
let resource_summary = status_request.resource.to_string();
|
||||
let mut client = self.get_client().await?;
|
||||
let args = serde_json::to_string(&bounded_status_request)
|
||||
let args = serde_json::to_string(&status_request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?;
|
||||
|
||||
// Try to acquire a very short-lived lock to test availability
|
||||
let mut req = Request::new(GenerallyLockRequest { args: args.clone() });
|
||||
req.set_timeout(rpc_timeout);
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
// A probe lock granted after the deadline must not linger on the peer.
|
||||
let late = self.late_release_hook(probe_lock_id.clone());
|
||||
let late_failure = self.late_release_failure_hook(vec![probe_lock_id]);
|
||||
let late = self.late_release_hook(lock_id.clone());
|
||||
|
||||
// Try exclusive lock first with very short timeout
|
||||
let resp = match self
|
||||
.execute_rpc(
|
||||
"check_status",
|
||||
&resource_summary,
|
||||
rpc_timeout,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.lock(req).await },
|
||||
RpcCleanup::new(late, late_failure, None),
|
||||
late,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -1358,7 +925,6 @@ impl LockClient for RemoteClient {
|
||||
// If we successfully acquired the lock, the resource was free.
|
||||
// Immediately release it on a best-effort basis.
|
||||
let mut release_req = Request::new(GenerallyLockRequest { args });
|
||||
release_req.set_timeout(rpc_timeout);
|
||||
attach_lock_mutation_body_digest(&mut release_req)?;
|
||||
if let Ok(mut client) = self.get_client().await {
|
||||
let _ = self
|
||||
@@ -1367,7 +933,7 @@ impl LockClient for RemoteClient {
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.un_lock(release_req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
@@ -1432,7 +998,7 @@ impl LockClient for RemoteClient {
|
||||
Self::ONLINE_CHECK_RESOURCE,
|
||||
online_timeout,
|
||||
async move { client.ping(ping_req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -1567,94 +1133,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_breaker_opens_after_three_timeouts_and_allows_one_probe() {
|
||||
let addr = "http://breaker-test";
|
||||
let client = RemoteClient::new(addr.to_string());
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
|
||||
for _ in 0..LOCK_RPC_REQUEST_BACKOFF_THRESHOLD {
|
||||
client.record_request_timeout("lock", Duration::from_millis(100));
|
||||
}
|
||||
|
||||
assert!(
|
||||
client.admit_lock_request("lock").is_none(),
|
||||
"open breaker must reject requests during backoff"
|
||||
);
|
||||
with_lock_peer_health(addr, |health| {
|
||||
health.request_backoff_until = Some(Instant::now() - Duration::from_millis(1));
|
||||
health.request_probe_generation = None;
|
||||
health.request_in_flight = 0;
|
||||
});
|
||||
|
||||
let probe = client
|
||||
.admit_lock_request("lock")
|
||||
.expect("half-open breaker must admit one probe");
|
||||
assert!(probe.is_probe());
|
||||
assert!(client.admit_lock_request("lock").is_none(), "only one half-open probe may run");
|
||||
client.release_lock_request("lock", Some(probe));
|
||||
assert!(lock_peer_health_for_test(addr).request_probe_generation.is_none());
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn old_request_completion_cannot_clear_a_new_probe() {
|
||||
let addr = "http://breaker-token-test";
|
||||
let client = RemoteClient::new(addr.to_string());
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
with_lock_peer_health(addr, |health| {
|
||||
health.request_backoff_until = Some(Instant::now() - Duration::from_millis(1));
|
||||
health.request_probe_generation = None;
|
||||
health.request_in_flight = 1;
|
||||
});
|
||||
|
||||
let probe = client.admit_lock_request("lock").expect("probe should be admitted");
|
||||
client.release_lock_request("lock", Some(LockRequestAdmission { probe_generation: None }));
|
||||
assert!(lock_peer_health_for_test(addr).request_probe_generation.is_some());
|
||||
client.record_rpc_success("lock", Some(LockRequestAdmission { probe_generation: None }));
|
||||
assert!(lock_peer_health_for_test(addr).request_probe_generation.is_some());
|
||||
client.release_lock_request("lock", Some(probe));
|
||||
assert!(lock_peer_health_for_test(addr).request_probe_generation.is_none());
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_probe_completion_cannot_clear_a_later_generation() {
|
||||
let addr = "http://breaker-generation-test";
|
||||
let client = RemoteClient::new(addr.to_string());
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
with_lock_peer_health(addr, |health| {
|
||||
health.request_backoff_until = Some(Instant::now() - Duration::from_millis(1));
|
||||
});
|
||||
|
||||
let first = client.admit_lock_request("lock").expect("first probe should be admitted");
|
||||
client.record_request_timeout("lock", Duration::from_millis(10));
|
||||
with_lock_peer_health(addr, |health| {
|
||||
health.request_backoff_until = Some(Instant::now() - Duration::from_millis(1));
|
||||
health.request_in_flight = 0;
|
||||
health.request_probe_generation = None;
|
||||
health.request_consecutive_timeouts = 0;
|
||||
});
|
||||
let second = client.admit_lock_request("lock").expect("second probe should be admitted");
|
||||
assert_ne!(first.probe_generation, second.probe_generation);
|
||||
|
||||
client.release_lock_request("lock", Some(first));
|
||||
assert_eq!(lock_peer_health_for_test(addr).request_probe_generation, second.probe_generation);
|
||||
client.release_lock_request("lock", Some(second));
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bounded_acquire_request_matches_transport_deadline() {
|
||||
let request = test_lock_request(Duration::from_secs(30));
|
||||
let bounded = RemoteClient::bounded_acquire_request(&request, Duration::from_secs(3));
|
||||
assert_eq!(bounded.acquire_timeout, Duration::from_secs(3));
|
||||
|
||||
let request = test_lock_request(Duration::from_secs(1));
|
||||
let bounded = RemoteClient::bounded_acquire_request(&request, Duration::from_secs(3));
|
||||
assert_eq!(bounded.acquire_timeout, Duration::from_secs(1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn acquired_lock_ids_picks_only_granted_batch_entries() {
|
||||
let lock_ids = vec![
|
||||
|
||||
+145
-1491
File diff suppressed because it is too large
Load Diff
@@ -481,9 +481,6 @@ mod decommission_lock_order_tests {
|
||||
if let Some(deployment_id) = other_store.ctx.deployment_id() {
|
||||
ctx.set_deployment_id(deployment_id);
|
||||
}
|
||||
for pool in &mut pools {
|
||||
Arc::make_mut(pool).set_instance_ctx_for_test(Arc::clone(&ctx));
|
||||
}
|
||||
let store = Arc::new(crate::store::ECStore {
|
||||
id: uuid::Uuid::new_v4(),
|
||||
disk_map: other_store.disk_map.clone(),
|
||||
@@ -2886,89 +2883,6 @@ mod decommission_lock_order_tests {
|
||||
assert!(upload_info.parts.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn unversioned_decommission_multipart_uses_reserved_target() {
|
||||
run_large_stack_current_thread_async_test(
|
||||
"unversioned-decommission-multipart-target",
|
||||
unversioned_decommission_multipart_uses_reserved_target_case,
|
||||
);
|
||||
}
|
||||
|
||||
async fn unversioned_decommission_multipart_uses_reserved_target_case() {
|
||||
let (_temp_dirs, store, _other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
|
||||
let bucket = test_bucket("unversioned-decommission-multipart-target");
|
||||
let object = "unversioned-existing-unreserved-target.bin";
|
||||
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
|
||||
let capacity_snapshot = || {
|
||||
vec![
|
||||
DecommissionPoolCapacityInfo::for_test(0, layout, 0, 1, 1),
|
||||
DecommissionPoolCapacityInfo::for_test(1, layout, 2, 2, 0),
|
||||
DecommissionPoolCapacityInfo::for_test(2, layout, 100, 100, 0),
|
||||
]
|
||||
};
|
||||
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create the unversioned multipart target bucket");
|
||||
let incarnation = store
|
||||
.bucket_incarnation_id(&bucket)
|
||||
.await
|
||||
.expect("load the unversioned multipart target incarnation");
|
||||
|
||||
// Force the ordinary pool selector to choose pool 2. A capacity-owned
|
||||
// upload must ignore this unreserved existing object and use pool 1.
|
||||
let mut existing = PutObjReader::from_vec(b"existing unreserved target".to_vec());
|
||||
store.pools[2]
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut existing,
|
||||
&ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed the unreserved target object");
|
||||
|
||||
set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot()]);
|
||||
store
|
||||
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
|
||||
.await
|
||||
.expect("activate the unversioned multipart reservation");
|
||||
{
|
||||
let pool_meta = store.pool_meta.read().await;
|
||||
let targets = &pool_meta.pools[0]
|
||||
.decommission
|
||||
.as_ref()
|
||||
.and_then(|info| info.capacity_reservation.as_ref())
|
||||
.expect("the unversioned multipart reservation should exist")
|
||||
.targets;
|
||||
assert_eq!(targets.len(), 1, "only pool 1 should be reserved");
|
||||
assert_eq!(targets[0].pool_index, 1);
|
||||
}
|
||||
|
||||
let owner = decommission_capacity_owner(&*store.pool_meta.read().await).with_mutation_id(uuid::Uuid::new_v4());
|
||||
let mut upload_opts = ObjectOptions {
|
||||
data_movement: true,
|
||||
src_pool_idx: 0,
|
||||
versioned: false,
|
||||
version_id: None,
|
||||
mod_time: Some(time::OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(97)),
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
..Default::default()
|
||||
};
|
||||
owner.apply_to(&mut upload_opts);
|
||||
|
||||
let (_, target_pool_idx, _) = store
|
||||
.handle_new_multipart_upload_with_pool_idx(&bucket, object, &upload_opts, None)
|
||||
.await
|
||||
.expect("unversioned decommission multipart must use the reserved target");
|
||||
assert_eq!(target_pool_idx, 1, "the reservation target must override the existing unreserved object");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn data_movement_multipart_abort_restart_reconciles_inflight_after_release_save_loss() {
|
||||
@@ -4253,13 +4167,6 @@ mod decommission_lock_order_tests {
|
||||
panic!("cancellation probe finished before observing target contention: {result:?}");
|
||||
}
|
||||
}
|
||||
// The mutation absorbs the gate contention inline before the outer wait
|
||||
// loop ever sees it, and that inline budget is bounded.
|
||||
assert_eq!(
|
||||
retry_observer.target_gate_inline_retries(),
|
||||
crate::core::pools::DECOMMISSION_MUTATION_GATE_MAX_INLINE_ATTEMPTS - 1,
|
||||
"inline target-gate retries must stop at their bounded attempt budget"
|
||||
);
|
||||
tokio::time::sleep(Duration::from_millis(800)).await;
|
||||
assert_eq!(
|
||||
retry_observer.target_gate_exact_reloads(),
|
||||
@@ -6655,19 +6562,7 @@ mod decommission_lock_order_tests {
|
||||
|
||||
let restore_get_barrier = if matches!(mutation, ExternalObjectMutation::Restore) {
|
||||
let tier_name = format!("ORDERRESTORE{}", &uuid::Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let source_tiers = store.tier_config_mgr();
|
||||
let restore_tiers = other_store.tier_config_mgr();
|
||||
let backend = register_mock_tier(&source_tiers, &tier_name).await;
|
||||
// Both peers must resolve the same persisted backend identity, including its prefix.
|
||||
let tier_config = source_tiers.read().await.tiers[&tier_name].clone();
|
||||
restore_tiers.write().await.tiers.insert(tier_name.clone(), tier_config);
|
||||
crate::services::tier::tier::TierConfigMgr::install_test_driver_in(
|
||||
&restore_tiers,
|
||||
&tier_name,
|
||||
Box::new(backend.clone()),
|
||||
)
|
||||
.await
|
||||
.expect("install the same mock tier on the restoring peer");
|
||||
let backend = register_mock_tier(&store.pools[2].instance_ctx().tier_config_mgr(), &tier_name).await;
|
||||
store.pools[2]
|
||||
.transition_object(
|
||||
&bucket,
|
||||
@@ -6941,10 +6836,7 @@ mod decommission_lock_order_tests {
|
||||
}
|
||||
});
|
||||
if let Some(get_barrier) = restore_get_barrier.as_ref() {
|
||||
tokio::select! {
|
||||
() = get_barrier.wait_until_paused() => {}
|
||||
result = &mut ordinary_mutation => panic!("restore finished before the tier GET barrier: {result:?}"),
|
||||
}
|
||||
get_barrier.wait_until_paused().await;
|
||||
let read_opts = ObjectOptions {
|
||||
skip_decommissioned: true,
|
||||
..Default::default()
|
||||
|
||||
@@ -324,15 +324,6 @@ impl Sets {
|
||||
&self.ctx
|
||||
}
|
||||
|
||||
/// Keep simulated peers' metadata ownership separate while sharing disks and lock clients.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn set_instance_ctx_for_test(&mut self, ctx: Arc<InstanceContext>) {
|
||||
for set in &mut self.disk_set {
|
||||
Arc::make_mut(set).set_instance_ctx_for_test(Arc::clone(&ctx));
|
||||
}
|
||||
self.ctx = ctx;
|
||||
}
|
||||
|
||||
async fn monitor_and_connect_endpoints_task(sets: Weak<Sets>, mut rx: Receiver<()>) {
|
||||
let startup_delay = tokio::time::sleep(Duration::from_secs(5));
|
||||
tokio::pin!(startup_delay);
|
||||
@@ -799,7 +790,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for Sets {
|
||||
version_id: dst_opts.version_id.clone(),
|
||||
mod_time: dst_opts.mod_time,
|
||||
http_preconditions: dst_opts.http_preconditions.clone(),
|
||||
quota_admission: dst_opts.quota_admission,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -1493,12 +1483,12 @@ impl Sets {
|
||||
object: &str,
|
||||
version_id: &str,
|
||||
opts: &HealOpts,
|
||||
proof: crate::set_disk::AbsenceProofRequest<'_>,
|
||||
retirement: Option<&crate::bucket::retirement::MarkerRetirementContext<'_>>,
|
||||
) -> Result<(HealResultItem, Option<Error>, Option<crate::set_disk::HealedObjectAbsence>)> {
|
||||
let mut absence = None;
|
||||
let (item, error) = self
|
||||
.get_disks_for_heal_object(object, opts)?
|
||||
.heal_object_with_retirement(bucket, object, version_id, opts, &mut absence, proof)
|
||||
.heal_object_with_retirement(bucket, object, version_id, opts, &mut absence, retirement)
|
||||
.await?;
|
||||
// A caller-owned lock does not expose its lease to this boundary.
|
||||
// Keep cleanup unverified when that lease cannot be checked here.
|
||||
|
||||
@@ -642,21 +642,6 @@ pub(crate) fn data_movement_stage_source(err: &Error) -> Option<&Error> {
|
||||
.downcast_ref::<Error>()
|
||||
}
|
||||
|
||||
/// Recover a concrete typed error wrapped by [`data_movement_stage_error`].
|
||||
pub(crate) fn data_movement_stage_source_as<T>(err: &Error) -> Option<&T>
|
||||
where
|
||||
T: std::error::Error + 'static,
|
||||
{
|
||||
let Error::Io(io_err) = err else {
|
||||
return None;
|
||||
};
|
||||
io_err
|
||||
.get_ref()?
|
||||
.downcast_ref::<DataMovementStageError>()?
|
||||
.source
|
||||
.downcast_ref::<T>()
|
||||
}
|
||||
|
||||
fn schedule_data_movement_multipart_abort_cleanup(
|
||||
store: Arc<ECStore>,
|
||||
target_pool_idx: usize,
|
||||
|
||||
@@ -12,7 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::data_movement::data_movement_context_error;
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::{Error, Result, is_err_object_not_found, is_err_version_not_found};
|
||||
use crate::object_api::ObjectOptions;
|
||||
@@ -28,28 +27,6 @@ use tokio::io::AsyncReadExt;
|
||||
pub const MAX_SCANNER_PAUSE_BACKLOG_BYTES: u64 = 64 * 1024;
|
||||
pub(crate) const SCANNER_PAUSE_BACKLOG_PATH: &str = "buckets/.scanner-pause-backlog.json";
|
||||
|
||||
/// Typed failure from the scanner-owned native retirement planner.
|
||||
///
|
||||
/// Decommission must preserve the source when the native authority cannot be
|
||||
/// proven. The handoff variant is deliberately narrower than other authority
|
||||
/// failures so the worker can retry without converting a transient membership
|
||||
/// transition into a terminal pool failure.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ScannerPauseBacklogRetirementError {
|
||||
#[error("scanner pause backlog retirement has a retryable handoff conflict: {reason}")]
|
||||
HandoffConflict { reason: String },
|
||||
#[error("scanner pause backlog retirement has an unsafe authority conflict: {reason}")]
|
||||
AuthorityConflict { reason: String },
|
||||
#[error("scanner pause backlog retirement found an invalid native record: {reason}")]
|
||||
InvalidRecord { reason: String },
|
||||
}
|
||||
|
||||
impl ScannerPauseBacklogRetirementError {
|
||||
pub fn is_retryable_handoff_conflict(&self) -> bool {
|
||||
matches!(self, Self::HandoffConflict { .. })
|
||||
}
|
||||
}
|
||||
|
||||
/// A bounded, storage-fenced native replica. Only a confirmed missing object
|
||||
/// has no payload; read failures never enter the Scanner verifier.
|
||||
pub struct ScannerPauseBacklogRetirementReplica {
|
||||
@@ -67,10 +44,7 @@ pub struct ScannerPauseBacklogRetirementPlan {
|
||||
}
|
||||
|
||||
pub type ScannerPauseBacklogRetirementPlanner =
|
||||
fn(
|
||||
usize,
|
||||
&[ScannerPauseBacklogRetirementReplica],
|
||||
) -> std::result::Result<Option<ScannerPauseBacklogRetirementPlan>, ScannerPauseBacklogRetirementError>;
|
||||
fn(usize, &[ScannerPauseBacklogRetirementReplica]) -> std::result::Result<Option<ScannerPauseBacklogRetirementPlan>, String>;
|
||||
|
||||
static RETIREMENT_PLANNER: OnceLock<ScannerPauseBacklogRetirementPlanner> = OnceLock::new();
|
||||
|
||||
@@ -187,8 +161,7 @@ pub(crate) fn plan_scanner_pause_backlog_retirement(
|
||||
data: read.replica.data.clone(),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
planner(source_pool_index, &snapshots)
|
||||
.map_err(|err| data_movement_context_error(format!("scanner pause backlog retirement planner failed: {err}"), err))
|
||||
planner(source_pool_index, &snapshots).map_err(Error::other)
|
||||
}
|
||||
|
||||
/// The native writer and retirement handoff use the same conditional, full-tail
|
||||
|
||||
@@ -30,8 +30,6 @@ use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
|
||||
use rustfs_madmin::{info_commands::DiskMetrics, metrics::TimedAction};
|
||||
#[cfg(not(test))]
|
||||
use std::sync::OnceLock;
|
||||
#[cfg(test)]
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
path::PathBuf,
|
||||
@@ -630,8 +628,6 @@ pub struct DiskHealthTracker {
|
||||
/// Authoritative atomically published runtime/status pair.
|
||||
state_snapshot: AtomicU64,
|
||||
transition_lock: std::sync::Mutex<()>,
|
||||
#[cfg(test)]
|
||||
test_forced_offline: AtomicBool,
|
||||
}
|
||||
|
||||
fn pack_health_state(runtime_state: RuntimeDriveHealthState, status: u32) -> u64 {
|
||||
@@ -979,8 +975,6 @@ impl DiskHealthTracker {
|
||||
last_capacity_probe_unix_secs: AtomicI64::new(0),
|
||||
state_snapshot: AtomicU64::new(pack_health_state(RuntimeDriveHealthState::Online, DISK_HEALTH_OK)),
|
||||
transition_lock: std::sync::Mutex::new(()),
|
||||
#[cfg(test)]
|
||||
test_forced_offline: AtomicBool::new(false),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1051,13 +1045,6 @@ impl DiskHealthTracker {
|
||||
self.publish_state(state, status);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn force_offline_for_test(&self) {
|
||||
self.test_forced_offline.store(true, Ordering::Release);
|
||||
let _guard = self.transition_lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
self.publish_state(RuntimeDriveHealthState::Offline, DISK_HEALTH_FAULTY);
|
||||
}
|
||||
|
||||
pub fn swap_ok_to_faulty(&self) -> bool {
|
||||
let _guard = self.transition_lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
let (_, status) = unpack_health_state(self.state_snapshot.load(Ordering::Acquire));
|
||||
@@ -1136,8 +1123,6 @@ impl DiskHealthTracker {
|
||||
/// Remote disks are marked faulty on timeout/network errors; the init loop retries with the
|
||||
/// same [`DiskStore`] handles, which would otherwise fail immediately at `is_faulty()`.
|
||||
pub fn reset_for_store_init_retry(&self, endpoint: &Endpoint) {
|
||||
#[cfg(test)]
|
||||
self.test_forced_offline.store(false, Ordering::Release);
|
||||
self.reset_for_store_init_retry_at(endpoint, current_unix_time());
|
||||
}
|
||||
|
||||
@@ -1157,10 +1142,6 @@ impl DiskHealthTracker {
|
||||
}
|
||||
|
||||
pub fn mark_recovery_success(&self, endpoint: &Endpoint, reason: &'static str) -> bool {
|
||||
#[cfg(test)]
|
||||
if self.test_forced_offline.load(Ordering::Acquire) {
|
||||
return false;
|
||||
}
|
||||
let _guard = self.transition_lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
let current = self.runtime_state();
|
||||
let next = match current {
|
||||
@@ -1459,11 +1440,6 @@ impl LocalDiskWrapper {
|
||||
self.health.force_runtime_state_for_test(state);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn force_offline_for_test(&self) {
|
||||
self.health.force_offline_for_test();
|
||||
}
|
||||
|
||||
/// Same as [`DiskHealthTracker::reset_for_store_init_retry`]: undo a transient faulty mark before another format load attempt.
|
||||
pub fn reset_health_for_store_init_retry(&self) {
|
||||
self.health.reset_for_store_init_retry(&self.disk.endpoint());
|
||||
|
||||
@@ -244,16 +244,13 @@ impl StdError for ConditionalFileNotCommittedError {
|
||||
}
|
||||
}
|
||||
|
||||
fn classify_internode_disk_error(error: &InternodeHttpError) -> Option<DiskError> {
|
||||
fn classify_internode_missing_error(error: &InternodeHttpError) -> Option<DiskError> {
|
||||
if error.is_remote_file_not_found() {
|
||||
return Some(DiskError::FileNotFound);
|
||||
}
|
||||
if error.is_remote_volume_not_found() {
|
||||
return Some(DiskError::VolumeNotFound);
|
||||
}
|
||||
if error.is_remote_file_corrupt() {
|
||||
return Some(DiskError::FileCorrupt);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
@@ -529,10 +526,13 @@ fn io_error_chain_contains_kind(io_error: &std::io::Error, kind: std::io::ErrorK
|
||||
|
||||
impl From<std::io::Error> for DiskError {
|
||||
fn from(e: std::io::Error) -> Self {
|
||||
if let Some(error) = e.get_ref().and_then(|source| source.downcast_ref::<InternodeHttpError>())
|
||||
&& let Some(classified) = classify_internode_disk_error(error)
|
||||
{
|
||||
return classified;
|
||||
if let Some(error) = e.get_ref().and_then(|source| source.downcast_ref::<InternodeHttpError>()) {
|
||||
if error.is_remote_file_not_found() {
|
||||
return DiskError::FileNotFound;
|
||||
}
|
||||
if error.is_remote_volume_not_found() {
|
||||
return DiskError::VolumeNotFound;
|
||||
}
|
||||
}
|
||||
let e = match e.downcast::<TerminalReadError>() {
|
||||
Ok(terminal_error) => {
|
||||
@@ -541,7 +541,7 @@ impl From<std::io::Error> for DiskError {
|
||||
&& let Some(internode_error) = io_error
|
||||
.get_ref()
|
||||
.and_then(|source| source.downcast_ref::<InternodeHttpError>())
|
||||
&& let Some(classified) = classify_internode_disk_error(internode_error)
|
||||
&& let Some(classified) = classify_internode_missing_error(internode_error)
|
||||
{
|
||||
return classified;
|
||||
}
|
||||
@@ -963,7 +963,6 @@ mod tests {
|
||||
for (remote_error, expected) in [
|
||||
(rustfs_rio::new_test_remote_file_not_found_http_io_error(), DiskError::FileNotFound),
|
||||
(rustfs_rio::new_test_remote_volume_not_found_http_io_error(), DiskError::VolumeNotFound),
|
||||
(rustfs_rio::new_test_remote_file_corrupt_http_io_error(), DiskError::FileCorrupt),
|
||||
] {
|
||||
let wrapped = terminal_read_error_to_io(DiskError::Io(remote_error));
|
||||
assert_eq!(DiskError::from(wrapped), expected);
|
||||
@@ -1534,23 +1533,25 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_internode_disk_errors_preserve_disk_error_types() {
|
||||
fn test_internode_missing_errors_preserve_disk_error_types() {
|
||||
let file_missing = DiskError::from(rustfs_rio::new_test_remote_file_not_found_http_io_error());
|
||||
let volume_missing = DiskError::from(rustfs_rio::new_test_remote_volume_not_found_http_io_error());
|
||||
let file_corrupt = DiskError::from(rustfs_rio::new_test_remote_file_corrupt_http_io_error());
|
||||
let unmarked_server_error = DiskError::from(rustfs_rio::new_test_internode_http_io_error(
|
||||
rustfs_rio::InternodeHttpErrorKind::HttpStatus(http::StatusCode::INTERNAL_SERVER_ERROR),
|
||||
));
|
||||
|
||||
assert_eq!(file_missing, DiskError::FileNotFound);
|
||||
assert_eq!(volume_missing, DiskError::VolumeNotFound);
|
||||
assert_eq!(file_corrupt, DiskError::FileCorrupt);
|
||||
assert!(matches!(unmarked_server_error, DiskError::Io(_)));
|
||||
for error in [file_missing, volume_missing, file_corrupt] {
|
||||
assert_eq!(error.clone(), error);
|
||||
for missing in [file_missing, volume_missing] {
|
||||
assert_eq!(missing.clone(), missing);
|
||||
assert_eq!(
|
||||
crate::disk::error_reduce::reduce_write_quorum_errs(&[Some(error.clone()), Some(error.clone()), None], &[], 2),
|
||||
Some(error)
|
||||
crate::disk::error_reduce::reduce_write_quorum_errs(
|
||||
&[Some(missing.clone()), Some(missing.clone()), None],
|
||||
&[],
|
||||
2
|
||||
),
|
||||
Some(missing)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
+121
-495
@@ -94,8 +94,6 @@ const STALE_TMP_OBJECT_EXPIRY: Duration = Duration::from_secs(24 * 60 * 60);
|
||||
#[cfg(test)]
|
||||
tokio::task_local! {
|
||||
static DIRECTORY_LISTING_ENTRY_PROBE_COUNT: Arc<AtomicUsize>;
|
||||
/// `(sampled, complete)` directory reads issued by the delete-residue probe.
|
||||
static DELETE_RESIDUE_PROBE_READS: Arc<(AtomicUsize, AtomicUsize)>;
|
||||
}
|
||||
const RUSTFS_META_TMP_OLD_BUCKET: &str = ".rustfs.sys/tmp-old";
|
||||
const INLINE_METADATA_ROLLBACK_DIR_XOR: u128 = 0x7275737466735f696e6c696e655f7262;
|
||||
@@ -107,21 +105,6 @@ pub(crate) const RESERVED_DELETE_DATA_DIR_MARKER_PREFIX: &str = "reserve-delete-
|
||||
/// under-filled batch settles the common case without materializing large
|
||||
/// child sets; a full batch cannot prove no listable child hides behind it.
|
||||
const DELETE_RESIDUE_PROBE_LIMIT: i32 = 8;
|
||||
/// Most directory reads one delete-residue probe spends walking down through
|
||||
/// the ancestors of a deleted key before it gives up and lets the prefix
|
||||
/// surface. A genuine prefix settles within its depth (the first object's
|
||||
/// `xl.meta` ends the walk), so the budget only caps the cost of hiding a
|
||||
/// large residue tree, which then hides one level at a time instead.
|
||||
const DELETE_RESIDUE_PROBE_READ_BUDGET: usize = 32;
|
||||
|
||||
/// One directory read owed by the delete-residue probe.
|
||||
enum ResidueProbeStep {
|
||||
/// Read a bounded batch of `dir` and probe what it holds.
|
||||
Sample(String),
|
||||
/// Read all of `dir` once every child in `sampled` proved unlistable, and
|
||||
/// probe the rest.
|
||||
Remainder { dir: String, sampled: Vec<String> },
|
||||
}
|
||||
|
||||
/// A `part.N` file with a positive part number, the shape erasure data takes
|
||||
/// inside a version data dir.
|
||||
@@ -1899,61 +1882,6 @@ impl AsyncWrite for DirectWriter {
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_write_vectored(
|
||||
self: std::pin::Pin<&mut Self>,
|
||||
cx: &mut std::task::Context<'_>,
|
||||
bufs: &[std::io::IoSlice<'_>],
|
||||
) -> std::task::Poll<std::io::Result<usize>> {
|
||||
// Bitrot writes arrive as [hash, data]. Coalesce both slices into the
|
||||
// aligned bounce buffer so the O_DIRECT path has the same byte-stream
|
||||
// contract as the buffered writer instead of relying on the trait's
|
||||
// first-slice fallback.
|
||||
let this = self.get_mut();
|
||||
loop {
|
||||
match &mut this.state {
|
||||
DirectWriteState::Busy(_) => {
|
||||
std::task::ready!(this.poll_drive_busy(cx))?;
|
||||
}
|
||||
DirectWriteState::Idle(inner_opt) => {
|
||||
let inner = inner_opt.as_mut().expect("idle direct writer must hold inner state");
|
||||
let capacity = inner.buf.len;
|
||||
let space = capacity - inner.filled;
|
||||
let mut remaining = space;
|
||||
let mut written = 0;
|
||||
for src in bufs {
|
||||
if remaining == 0 {
|
||||
break;
|
||||
}
|
||||
let take = src.len().min(remaining);
|
||||
let start = inner.filled + written;
|
||||
inner.buf.as_mut_slice()[start..start + take].copy_from_slice(&src[..take]);
|
||||
written += take;
|
||||
remaining -= take;
|
||||
}
|
||||
|
||||
if written == 0 {
|
||||
return std::task::Poll::Ready(Ok(0));
|
||||
}
|
||||
inner.filled += written;
|
||||
|
||||
if inner.filled == capacity {
|
||||
let mut inner = inner_opt.take().expect("idle direct writer must hold inner state");
|
||||
let handle = tokio::task::spawn_blocking(move || {
|
||||
let res = inner.flush_batch();
|
||||
(inner, res)
|
||||
});
|
||||
this.state = DirectWriteState::Busy(handle);
|
||||
}
|
||||
return std::task::Poll::Ready(Ok(written));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn is_write_vectored(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn poll_flush(self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> std::task::Poll<std::io::Result<()>> {
|
||||
// Only drive an in-flight batch to completion. Sub-alignment staged
|
||||
// bytes cannot be flushed mid-stream (they would misalign the next
|
||||
@@ -7611,7 +7539,7 @@ impl LocalDisk {
|
||||
objs_returned: &mut i32,
|
||||
skip_current_dir_object: bool,
|
||||
multipart_dir_to_skip: Option<HashSet<String>>,
|
||||
) -> Result<bool>
|
||||
) -> Result<()>
|
||||
where
|
||||
W: AsyncWrite + Unpin + Send,
|
||||
{
|
||||
@@ -7629,7 +7557,7 @@ impl LocalDisk {
|
||||
};
|
||||
|
||||
if opts.limit > 0 && *objs_returned >= opts.limit {
|
||||
return Ok(true);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// TODO(backlog): add directory listing lock to prevent concurrent enumeration
|
||||
@@ -7690,12 +7618,12 @@ impl LocalDisk {
|
||||
return Err(DiskError::FileNotFound);
|
||||
}
|
||||
|
||||
return Ok(false);
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
|
||||
if entries.is_empty() {
|
||||
return Ok(false);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
current = current.trim_matches('/').to_owned();
|
||||
@@ -7709,7 +7637,7 @@ impl LocalDisk {
|
||||
let entry = item.clone();
|
||||
// check limit
|
||||
if opts.limit > 0 && *objs_returned >= opts.limit {
|
||||
return Ok(true);
|
||||
return Ok(());
|
||||
}
|
||||
// check multipart dir
|
||||
if skip_current_dir_object
|
||||
@@ -7814,12 +7742,12 @@ impl LocalDisk {
|
||||
prefix = "".to_owned();
|
||||
|
||||
for entry in entries.iter() {
|
||||
if entry.is_empty() {
|
||||
continue;
|
||||
if opts.limit > 0 && *objs_returned >= opts.limit {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if opts.limit > 0 && *objs_returned >= opts.limit {
|
||||
return Ok(true);
|
||||
if entry.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let name = path_join_buf(&[current.as_str(), entry.as_str()]);
|
||||
@@ -7835,7 +7763,7 @@ impl LocalDisk {
|
||||
// moment the limit is reached, same as the check below this
|
||||
// loop guards against for the current entry itself.
|
||||
if opts.limit > 0 && *objs_returned >= opts.limit {
|
||||
return Ok(true);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let (pop, skip_object, dir_to_skip, scan_required) = dir_stack.pop().expect("operation should succeed");
|
||||
@@ -7849,25 +7777,23 @@ impl LocalDisk {
|
||||
.await?;
|
||||
|
||||
let scan_path = pop.clone();
|
||||
if opts.recursive && scan_required {
|
||||
match Box::pin(self.scan_dir(pop, prefix.clone(), opts, out, objs_returned, skip_object, dir_to_skip)).await {
|
||||
Ok(true) => return Ok(true),
|
||||
Ok(false) => {}
|
||||
Err(er) => {
|
||||
if !er.is_metacache_output_stream_closed() {
|
||||
error!(
|
||||
event = EVENT_DISK_LOCAL_SCAN_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
path = %scan_path,
|
||||
operation = "scan_dir",
|
||||
error = ?er,
|
||||
"Disk local scan failed"
|
||||
);
|
||||
}
|
||||
return Err(er);
|
||||
}
|
||||
if opts.recursive
|
||||
&& scan_required
|
||||
&& let Err(er) =
|
||||
Box::pin(self.scan_dir(pop, prefix.clone(), opts, out, objs_returned, skip_object, dir_to_skip)).await
|
||||
{
|
||||
if !er.is_metacache_output_stream_closed() {
|
||||
error!(
|
||||
event = EVENT_DISK_LOCAL_SCAN_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
path = %scan_path,
|
||||
operation = "scan_dir",
|
||||
error = ?er,
|
||||
"Disk local scan failed"
|
||||
);
|
||||
}
|
||||
return Err(er);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7878,7 +7804,7 @@ impl LocalDisk {
|
||||
// tail, permanently skipping those keys on the next page instead
|
||||
// of just deferring them to it.
|
||||
if opts.limit > 0 && *objs_returned >= opts.limit {
|
||||
return Ok(true);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut meta = MetaCacheEntry {
|
||||
@@ -8021,7 +7947,7 @@ impl LocalDisk {
|
||||
|
||||
while let Some((dir, skip_object, dir_to_skip, scan_required)) = dir_stack.pop() {
|
||||
if opts.limit > 0 && *objs_returned >= opts.limit {
|
||||
return Ok(true);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
write_metacache_obj(
|
||||
@@ -8034,29 +7960,27 @@ impl LocalDisk {
|
||||
.await?;
|
||||
|
||||
let scan_path = dir.clone();
|
||||
if opts.recursive && scan_required {
|
||||
match Box::pin(self.scan_dir(dir, prefix.clone(), opts, out, objs_returned, skip_object, dir_to_skip)).await {
|
||||
Ok(true) => return Ok(true),
|
||||
Ok(false) => {}
|
||||
Err(er) => {
|
||||
if !er.is_metacache_output_stream_closed() {
|
||||
error!(
|
||||
event = EVENT_DISK_LOCAL_SCAN_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
path = %scan_path,
|
||||
operation = "scan_dir",
|
||||
error = ?er,
|
||||
"Disk local recursive scan failed"
|
||||
);
|
||||
}
|
||||
return Err(er);
|
||||
}
|
||||
if opts.recursive
|
||||
&& scan_required
|
||||
&& let Err(er) =
|
||||
Box::pin(self.scan_dir(dir, prefix.clone(), opts, out, objs_returned, skip_object, dir_to_skip)).await
|
||||
{
|
||||
if !er.is_metacache_output_stream_closed() {
|
||||
error!(
|
||||
event = EVENT_DISK_LOCAL_SCAN_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
path = %scan_path,
|
||||
operation = "scan_dir",
|
||||
error = ?er,
|
||||
"Disk local recursive scan failed"
|
||||
);
|
||||
}
|
||||
return Err(er);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(false)
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Whether the backing directory of plain object `object_name` also holds
|
||||
@@ -8123,116 +8047,72 @@ impl LocalDisk {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Whether the metadata-less directory `dir_name` holds nothing that a
|
||||
/// listing could show: every leaf under it is the data dir of a deleted
|
||||
/// version (a non-nil UUID directory of `part.N` files and
|
||||
/// delete-transaction markers) or an empty directory. That is what an
|
||||
/// interrupted or deferred version delete leaves behind once the `xl.meta`
|
||||
/// is gone, and neither the object directory nor the date-style ancestors
|
||||
/// above it may surface as prefixes (#6898). Real objects are directories
|
||||
/// carrying their own `xl.meta`, so the first file met outside a UUID data
|
||||
/// dir, or any stray entry inside one, proves a genuine prefix and ends the
|
||||
/// walk at the depth of the first object.
|
||||
///
|
||||
/// Reads stay small on genuine prefixes: each directory is sampled with one
|
||||
/// bounded batch, the sampled children are probed depth-first, and the
|
||||
/// directory is only read in full once every sampled child proved to hold
|
||||
/// nothing listable. A total read budget makes a residue tree larger than
|
||||
/// the budget surface and be hidden one level down instead, so the cost on
|
||||
/// a genuine prefix never exceeds its depth in small directory reads. A
|
||||
/// Whether the metadata-less directory `dir_name` holds nothing but the
|
||||
/// data dirs of deleted versions: it is itself a non-nil UUID directory of
|
||||
/// `part.N` files and delete-transaction markers, or every child is one.
|
||||
/// That is what an interrupted or deferred version delete leaves behind
|
||||
/// once the `xl.meta` is gone, and it must not surface as a prefix. Real
|
||||
/// object children are directories carrying their own `xl.meta`, so the
|
||||
/// first non-UUID child, stray file, or subdirectory inside a UUID child
|
||||
/// proves the directory is a genuine prefix. Reads are bounded: a
|
||||
/// directory that vanishes mid-probe holds nothing listable.
|
||||
async fn directory_is_delete_residue(&self, bucket: &str, dir_name: &str, stall: Option<Duration>) -> Result<bool> {
|
||||
let dir_name = dir_name.trim_end_matches(SLASH_SEPARATOR);
|
||||
let mut reads = 0usize;
|
||||
let mut pending = vec![ResidueProbeStep::Sample(dir_name.to_owned())];
|
||||
while let Some(step) = pending.pop() {
|
||||
if reads >= DELETE_RESIDUE_PROBE_READ_BUDGET {
|
||||
let Some(entries) = self.read_dir_for_residue_probe(bucket, dir_name, stall).await? else {
|
||||
return Ok(false);
|
||||
};
|
||||
if entries.is_empty() {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let is_data_dir = dir_name
|
||||
.rsplit(SLASH_SEPARATOR)
|
||||
.next()
|
||||
.is_some_and(|name| Uuid::parse_str(name).is_ok_and(|uuid| !uuid.is_nil()));
|
||||
if is_data_dir && entries.iter().all(|entry| is_metadata_less_data_dir_entry(entry)) {
|
||||
return Ok(true);
|
||||
}
|
||||
|
||||
for entry in entries {
|
||||
let Some(child) = entry.strip_suffix(SLASH_SEPARATOR) else {
|
||||
return Ok(false);
|
||||
};
|
||||
if !Uuid::parse_str(child).is_ok_and(|uuid| !uuid.is_nil()) {
|
||||
return Ok(false);
|
||||
}
|
||||
reads += 1;
|
||||
|
||||
let (dir, entries, complete) = match step {
|
||||
ResidueProbeStep::Sample(dir) => {
|
||||
let Some((entries, complete)) = self.read_dir_for_residue_probe(bucket, &dir, stall, false).await? else {
|
||||
continue;
|
||||
};
|
||||
(dir, entries, complete)
|
||||
}
|
||||
ResidueProbeStep::Remainder { dir, sampled } => {
|
||||
let Some((entries, _)) = self.read_dir_for_residue_probe(bucket, &dir, stall, true).await? else {
|
||||
continue;
|
||||
};
|
||||
let entries = entries
|
||||
.into_iter()
|
||||
.filter(|entry| !sampled.contains(entry))
|
||||
.collect::<Vec<_>>();
|
||||
(dir, entries, true)
|
||||
}
|
||||
};
|
||||
|
||||
let is_data_dir = dir
|
||||
.rsplit(SLASH_SEPARATOR)
|
||||
.next()
|
||||
.is_some_and(|name| Uuid::parse_str(name).is_ok_and(|uuid| !uuid.is_nil()));
|
||||
if is_data_dir {
|
||||
if !entries.iter().all(|entry| is_metadata_less_data_dir_entry(entry)) {
|
||||
// A subdirectory, an `xl.meta`, or an unknown file inside a
|
||||
// UUID directory: not plain delete residue.
|
||||
return Ok(false);
|
||||
}
|
||||
if !complete {
|
||||
// Only the sampled part files were seen; the rest must be
|
||||
// read before the data dir counts as plain residue.
|
||||
pending.push(ResidueProbeStep::Remainder { dir, sampled: entries });
|
||||
}
|
||||
let child_path = path_join_buf(&[dir_name, child]);
|
||||
let Some(child_entries) = self.read_dir_for_residue_probe(bucket, &child_path, stall).await? else {
|
||||
continue;
|
||||
};
|
||||
if !child_entries.iter().all(|entry| is_metadata_less_data_dir_entry(entry)) {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let mut children = Vec::with_capacity(entries.len());
|
||||
for entry in &entries {
|
||||
let Some(child) = entry.strip_suffix(SLASH_SEPARATOR) else {
|
||||
// A file outside a plain data dir: `xl.meta` or something
|
||||
// this probe does not understand. Either way, not residue.
|
||||
return Ok(false);
|
||||
};
|
||||
children.push(path_join_buf(&[&dir, child]));
|
||||
}
|
||||
if !complete {
|
||||
// Revisit the unsampled siblings only after every sampled
|
||||
// child, probed first, turned out to hold nothing listable.
|
||||
pending.push(ResidueProbeStep::Remainder { dir, sampled: entries });
|
||||
}
|
||||
pending.extend(children.into_iter().map(ResidueProbeStep::Sample));
|
||||
}
|
||||
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Read `dir` for the residue probe: one bounded batch, or the complete
|
||||
/// directory when `complete` is set. The flag in the result says whether
|
||||
/// the batch held the whole directory. `None` when the directory does not
|
||||
/// exist any more.
|
||||
async fn read_dir_for_residue_probe(
|
||||
&self,
|
||||
bucket: &str,
|
||||
dir: &str,
|
||||
stall: Option<Duration>,
|
||||
complete: bool,
|
||||
) -> Result<Option<(Vec<String>, bool)>> {
|
||||
let count = if complete { -1 } else { DELETE_RESIDUE_PROBE_LIMIT };
|
||||
#[cfg(test)]
|
||||
let _ = DELETE_RESIDUE_PROBE_READS.try_with(|reads| {
|
||||
let counter = if complete { &reads.1 } else { &reads.0 };
|
||||
counter.fetch_add(1, Ordering::Relaxed)
|
||||
});
|
||||
match with_walk_stall_timeout(stall, self.list_dir("", bucket, dir, count)).await {
|
||||
Ok(entries) => {
|
||||
let complete = complete || entries.len() < DELETE_RESIDUE_PROBE_LIMIT as usize;
|
||||
Ok(Some((entries, complete)))
|
||||
/// Read `dir` with a bounded batch first and a complete read only when the
|
||||
/// batch was full. `None` when the directory does not exist any more.
|
||||
async fn read_dir_for_residue_probe(&self, bucket: &str, dir: &str, stall: Option<Duration>) -> Result<Option<Vec<String>>> {
|
||||
for count in [DELETE_RESIDUE_PROBE_LIMIT, -1] {
|
||||
let entries = match with_walk_stall_timeout(stall, self.list_dir("", bucket, dir, count)).await {
|
||||
Ok(entries) => entries,
|
||||
Err(err) => {
|
||||
if err == DiskError::VolumeNotFound || err == Error::FileNotFound {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
if count < 0 || entries.len() < count as usize {
|
||||
return Ok(Some(entries));
|
||||
}
|
||||
Err(err) if err == DiskError::VolumeNotFound || err == Error::FileNotFound => Ok(None),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Whether anything under `dir_name` would appear in a listing. With
|
||||
@@ -10180,25 +10060,21 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
}
|
||||
|
||||
let limit_reached = self
|
||||
.scan_dir(
|
||||
opts.base_dir.clone(),
|
||||
opts.filter_prefix.clone().unwrap_or_default(),
|
||||
&opts,
|
||||
&mut out,
|
||||
&mut objs_returned,
|
||||
skip_current_dir_object,
|
||||
if multipart_dir_to_skip.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(multipart_dir_to_skip)
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
self.scan_dir(
|
||||
opts.base_dir.clone(),
|
||||
opts.filter_prefix.clone().unwrap_or_default(),
|
||||
&opts,
|
||||
&mut out,
|
||||
&mut objs_returned,
|
||||
skip_current_dir_object,
|
||||
if multipart_dir_to_skip.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(multipart_dir_to_skip)
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
if let Some(flag) = opts.producer_limit_reached.as_ref() {
|
||||
flag.store(limit_reached, std::sync::atomic::Ordering::Release);
|
||||
}
|
||||
out.close().await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -18843,9 +18719,9 @@ mod test {
|
||||
let (fast_path_names, fast_path_probes) = scan_prefixes(&disk, bucket, true).await;
|
||||
|
||||
assert_eq!(conservative_names, expected_names);
|
||||
// The fast path hides the empty `stale/` chain too, at the cost of a
|
||||
// few bounded directory reads rather than the metadata probes.
|
||||
assert_eq!(fast_path_names, expected_names);
|
||||
let mut expected_fast_path_names = expected_names.clone();
|
||||
expected_fast_path_names.push("stale/".to_owned());
|
||||
assert_eq!(fast_path_names, expected_fast_path_names);
|
||||
let expected_probes = PREFIX_COUNT * 3 + 3;
|
||||
assert_eq!(conservative_probes, expected_probes);
|
||||
assert_eq!(fast_path_probes, 0);
|
||||
@@ -18956,9 +18832,9 @@ mod test {
|
||||
}
|
||||
|
||||
// Directories whose only content is a deleted version's data dir are
|
||||
// not prefixes, and neither are their ancestors.
|
||||
// not prefixes; their ancestors stay ordinary directories until an
|
||||
// empty listing reclaims them.
|
||||
assert_eq!(scan_names(&disk, bucket, "residue/2026/").await, Vec::<String>::new());
|
||||
assert_eq!(scan_names(&disk, bucket, "residue/").await, Vec::<String>::new());
|
||||
assert_eq!(scan_names(&disk, bucket, "committed/").await, Vec::<String>::new());
|
||||
|
||||
// UUID-named directories holding real objects, an object keyed by a
|
||||
@@ -18966,258 +18842,18 @@ mod test {
|
||||
assert_eq!(scan_names(&disk, bucket, "uploads/").await, vec![format!("uploads/{upload}/")]);
|
||||
assert_eq!(scan_names(&disk, bucket, "named/").await, vec![format!("named/{named}")]);
|
||||
assert_eq!(scan_names(&disk, bucket, "mixed/").await, vec!["mixed/child".to_owned()]);
|
||||
assert_eq!(
|
||||
scan_names(&disk, bucket, "").await,
|
||||
vec!["mixed/".to_owned(), "named/".to_owned(), "uploads/".to_owned()]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_scan_dir_nonrecursive_fast_path_hides_delete_residue_ancestors() {
|
||||
use rustfs_filemeta::MetacacheReader;
|
||||
use tempfile::tempdir;
|
||||
|
||||
let dir = tempdir().expect("tempdir should be created");
|
||||
let bucket = "test-bucket";
|
||||
let bucket_dir = dir.path().join(bucket);
|
||||
|
||||
async fn write_object(object_dir: &Path, object_name: &str) {
|
||||
fs::create_dir_all(object_dir)
|
||||
.await
|
||||
.expect("object directory should be created");
|
||||
let mut metadata = FileMeta::default();
|
||||
let mut file_info = FileInfo::new(object_name, 1, 1);
|
||||
file_info.mod_time = Some(OffsetDateTime::now_utc());
|
||||
metadata.add_version(file_info).expect("metadata should be valid");
|
||||
fs::write(
|
||||
object_dir.join(STORAGE_FORMAT_FILE),
|
||||
metadata.marshal_msg().expect("metadata should encode"),
|
||||
)
|
||||
.await
|
||||
.expect("object metadata should be written");
|
||||
}
|
||||
|
||||
async fn write_residue(object_dir: &Path, committed: bool) {
|
||||
let residue = object_dir.join(Uuid::new_v4().to_string());
|
||||
fs::create_dir_all(&residue).await.expect("residue should be created");
|
||||
fs::write(residue.join("part.1"), b"stale")
|
||||
.await
|
||||
.expect("stale part should be written");
|
||||
if committed {
|
||||
fs::write(residue.join(format!("{DELETE_DATA_DIR_MARKER_PREFIX}{}", Uuid::new_v4())), [])
|
||||
.await
|
||||
.expect("delete marker should be written");
|
||||
}
|
||||
}
|
||||
|
||||
// The reported shape: a date-partitioned key deleted on an older build,
|
||||
// whose data dir survived without a delete-transaction marker. Every
|
||||
// ancestor up to `metrics/` holds nothing else.
|
||||
write_residue(&bucket_dir.join("metrics/kubelet/2026/08/28/23/74992556388248657933757.parquet"), false).await;
|
||||
// Several committed residues under one hour plus an empty sibling hour.
|
||||
write_residue(&bucket_dir.join("metrics/cpu/2026/08/28/22/a.parquet"), true).await;
|
||||
write_residue(&bucket_dir.join("metrics/cpu/2026/08/28/22/b.parquet"), true).await;
|
||||
fs::create_dir_all(bucket_dir.join("metrics/cpu/2026/08/28/21"))
|
||||
.await
|
||||
.expect("empty hour directory should be created");
|
||||
|
||||
// A live object deep under an otherwise identical tree keeps every
|
||||
// ancestor visible, even beside residue.
|
||||
write_residue(&bucket_dir.join("logs/default/2026/08/28/23/old.parquet"), false).await;
|
||||
write_object(
|
||||
&bucket_dir.join("logs/default/2026/08/28/23/live.parquet"),
|
||||
"logs/default/2026/08/28/23/live.parquet",
|
||||
)
|
||||
.await;
|
||||
|
||||
// A prefix with more residue directories than the probe budget stays
|
||||
// visible rather than costing an unbounded walk.
|
||||
for hour in 0..(DELETE_RESIDUE_PROBE_READ_BUDGET + 1) {
|
||||
write_residue(&bucket_dir.join(format!("bulk/2026/08/28/{hour:02}/a.parquet")), false).await;
|
||||
}
|
||||
// Exactly at the budget: `edge/` with N object dirs costs one sampled
|
||||
// read of `edge`, two reads per object dir (its own and its data dir),
|
||||
// and one remainder read of `edge` once its 8-entry sample was full.
|
||||
for object in 0..15 {
|
||||
write_residue(&bucket_dir.join(format!("edge-hide/{object:02}.parquet")), false).await;
|
||||
}
|
||||
for object in 0..16 {
|
||||
write_residue(&bucket_dir.join(format!("edge-show/{object:02}.parquet")), false).await;
|
||||
}
|
||||
|
||||
// A file that is not `xl.meta` outside a UUID data dir is not residue
|
||||
// either: the probe does not guess, the prefix surfaces.
|
||||
fs::create_dir_all(bucket_dir.join("stray/2026/08"))
|
||||
.await
|
||||
.expect("stray directory should be created");
|
||||
fs::write(bucket_dir.join("stray/2026/08/notes.txt"), b"?")
|
||||
.await
|
||||
.expect("stray file should be written");
|
||||
|
||||
// A UUID directory holding a subdirectory is not a data dir shape the
|
||||
// probe understands, so it surfaces even when the subdirectory is empty.
|
||||
let odd = Uuid::new_v4().to_string();
|
||||
fs::create_dir_all(bucket_dir.join("odd").join(&odd).join("sub"))
|
||||
.await
|
||||
.expect("odd directory should be created");
|
||||
|
||||
let endpoint =
|
||||
Endpoint::try_from(dir.path().to_str().expect("tempdir path should be UTF-8")).expect("endpoint should parse");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should initialize");
|
||||
|
||||
async fn scan_names(disk: &LocalDisk, bucket: &str, current: &str) -> Vec<String> {
|
||||
let (reader, mut writer) = tokio::io::duplex(64 * 1024);
|
||||
let mut output = MetacacheWriter::new(&mut writer);
|
||||
let opts = WalkDirOptions {
|
||||
bucket: bucket.to_string(),
|
||||
base_dir: current.to_string(),
|
||||
skip_hidden_prefix_check: true,
|
||||
..Default::default()
|
||||
};
|
||||
let mut objects_returned = 0;
|
||||
disk.scan_dir(
|
||||
current.to_string(),
|
||||
"".to_string(),
|
||||
&opts,
|
||||
&mut output,
|
||||
&mut objects_returned,
|
||||
false,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("scan_dir should succeed");
|
||||
output.close().await.expect("metacache writer should close");
|
||||
drop(output);
|
||||
drop(writer);
|
||||
|
||||
let mut names = MetacacheReader::new(reader)
|
||||
.read_all()
|
||||
.await
|
||||
.expect("scan output should decode")
|
||||
.into_iter()
|
||||
.map(|entry| entry.name)
|
||||
.collect::<Vec<_>>();
|
||||
names.sort();
|
||||
names
|
||||
}
|
||||
|
||||
// Every level of a tree whose only leaves are deleted data dirs is
|
||||
// hidden, not just the object directory itself.
|
||||
assert_eq!(scan_names(&disk, bucket, "metrics/kubelet/2026/08/28/").await, Vec::<String>::new());
|
||||
assert_eq!(scan_names(&disk, bucket, "metrics/kubelet/").await, Vec::<String>::new());
|
||||
assert_eq!(scan_names(&disk, bucket, "metrics/cpu/2026/08/28/").await, Vec::<String>::new());
|
||||
assert_eq!(scan_names(&disk, bucket, "metrics/").await, Vec::<String>::new());
|
||||
assert_eq!(
|
||||
scan_names(&disk, bucket, "logs/default/2026/08/28/").await,
|
||||
vec!["logs/default/2026/08/28/23/".to_owned()]
|
||||
);
|
||||
assert_eq!(scan_names(&disk, bucket, "logs/").await, vec!["logs/default/".to_owned()]);
|
||||
assert_eq!(scan_names(&disk, bucket, "bulk/2026/08/").await, vec!["bulk/2026/08/28/".to_owned()]);
|
||||
// Listed directly, every object dir is probed on its own and hidden;
|
||||
// the budget only bites where the whole tree hangs off one entry, so
|
||||
// the root listing below shows `edge-show/` but not `edge-hide/`.
|
||||
assert_eq!(scan_names(&disk, bucket, "edge-hide/").await, Vec::<String>::new());
|
||||
assert_eq!(scan_names(&disk, bucket, "edge-show/").await, Vec::<String>::new());
|
||||
assert_eq!(scan_names(&disk, bucket, "stray/").await, vec!["stray/2026/".to_owned()]);
|
||||
assert_eq!(scan_names(&disk, bucket, "odd/").await, vec![format!("odd/{odd}/")]);
|
||||
assert_eq!(
|
||||
scan_names(&disk, bucket, "").await,
|
||||
vec![
|
||||
"bulk/".to_owned(),
|
||||
"edge-show/".to_owned(),
|
||||
"logs/".to_owned(),
|
||||
"odd/".to_owned(),
|
||||
"stray/".to_owned(),
|
||||
"committed/".to_owned(),
|
||||
"mixed/".to_owned(),
|
||||
"named/".to_owned(),
|
||||
"residue/".to_owned(),
|
||||
"uploads/".to_owned(),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_scan_dir_nonrecursive_fast_path_residue_probe_reads_stay_within_prefix_depth() {
|
||||
use rustfs_filemeta::MetacacheReader;
|
||||
use tempfile::tempdir;
|
||||
|
||||
let dir = tempdir().expect("tempdir should be created");
|
||||
let bucket = "test-bucket";
|
||||
let bucket_dir = dir.path().join(bucket);
|
||||
|
||||
async fn write_object(object_dir: &Path, object_name: &str) {
|
||||
fs::create_dir_all(object_dir)
|
||||
.await
|
||||
.expect("object directory should be created");
|
||||
let mut metadata = FileMeta::default();
|
||||
let mut file_info = FileInfo::new(object_name, 1, 1);
|
||||
file_info.mod_time = Some(OffsetDateTime::now_utc());
|
||||
metadata.add_version(file_info).expect("metadata should be valid");
|
||||
fs::write(
|
||||
object_dir.join(STORAGE_FORMAT_FILE),
|
||||
metadata.marshal_msg().expect("metadata should encode"),
|
||||
)
|
||||
.await
|
||||
.expect("object metadata should be written");
|
||||
}
|
||||
|
||||
// A genuine date-partitioned prefix wider than the probe batch at two
|
||||
// levels: 20 days, each hour holding 100 objects.
|
||||
const DAYS: usize = 20;
|
||||
const OBJECTS: usize = 100;
|
||||
for day in 0..DAYS {
|
||||
for object in 0..OBJECTS {
|
||||
let key = format!("data/stream/2026/08/{day:02}/23/{object:03}.parquet");
|
||||
write_object(&bucket_dir.join(&key), &key).await;
|
||||
}
|
||||
}
|
||||
|
||||
let endpoint =
|
||||
Endpoint::try_from(dir.path().to_str().expect("tempdir path should be UTF-8")).expect("endpoint should parse");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should initialize");
|
||||
|
||||
let reads = Arc::new((AtomicUsize::new(0), AtomicUsize::new(0)));
|
||||
let (reader, mut writer) = tokio::io::duplex(64 * 1024);
|
||||
let mut output = MetacacheWriter::new(&mut writer);
|
||||
let opts = WalkDirOptions {
|
||||
bucket: bucket.to_string(),
|
||||
base_dir: "data/".to_string(),
|
||||
skip_hidden_prefix_check: true,
|
||||
..Default::default()
|
||||
};
|
||||
let mut objects_returned = 0;
|
||||
DELETE_RESIDUE_PROBE_READS
|
||||
.scope(
|
||||
Arc::clone(&reads),
|
||||
disk.scan_dir(
|
||||
"data/".to_string(),
|
||||
"".to_string(),
|
||||
&opts,
|
||||
&mut output,
|
||||
&mut objects_returned,
|
||||
false,
|
||||
None,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("scan_dir should succeed");
|
||||
output.close().await.expect("metacache writer should close");
|
||||
drop(output);
|
||||
drop(writer);
|
||||
|
||||
let names = MetacacheReader::new(reader)
|
||||
.read_all()
|
||||
.await
|
||||
.expect("scan output should decode")
|
||||
.into_iter()
|
||||
.map(|entry| entry.name)
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(names, vec!["data/stream/".to_owned()]);
|
||||
|
||||
// stream, 2026, 08, one sampled day, its hour, one sampled object:
|
||||
// the walk ends at the first `xl.meta`, one bounded read per level,
|
||||
// without ever reading a wide directory in full.
|
||||
let (sampled, complete) = (reads.0.load(Ordering::Relaxed), reads.1.load(Ordering::Relaxed));
|
||||
assert_eq!(sampled, 6, "one sampled read per level down to the first object");
|
||||
assert_eq!(complete, 0, "a genuine prefix must never cost a complete directory read");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_scan_dir_nonrecursive_skips_dirs_with_only_hidden_delete_markers() {
|
||||
use rustfs_filemeta::MetacacheReader;
|
||||
@@ -22197,7 +21833,7 @@ mod test {
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test]
|
||||
async fn direct_writer_state_machine_round_trips_over_plain_file() {
|
||||
use std::io::{IoSlice, Read};
|
||||
use std::io::Read;
|
||||
use tempfile::tempdir;
|
||||
|
||||
let dir = tempdir().expect("tempdir");
|
||||
@@ -22216,16 +21852,6 @@ mod test {
|
||||
|
||||
let mut writer = DirectWriter::from_std_file_for_test(file, align, capacity);
|
||||
let mut off = 0;
|
||||
if content.len() >= 2 {
|
||||
let split = content.len().min(300);
|
||||
let first = split / 2;
|
||||
let written = writer
|
||||
.write_vectored(&[IoSlice::new(&content[..first]), IoSlice::new(&content[first..split])])
|
||||
.await
|
||||
.expect("vectored write");
|
||||
assert_eq!(written, split, "vectored write must consume both hash/data-like slices");
|
||||
off = split;
|
||||
}
|
||||
while off < content.len() {
|
||||
let end = (off + 300).min(content.len());
|
||||
writer.write_all(&content[off..end]).await.expect("write_all");
|
||||
|
||||
@@ -70,12 +70,7 @@ use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
|
||||
use rustfs_madmin::info_commands::DiskMetrics;
|
||||
use rustfs_rio::ChunkReaderBox;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
fmt::Debug,
|
||||
path::PathBuf,
|
||||
sync::{Arc, atomic::AtomicBool},
|
||||
time::Duration,
|
||||
};
|
||||
use std::{fmt::Debug, path::PathBuf, sync::Arc, time::Duration};
|
||||
use time::OffsetDateTime;
|
||||
use tokio::io::{AsyncRead, AsyncWrite};
|
||||
use uuid::Uuid;
|
||||
@@ -825,7 +820,7 @@ impl Disk {
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
) -> Result<RenameDataResp> {
|
||||
let Some(scope) = crate::store::bucket_heal_scope_for_object(dst_volume, dst_path) else {
|
||||
let Some(scope) = crate::store::bucket_heal_scope(dst_volume) else {
|
||||
return self
|
||||
.rename_data_borrowed_with_fence(src_volume, src_path, fi, dst_volume, dst_path, None)
|
||||
.await;
|
||||
@@ -984,14 +979,6 @@ impl Disk {
|
||||
Disk::Remote(remote_disk) => remote_disk.force_runtime_state_for_test(state),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn force_offline_for_test(&self) {
|
||||
match self {
|
||||
Disk::Local(local_disk) => local_disk.force_offline_for_test(),
|
||||
Disk::Remote(remote_disk) => remote_disk.force_offline_for_test(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -1484,12 +1471,6 @@ pub struct WalkDirOptions {
|
||||
// Override the remote stream stall timeout for long background walks.
|
||||
#[serde(default)]
|
||||
pub stall_timeout_ms: Option<u64>,
|
||||
|
||||
/// In-process completion state for bounded local walks. This is skipped
|
||||
/// from RPC serialization; remote peers retain the legacy natural-EOF
|
||||
/// behavior until they support an explicit capability.
|
||||
#[serde(skip)]
|
||||
pub producer_limit_reached: Option<Arc<AtomicBool>>,
|
||||
}
|
||||
|
||||
impl WalkDirOptions {
|
||||
@@ -1813,7 +1794,6 @@ mod tests {
|
||||
skip_total_timeout: false,
|
||||
timeout_ms: Some(10_000),
|
||||
stall_timeout_ms: Some(20_000),
|
||||
producer_limit_reached: None,
|
||||
};
|
||||
|
||||
assert_eq!(opts.bucket, "test-bucket");
|
||||
|
||||
@@ -12,7 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::disk::error::Error as DiskError;
|
||||
use crate::erasure::codec::buffer_pool::{get_ec_buffer, return_ec_buffer};
|
||||
use pin_project_lite::pin_project;
|
||||
use rustfs_utils::HashAlgorithm;
|
||||
@@ -240,7 +239,7 @@ fn short_shard_read(got: usize, want: usize) -> std::io::Error {
|
||||
want,
|
||||
"short shard read: got {got} of {want} bytes"
|
||||
);
|
||||
std::io::Error::new(std::io::ErrorKind::UnexpectedEof, DiskError::FileCorrupt)
|
||||
std::io::Error::new(std::io::ErrorKind::UnexpectedEof, format!("short shard read: got {got} of {want} bytes"))
|
||||
}
|
||||
|
||||
/// Split a `[hash][data]` block, verify the hash (unless `skip_verify`), and
|
||||
@@ -266,7 +265,7 @@ fn split_and_verify<'a>(hash_algo: &HashAlgorithm, skip_verify: bool, block: &'a
|
||||
data_len = data.len(),
|
||||
"bitrot hash mismatch"
|
||||
);
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, DiskError::FileCorrupt));
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "bitrot hash mismatch"));
|
||||
}
|
||||
Ok((data, verify))
|
||||
}
|
||||
@@ -435,7 +434,7 @@ where
|
||||
data_len = want,
|
||||
"bitrot hash mismatch"
|
||||
);
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, DiskError::FileCorrupt));
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "bitrot hash mismatch"));
|
||||
}
|
||||
hash_offset += take;
|
||||
remaining -= take;
|
||||
@@ -1142,26 +1141,6 @@ mod tests {
|
||||
use std::time::Duration;
|
||||
use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt, ReadBuf};
|
||||
|
||||
#[test]
|
||||
fn bitrot_damage_preserves_typed_repair_evidence() {
|
||||
use crate::disk::error::Error as DiskError;
|
||||
let algo = HashAlgorithm::HighwayHash256;
|
||||
let data = b"repair evidence";
|
||||
let mut frame = algo.hash_encode(data).as_ref().to_vec();
|
||||
frame.extend_from_slice(data);
|
||||
*frame.last_mut().expect("test frame has payload") ^= 1;
|
||||
let error = super::split_and_verify(&algo, false, &frame).expect_err("a hash mismatch must reject the shard");
|
||||
assert_eq!(error.kind(), io::ErrorKind::InvalidData);
|
||||
assert_eq!(DiskError::from(error), DiskError::FileCorrupt);
|
||||
let error = super::short_shard_read(3, 4);
|
||||
assert_eq!(error.kind(), io::ErrorKind::UnexpectedEof);
|
||||
assert_eq!(DiskError::from(error), DiskError::FileCorrupt);
|
||||
assert!(matches!(
|
||||
DiskError::from(io::Error::new(io::ErrorKind::InvalidData, "unrelated invalid data")),
|
||||
DiskError::Io(_)
|
||||
));
|
||||
}
|
||||
|
||||
struct FragmentedSource {
|
||||
chunks: VecDeque<Bytes>,
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ use crate::erasure::codec::workspace::ShardBufferPool;
|
||||
use crate::erasure::coding::{BitrotReader, Erasure};
|
||||
use crate::io_support::bitrot::DeferredReaderStripeHandle;
|
||||
use crate::set_disk::shard_source::{
|
||||
INLINE_SHARD_SLOTS, ShardBuffers, ShardErrors, ShardReadCost, ShardReadRepair, ShardStripeSource, StripeReadState,
|
||||
INLINE_SHARD_SLOTS, ShardBuffers, ShardErrors, ShardReadCost, ShardStripeSource, StripeReadState,
|
||||
};
|
||||
use futures::FutureExt;
|
||||
use futures::stream::{FuturesUnordered, StreamExt};
|
||||
@@ -46,13 +46,7 @@ type ShardReadFuture<'a> = Pin<Box<dyn Future<Output = (usize, ShardReadCost, Re
|
||||
type OwnedShardReadFuture<'a, R> =
|
||||
Pin<Box<dyn Future<Output = (usize, ShardReadCost, Result<Vec<u8>, Error>, Option<BitrotReader<R>>, bool)> + Send + 'a>>;
|
||||
pub(crate) type DeferredReaderReopener<R> = Arc<dyn Fn(usize) -> Option<BitrotReader<R>> + Send + Sync>;
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct DecodeOutcome {
|
||||
pub(crate) written: usize,
|
||||
pub(crate) error: Option<std::io::Error>,
|
||||
pub(crate) exact_quorum: bool,
|
||||
pub(crate) repair: ShardReadRepair,
|
||||
}
|
||||
pub(crate) type DecodeOutcome = (usize, Option<std::io::Error>, bool);
|
||||
|
||||
type ShardIndexes = SmallVec<[usize; INLINE_SHARD_SLOTS]>;
|
||||
type ActiveReaders = SmallVec<[bool; INLINE_SHARD_SLOTS]>;
|
||||
@@ -602,7 +596,6 @@ pub(crate) struct ParallelReader<R> {
|
||||
hedged_readers: ActiveReaders,
|
||||
stripe_index: usize,
|
||||
integrity: Option<std::sync::Arc<crate::io_support::shard_integrity::PartProofReader>>,
|
||||
repair_evidence: Option<Arc<ShardReadRepair>>,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -815,15 +808,9 @@ where
|
||||
deferred_reopeners: Vec::new(),
|
||||
hedged_readers,
|
||||
stripe_index: 0,
|
||||
repair_evidence: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn with_repair_evidence(mut self, evidence: Arc<ShardReadRepair>) -> Self {
|
||||
self.repair_evidence = Some(evidence);
|
||||
self
|
||||
}
|
||||
|
||||
/// Attach the per-slot deferred stripe handles produced during bitrot
|
||||
/// reader setup. Only parity slots are consulted: a handle lets the
|
||||
/// lockstep path open a parity shard aligned to the stripe where a data
|
||||
@@ -2040,9 +2027,6 @@ where
|
||||
.take()
|
||||
.unwrap_or_else(|| Box::new(StripeReadState::with_slot_count(self.readers.len(), self.data_shards)));
|
||||
self.read_into_state(&mut state).await;
|
||||
if let Some(evidence) = &self.repair_evidence {
|
||||
evidence.observe(state.parts_mut().1);
|
||||
}
|
||||
if state.can_decode()
|
||||
&& let Some(proof) = &self.integrity
|
||||
{
|
||||
@@ -2247,10 +2231,10 @@ impl Erasure {
|
||||
W: AsyncWrite + Send + Sync + Unpin,
|
||||
R: crate::erasure::coding::ShardSource,
|
||||
{
|
||||
let outcome = self
|
||||
let (written, error, _) = self
|
||||
.decode_inner(writer, readers, offset, length, total_length, None, Vec::new(), Vec::new())
|
||||
.await;
|
||||
(outcome.written, outcome.error)
|
||||
(written, error)
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "read-cost decode path asserted by this file's tests (backlog#1823)")]
|
||||
@@ -2267,10 +2251,10 @@ impl Erasure {
|
||||
W: AsyncWrite + Send + Sync + Unpin,
|
||||
R: crate::erasure::coding::ShardSource,
|
||||
{
|
||||
let outcome = self
|
||||
let (written, error, _) = self
|
||||
.decode_inner(writer, readers, offset, length, total_length, Some(read_costs), Vec::new(), Vec::new())
|
||||
.await;
|
||||
(outcome.written, outcome.error)
|
||||
(written, error)
|
||||
}
|
||||
|
||||
/// GET decode entry point that also carries the deferred-parity stripe
|
||||
@@ -2327,7 +2311,7 @@ impl Erasure {
|
||||
W: AsyncWrite + Send + Sync + Unpin,
|
||||
R: crate::erasure::coding::ShardSource,
|
||||
{
|
||||
let outcome = self
|
||||
let (written, error, _) = self
|
||||
.decode_inner(
|
||||
writer,
|
||||
readers,
|
||||
@@ -2339,7 +2323,7 @@ impl Erasure {
|
||||
deferred_reopeners,
|
||||
)
|
||||
.await;
|
||||
(outcome.written, outcome.error)
|
||||
(written, error)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
@@ -2390,7 +2374,6 @@ impl Erasure {
|
||||
block_length: usize,
|
||||
written: &mut usize,
|
||||
ret_err: &mut Option<std::io::Error>,
|
||||
repair: &ShardReadRepair,
|
||||
stage_metrics_enabled: bool,
|
||||
require_surplus_source: bool,
|
||||
integrity: Option<&std::sync::Arc<crate::io_support::shard_integrity::PartProofReader>>,
|
||||
@@ -2399,7 +2382,6 @@ impl Erasure {
|
||||
where
|
||||
W: AsyncWrite + Send + Sync + Unpin,
|
||||
{
|
||||
repair.observe(errs);
|
||||
if ret_err.is_none()
|
||||
&& let (_, Some(err)) = reduce_errs(errs, &[])
|
||||
&& (err == Error::FileNotFound || err == Error::FileCorrupt)
|
||||
@@ -2534,42 +2516,41 @@ impl Erasure {
|
||||
{
|
||||
if readers.len() != self.data_shards + self.parity_shards {
|
||||
record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid);
|
||||
return DecodeOutcome {
|
||||
error: Some(io::Error::new(ErrorKind::InvalidInput, "Invalid number of readers")),
|
||||
..Default::default()
|
||||
};
|
||||
return (0, Some(io::Error::new(ErrorKind::InvalidInput, "Invalid number of readers")), false);
|
||||
}
|
||||
|
||||
// block_size/data_shards come from on-disk metadata; a corrupt FileInfo with a
|
||||
// zero here must surface as an error, not a divide-by-zero panic on every GET.
|
||||
if self.block_size == 0 || self.data_shards == 0 {
|
||||
record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid);
|
||||
return DecodeOutcome {
|
||||
error: Some(io::Error::new(ErrorKind::InvalidInput, "Invalid erasure coding parameters")),
|
||||
..Default::default()
|
||||
};
|
||||
return (
|
||||
0,
|
||||
Some(io::Error::new(ErrorKind::InvalidInput, "Invalid erasure coding parameters")),
|
||||
false,
|
||||
);
|
||||
}
|
||||
|
||||
let Some(end_offset) = offset.checked_add(length) else {
|
||||
record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid);
|
||||
return DecodeOutcome {
|
||||
error: Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length")),
|
||||
..Default::default()
|
||||
};
|
||||
return (
|
||||
0,
|
||||
Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length")),
|
||||
false,
|
||||
);
|
||||
};
|
||||
if end_offset > total_length {
|
||||
record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid);
|
||||
return DecodeOutcome {
|
||||
error: Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length")),
|
||||
..Default::default()
|
||||
};
|
||||
return (
|
||||
0,
|
||||
Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length")),
|
||||
false,
|
||||
);
|
||||
}
|
||||
|
||||
let mut ret_err = None;
|
||||
let repair = ShardReadRepair::default();
|
||||
|
||||
if length == 0 {
|
||||
return DecodeOutcome::default();
|
||||
return (0, ret_err, false);
|
||||
}
|
||||
|
||||
let mut written = 0;
|
||||
@@ -2592,11 +2573,6 @@ impl Erasure {
|
||||
.with_deferred_parity_reopeners(deferred_reopeners);
|
||||
|
||||
let integrity = reader.integrity.clone();
|
||||
// Copy sources use demand-bound scheduling for backpressure, but retain
|
||||
// the ordinary GET read quorum. Only the opt-in GET rollout requires a
|
||||
// surplus source; available surplus shards and integrity proofs are
|
||||
// still verified before any reconstructed copy bytes are emitted.
|
||||
let require_surplus_source = reader.demand_bound_lockstep && matches!(decode_read_policy(), DecodeReadPolicy::Default);
|
||||
|
||||
let start = offset / self.block_size;
|
||||
let end = end_offset.saturating_sub(1) / self.block_size;
|
||||
@@ -2691,6 +2667,7 @@ impl Erasure {
|
||||
// `shards` are borrowed again below. In the `Stop` case that
|
||||
// drop is what cancels the still-in-flight prefetch read.
|
||||
let (flow, next): (Option<StripeFlow>, Option<StripeReadOutput>) = {
|
||||
let require_surplus_source = reader.demand_bound_lockstep;
|
||||
let read_fut = read_stripe_timed(&mut reader, stage_metrics_enabled);
|
||||
let emit_fut = self.emit_decoded_stripe(
|
||||
writer,
|
||||
@@ -2700,7 +2677,6 @@ impl Erasure {
|
||||
block_length,
|
||||
&mut written,
|
||||
&mut ret_err,
|
||||
&repair,
|
||||
stage_metrics_enabled,
|
||||
require_surplus_source,
|
||||
integrity.as_ref(),
|
||||
@@ -2751,9 +2727,8 @@ impl Erasure {
|
||||
block_length,
|
||||
&mut written,
|
||||
&mut ret_err,
|
||||
&repair,
|
||||
stage_metrics_enabled,
|
||||
require_surplus_source,
|
||||
reader.demand_bound_lockstep,
|
||||
integrity.as_ref(),
|
||||
offset,
|
||||
)
|
||||
@@ -2795,9 +2770,8 @@ impl Erasure {
|
||||
block_length,
|
||||
&mut written,
|
||||
&mut ret_err,
|
||||
&repair,
|
||||
stage_metrics_enabled,
|
||||
require_surplus_source,
|
||||
reader.demand_bound_lockstep,
|
||||
integrity.as_ref(),
|
||||
offset,
|
||||
)
|
||||
@@ -2811,16 +2785,15 @@ impl Erasure {
|
||||
}
|
||||
}
|
||||
|
||||
if ret_err.is_none() && written < length {
|
||||
if ret_err.is_some() {
|
||||
return (written, ret_err, exact_quorum);
|
||||
}
|
||||
|
||||
if written < length {
|
||||
ret_err = Some(Error::LessData.into());
|
||||
}
|
||||
|
||||
DecodeOutcome {
|
||||
written,
|
||||
error: ret_err,
|
||||
exact_quorum,
|
||||
repair,
|
||||
}
|
||||
(written, ret_err, exact_quorum)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3476,150 +3449,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
async fn copy_source_bitrot_shards(erasure: &Erasure, payload: &[u8], corrupt_parity: bool) -> Vec<Vec<u8>> {
|
||||
let mut writers: Vec<_> = (0..erasure.data_shards + erasure.parity_shards)
|
||||
.map(|_| BitrotWriter::new(Cursor::new(Vec::new()), erasure.shard_size(), HashAlgorithm::HighwayHash256))
|
||||
.collect();
|
||||
for block in payload.chunks(erasure.block_size) {
|
||||
let shards = erasure.encode_data(block).expect("copy source should encode");
|
||||
for (index, shard) in shards.iter().enumerate() {
|
||||
let mut bytes = shard.to_vec();
|
||||
if corrupt_parity && index == erasure.data_shards {
|
||||
// A stale source can have a valid per-shard bitrot checksum.
|
||||
// Reframe the changed parity to exercise reconstruction verification.
|
||||
bytes[0] ^= 0x80;
|
||||
}
|
||||
writers[index].write(&bytes).await.expect("copy source should frame");
|
||||
}
|
||||
}
|
||||
writers.into_iter().map(|writer| writer.into_inner().into_inner()).collect()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn copy_source_reads_exact_quorum_across_stripes_and_ranges() {
|
||||
const BLOCK_SIZE: usize = 768;
|
||||
let payload: Vec<_> = (0..BLOCK_SIZE * 3 + 37)
|
||||
.map(|index| (index.wrapping_mul(31) % 251) as u8)
|
||||
.collect();
|
||||
|
||||
for uses_legacy in [false, true] {
|
||||
let erasure = Erasure::new_with_options(12, 4, BLOCK_SIZE, uses_legacy);
|
||||
let buffers = copy_source_bitrot_shards(&erasure, &payload, false).await;
|
||||
for offline in [[0, 1, 2, 3], [0, 5, 12, 15]] {
|
||||
for (offset, length) in [(0, payload.len()), (BLOCK_SIZE + 17, payload.len() - BLOCK_SIZE - 17)] {
|
||||
let frame_offset = offset / BLOCK_SIZE * (erasure.shard_size() + HashAlgorithm::HighwayHash256.size());
|
||||
let positioned: Vec<_> = buffers.iter().map(|bytes| bytes[frame_offset..].to_vec()).collect();
|
||||
let (mut readers, handles) =
|
||||
readers_with_deferred_parity(&positioned, 12, erasure.shard_size(), &HashAlgorithm::HighwayHash256, &[]);
|
||||
for index in offline {
|
||||
readers[index] = None;
|
||||
}
|
||||
let mut output = Vec::new();
|
||||
let (written, error) = crate::set_disk::with_get_object_read_policy(
|
||||
crate::set_disk::GetObjectReadPolicy::CopySource,
|
||||
erasure.decode_with_stripe_handles(&mut output, readers, offset, length, payload.len(), None, handles),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(error.is_none(), "legacy={uses_legacy}, offline={offline:?}, offset={offset}: {error:?}");
|
||||
assert_eq!(written, length);
|
||||
assert_eq!(output, payload[offset..offset + length]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn copy_source_recovers_exact_quorum_after_first_stripe() {
|
||||
const BLOCK_SIZE: usize = 768;
|
||||
let payload: Vec<_> = (0..BLOCK_SIZE * 3 + 37)
|
||||
.map(|index| (index.wrapping_mul(31) % 251) as u8)
|
||||
.collect();
|
||||
for uses_legacy in [false, true] {
|
||||
let erasure = Erasure::new_with_options(12, 4, BLOCK_SIZE, uses_legacy);
|
||||
let buffers = copy_source_bitrot_shards(&erasure, &payload, false).await;
|
||||
let first_frame = erasure.shard_size() + HashAlgorithm::HighwayHash256.size();
|
||||
let (readers, handles) = readers_with_deferred_parity(
|
||||
&buffers,
|
||||
12,
|
||||
erasure.shard_size(),
|
||||
&HashAlgorithm::HighwayHash256,
|
||||
&[(0, first_frame), (1, first_frame), (2, first_frame), (3, first_frame)],
|
||||
);
|
||||
let mut output = Vec::new();
|
||||
let (written, error) = crate::set_disk::with_get_object_read_policy(
|
||||
crate::set_disk::GetObjectReadPolicy::CopySource,
|
||||
erasure.decode_with_stripe_handles(&mut output, readers, 0, payload.len(), payload.len(), None, handles),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(error.is_none(), "legacy={uses_legacy}: late shard loss should recover: {error:?}");
|
||||
assert_eq!(written, payload.len());
|
||||
assert_eq!(output, payload);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn copy_source_below_read_quorum_fails_before_output() {
|
||||
const BLOCK_SIZE: usize = 768;
|
||||
let payload = vec![0x5a; BLOCK_SIZE * 2 + 37];
|
||||
for uses_legacy in [false, true] {
|
||||
let erasure = Erasure::new_with_options(12, 4, BLOCK_SIZE, uses_legacy);
|
||||
let buffers = copy_source_bitrot_shards(&erasure, &payload, false).await;
|
||||
let (mut readers, handles) =
|
||||
readers_with_deferred_parity(&buffers, 12, erasure.shard_size(), &HashAlgorithm::HighwayHash256, &[]);
|
||||
for reader in readers.iter_mut().take(5) {
|
||||
*reader = None;
|
||||
}
|
||||
let mut output = Vec::new();
|
||||
let (written, error) = crate::set_disk::with_get_object_read_policy(
|
||||
crate::set_disk::GetObjectReadPolicy::CopySource,
|
||||
erasure.decode_with_stripe_handles(&mut output, readers, 0, payload.len(), payload.len(), None, handles),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(written, 0);
|
||||
assert!(output.is_empty());
|
||||
assert_eq!(
|
||||
error
|
||||
.expect("eleven shards cannot reconstruct twelve data shards")
|
||||
.to_string(),
|
||||
Error::ErasureReadQuorum.to_string()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn copy_source_rejects_corrupt_surplus_parity_before_output() {
|
||||
const BLOCK_SIZE: usize = 768;
|
||||
let payload = vec![0x5a; BLOCK_SIZE * 2 + 37];
|
||||
for uses_legacy in [false, true] {
|
||||
let erasure = Erasure::new_with_options(12, 4, BLOCK_SIZE, uses_legacy);
|
||||
let buffers = copy_source_bitrot_shards(&erasure, &payload, true).await;
|
||||
let (mut readers, handles) =
|
||||
readers_with_deferred_parity(&buffers, 12, erasure.shard_size(), &HashAlgorithm::HighwayHash256, &[]);
|
||||
for reader in readers.iter_mut().take(3) {
|
||||
*reader = None;
|
||||
}
|
||||
let mut output = Vec::new();
|
||||
let (written, error) = crate::set_disk::with_get_object_read_policy(
|
||||
crate::set_disk::GetObjectReadPolicy::CopySource,
|
||||
erasure.decode_with_stripe_handles(&mut output, readers, 0, payload.len(), payload.len(), None, handles),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(written, 0);
|
||||
assert!(output.is_empty());
|
||||
let error = error.expect("copy must reject inconsistent surplus parity despite valid bitrot framing");
|
||||
assert_eq!(error.kind(), ErrorKind::InvalidData);
|
||||
assert!(error.to_string().contains("inconsistent read source shards"));
|
||||
}
|
||||
}
|
||||
|
||||
/// Guards the shard-buffer reuse in `ParallelReader`: a multi-stripe
|
||||
/// `Erasure::decode` that reconstructs missing data shards on every stripe
|
||||
/// must still return byte-exact output. Reconstructed and parity buffers are
|
||||
@@ -4005,45 +3834,10 @@ mod tests {
|
||||
temp_env::async_with_vars(vars, async {
|
||||
let readers = build_prefetch_readers(&shard_bufs, shard_size, &hash_algo, BLOCK_SIZE, 0, &[], &[1]);
|
||||
let mut output = Vec::new();
|
||||
let outcome = erasure
|
||||
.decode_with_stripe_handles_and_reopeners_with_diagnostics(
|
||||
&mut output,
|
||||
readers,
|
||||
0,
|
||||
total_len,
|
||||
total_len,
|
||||
None,
|
||||
Vec::new(),
|
||||
Vec::new(),
|
||||
)
|
||||
.await;
|
||||
assert!(outcome.error.is_none(), "{label}: corrupt-shard read errored {:?}", outcome.error);
|
||||
assert_eq!(outcome.written, total_len, "{label}: corrupt-shard short write");
|
||||
let (written, err) = erasure.decode(&mut output, readers, 0, total_len, total_len).await;
|
||||
assert!(err.is_none(), "{label}: corrupt-shard read errored {err:?}");
|
||||
assert_eq!(written, total_len, "{label}: corrupt-shard short write");
|
||||
assert_eq!(output, total_data, "{label}: corrupt bytes leaked into output");
|
||||
assert!(
|
||||
outcome.repair.needed(false),
|
||||
"{label}: minority corruption must survive successful reconstruction"
|
||||
);
|
||||
let readers = build_prefetch_readers(&shard_bufs, shard_size, &hash_algo, BLOCK_SIZE, 0, &[], &[]);
|
||||
let mut healthy_output = Vec::new();
|
||||
let healthy = erasure
|
||||
.decode_with_stripe_handles_and_reopeners_with_diagnostics(
|
||||
&mut healthy_output,
|
||||
readers,
|
||||
0,
|
||||
total_len,
|
||||
total_len,
|
||||
None,
|
||||
Vec::new(),
|
||||
Vec::new(),
|
||||
)
|
||||
.await;
|
||||
assert!(healthy.error.is_none());
|
||||
assert_eq!(healthy_output, total_data);
|
||||
assert!(
|
||||
!healthy.repair.needed(true),
|
||||
"{label}: healthy/deferred shards cannot create repair evidence"
|
||||
);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
@@ -4841,38 +4635,40 @@ mod tests {
|
||||
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
|
||||
],
|
||||
async {
|
||||
const NUM_SHARDS: usize = 1;
|
||||
const BLOCK_SIZE: usize = 64;
|
||||
const DATA_SHARDS: usize = 4;
|
||||
const PARITY_SHARDS: usize = 2;
|
||||
const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
|
||||
const NUM_SHARDS: usize = 1;
|
||||
const BLOCK_SIZE: usize = 64;
|
||||
const DATA_SHARDS: usize = 4;
|
||||
const PARITY_SHARDS: usize = 2;
|
||||
const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
|
||||
|
||||
let hash_algo = HashAlgorithm::HighwayHash256;
|
||||
let readers = make_test_readers(DATA_SHARDS + PARITY_SHARDS, SHARD_SIZE, NUM_SHARDS, &hash_algo, &[], &[1]).await;
|
||||
let read_costs = vec![
|
||||
ShardReadCost::Local,
|
||||
ShardReadCost::Local,
|
||||
ShardReadCost::Local,
|
||||
ShardReadCost::Local,
|
||||
ShardReadCost::Remote,
|
||||
ShardReadCost::Remote,
|
||||
];
|
||||
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
|
||||
let mut parallel_reader = ParallelReader::new_with_metrics_path_and_read_costs(
|
||||
readers,
|
||||
erasure,
|
||||
0,
|
||||
NUM_SHARDS * BLOCK_SIZE,
|
||||
None,
|
||||
read_costs,
|
||||
);
|
||||
let hash_algo = HashAlgorithm::HighwayHash256;
|
||||
let readers = make_test_readers(DATA_SHARDS + PARITY_SHARDS, SHARD_SIZE, NUM_SHARDS, &hash_algo, &[], &[1]).await;
|
||||
let read_costs = vec![
|
||||
ShardReadCost::Local,
|
||||
ShardReadCost::Local,
|
||||
ShardReadCost::Local,
|
||||
ShardReadCost::Local,
|
||||
ShardReadCost::Remote,
|
||||
ShardReadCost::Remote,
|
||||
];
|
||||
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
|
||||
let mut parallel_reader = ParallelReader::new_with_metrics_path_and_read_costs(
|
||||
readers,
|
||||
erasure,
|
||||
0,
|
||||
NUM_SHARDS * BLOCK_SIZE,
|
||||
None,
|
||||
read_costs,
|
||||
);
|
||||
|
||||
let (bufs, errs) = parallel_reader.read().await;
|
||||
let (bufs, errs) = parallel_reader.read().await;
|
||||
|
||||
assert_eq!(DATA_SHARDS, bufs.iter().filter(|buf| buf.is_some()).count());
|
||||
assert_eq!(errs[1], Some(DiskError::FileCorrupt));
|
||||
assert_eq!(bufs[4].as_deref(), Some(&[4u8; SHARD_SIZE][..]));
|
||||
assert!(bufs[5].is_none());
|
||||
assert_eq!(DATA_SHARDS, bufs.iter().filter(|buf| buf.is_some()).count());
|
||||
assert!(
|
||||
matches!(&errs[1], Some(DiskError::Io(err)) if err.kind() == ErrorKind::InvalidData && err.to_string().contains("bitrot"))
|
||||
);
|
||||
assert_eq!(bufs[4].as_deref(), Some(&[4u8; SHARD_SIZE][..]));
|
||||
assert!(bufs[5].is_none());
|
||||
},
|
||||
)
|
||||
.await;
|
||||
@@ -5018,7 +4814,16 @@ mod tests {
|
||||
assert_eq!(DATA_SHARDS, bufs.iter().filter(|buf| buf.is_some()).count());
|
||||
assert_eq!(
|
||||
BITROT_DISKS,
|
||||
errs.iter().filter(|err| matches!(err, Some(DiskError::FileCorrupt))).count()
|
||||
errs.iter()
|
||||
.filter(|err| {
|
||||
match err {
|
||||
Some(DiskError::Io(err)) => {
|
||||
err.kind() == std::io::ErrorKind::InvalidData && err.to_string().contains("bitrot")
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
})
|
||||
.count()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -587,9 +587,7 @@ pub(crate) async fn verify_deep_parts(
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
let part_status = statuses.get_mut(&part_index).ok_or(DiskError::FileCorrupt)?;
|
||||
// Deep verification must consume every encoded stripe, including a
|
||||
// partial final stripe, before certifying the shard's integrity.
|
||||
let length = usize::try_from(erasure.shard_file_size(part.size as i64)).map_err(|_| DiskError::FileCorrupt)?;
|
||||
let length = erasure.shard_file_offset(0, part.size, part.size);
|
||||
let mut readers = Vec::with_capacity(disks.len());
|
||||
for (index, disk) in disks.iter().enumerate() {
|
||||
if part_status[index] != CHECK_PART_UNKNOWN {
|
||||
@@ -633,9 +631,6 @@ pub(crate) async fn verify_deep_parts(
|
||||
part_status[index] = CHECK_PART_FILE_CORRUPT;
|
||||
}
|
||||
Err(error) => {
|
||||
if !matches!(error, DiskError::FileNotFound | DiskError::FileVersionNotFound | DiskError::FileCorrupt) {
|
||||
return Err(error);
|
||||
}
|
||||
readers.push(None);
|
||||
part_status[index] = conv_part_err_to_int(&Some(error));
|
||||
}
|
||||
|
||||
@@ -41,7 +41,7 @@ impl Debug for NamespaceLockFence {
|
||||
}
|
||||
|
||||
impl NamespaceLockFence {
|
||||
pub(crate) fn new() -> Self {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
signals: Arc::default(),
|
||||
#[cfg(test)]
|
||||
@@ -153,7 +153,7 @@ pub struct ObjectLockConfigSnapshot {
|
||||
state: crate::bucket::metadata_sys::ObjectLockConfigState,
|
||||
lifecycle_fence: NamespaceLockFence,
|
||||
_lifecycle_guard: Option<rustfs_lock::NamespaceLockGuard>,
|
||||
metadata_transaction_guard: Option<Arc<rustfs_lock::NamespaceLockGuard>>,
|
||||
metadata_transaction_guard: Option<rustfs_lock::NamespaceLockGuard>,
|
||||
}
|
||||
|
||||
impl ObjectLockConfigSnapshot {
|
||||
@@ -170,7 +170,6 @@ impl ObjectLockConfigSnapshot {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn for_store_bucket(
|
||||
store_id: Uuid,
|
||||
bucket: &str,
|
||||
@@ -211,7 +210,7 @@ impl ObjectLockConfigSnapshot {
|
||||
state,
|
||||
lifecycle_fence,
|
||||
_lifecycle_guard: Some(lifecycle_guard),
|
||||
metadata_transaction_guard: Some(Arc::new(metadata_transaction_guard)),
|
||||
metadata_transaction_guard: Some(metadata_transaction_guard),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -222,7 +221,7 @@ impl ObjectLockConfigSnapshot {
|
||||
config_revision: OffsetDateTime,
|
||||
state: crate::bucket::metadata_sys::ObjectLockConfigState,
|
||||
lifecycle_fence: NamespaceLockFence,
|
||||
metadata_transaction_guard: Arc<rustfs_lock::NamespaceLockGuard>,
|
||||
metadata_transaction_guard: rustfs_lock::NamespaceLockGuard,
|
||||
) -> Self {
|
||||
Self {
|
||||
store_id: Some(store_id),
|
||||
@@ -266,23 +265,6 @@ impl ObjectLockConfigSnapshot {
|
||||
.is_some_and(|guard| !guard.is_lock_lost())
|
||||
}
|
||||
|
||||
/// Share the held transaction lock without queuing another reader behind
|
||||
/// a metadata writer that is itself waiting for this snapshot to drop.
|
||||
pub(crate) fn metadata_transaction_guard_for(
|
||||
&self,
|
||||
store_id: Uuid,
|
||||
bucket: &str,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
) -> Option<Arc<rustfs_lock::NamespaceLockGuard>> {
|
||||
let bucket_incarnation_id = self.bucket_incarnation_id?;
|
||||
if expected_incarnation_id.is_some_and(|expected| expected != bucket_incarnation_id)
|
||||
|| !self.is_valid_for_destructive_put(store_id, bucket, bucket_incarnation_id)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
self.metadata_transaction_guard.clone()
|
||||
}
|
||||
|
||||
pub(crate) fn add_lock_fences(&self, opts: &mut ObjectOptions) {
|
||||
opts.bucket_lifecycle_lock_fence
|
||||
.get_or_insert_with(NamespaceLockFence::new)
|
||||
@@ -935,8 +917,6 @@ pub struct ObjectOptions {
|
||||
/// Persisted bucket incarnation observed before authorization.
|
||||
pub expected_bucket_incarnation_id: Option<Uuid>,
|
||||
pub no_lock: bool,
|
||||
/// Internal read-only inspection must not enqueue metadata or payload repairs.
|
||||
pub suppress_read_repair: bool,
|
||||
/// Control-plane writers that immediately read or CAS the same namespace
|
||||
/// key use TailDrained without changing namespace lock ownership.
|
||||
#[doc(hidden)]
|
||||
|
||||
@@ -1,290 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
//! Explicit, bounded integrity operations. This service does not participate in ordinary GET or Heal.
|
||||
//! Lock order: cluster worker lock, bucket incarnation fence, object commit lock.
|
||||
//! Control requests use checkpoint CAS without taking the worker lock; a worker that loses CAS stops.
|
||||
|
||||
mod model;
|
||||
mod runner;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
pub use model::{
|
||||
InventoryItem, InventoryPage, ItemRequest, ItemResult, ItemState, Job, JobMode, JobRequest, JobState, Protection, Readiness,
|
||||
readiness,
|
||||
};
|
||||
|
||||
use crate::config::com::{read_config_limited_preserve_empty_with_metadata, save_config_with_opts_quiet};
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::Error as StorageError;
|
||||
use crate::object_api::{ObjectOptions, WriteCompletion};
|
||||
use crate::storage_api_contracts::{
|
||||
list::ListOperations,
|
||||
namespace::NamespaceLocking,
|
||||
object::{HTTPPreconditions, ObjectOperations},
|
||||
};
|
||||
use crate::store::ECStore;
|
||||
use model::InventoryItem as Item;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use uuid::Uuid;
|
||||
|
||||
type Result<T> = std::result::Result<T, IntegrityError>;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum IntegrityError {
|
||||
#[error("invalid integrity request: {0}")]
|
||||
Invalid(&'static str),
|
||||
#[error("integrity job or target changed concurrently")]
|
||||
Conflict,
|
||||
#[error("an integrity worker is already active")]
|
||||
Busy,
|
||||
#[error("integrity job not found")]
|
||||
NotFound,
|
||||
#[error("integrity operation was stopped")]
|
||||
Stopped,
|
||||
#[error("protected writes are not enabled on this coordinator")]
|
||||
NotActivated,
|
||||
#[error("unsupported migration bucket configuration")]
|
||||
UnsupportedBucket,
|
||||
#[error("integrity storage operation failed: {0}")]
|
||||
Storage(#[from] StorageError),
|
||||
#[error("integrity stream operation failed: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
#[error("integrity checkpoint is malformed: {0}")]
|
||||
Json(#[from] serde_json::Error),
|
||||
}
|
||||
|
||||
fn check_bucket(bucket: &str) -> Result<()> {
|
||||
crate::bucket::utils::check_valid_bucket_name_strict(bucket).map_err(|_| IntegrityError::Invalid("invalid source bucket"))
|
||||
}
|
||||
|
||||
fn path(bucket: &str, id: Uuid) -> String {
|
||||
format!("buckets/{bucket}/integrity-jobs/{id}.json")
|
||||
}
|
||||
|
||||
pub async fn inventory(
|
||||
api: Arc<ECStore>,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
key_marker: Option<String>,
|
||||
version_marker: Option<String>,
|
||||
limit: i32,
|
||||
) -> Result<InventoryPage> {
|
||||
check_bucket(bucket)?;
|
||||
if !(1..=100).contains(&limit) || prefix.len() > 1024 || (version_marker.is_some() && key_marker.is_none()) {
|
||||
return Err(IntegrityError::Invalid("invalid inventory page"));
|
||||
}
|
||||
let incarnation = api.bucket_incarnation_id_from_disk(bucket).await?;
|
||||
let page = api
|
||||
.clone()
|
||||
.list_object_versions(bucket, prefix, key_marker.clone(), version_marker.clone(), None, limit)
|
||||
.await?;
|
||||
if page.is_truncated && (page.next_marker.clone(), page.next_version_idmarker.clone()) == (key_marker, version_marker) {
|
||||
return Err(IntegrityError::Invalid("inventory cursor did not advance"));
|
||||
}
|
||||
let mut items = Vec::with_capacity(page.objects.len());
|
||||
for listed in page.objects {
|
||||
if listed.delete_marker {
|
||||
items.push(Item::from_info(&listed));
|
||||
continue;
|
||||
}
|
||||
let opts = ObjectOptions {
|
||||
version_id: Some(listed.version_id.unwrap_or_else(Uuid::nil).to_string()),
|
||||
include_part_checksums: true,
|
||||
suppress_read_repair: true,
|
||||
..Default::default()
|
||||
};
|
||||
match api.get_object_info(bucket, &listed.name, &opts).await {
|
||||
Ok(info) => items.push(Item::from_info(&info)),
|
||||
Err(_) => {
|
||||
let mut item = Item::from_info(&listed);
|
||||
item.observation_error = Some("metadata_unavailable_or_changed".into());
|
||||
item.protection = Protection::Unknown;
|
||||
item.audit_unsupported = Some("metadata_unavailable_or_changed".into());
|
||||
items.push(item);
|
||||
}
|
||||
}
|
||||
}
|
||||
if api.bucket_incarnation_id_from_disk(bucket).await? != incarnation {
|
||||
return Err(IntegrityError::Conflict);
|
||||
}
|
||||
Ok(InventoryPage {
|
||||
observed_at: time::OffsetDateTime::now_utc().to_string(),
|
||||
bucket_incarnation: incarnation,
|
||||
items,
|
||||
is_truncated: page.is_truncated,
|
||||
next_key_marker: page.next_marker,
|
||||
next_version_marker: page.next_version_idmarker,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn create_job(api: Arc<ECStore>, bucket: &str, request: JobRequest) -> Result<Job> {
|
||||
check_bucket(bucket)?;
|
||||
request.validate()?;
|
||||
if request.mode == JobMode::Migrate && !readiness().new_writes_enabled_here {
|
||||
return Err(IntegrityError::NotActivated);
|
||||
}
|
||||
let incarnation = api.bucket_incarnation_id_from_disk(bucket).await?;
|
||||
let job = Job {
|
||||
format_version: 1,
|
||||
id: Uuid::new_v4(),
|
||||
bucket: bucket.into(),
|
||||
bucket_incarnation: incarnation,
|
||||
results: vec![ItemResult::default(); request.items.len()],
|
||||
request,
|
||||
state: JobState::Paused,
|
||||
revision: 0,
|
||||
last_error: None,
|
||||
};
|
||||
save(&api, &job, None, None).await?;
|
||||
Ok(job)
|
||||
}
|
||||
|
||||
pub async fn get_job(api: Arc<ECStore>, bucket: &str, id: Uuid) -> Result<Job> {
|
||||
Ok(load(&api, bucket, id).await?.0)
|
||||
}
|
||||
|
||||
async fn load(api: &Arc<ECStore>, bucket: &str, id: Uuid) -> Result<(Job, String)> {
|
||||
check_bucket(bucket)?;
|
||||
let (bytes, info) = match read_config_limited_preserve_empty_with_metadata(api.clone(), &path(bucket, id), 1024 * 1024).await
|
||||
{
|
||||
Ok(value) => value,
|
||||
Err(StorageError::ConfigNotFound | StorageError::FileNotFound) => return Err(IntegrityError::NotFound),
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
let job: Job = serde_json::from_slice(&bytes)?;
|
||||
job.request.validate()?;
|
||||
if job.format_version != 1
|
||||
|| job.bucket != bucket
|
||||
|| job.id != id
|
||||
|| job.results.len() != job.request.items.len()
|
||||
|| api.bucket_incarnation_id_from_disk(bucket).await? != job.bucket_incarnation
|
||||
{
|
||||
return Err(IntegrityError::Conflict);
|
||||
}
|
||||
Ok((job, info.etag.ok_or(IntegrityError::Conflict)?))
|
||||
}
|
||||
|
||||
async fn save(
|
||||
api: &Arc<ECStore>,
|
||||
job: &Job,
|
||||
etag: Option<&str>,
|
||||
worker: Option<&rustfs_lock::NamespaceLockGuard>,
|
||||
) -> Result<String> {
|
||||
let fence = api
|
||||
.acquire_bucket_incarnation_fence(&job.bucket, job.bucket_incarnation)
|
||||
.await?;
|
||||
let mut opts = ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(match etag {
|
||||
Some(value) => HTTPPreconditions {
|
||||
if_match: Some(value.into()),
|
||||
..Default::default()
|
||||
},
|
||||
None => HTTPPreconditions {
|
||||
if_none_match: Some("*".into()),
|
||||
..Default::default()
|
||||
},
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
fence.attach_to_object_options(&mut opts);
|
||||
if let Some(guard) = worker {
|
||||
opts.add_namespace_lock_guard(guard);
|
||||
}
|
||||
let api = api.clone();
|
||||
let job = job.clone();
|
||||
// Keep the lifecycle guard through the drained commit even if the HTTP waiter is cancelled.
|
||||
tokio::spawn(async move {
|
||||
save_config_with_opts_quiet(api.clone(), &path(&job.bucket, job.id), serde_json::to_vec(&job)?, &opts).await?;
|
||||
let (stored, next) = load(&api, &job.bucket, job.id).await?;
|
||||
if stored.revision != job.revision {
|
||||
return Err(IntegrityError::Conflict);
|
||||
}
|
||||
drop(fence);
|
||||
Ok(next)
|
||||
})
|
||||
.await
|
||||
.map_err(|_| IntegrityError::Invalid("checkpoint commit task failed"))?
|
||||
}
|
||||
|
||||
/// Pause/cancel acknowledge intent. An in-flight conditional publication is allowed to drain.
|
||||
pub async fn control_job(api: Arc<ECStore>, bucket: &str, id: Uuid, cancel: bool) -> Result<Job> {
|
||||
let (mut job, etag) = load(&api, bucket, id).await?;
|
||||
match job.state {
|
||||
JobState::Complete | JobState::Cancelled | JobState::CancelRequested => return Ok(job),
|
||||
JobState::Paused | JobState::Failed => job.state = if cancel { JobState::Cancelled } else { JobState::Paused },
|
||||
_ => {
|
||||
job.state = if cancel {
|
||||
JobState::CancelRequested
|
||||
} else {
|
||||
JobState::PauseRequested
|
||||
}
|
||||
}
|
||||
}
|
||||
job.revision = job.revision.checked_add(1).ok_or(IntegrityError::Conflict)?;
|
||||
save(&api, &job, Some(&etag), None).await?;
|
||||
Ok(job)
|
||||
}
|
||||
|
||||
/// Resume is explicit after restart. The distributed lock excludes another worker, including another node.
|
||||
pub async fn resume_job(api: Arc<ECStore>, bucket: &str, id: Uuid) -> Result<Job> {
|
||||
check_bucket(bucket)?;
|
||||
let lock = api.new_ns_lock(RUSTFS_META_BUCKET, "integrity-jobs/worker.lock").await?;
|
||||
let guard = lock
|
||||
.get_write_lock_quiet(Duration::from_secs(1))
|
||||
.await
|
||||
.map_err(|_| IntegrityError::Busy)?;
|
||||
let (mut job, etag) = load(&api, bucket, id).await?;
|
||||
if matches!(job.state, JobState::Complete | JobState::Cancelled) {
|
||||
return Ok(job);
|
||||
}
|
||||
if job.state == JobState::CancelRequested {
|
||||
job.state = JobState::Cancelled;
|
||||
job.revision = job.revision.checked_add(1).ok_or(IntegrityError::Conflict)?;
|
||||
save(&api, &job, Some(&etag), Some(&guard)).await?;
|
||||
return Ok(job);
|
||||
}
|
||||
if job.request.mode == JobMode::Migrate && !readiness().new_writes_enabled_here {
|
||||
return Err(IntegrityError::NotActivated);
|
||||
}
|
||||
job.state = JobState::Running;
|
||||
job.last_error = None;
|
||||
job.revision = job.revision.checked_add(1).ok_or(IntegrityError::Conflict)?;
|
||||
let etag = save(&api, &job, Some(&etag), Some(&guard)).await?;
|
||||
let response = job.clone();
|
||||
tokio::spawn(async move {
|
||||
let result = runner::run(&api, &mut job, etag, &guard).await;
|
||||
if let Err(error) = result {
|
||||
// A failed write/CAS stops publication. Never overwrite a concurrent control request.
|
||||
if let Ok((mut latest, etag)) = load(&api, &job.bucket, job.id).await {
|
||||
match latest.state {
|
||||
JobState::PauseRequested => latest.state = JobState::Paused,
|
||||
JobState::CancelRequested => latest.state = JobState::Cancelled,
|
||||
JobState::Running if latest.revision <= job.revision => {
|
||||
latest.state = JobState::Failed;
|
||||
latest.last_error = Some(
|
||||
match error {
|
||||
IntegrityError::Conflict => "concurrent_change",
|
||||
IntegrityError::Stopped => "worker_stopped",
|
||||
IntegrityError::UnsupportedBucket => "unsupported_destination_bucket_configuration",
|
||||
IntegrityError::NotActivated => "protected_writes_not_activated",
|
||||
_ => "operation_failed_retry_requires_resume",
|
||||
}
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
_ => return,
|
||||
}
|
||||
if let Some(revision) = latest.revision.checked_add(1) {
|
||||
latest.revision = revision;
|
||||
let _ = save(&api, &latest, Some(&etag), Some(&guard)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
Ok(response)
|
||||
}
|
||||
@@ -1,343 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::{IntegrityError, Result};
|
||||
use crate::object_api::ObjectInfo;
|
||||
use rustfs_rio::{Checksum, read_checksums};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::{BTreeMap, HashSet};
|
||||
use uuid::Uuid;
|
||||
|
||||
pub const MAX_ITEMS: usize = 64;
|
||||
pub const MAX_OBJECT_BYTES: u64 = 5 * 1024 * 1024 * 1024;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ItemRequest {
|
||||
pub key: String,
|
||||
pub version_id: Option<String>,
|
||||
/// Base64 SHA-256 supplied by the administrator from an independent source.
|
||||
pub expected_sha256: Option<String>,
|
||||
pub target_key: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum JobMode {
|
||||
Audit,
|
||||
Migrate,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct JobRequest {
|
||||
pub mode: JobMode,
|
||||
pub items: Vec<ItemRequest>,
|
||||
/// Bounds each source read, destination PUT, and verification stream; one worker runs per cluster.
|
||||
pub bytes_per_second: u64,
|
||||
pub max_object_bytes: u64,
|
||||
}
|
||||
|
||||
impl JobRequest {
|
||||
pub(crate) fn validate(&self) -> Result<()> {
|
||||
if self.items.is_empty() || self.items.len() > MAX_ITEMS {
|
||||
return Err(IntegrityError::Invalid("a job requires 1..=64 explicit source versions"));
|
||||
}
|
||||
if !(64 * 1024..=1024 * 1024 * 1024).contains(&self.bytes_per_second)
|
||||
|| self.max_object_bytes == 0
|
||||
|| self.max_object_bytes > MAX_OBJECT_BYTES
|
||||
{
|
||||
return Err(IntegrityError::Invalid("invalid byte rate or staging limit"));
|
||||
}
|
||||
let mut targets = HashSet::new();
|
||||
let sources: HashSet<_> = self.items.iter().map(|i| i.key.as_str()).collect();
|
||||
for item in &self.items {
|
||||
validate_key(&item.key)?;
|
||||
if let Some(version) = &item.version_id
|
||||
&& version != "null"
|
||||
&& Uuid::parse_str(version).ok().is_none_or(|id| id.is_nil())
|
||||
{
|
||||
return Err(IntegrityError::Invalid("invalid source version"));
|
||||
}
|
||||
if let Some(value) = &item.expected_sha256 {
|
||||
sha256_bytes(value)?;
|
||||
}
|
||||
match (self.mode, &item.target_key) {
|
||||
(JobMode::Audit, None) => {}
|
||||
(JobMode::Migrate, Some(target)) => {
|
||||
validate_key(target)?;
|
||||
if sources.contains(target.as_str()) || !targets.insert(target) {
|
||||
return Err(IntegrityError::Invalid("targets must be distinct from every source and target"));
|
||||
}
|
||||
}
|
||||
_ => return Err(IntegrityError::Invalid("only migrations require an explicit target key")),
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_key(key: &str) -> Result<()> {
|
||||
if key.is_empty() || key.len() > 1024 || key.ends_with('/') || key.contains('\0') {
|
||||
return Err(IntegrityError::Invalid("invalid object key"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn sha256_bytes(value: &str) -> Result<Vec<u8>> {
|
||||
let raw = base64_simd::STANDARD
|
||||
.decode_to_vec(value)
|
||||
.map_err(|_| IntegrityError::Invalid("SHA256 must be canonical Base64"))?;
|
||||
if raw.len() != 32 || base64_simd::STANDARD.encode_to_string(&raw) != value {
|
||||
return Err(IntegrityError::Invalid("SHA256 must be canonical Base64 of 32 bytes"));
|
||||
}
|
||||
Ok(raw)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum Protection {
|
||||
Unknown,
|
||||
Legacy,
|
||||
IndependentCommitment,
|
||||
InvalidDeclaration,
|
||||
NoPayload,
|
||||
}
|
||||
|
||||
pub(crate) fn protection(info: &ObjectInfo) -> Protection {
|
||||
if info.delete_marker {
|
||||
return Protection::NoPayload;
|
||||
}
|
||||
match rustfs_filemeta::shard_integrity::descriptor_from_metadata(&info.user_defined) {
|
||||
Ok(None) if info.parts.iter().all(|p| p.integrity.is_none()) => Protection::Legacy,
|
||||
Ok(Some(parts))
|
||||
if parts.len() == info.parts.len()
|
||||
&& !parts.is_empty()
|
||||
&& parts
|
||||
.iter()
|
||||
.zip(info.parts.iter())
|
||||
.all(|(declared, part)| part.integrity.as_ref() == Some(declared)) =>
|
||||
{
|
||||
Protection::IndependentCommitment
|
||||
}
|
||||
_ => Protection::InvalidDeclaration,
|
||||
}
|
||||
}
|
||||
|
||||
/// Checksums shown in inventory are not automatically accepted as migration evidence.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct InventoryItem {
|
||||
pub key: String,
|
||||
/// S3 selector, including the literal "null" for the null version slot.
|
||||
pub version_id: String,
|
||||
pub data_dir: Option<Uuid>,
|
||||
pub size: i64,
|
||||
pub protection: Protection,
|
||||
pub checksums: BTreeMap<String, String>,
|
||||
pub checksum_is_multipart: bool,
|
||||
pub audit_unsupported: Option<String>,
|
||||
pub source_fingerprint: String,
|
||||
pub observation_error: Option<String>,
|
||||
}
|
||||
|
||||
impl InventoryItem {
|
||||
pub(crate) fn from_info(info: &ObjectInfo) -> Self {
|
||||
let (checksums, checksum_is_multipart) = info.checksum.as_ref().map_or_else(Default::default, |b| read_checksums(b, 0));
|
||||
Self {
|
||||
key: info.name.clone(),
|
||||
version_id: info
|
||||
.version_id
|
||||
.filter(|id| !id.is_nil())
|
||||
.map_or_else(|| "null".into(), |id| id.to_string()),
|
||||
data_dir: info.data_dir,
|
||||
size: info.size,
|
||||
protection: protection(info),
|
||||
checksums: checksums.into_iter().collect(),
|
||||
checksum_is_multipart,
|
||||
audit_unsupported: unsupported(info).map(str::to_owned),
|
||||
source_fingerprint: fingerprint(info),
|
||||
observation_error: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn unsupported(info: &ObjectInfo) -> Option<&'static str> {
|
||||
if info.delete_marker {
|
||||
Some("delete_marker")
|
||||
} else if !info.transitioned_object.status.is_empty() {
|
||||
Some("transitioned_object")
|
||||
} else if info.is_encrypted() || info.is_compressed() {
|
||||
Some("transformed_object")
|
||||
} else if info.is_multipart() {
|
||||
Some("multipart_object")
|
||||
} else if protection(info) == Protection::InvalidDeclaration {
|
||||
Some("invalid_protection_declaration")
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn stored_sha256(info: &ObjectInfo) -> Option<String> {
|
||||
let bytes = info.checksum.as_ref()?;
|
||||
let (sums, multipart) = read_checksums(bytes, 0);
|
||||
let value = sums.get("SHA256")?;
|
||||
let canonical = Checksum::new_from_string("SHA256", value)?.to_bytes(&[]);
|
||||
// The general display decoder tolerates truncated suffixes. Eligibility does not.
|
||||
(!multipart && canonical.as_ref() == bytes.as_ref()).then(|| value.clone())
|
||||
}
|
||||
|
||||
pub(crate) fn fingerprint(info: &ObjectInfo) -> String {
|
||||
let mut hash = Sha256::new();
|
||||
// Ordered metadata includes transformation and protection declarations as well as user metadata.
|
||||
for value in [
|
||||
info.bucket.clone(),
|
||||
info.name.clone(),
|
||||
format!("{:?}", info.version_id),
|
||||
format!("{:?}", info.data_dir),
|
||||
format!("{:?}", info.mod_time),
|
||||
info.size.to_string(),
|
||||
info.actual_size.to_string(),
|
||||
format!("{:?}", info.etag),
|
||||
format!("{:?}", info.checksum),
|
||||
format!("{:?}", info.transitioned_object),
|
||||
info.user_tags.to_string(),
|
||||
format!("{:?}", info.expires),
|
||||
format!("{:?}", info.content_type),
|
||||
format!("{:?}", info.content_encoding),
|
||||
format!("{:?}", info.storage_class),
|
||||
] {
|
||||
hash.update(value.len().to_le_bytes());
|
||||
hash.update(value.as_bytes());
|
||||
}
|
||||
for part in info.parts.iter() {
|
||||
let values = format!(
|
||||
"{}:{}:{}:{:?}:{:?}:{:?}:{:?}",
|
||||
part.number, part.size, part.actual_size, part.etag, part.mod_time, part.index, part.integrity
|
||||
);
|
||||
hash.update(values.len().to_le_bytes());
|
||||
hash.update(values.as_bytes());
|
||||
if let Some(checksums) = &part.checksums {
|
||||
let mut checksums: Vec<_> = checksums.iter().collect();
|
||||
checksums.sort_unstable_by_key(|(key, _)| *key);
|
||||
for (key, value) in checksums {
|
||||
hash.update(key.len().to_le_bytes());
|
||||
hash.update(key.as_bytes());
|
||||
hash.update(value.len().to_le_bytes());
|
||||
hash.update(value.as_bytes());
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut metadata: Vec<_> = info.user_defined.iter().collect();
|
||||
metadata.sort_unstable_by_key(|(key, _)| *key);
|
||||
for (key, value) in metadata {
|
||||
hash.update(key.len().to_le_bytes());
|
||||
hash.update(key.as_bytes());
|
||||
hash.update(value.len().to_le_bytes());
|
||||
hash.update(value.as_bytes());
|
||||
}
|
||||
hex_simd::encode_to_string(hash.finalize(), hex_simd::AsciiCase::Lower)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum JobState {
|
||||
Paused,
|
||||
Running,
|
||||
PauseRequested,
|
||||
CancelRequested,
|
||||
Cancelled,
|
||||
Complete,
|
||||
Failed,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ItemState {
|
||||
Pending,
|
||||
Prepared,
|
||||
Verified,
|
||||
Migrated,
|
||||
Unsupported,
|
||||
Unavailable,
|
||||
Mismatch,
|
||||
Stale,
|
||||
Conflict,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ItemResult {
|
||||
pub state: ItemState,
|
||||
pub source_fingerprint: Option<String>,
|
||||
pub expected_sha256: Option<String>,
|
||||
pub evidence_source: Option<String>,
|
||||
pub source_size: Option<u64>,
|
||||
pub target_version_id: Option<Uuid>,
|
||||
pub detail: Option<String>,
|
||||
}
|
||||
|
||||
impl Default for ItemResult {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
state: ItemState::Pending,
|
||||
source_fingerprint: None,
|
||||
expected_sha256: None,
|
||||
evidence_source: None,
|
||||
source_size: None,
|
||||
target_version_id: None,
|
||||
detail: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Job {
|
||||
pub format_version: u32,
|
||||
pub id: Uuid,
|
||||
pub bucket: String,
|
||||
pub bucket_incarnation: Uuid,
|
||||
pub request: JobRequest,
|
||||
pub state: JobState,
|
||||
pub revision: u64,
|
||||
pub results: Vec<ItemResult>,
|
||||
pub last_error: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct InventoryPage {
|
||||
pub observed_at: String,
|
||||
pub bucket_incarnation: Uuid,
|
||||
pub items: Vec<InventoryItem>,
|
||||
pub is_truncated: bool,
|
||||
pub next_key_marker: Option<String>,
|
||||
pub next_version_marker: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct Readiness {
|
||||
pub local_reader_supported: bool,
|
||||
pub write_requested: bool,
|
||||
pub fleet_operator_attested: bool,
|
||||
pub new_writes_enabled_here: bool,
|
||||
pub fleet_capability_verified: bool,
|
||||
pub unverified_requirements: [&'static str; 3],
|
||||
}
|
||||
|
||||
pub fn readiness() -> Readiness {
|
||||
let write = rustfs_utils::get_env_bool(rustfs_config::ENV_SHARD_INTEGRITY_WRITE, false);
|
||||
let fleet = rustfs_utils::get_env_bool(rustfs_config::ENV_SHARD_INTEGRITY_FLEET_CONFIRMED, false);
|
||||
Readiness {
|
||||
local_reader_supported: true,
|
||||
write_requested: write,
|
||||
fleet_operator_attested: fleet,
|
||||
new_writes_enabled_here: write && fleet,
|
||||
fleet_capability_verified: false,
|
||||
unverified_requirements: [
|
||||
"peer_reader_writer_and_repair_capabilities",
|
||||
"old_process_storage_admission",
|
||||
"deployment_failure_and_performance_qualification",
|
||||
],
|
||||
}
|
||||
}
|
||||
@@ -1,468 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::model::{Protection, fingerprint, protection, sha256_bytes, stored_sha256, unsupported};
|
||||
use super::{IntegrityError, ItemState, Job, JobMode, JobState, Result, load, readiness, save};
|
||||
use crate::object_api::{
|
||||
ObjectInfo, ObjectOptions, PutObjReader, ShardIntegrityWriteMode, WriteCompletion, without_get_object_body_cache_hook,
|
||||
};
|
||||
use crate::storage_api_contracts::object::{HTTPPreconditions, ObjectIO, ObjectOperations};
|
||||
use crate::store::ECStore;
|
||||
use rustfs_rio::{Checksum, HashReader};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::{collections::HashMap, sync::Arc, time::Duration};
|
||||
use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
|
||||
|
||||
const MARKER: &str = "integrity-migration-job-v1";
|
||||
|
||||
pub(super) async fn run(
|
||||
api: &Arc<ECStore>,
|
||||
job: &mut Job,
|
||||
mut etag: String,
|
||||
guard: &rustfs_lock::NamespaceLockGuard,
|
||||
) -> Result<()> {
|
||||
without_get_object_body_cache_hook(async {
|
||||
for index in 0..job.results.len() {
|
||||
ensure_running(api, job, guard).await?;
|
||||
if !matches!(
|
||||
job.results[index].state,
|
||||
ItemState::Pending | ItemState::Prepared | ItemState::Unavailable
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
if job.request.mode == JobMode::Migrate {
|
||||
validate_destination(api, job).await?;
|
||||
}
|
||||
if job.results[index].state == ItemState::Prepared
|
||||
&& job.request.mode == JobMode::Migrate
|
||||
&& reconcile_target(api, job, index, guard).await?
|
||||
{
|
||||
etag = checkpoint(api, job, &etag, guard).await?;
|
||||
continue;
|
||||
}
|
||||
let request = job.request.items[index].clone();
|
||||
let opts = ObjectOptions {
|
||||
version_id: request.version_id.as_ref().map(|version| {
|
||||
if version == "null" {
|
||||
uuid::Uuid::nil().to_string()
|
||||
} else {
|
||||
version.clone()
|
||||
}
|
||||
}),
|
||||
include_part_checksums: true,
|
||||
suppress_read_repair: true,
|
||||
..Default::default()
|
||||
};
|
||||
let info = match api.get_object_info(&job.bucket, &request.key, &opts).await {
|
||||
Ok(info) => info,
|
||||
Err(_) => {
|
||||
job.results[index].state = ItemState::Unavailable;
|
||||
job.results[index].detail = Some("source_metadata_unavailable".into());
|
||||
etag = checkpoint(api, job, &etag, guard).await?;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if let Some(reason) = unsupported(&info) {
|
||||
job.results[index].state = ItemState::Unsupported;
|
||||
job.results[index].detail = Some(reason.into());
|
||||
etag = checkpoint(api, job, &etag, guard).await?;
|
||||
continue;
|
||||
}
|
||||
let identity = fingerprint(&info);
|
||||
if job.results[index]
|
||||
.source_fingerprint
|
||||
.as_ref()
|
||||
.is_some_and(|old| old != &identity)
|
||||
{
|
||||
job.results[index].state = ItemState::Stale;
|
||||
etag = checkpoint(api, job, &etag, guard).await?;
|
||||
continue;
|
||||
}
|
||||
let expected = request.expected_sha256.clone().or_else(|| stored_sha256(&info));
|
||||
let Some(expected) = expected else {
|
||||
job.results[index].state = ItemState::Unsupported;
|
||||
job.results[index].detail = Some("no_canonical_single_part_sha256".into());
|
||||
etag = checkpoint(api, job, &etag, guard).await?;
|
||||
continue;
|
||||
};
|
||||
let size = u64::try_from(info.size).map_err(|_| IntegrityError::Invalid("negative source size"))?;
|
||||
if size > job.request.max_object_bytes {
|
||||
job.results[index].state = ItemState::Unsupported;
|
||||
job.results[index].detail = Some("object_exceeds_staging_budget".into());
|
||||
etag = checkpoint(api, job, &etag, guard).await?;
|
||||
continue;
|
||||
}
|
||||
let result = &mut job.results[index];
|
||||
result.source_fingerprint = Some(identity);
|
||||
result.expected_sha256 = Some(expected.clone());
|
||||
result.evidence_source = Some(
|
||||
if request.expected_sha256.is_some() {
|
||||
"administrator_supplied_sha256"
|
||||
} else {
|
||||
"stored_sha256"
|
||||
}
|
||||
.into(),
|
||||
);
|
||||
result.source_size = Some(size);
|
||||
result.state = ItemState::Prepared;
|
||||
result.detail = None;
|
||||
etag = checkpoint(api, job, &etag, guard).await?;
|
||||
let mut file = match stage(api, job, index, &info, &opts, guard).await {
|
||||
Ok(file) => file,
|
||||
Err(IntegrityError::Conflict) => {
|
||||
job.results[index].state = ItemState::Stale;
|
||||
etag = checkpoint(api, job, &etag, guard).await?;
|
||||
continue;
|
||||
}
|
||||
Err(IntegrityError::Invalid("content_sha256_mismatch")) => {
|
||||
job.results[index].state = ItemState::Mismatch;
|
||||
etag = checkpoint(api, job, &etag, guard).await?;
|
||||
continue;
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
ensure_running(api, job, guard).await?;
|
||||
if job.request.mode == JobMode::Audit {
|
||||
job.results[index].state = ItemState::Verified;
|
||||
} else {
|
||||
let snapshot = validate_destination(api, job).await?;
|
||||
file.rewind().await?;
|
||||
let raw = sha256_bytes(&expected)?;
|
||||
let expected_hex = hex_simd::encode_to_string(raw, hex_simd::AsciiCase::Lower);
|
||||
let mut reader = HashReader::from_stream(
|
||||
ThrottledFile::new(file, job.request.bytes_per_second),
|
||||
info.size,
|
||||
info.size,
|
||||
None,
|
||||
Some(expected_hex),
|
||||
false,
|
||||
)?;
|
||||
reader.add_non_trailing_checksum(Checksum::new_from_string("SHA256", &expected), false)?;
|
||||
let mut body = PutObjReader::new(reader);
|
||||
let mut metadata: HashMap<String, String> = info
|
||||
.user_defined
|
||||
.iter()
|
||||
.filter(|(key, _)| {
|
||||
key.starts_with("x-amz-meta-")
|
||||
|| matches!(key.as_str(), "cache-control" | "content-disposition" | "content-language" | "expires")
|
||||
})
|
||||
.map(|(key, value)| (key.clone(), value.clone()))
|
||||
.collect();
|
||||
if let Some(content_type) = &info.content_type {
|
||||
metadata.insert("content-type".into(), content_type.clone());
|
||||
}
|
||||
if let Some(content_encoding) = &info.content_encoding {
|
||||
metadata.insert("content-encoding".into(), content_encoding.clone());
|
||||
}
|
||||
if let Some(expires) = info.expires {
|
||||
metadata.insert(
|
||||
"expires".into(),
|
||||
expires
|
||||
.format(&time::format_description::well_known::Rfc3339)
|
||||
.map_err(|_| IntegrityError::Invalid("invalid source expiration"))?,
|
||||
);
|
||||
}
|
||||
if let Some(storage_class) = &info.storage_class {
|
||||
metadata.insert("x-amz-storage-class".into(), storage_class.clone());
|
||||
}
|
||||
if !info.user_tags.is_empty() {
|
||||
metadata.insert(rustfs_utils::http::headers::AMZ_OBJECT_TAGGING.into(), info.user_tags.to_string());
|
||||
}
|
||||
rustfs_utils::http::insert_str(&mut metadata, MARKER, marker(job, index)?);
|
||||
let mut write = ObjectOptions {
|
||||
user_defined: metadata,
|
||||
preserve_delete_marker: true,
|
||||
shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Protected),
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_none_match: Some("*".into()),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
write.add_namespace_lock_guard(guard);
|
||||
write.expected_bucket_incarnation_id = Some(job.bucket_incarnation);
|
||||
snapshot.add_lock_fences(&mut write);
|
||||
write.object_lock_config_snapshot = Some(snapshot.clone());
|
||||
let target = request
|
||||
.target_key
|
||||
.as_deref()
|
||||
.ok_or(IntegrityError::Invalid("missing target"))?;
|
||||
// Even an error may follow a committed PUT. Reconcile before retrying; never overwrite.
|
||||
let put = api.put_object(&job.bucket, target, &mut body, &write).await;
|
||||
drop(write);
|
||||
drop(snapshot);
|
||||
if !reconcile_target(api, job, index, guard).await? {
|
||||
put?;
|
||||
return Err(IntegrityError::Conflict);
|
||||
}
|
||||
}
|
||||
etag = checkpoint(api, job, &etag, guard).await?;
|
||||
}
|
||||
job.state = if job.results.iter().any(|item| item.state == ItemState::Unavailable) {
|
||||
job.last_error = Some("source_unavailable_retry_requires_resume".into());
|
||||
JobState::Failed
|
||||
} else {
|
||||
JobState::Complete
|
||||
};
|
||||
checkpoint(api, job, &etag, guard).await?;
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn checkpoint(api: &Arc<ECStore>, job: &mut Job, etag: &str, guard: &rustfs_lock::NamespaceLockGuard) -> Result<String> {
|
||||
ensure_running(api, job, guard).await?;
|
||||
job.revision = job.revision.checked_add(1).ok_or(IntegrityError::Conflict)?;
|
||||
save(api, job, Some(etag), Some(guard)).await
|
||||
}
|
||||
|
||||
async fn ensure_running(api: &Arc<ECStore>, job: &Job, guard: &rustfs_lock::NamespaceLockGuard) -> Result<()> {
|
||||
if guard.lock_lost_signal().is_some_and(|signal| signal.is_lost()) {
|
||||
return Err(IntegrityError::Stopped);
|
||||
}
|
||||
let (current, _) = load(api, &job.bucket, job.id).await?;
|
||||
if current.state != JobState::Running {
|
||||
return Err(IntegrityError::Stopped);
|
||||
}
|
||||
if current.revision != job.revision {
|
||||
return Err(IntegrityError::Conflict);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn stage(
|
||||
api: &Arc<ECStore>,
|
||||
job: &Job,
|
||||
index: usize,
|
||||
info: &ObjectInfo,
|
||||
opts: &ObjectOptions,
|
||||
guard: &rustfs_lock::NamespaceLockGuard,
|
||||
) -> Result<tokio::fs::File> {
|
||||
let mut reader = api
|
||||
.get_object_reader(&job.bucket, &job.request.items[index].key, None, Default::default(), opts)
|
||||
.await?;
|
||||
if fingerprint(&reader.object_info) != fingerprint(info) {
|
||||
return Err(IntegrityError::Conflict);
|
||||
}
|
||||
let std_file = tokio::task::spawn_blocking(tempfile::tempfile)
|
||||
.await
|
||||
.map_err(|_| IntegrityError::Invalid("staging task failed"))??;
|
||||
let mut file = tokio::fs::File::from_std(std_file);
|
||||
let expected = job.results[index]
|
||||
.expected_sha256
|
||||
.as_deref()
|
||||
.ok_or(IntegrityError::Invalid("missing expected digest"))?;
|
||||
copy_verified(api, job, &mut reader.stream, Some(&mut file), info.size, expected, guard).await?;
|
||||
file.flush().await?;
|
||||
let after = api.get_object_info(&job.bucket, &job.request.items[index].key, opts).await?;
|
||||
if fingerprint(&after) != fingerprint(info) {
|
||||
return Err(IntegrityError::Conflict);
|
||||
}
|
||||
Ok(file)
|
||||
}
|
||||
|
||||
async fn copy_verified(
|
||||
api: &Arc<ECStore>,
|
||||
job: &Job,
|
||||
input: &mut (dyn tokio::io::AsyncRead + Unpin + Send + Sync),
|
||||
mut output: Option<&mut tokio::fs::File>,
|
||||
size: i64,
|
||||
expected: &str,
|
||||
guard: &rustfs_lock::NamespaceLockGuard,
|
||||
) -> Result<()> {
|
||||
let expected = sha256_bytes(expected)?;
|
||||
let expected_size = u64::try_from(size).map_err(|_| IntegrityError::Invalid("negative object size"))?;
|
||||
let mut buffer = vec![0u8; 64 * 1024];
|
||||
let mut count = 0u64;
|
||||
let mut hash = Sha256::new();
|
||||
let started = tokio::time::Instant::now();
|
||||
let mut checked = started;
|
||||
loop {
|
||||
if checked.elapsed() >= Duration::from_secs(1) {
|
||||
ensure_running(api, job, guard).await?;
|
||||
checked = tokio::time::Instant::now();
|
||||
}
|
||||
let n = tokio::time::timeout(Duration::from_secs(30), input.read(&mut buffer))
|
||||
.await
|
||||
.map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "integrity read timed out"))??;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
count = count
|
||||
.checked_add(u64::try_from(n).map_err(|_| IntegrityError::Invalid("read size overflow"))?)
|
||||
.ok_or(IntegrityError::Invalid("read size overflow"))?;
|
||||
if count > expected_size || count > job.request.max_object_bytes {
|
||||
return Err(IntegrityError::Invalid("content_sha256_mismatch"));
|
||||
}
|
||||
hash.update(&buffer[..n]);
|
||||
if let Some(file) = output.as_deref_mut() {
|
||||
file.write_all(&buffer[..n]).await?;
|
||||
}
|
||||
let delay = Duration::from_secs_f64(count as f64 / job.request.bytes_per_second as f64);
|
||||
tokio::time::sleep_until(started + delay).await;
|
||||
}
|
||||
if count != expected_size || hash.finalize().as_slice() != expected {
|
||||
return Err(IntegrityError::Invalid("content_sha256_mismatch"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn marker(job: &Job, index: usize) -> Result<String> {
|
||||
let result = &job.results[index];
|
||||
Ok(format!(
|
||||
"{}:{}:{}:{}",
|
||||
job.id,
|
||||
index,
|
||||
result
|
||||
.source_fingerprint
|
||||
.as_deref()
|
||||
.ok_or(IntegrityError::Invalid("missing source identity"))?,
|
||||
result
|
||||
.expected_sha256
|
||||
.as_deref()
|
||||
.ok_or(IntegrityError::Invalid("missing source digest"))?
|
||||
))
|
||||
}
|
||||
|
||||
async fn reconcile_target(
|
||||
api: &Arc<ECStore>,
|
||||
job: &mut Job,
|
||||
index: usize,
|
||||
guard: &rustfs_lock::NamespaceLockGuard,
|
||||
) -> Result<bool> {
|
||||
let key = job.request.items[index]
|
||||
.target_key
|
||||
.as_deref()
|
||||
.ok_or(IntegrityError::Invalid("missing target"))?;
|
||||
let opts = ObjectOptions {
|
||||
include_part_checksums: true,
|
||||
suppress_read_repair: true,
|
||||
..Default::default()
|
||||
};
|
||||
let target = match api.get_object_info(&job.bucket, key, &opts).await {
|
||||
Ok(info) => info,
|
||||
Err(crate::error::Error::FileNotFound | crate::error::Error::ObjectNotFound(_, _)) => return Ok(false),
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
let expected_marker = marker(job, index)?;
|
||||
if rustfs_utils::http::get_consistent_str(&target.user_defined, MARKER) != Some(expected_marker.as_str()) {
|
||||
job.results[index].state = ItemState::Conflict;
|
||||
job.results[index].detail = Some("target_owned_by_another_operation".into());
|
||||
return Ok(true);
|
||||
}
|
||||
if protection(&target) != Protection::IndependentCommitment {
|
||||
return Err(IntegrityError::Invalid("target_not_protected"));
|
||||
}
|
||||
let mut reader = api
|
||||
.get_object_reader(&job.bucket, key, None, Default::default(), &opts)
|
||||
.await?;
|
||||
if fingerprint(&target) != fingerprint(&reader.object_info) {
|
||||
return Err(IntegrityError::Conflict);
|
||||
}
|
||||
let size = job.results[index]
|
||||
.source_size
|
||||
.ok_or(IntegrityError::Invalid("missing source size"))?;
|
||||
let expected = job.results[index]
|
||||
.expected_sha256
|
||||
.clone()
|
||||
.ok_or(IntegrityError::Invalid("missing source digest"))?;
|
||||
copy_verified(
|
||||
api,
|
||||
job,
|
||||
&mut reader.stream,
|
||||
None,
|
||||
i64::try_from(size).map_err(|_| IntegrityError::Invalid("source size overflow"))?,
|
||||
&expected,
|
||||
guard,
|
||||
)
|
||||
.await?;
|
||||
let after = api.get_object_info(&job.bucket, key, &opts).await?;
|
||||
if fingerprint(&after) != fingerprint(&target) {
|
||||
return Err(IntegrityError::Conflict);
|
||||
}
|
||||
job.results[index].state = ItemState::Migrated;
|
||||
job.results[index].target_version_id = target.version_id;
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn validate_destination(api: &Arc<ECStore>, job: &Job) -> Result<Arc<crate::object_api::ObjectLockConfigSnapshot>> {
|
||||
if !readiness().new_writes_enabled_here {
|
||||
return Err(IntegrityError::NotActivated);
|
||||
}
|
||||
let snapshot = api.object_lock_config_snapshot(&job.bucket).await?;
|
||||
if snapshot
|
||||
.metadata_transaction_guard_for(api.id, &job.bucket, Some(job.bucket_incarnation))
|
||||
.is_none()
|
||||
{
|
||||
return Err(IntegrityError::Conflict);
|
||||
}
|
||||
let metadata = crate::bucket::metadata_sys::integrity_migration_metadata_in(&api.ctx, &job.bucket).await?;
|
||||
// The direct storage writer is not the S3 orchestration layer. Reject configurations it cannot preserve.
|
||||
if metadata.bucket_incarnation_id != job.bucket_incarnation {
|
||||
return Err(IntegrityError::Conflict);
|
||||
}
|
||||
if !metadata.versioning_config_xml.is_empty()
|
||||
|| !metadata.encryption_config_xml.is_empty()
|
||||
|| !metadata.object_lock_config_xml.is_empty()
|
||||
|| !metadata.replication_config_xml.is_empty()
|
||||
|| !metadata.notification_config_xml.is_empty()
|
||||
|| !metadata.quota_config_json.is_empty()
|
||||
|| !metadata.on_demand_migration_config_json.is_empty()
|
||||
|| !metadata.lifecycle_config_xml.is_empty()
|
||||
|| !metadata.table_bucket_config_json.is_empty()
|
||||
|| !metadata.bucket_acl_config_json.is_empty()
|
||||
|| !metadata.logging_config_xml.is_empty()
|
||||
{
|
||||
return Err(IntegrityError::UnsupportedBucket);
|
||||
}
|
||||
Ok(snapshot)
|
||||
}
|
||||
|
||||
/// Bound publication throughput as well as the source and destination verification reads.
|
||||
struct ThrottledFile {
|
||||
file: tokio::fs::File,
|
||||
started: tokio::time::Instant,
|
||||
count: u64,
|
||||
rate: u64,
|
||||
sleep: std::pin::Pin<Box<tokio::time::Sleep>>,
|
||||
}
|
||||
|
||||
impl ThrottledFile {
|
||||
fn new(file: tokio::fs::File, rate: u64) -> Self {
|
||||
let started = tokio::time::Instant::now();
|
||||
Self {
|
||||
file,
|
||||
started,
|
||||
count: 0,
|
||||
rate,
|
||||
sleep: Box::pin(tokio::time::sleep_until(started)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl tokio::io::AsyncRead for ThrottledFile {
|
||||
fn poll_read(
|
||||
mut self: std::pin::Pin<&mut Self>,
|
||||
cx: &mut std::task::Context<'_>,
|
||||
output: &mut tokio::io::ReadBuf<'_>,
|
||||
) -> std::task::Poll<std::io::Result<()>> {
|
||||
use std::future::Future;
|
||||
if self.sleep.as_mut().poll(cx).is_pending() {
|
||||
return std::task::Poll::Pending;
|
||||
}
|
||||
let capacity = output.remaining().min(64 * 1024);
|
||||
let mut buffer = tokio::io::ReadBuf::new(output.initialize_unfilled_to(capacity));
|
||||
match std::pin::Pin::new(&mut self.file).poll_read(cx, &mut buffer) {
|
||||
std::task::Poll::Ready(Ok(())) => {
|
||||
let count = buffer.filled().len();
|
||||
output.advance(count);
|
||||
self.count += count as u64;
|
||||
let deadline = self.started + Duration::from_secs_f64(self.count as f64 / self.rate as f64);
|
||||
self.sleep.as_mut().reset(deadline);
|
||||
std::task::Poll::Ready(Ok(()))
|
||||
}
|
||||
result => result,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,387 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
use crate::object_api::{ObjectOptions, PutObjReader, ShardIntegrityWriteMode};
|
||||
use crate::storage_api_contracts::{
|
||||
bucket::{BucketOperations, MakeBucketOptions},
|
||||
object::{ObjectIO, ObjectOperations},
|
||||
};
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
async fn fixture() -> (Vec<tempfile::TempDir>, Arc<crate::store::ECStore>, String) {
|
||||
let (dirs, store) = crate::services::rebalance::test_store_with_persisted_rebalance_meta(Default::default()).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("integrity-{}", Uuid::new_v4());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("fixture bucket");
|
||||
(dirs, store, bucket)
|
||||
}
|
||||
|
||||
fn request(mode: JobMode, digest: String) -> JobRequest {
|
||||
JobRequest {
|
||||
mode,
|
||||
items: vec![ItemRequest {
|
||||
key: "source".into(),
|
||||
version_id: Some("null".into()),
|
||||
expected_sha256: Some(digest),
|
||||
target_key: (mode == JobMode::Migrate).then(|| "target".into()),
|
||||
}],
|
||||
bytes_per_second: 1024 * 1024,
|
||||
max_object_bytes: 16 * 1024 * 1024,
|
||||
}
|
||||
}
|
||||
|
||||
async fn finish(store: Arc<crate::store::ECStore>, bucket: &str, id: Uuid) -> Job {
|
||||
tokio::time::timeout(std::time::Duration::from_secs(60), async {
|
||||
loop {
|
||||
let job = get_job(store.clone(), bucket, id).await.expect("durable job");
|
||||
if job.state != JobState::Running {
|
||||
return job;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("job must finish")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(shard_integrity_rollout)]
|
||||
async fn audit_migration_and_ack_recovery_preserve_source_and_reject_conflicts() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_SHARD_INTEGRITY_WRITE, Some("true")),
|
||||
(rustfs_config::ENV_SHARD_INTEGRITY_FLEET_CONFIRMED, Some("true")),
|
||||
],
|
||||
async {
|
||||
let (_dirs, store, bucket) = fixture().await;
|
||||
let payload = vec![42; 256 * 1024 + 17];
|
||||
let expected = rustfs_rio::Checksum::new_from_data(rustfs_rio::ChecksumType::SHA256, &payload)
|
||||
.expect("digest")
|
||||
.encoded;
|
||||
store
|
||||
.put_object(
|
||||
&bucket,
|
||||
"source",
|
||||
&mut PutObjReader::from_vec(payload.clone()),
|
||||
&ObjectOptions {
|
||||
user_defined: [
|
||||
("cache-control".into(), "max-age=300".into()),
|
||||
("content-disposition".into(), "attachment; filename=sample.bin".into()),
|
||||
("expires".into(), "2030-01-01T00:00:00Z".into()),
|
||||
(rustfs_utils::http::headers::AMZ_OBJECT_TAGGING.into(), "kind=archive".into()),
|
||||
]
|
||||
.into(),
|
||||
shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Legacy),
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("source");
|
||||
let source = store
|
||||
.get_object_info(&bucket, "source", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("source metadata");
|
||||
assert_eq!(source.user_tags.as_str(), "kind=archive", "source fixture tags");
|
||||
let before = model::fingerprint(&source);
|
||||
let page = inventory(store.clone(), &bucket, "", None, None, 1).await.expect("inventory");
|
||||
assert_eq!(page.items.len(), 1);
|
||||
assert_eq!(page.items[0].protection, Protection::Legacy);
|
||||
assert_eq!(page.items[0].version_id, "null", "inventory must round-trip an explicit null selector");
|
||||
let audit = create_job(store.clone(), &bucket, request(JobMode::Audit, expected.clone()))
|
||||
.await
|
||||
.expect("create audit");
|
||||
resume_job(store.clone(), &bucket, audit.id).await.expect("resume audit");
|
||||
let audited = finish(store.clone(), &bucket, audit.id).await;
|
||||
assert_eq!(audited.state, JobState::Complete, "{audited:?}");
|
||||
assert_eq!(audited.results[0].state, ItemState::Verified);
|
||||
|
||||
let migration = create_job(store.clone(), &bucket, request(JobMode::Migrate, expected.clone()))
|
||||
.await
|
||||
.expect("create migration");
|
||||
resume_job(store.clone(), &bucket, migration.id)
|
||||
.await
|
||||
.expect("resume migration");
|
||||
let migrated = finish(store.clone(), &bucket, migration.id).await;
|
||||
assert_eq!(migrated.state, JobState::Complete, "{migrated:?}");
|
||||
assert_eq!(migrated.results[0].state, ItemState::Migrated);
|
||||
let mut reader = store
|
||||
.get_object_reader(&bucket, "target", None, Default::default(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("target");
|
||||
assert_eq!(model::protection(&reader.object_info), Protection::IndependentCommitment);
|
||||
assert_eq!(reader.object_info.user_tags.as_str(), "kind=archive");
|
||||
assert_eq!(reader.object_info.expires, source.expires);
|
||||
assert_eq!(
|
||||
reader.object_info.user_defined.get("cache-control"),
|
||||
source.user_defined.get("cache-control")
|
||||
);
|
||||
assert_eq!(
|
||||
reader.object_info.user_defined.get("content-disposition"),
|
||||
source.user_defined.get("content-disposition")
|
||||
);
|
||||
let mut actual = Vec::new();
|
||||
reader.stream.read_to_end(&mut actual).await.expect("target content");
|
||||
assert_eq!(actual, payload);
|
||||
let after = store
|
||||
.get_object_info(&bucket, "source", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("source after");
|
||||
assert_eq!(model::fingerprint(&after), before);
|
||||
|
||||
// Simulate a lost completion checkpoint after the create-only PUT committed.
|
||||
let (mut replay, etag) = load(&store, &bucket, migration.id).await.expect("load receipt");
|
||||
replay.state = JobState::Paused;
|
||||
replay.results[0].state = ItemState::Prepared;
|
||||
replay.revision += 1;
|
||||
save(&store, &replay, Some(&etag), None)
|
||||
.await
|
||||
.expect("lost acknowledgement fixture");
|
||||
resume_job(store.clone(), &bucket, replay.id)
|
||||
.await
|
||||
.expect("reconcile after restart");
|
||||
let reconciled = finish(store.clone(), &bucket, replay.id).await;
|
||||
assert_eq!(reconciled.results[0].state, ItemState::Migrated, "{reconciled:?}");
|
||||
|
||||
let conflict = create_job(store.clone(), &bucket, request(JobMode::Migrate, expected))
|
||||
.await
|
||||
.expect("conflicting job");
|
||||
resume_job(store.clone(), &bucket, conflict.id)
|
||||
.await
|
||||
.expect("resume conflict");
|
||||
let conflict = finish(store.clone(), &bucket, conflict.id).await;
|
||||
assert_eq!(conflict.results[0].state, ItemState::Conflict, "{conflict:?}");
|
||||
let target = store
|
||||
.get_object_info(&bucket, "target", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("unchanged target");
|
||||
assert_eq!(target.data_dir, reader.object_info.data_dir);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mismatch_never_publishes_and_paused_job_survives_reload() {
|
||||
let (_dirs, store, bucket) = fixture().await;
|
||||
store
|
||||
.put_object(
|
||||
&bucket,
|
||||
"source",
|
||||
&mut PutObjReader::from_vec(b"actual".to_vec()),
|
||||
&ObjectOptions {
|
||||
shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Legacy),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("source");
|
||||
let wrong = rustfs_rio::Checksum::new_from_data(rustfs_rio::ChecksumType::SHA256, b"different")
|
||||
.expect("digest")
|
||||
.encoded;
|
||||
let job = create_job(store.clone(), &bucket, request(JobMode::Audit, wrong))
|
||||
.await
|
||||
.expect("job");
|
||||
assert_eq!(get_job(store.clone(), &bucket, job.id).await.expect("reload").state, JobState::Paused);
|
||||
resume_job(store.clone(), &bucket, job.id).await.expect("resume");
|
||||
let job = finish(store.clone(), &bucket, job.id).await;
|
||||
assert_eq!(job.results[0].state, ItemState::Mismatch, "{job:?}");
|
||||
assert!(
|
||||
store
|
||||
.get_object_info(&bucket, "target", &ObjectOptions::default())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fingerprint_is_independent_of_metadata_insertion_order() {
|
||||
let mut a = crate::object_api::ObjectInfo::default();
|
||||
let mut first = std::collections::HashMap::new();
|
||||
first.insert("a".into(), "one".into());
|
||||
first.insert("b".into(), "two".into());
|
||||
a.user_defined = Arc::new(first);
|
||||
let mut b = a.clone();
|
||||
let mut second = std::collections::HashMap::new();
|
||||
second.insert("b".into(), "two".into());
|
||||
second.insert("a".into(), "one".into());
|
||||
b.user_defined = Arc::new(second);
|
||||
assert_eq!(model::fingerprint(&a), model::fingerprint(&b));
|
||||
b.data_dir = Some(Uuid::new_v4());
|
||||
assert_ne!(model::fingerprint(&a), model::fingerprint(&b));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fingerprint_is_independent_of_part_checksum_insertion_order() {
|
||||
let mut checksums = [
|
||||
("SHA256", "sha256"),
|
||||
("SHA1", "sha1"),
|
||||
("CRC32", "crc32"),
|
||||
("CRC32C", "crc32c"),
|
||||
];
|
||||
let a = crate::object_api::ObjectInfo {
|
||||
parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo {
|
||||
checksums: Some(checksums.map(|(key, value)| (key.to_string(), value.to_string())).into()),
|
||||
..Default::default()
|
||||
}]),
|
||||
..Default::default()
|
||||
};
|
||||
let mut b = a.clone();
|
||||
let expected = model::fingerprint(&a);
|
||||
for _ in 0..checksums.len() {
|
||||
checksums.rotate_left(1);
|
||||
Arc::make_mut(&mut b.parts)[0].checksums =
|
||||
Some(checksums.map(|(key, value)| (key.to_string(), value.to_string())).into());
|
||||
assert_eq!(expected, model::fingerprint(&b));
|
||||
}
|
||||
Arc::make_mut(&mut b.parts)[0]
|
||||
.checksums
|
||||
.as_mut()
|
||||
.expect("part checksums")
|
||||
.insert("SHA256".into(), "changed".into());
|
||||
assert_ne!(expected, model::fingerprint(&b));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_migration_into_a_source_key() {
|
||||
let request = JobRequest {
|
||||
mode: JobMode::Migrate,
|
||||
items: vec![ItemRequest {
|
||||
key: "source".into(),
|
||||
version_id: None,
|
||||
expected_sha256: None,
|
||||
target_key: Some("source".into()),
|
||||
}],
|
||||
bytes_per_second: 1024 * 1024,
|
||||
max_object_bytes: 1024,
|
||||
};
|
||||
assert!(request.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_truncated_checksum_suffix_as_migration_evidence() {
|
||||
let checksum = rustfs_rio::Checksum::new_from_data(rustfs_rio::ChecksumType::SHA256, b"payload").expect("checksum");
|
||||
let mut info = crate::object_api::ObjectInfo {
|
||||
checksum: Some(checksum.to_bytes(&[])),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(model::stored_sha256(&info), Some(checksum.encoded));
|
||||
let mut bytes = info.checksum.take().expect("bytes").to_vec();
|
||||
bytes.push(0x80);
|
||||
info.checksum = Some(bytes.into());
|
||||
assert!(model::stored_sha256(&info).is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unavailable_source_can_resume_and_cancel_intent_cannot_restart() {
|
||||
let (_dirs, store, bucket) = fixture().await;
|
||||
let payload = b"recovered".to_vec();
|
||||
let digest = rustfs_rio::Checksum::new_from_data(rustfs_rio::ChecksumType::SHA256, &payload)
|
||||
.expect("digest")
|
||||
.encoded;
|
||||
let job = create_job(store.clone(), &bucket, request(JobMode::Audit, digest))
|
||||
.await
|
||||
.expect("create");
|
||||
resume_job(store.clone(), &bucket, job.id).await.expect("start");
|
||||
let failed = finish(store.clone(), &bucket, job.id).await;
|
||||
assert_eq!(failed.state, JobState::Failed);
|
||||
assert_eq!(failed.results[0].state, ItemState::Unavailable);
|
||||
store
|
||||
.put_object(
|
||||
&bucket,
|
||||
"source",
|
||||
&mut PutObjReader::from_vec(payload),
|
||||
&ObjectOptions {
|
||||
shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Legacy),
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("recovered source");
|
||||
resume_job(store.clone(), &bucket, job.id).await.expect("retry");
|
||||
let completed = finish(store.clone(), &bucket, job.id).await;
|
||||
assert_eq!(completed.state, JobState::Complete, "{completed:?}");
|
||||
assert_eq!(completed.results[0].state, ItemState::Verified);
|
||||
|
||||
// A process restart may leave the durable cancel intent without a live worker.
|
||||
let (mut interrupted, etag) = load(&store, &bucket, job.id).await.expect("load");
|
||||
interrupted.state = JobState::CancelRequested;
|
||||
interrupted.revision += 1;
|
||||
save(&store, &interrupted, Some(&etag), None).await.expect("persist intent");
|
||||
assert_eq!(
|
||||
control_job(store.clone(), &bucket, job.id, false).await.expect("pause").state,
|
||||
JobState::CancelRequested
|
||||
);
|
||||
assert_eq!(
|
||||
resume_job(store.clone(), &bucket, job.id)
|
||||
.await
|
||||
.expect("recover cancel")
|
||||
.state,
|
||||
JobState::Cancelled
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pause_and_cancel_are_durable_during_content_read() {
|
||||
let (_dirs, store, bucket) = fixture().await;
|
||||
let payload = vec![7; 512 * 1024];
|
||||
let digest = rustfs_rio::Checksum::new_from_data(rustfs_rio::ChecksumType::SHA256, &payload)
|
||||
.expect("digest")
|
||||
.encoded;
|
||||
store
|
||||
.put_object(
|
||||
&bucket,
|
||||
"source",
|
||||
&mut PutObjReader::from_vec(payload),
|
||||
&ObjectOptions {
|
||||
shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Legacy),
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("source");
|
||||
let mut req = request(JobMode::Audit, digest);
|
||||
req.bytes_per_second = 64 * 1024;
|
||||
let job = create_job(store.clone(), &bucket, req).await.expect("create");
|
||||
resume_job(store.clone(), &bucket, job.id).await.expect("start");
|
||||
tokio::time::timeout(std::time::Duration::from_secs(20), async {
|
||||
loop {
|
||||
if get_job(store.clone(), &bucket, job.id).await.expect("status").results[0].state == ItemState::Prepared {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("prepared");
|
||||
let intent = control_job(store.clone(), &bucket, job.id, false).await.expect("pause");
|
||||
assert_eq!(intent.state, JobState::PauseRequested);
|
||||
tokio::time::timeout(std::time::Duration::from_secs(20), async {
|
||||
loop {
|
||||
let paused = get_job(store.clone(), &bucket, job.id).await.expect("status");
|
||||
if paused.state == JobState::Paused {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("pause acknowledged");
|
||||
assert_eq!(
|
||||
control_job(store.clone(), &bucket, job.id, true).await.expect("cancel").state,
|
||||
JobState::Cancelled
|
||||
);
|
||||
assert_eq!(
|
||||
resume_job(store.clone(), &bucket, job.id)
|
||||
.await
|
||||
.expect("terminal resume")
|
||||
.state,
|
||||
JobState::Cancelled
|
||||
);
|
||||
}
|
||||
@@ -16,7 +16,6 @@
|
||||
|
||||
pub(crate) mod batch_processor;
|
||||
pub(crate) mod event_notification;
|
||||
pub(crate) mod integrity;
|
||||
pub(crate) mod metrics_realtime;
|
||||
pub(crate) mod notification_sys;
|
||||
pub(crate) mod rebalance;
|
||||
|
||||
@@ -42,7 +42,6 @@ use std::sync::{
|
||||
atomic::{AtomicBool, AtomicUsize, Ordering},
|
||||
};
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
use tokio::sync::Notify;
|
||||
use tokio::time::{sleep, timeout};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, info, warn};
|
||||
@@ -62,14 +61,6 @@ const TIER_CONFIG_RELOAD_RETRY_CAP: Duration = Duration::from_secs(5);
|
||||
const REMOTE_VERSION_STATE_PROBE_INTERVAL: Duration = Duration::from_secs(10);
|
||||
const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30);
|
||||
/// Fallback poll cadence while startup has not yet published the notification
|
||||
/// system. IAM finalization starts the probe before `init_notification_runtime`
|
||||
/// runs, so the first pass always fails closed; waiting a full probe interval
|
||||
/// there left a single node without its durable quota capability for ten
|
||||
/// seconds after `/health` reported ok (rustfs/rustfs#8014). Publication wakes
|
||||
/// the probe immediately through `NOTIFICATION_SYS_PUBLISHED`; this bound only
|
||||
/// covers a wakeup that races the availability check.
|
||||
const REMOTE_VERSION_STATE_PROBE_BOOTSTRAP_INTERVAL: Duration = Duration::from_millis(100);
|
||||
const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 2;
|
||||
const TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION: u32 = 3;
|
||||
const DECOMMISSION_TARGET_FENCE_POLICY_SUPPORTED_VERSION: u32 = 4;
|
||||
@@ -347,9 +338,6 @@ static LEGACY_TRANSITION_STATE_RECONCILE_FLEET_PROOF: OnceLock<std::sync::RwLock
|
||||
static ILM_RECOVERY_EXPORT_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static TRANSITION_TRANSACTION_COMPACTION_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock<String> = OnceLock::new();
|
||||
/// Signalled once `GLOBAL_NOTIFICATION_SYS` is published so the fleet probe can
|
||||
/// run its first real pass without waiting for the bootstrap poll.
|
||||
static NOTIFICATION_SYS_PUBLISHED: Notify = Notify::const_new();
|
||||
static ILM_RECOVERY_EXPORT_LOCAL_PROCESS_EPOCH: LazyLock<Uuid> = LazyLock::new(Uuid::new_v4);
|
||||
|
||||
fn cross_pool_fence_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
|
||||
@@ -380,24 +368,6 @@ fn transition_transaction_compaction_fleet_proof_slot() -> &'static std::sync::R
|
||||
TRANSITION_TRANSACTION_COMPACTION_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
|
||||
}
|
||||
|
||||
fn all_fleet_capability_proof_slots() -> [&'static std::sync::RwLock<FleetCapabilityProofState>; 7] {
|
||||
[
|
||||
remote_version_state_fleet_proof_slot(),
|
||||
cross_pool_fence_fleet_proof_slot(),
|
||||
tier_delete_journal_fleet_proof_slot(),
|
||||
decommission_target_fence_fleet_proof_slot(),
|
||||
legacy_transition_state_reconcile_fleet_proof_slot(),
|
||||
ilm_recovery_export_fleet_proof_slot(),
|
||||
transition_transaction_compaction_fleet_proof_slot(),
|
||||
]
|
||||
}
|
||||
|
||||
fn revoke_all_fleet_capability_proofs() {
|
||||
for slot in all_fleet_capability_proof_slots() {
|
||||
revoke_fleet_capability_proof(slot);
|
||||
}
|
||||
}
|
||||
|
||||
fn revoke_fleet_capability_proof_state(state: &mut FleetCapabilityProofState) {
|
||||
if let Some(proof) = state.proof.take() {
|
||||
proof.generation.revoke();
|
||||
@@ -483,20 +453,6 @@ pub(crate) fn acquire_remote_version_state_fleet_proof() -> Option<RemoteVersion
|
||||
acquire_fleet_capability_proof_from(&state, expected_topology, Instant::now()).map(RemoteVersionStateFleetProofToken)
|
||||
}
|
||||
|
||||
pub(crate) fn acquire_remote_version_state_writer_fleet_proof() -> Option<RemoteVersionStateFleetProofToken> {
|
||||
let requested = rustfs_utils::get_env_bool(
|
||||
rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_WRITE,
|
||||
rustfs_config::DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE,
|
||||
);
|
||||
let fleet_confirmed = rustfs_utils::get_env_bool(
|
||||
rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
|
||||
rustfs_config::DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
|
||||
);
|
||||
(requested && fleet_confirmed)
|
||||
.then(acquire_remote_version_state_fleet_proof)
|
||||
.flatten()
|
||||
}
|
||||
|
||||
fn acquire_fleet_capability_proof_from(
|
||||
state: &FleetCapabilityProofState,
|
||||
expected_topology: &str,
|
||||
@@ -1256,7 +1212,15 @@ fn insert_remote_version_state_peer(peer_epochs: &mut BTreeMap<String, Uuid>, pe
|
||||
pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.set(topology_fingerprint.clone()).is_err() {
|
||||
if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() != Some(&topology_fingerprint) {
|
||||
for slot in all_fleet_capability_proof_slots() {
|
||||
for slot in [
|
||||
remote_version_state_fleet_proof_slot(),
|
||||
cross_pool_fence_fleet_proof_slot(),
|
||||
tier_delete_journal_fleet_proof_slot(),
|
||||
decommission_target_fence_fleet_proof_slot(),
|
||||
legacy_transition_state_reconcile_fleet_proof_slot(),
|
||||
ilm_recovery_export_fleet_proof_slot(),
|
||||
transition_transaction_compaction_fleet_proof_slot(),
|
||||
] {
|
||||
mark_fleet_capability_topology_conflict(slot);
|
||||
}
|
||||
}
|
||||
@@ -1264,59 +1228,53 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
}
|
||||
|
||||
tokio::spawn(async move {
|
||||
let mut notification_sys_unavailable_logged = false;
|
||||
loop {
|
||||
let Some(notification_sys) = get_global_notification_sys() else {
|
||||
// Startup publishes the notification system after this probe
|
||||
// is started. Stay failed closed, but retry on the bootstrap
|
||||
// cadence so a single node gains its capabilities as soon as
|
||||
// the system appears instead of one full probe interval later.
|
||||
revoke_all_fleet_capability_proofs();
|
||||
if !notification_sys_unavailable_logged {
|
||||
notification_sys_unavailable_logged = true;
|
||||
debug!(
|
||||
event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||
state = "waiting_for_notification_system",
|
||||
"notification capability probe"
|
||||
);
|
||||
}
|
||||
let _ = timeout(REMOTE_VERSION_STATE_PROBE_BOOTSTRAP_INTERVAL, NOTIFICATION_SYS_PUBLISHED.notified()).await;
|
||||
continue;
|
||||
};
|
||||
notification_sys_unavailable_logged = false;
|
||||
let notification_sys = get_global_notification_sys();
|
||||
let remote_version_state_probe = async {
|
||||
timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_remote_version_state_fleet(&topology_fingerprint),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other("remote version state fleet capability probe timed out")))
|
||||
match notification_sys.as_ref() {
|
||||
Some(notification_sys) => timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_remote_version_state_fleet(&topology_fingerprint),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other("remote version state fleet capability probe timed out"))),
|
||||
None => Err(Error::other("remote version state fleet capability notification system is unavailable")),
|
||||
}
|
||||
};
|
||||
let cross_pool_fence_probe = async {
|
||||
timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_cross_pool_fence_fleet(&topology_fingerprint),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other("cross-pool fence fleet capability probe timed out")))
|
||||
match notification_sys.as_ref() {
|
||||
Some(notification_sys) => timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_cross_pool_fence_fleet(&topology_fingerprint),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other("cross-pool fence fleet capability probe timed out"))),
|
||||
None => Err(Error::other("cross-pool fence fleet capability notification system is unavailable")),
|
||||
}
|
||||
};
|
||||
let recovery_export_probe = async {
|
||||
timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_ilm_recovery_export_fleet(&topology_fingerprint),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other("ILM recovery export fleet capability probe timed out")))
|
||||
match notification_sys.as_ref() {
|
||||
Some(notification_sys) => timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_ilm_recovery_export_fleet(&topology_fingerprint),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other("ILM recovery export fleet capability probe timed out"))),
|
||||
None => Err(Error::other("ILM recovery export fleet capability notification system is unavailable")),
|
||||
}
|
||||
};
|
||||
let transition_transaction_compaction_probe = async {
|
||||
timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_transition_transaction_compaction_fleet(&topology_fingerprint),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other("transition transaction compaction fleet capability probe timed out")))
|
||||
match notification_sys.as_ref() {
|
||||
Some(notification_sys) => timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_transition_transaction_compaction_fleet(&topology_fingerprint),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other("transition transaction compaction fleet capability probe timed out"))),
|
||||
None => Err(Error::other(
|
||||
"transition transaction compaction fleet capability notification system is unavailable",
|
||||
)),
|
||||
}
|
||||
};
|
||||
let (result, fence_probe, recovery_export_result, transition_transaction_compaction_result) = tokio::join!(
|
||||
remote_version_state_probe,
|
||||
@@ -1341,7 +1299,13 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.topology_conflict;
|
||||
if topology_conflict {
|
||||
revoke_all_fleet_capability_proofs();
|
||||
revoke_fleet_capability_proof(remote_version_state_fleet_proof_slot());
|
||||
revoke_fleet_capability_proof(cross_pool_fence_fleet_proof_slot());
|
||||
revoke_fleet_capability_proof(tier_delete_journal_fleet_proof_slot());
|
||||
revoke_fleet_capability_proof(decommission_target_fence_fleet_proof_slot());
|
||||
revoke_fleet_capability_proof(legacy_transition_state_reconcile_fleet_proof_slot());
|
||||
revoke_fleet_capability_proof(ilm_recovery_export_fleet_proof_slot());
|
||||
revoke_fleet_capability_proof(transition_transaction_compaction_fleet_proof_slot());
|
||||
} else if let Some(err) = publish_fleet_capability_probe_result(
|
||||
remote_version_state_fleet_proof_slot(),
|
||||
&topology_fingerprint,
|
||||
@@ -1464,11 +1428,9 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
}
|
||||
|
||||
pub async fn new_global_notification_sys(eps: EndpointServerPools) -> Result<()> {
|
||||
if GLOBAL_NOTIFICATION_SYS.set(Arc::new(NotificationSys::new(eps).await)).is_ok() {
|
||||
// `notify_one` stores a permit, so a probe that checks availability
|
||||
// just before this publication still wakes without losing the signal.
|
||||
NOTIFICATION_SYS_PUBLISHED.notify_one();
|
||||
}
|
||||
let _ = GLOBAL_NOTIFICATION_SYS
|
||||
.set(Arc::new(NotificationSys::new(eps).await))
|
||||
.map_err(|_| Error::other("init notification_sys fail"));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -3943,49 +3905,6 @@ mod tests {
|
||||
stats
|
||||
}
|
||||
|
||||
/// rustfs/rustfs#8014: IAM finalization starts the fleet probe before the
|
||||
/// notification system is published. The first probe therefore fails
|
||||
/// closed, and a single node used to wait a full probe interval before the
|
||||
/// durable quota capability appeared, even though `/health` was already
|
||||
/// reporting ok. The probe must retry promptly while it waits for startup
|
||||
/// to publish the notification system.
|
||||
#[tokio::test]
|
||||
async fn fleet_probe_publishes_promptly_once_the_notification_system_appears() {
|
||||
if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get().is_some() || GLOBAL_NOTIFICATION_SYS.get().is_some() {
|
||||
// Process-wide state was bound by another test in this binary. The
|
||||
// startup ordering under test needs a fresh process; nextest
|
||||
// (the authoritative runner) always provides one.
|
||||
eprintln!("skipping: fleet probe globals already bound in this process");
|
||||
return;
|
||||
}
|
||||
|
||||
// Share the fingerprint every other proof-installing test in this crate
|
||||
// binds, so the plain `cargo test` fallback cannot see two topologies.
|
||||
start_remote_version_state_fleet_probe("object-transaction-fencing-test".to_string());
|
||||
sleep(Duration::from_millis(300)).await;
|
||||
assert!(
|
||||
acquire_cross_pool_fence_fleet_proof().is_none(),
|
||||
"no capability proof may exist before the notification system is published"
|
||||
);
|
||||
|
||||
let published_at = Instant::now();
|
||||
new_global_notification_sys(EndpointServerPools::default())
|
||||
.await
|
||||
.expect("single-node notification system initializes");
|
||||
|
||||
let observed = timeout(Duration::from_secs(2), async {
|
||||
while acquire_cross_pool_fence_fleet_proof().is_none() {
|
||||
sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
observed.is_ok(),
|
||||
"single-node cross-pool fence proof should publish within 2s of the notification system, waited {:?}",
|
||||
published_at.elapsed()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merging_peer_rings_sums_each_tier_without_double_counting() {
|
||||
let mut cluster = DailyAllTierStats::from([("WARM".to_string(), ring(10))]);
|
||||
|
||||
@@ -9,12 +9,11 @@ use super::meta::{
|
||||
};
|
||||
use super::worker::{
|
||||
rebalance_max_attempts, rebalance_meta_lock_error, resolve_load_rebalance_stats_update_result,
|
||||
resolve_rebalance_deferred_last_error, resolve_rebalance_meta_load_result, resolve_rebalance_meta_save_result,
|
||||
retry_rebalance_metadata_access,
|
||||
resolve_rebalance_meta_load_result, resolve_rebalance_meta_save_result, retry_rebalance_metadata_access,
|
||||
};
|
||||
use super::{
|
||||
DiskStat, EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE, REBAL_META_NAME,
|
||||
RebalStatus, RebalanceDeferKind, RebalanceInfo, RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord,
|
||||
RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord,
|
||||
encode_rebalance_stop_propagation_record,
|
||||
};
|
||||
use crate::core::pools::{
|
||||
@@ -1104,7 +1103,6 @@ impl ECStore {
|
||||
bucket: String,
|
||||
last_error: String,
|
||||
expected_id: &str,
|
||||
kind: RebalanceDeferKind,
|
||||
) -> Result<()> {
|
||||
let mut rebalance_meta = self.rebalance_meta.write().await;
|
||||
ensure_rebalance_worker_active(rebalance_meta.as_ref(), expected_id, "defer rebalance bucket")?;
|
||||
@@ -1118,8 +1116,7 @@ impl ECStore {
|
||||
};
|
||||
|
||||
defer_bucket_in_rebalance_queue(pool_stat, &bucket)?;
|
||||
let pending_entry_defer = pool_stat.info.last_error.clone();
|
||||
pool_stat.info.last_error = resolve_rebalance_deferred_last_error(kind, pending_entry_defer.as_deref(), &last_error);
|
||||
pool_stat.info.last_error = Some(last_error);
|
||||
meta.last_refreshed_at = Some(OffsetDateTime::now_utc());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1209,7 +1209,7 @@ mod tests {
|
||||
cancel: &warning_cancel,
|
||||
};
|
||||
let warning_result = store
|
||||
.finish_rebalance_entry_after_cleanup(&warning_cleanup_context, async { Err(Error::FileAccessDenied.into()) })
|
||||
.finish_rebalance_entry_after_cleanup(&warning_cleanup_context, async { Err(Error::SlowDown.into()) })
|
||||
.await
|
||||
.expect("cleanup warnings should not fail the completed migration");
|
||||
assert!(matches!(warning_result, RebalanceEntryCleanupResult::Completed { warning: Some(_) }));
|
||||
@@ -1250,35 +1250,6 @@ mod tests {
|
||||
let pool_stats = &meta.as_ref().expect("rebalance metadata should exist").pool_stats[0];
|
||||
assert_eq!(pool_stats.bytes, 0, "deferred cleanup must not commit completion stats");
|
||||
assert_eq!(pool_stats.cleanup_warnings.count, 1, "deferred cleanup must not add a permanent warning");
|
||||
drop(meta);
|
||||
|
||||
let transient_guard = store
|
||||
.rebalance_run_guard(rebalance_id, "rebalance transient cleanup deferral test")
|
||||
.await
|
||||
.expect("rebalance transient cleanup deferral test guard should be acquired");
|
||||
let transient_cancel = CancellationToken::new();
|
||||
let transient_stats_updates = [&warning_version];
|
||||
let transient_cleanup_context = RebalanceEntryCleanupContext {
|
||||
run_guard: &transient_guard,
|
||||
pool_index: 0,
|
||||
bucket: "bucket",
|
||||
object: "object.bin",
|
||||
stats_updates: &transient_stats_updates,
|
||||
expected_id: rebalance_id,
|
||||
cancel: &transient_cancel,
|
||||
};
|
||||
let transient = store
|
||||
.finish_rebalance_entry_after_cleanup(&transient_cleanup_context, async { Err(Error::SlowDown.into()) })
|
||||
.await
|
||||
.expect("retryable cleanup failures should defer without failing the worker");
|
||||
assert!(matches!(transient, RebalanceEntryCleanupResult::Deferred { .. }));
|
||||
let meta = store.rebalance_meta.read().await;
|
||||
let pool_stats = &meta.as_ref().expect("rebalance metadata should exist").pool_stats[0];
|
||||
assert_eq!(pool_stats.bytes, 0, "retryable cleanup failures must not commit completion stats");
|
||||
assert_eq!(
|
||||
pool_stats.cleanup_warnings.count, 1,
|
||||
"retryable cleanup failures must not add a permanent warning"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -29,7 +29,7 @@ const REBAL_META_FMT: u16 = 1; // Replace with actual format value
|
||||
const REBAL_META_VER: u16 = 1; // Replace with actual version value
|
||||
pub(crate) const REBAL_META_NAME: &str = "rebalance.bin";
|
||||
const DEFAULT_REBALANCE_MAX_ATTEMPTS: usize = 3;
|
||||
pub(crate) const REBALANCE_SOURCE_CLEANUP_MAX_DEFERS: usize = 8;
|
||||
pub(crate) const REBALANCE_SOURCE_CLEANUP_MAX_DEFERS: usize = 3;
|
||||
const REBALANCE_MAX_ATTEMPTS_ENV: &str = "RUSTFS_REBALANCE_MAX_ATTEMPTS";
|
||||
const REBALANCE_STOP_PROPAGATION_ERROR_PREFIX: &str = "rebalance stop propagation incomplete: ";
|
||||
const REBALANCE_LISTING_RETRY_BASE_DELAY: Duration = Duration::from_millis(250);
|
||||
@@ -55,7 +55,7 @@ pub use types::{
|
||||
DiskStat, RebalSaveOpt, RebalStatus, RebalanceCleanupWarningEntry, RebalanceCleanupWarnings, RebalanceInfo, RebalanceMeta,
|
||||
RebalanceStats, RebalanceStopPropagationRecord,
|
||||
};
|
||||
use types::{RebalanceBucketConfigs, RebalanceBucketOutcome, RebalanceDeferKind, RebalanceEntryOutcome};
|
||||
use types::{RebalanceBucketConfigs, RebalanceBucketOutcome, RebalanceEntryOutcome};
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub async fn test_store_with_persisted_rebalance_meta(
|
||||
@@ -222,14 +222,10 @@ async fn test_pool_stores_with_contexts(
|
||||
std::sync::Arc::clone(&ctx)
|
||||
};
|
||||
let make_store = |store_ctx: std::sync::Arc<crate::runtime::instance::InstanceContext>| {
|
||||
let mut store_pools = pools.clone();
|
||||
for pool in &mut store_pools {
|
||||
std::sync::Arc::make_mut(pool).set_instance_ctx_for_test(std::sync::Arc::clone(&store_ctx));
|
||||
}
|
||||
std::sync::Arc::new(crate::store::ECStore {
|
||||
id: uuid::Uuid::new_v4(),
|
||||
disk_map: std::collections::HashMap::new(),
|
||||
pools: store_pools,
|
||||
pools: pools.clone(),
|
||||
peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoint_pools, std::sync::Arc::clone(&store_ctx)),
|
||||
pool_meta: tokio::sync::RwLock::new(pool_meta.clone()),
|
||||
rebalance_meta: tokio::sync::RwLock::new(rebalance_meta.clone()),
|
||||
|
||||
@@ -34,29 +34,27 @@ use super::migration::{
|
||||
};
|
||||
use super::runtime::{
|
||||
RebalanceLocalActivationOutcome, commit_local_rebalance_worker_activation,
|
||||
commit_local_rebalance_worker_activation_candidate, reached_rebalance_source_cleanup_defer_limit,
|
||||
should_fail_repeated_rebalance_bucket_defer, source_cleanup_defer_attempt, stage_local_rebalance_worker_activation,
|
||||
commit_local_rebalance_worker_activation_candidate, should_fail_repeated_rebalance_bucket_defer,
|
||||
source_cleanup_defer_attempt, stage_local_rebalance_worker_activation,
|
||||
};
|
||||
use super::worker::{
|
||||
RebalanceEntryCleanupResult, ensure_rebalance_listing_disks_available, is_transient_rebalance_error,
|
||||
parse_rebalance_max_attempts, rebalance_listing_retry_delay, rebalance_migration_retry_delay,
|
||||
resolve_load_rebalance_stats_update_result, resolve_rebalance_bucket_error, resolve_rebalance_bucket_result,
|
||||
resolve_rebalance_deferred_last_error, resolve_rebalance_entry_cleanup_delete_result,
|
||||
resolve_rebalance_file_info_versions_result, resolve_rebalance_meta_load_result, resolve_rebalance_meta_save_result,
|
||||
resolve_rebalance_migrate_result_error, resolve_rebalance_optional_bucket_config_result, resolve_rebalance_save_task_result,
|
||||
resolve_rebalance_stats_update_result, resolve_rebalance_terminal_error, resolve_rebalance_worker_result,
|
||||
run_rebalance_listing_with_retry, send_rebalance_done_signal, should_cleanup_rebalance_source_entry,
|
||||
should_count_rebalance_version_complete, should_defer_rebalance_entry_failure, should_retry_rebalance_listing,
|
||||
should_skip_rebalance_delete_marker, wait_rebalance_entry_tasks, wait_rebalance_listing_retry, with_rebalance_entry_context,
|
||||
resolve_rebalance_entry_cleanup_delete_result, resolve_rebalance_file_info_versions_result,
|
||||
resolve_rebalance_meta_load_result, resolve_rebalance_meta_save_result, resolve_rebalance_migrate_result_error,
|
||||
resolve_rebalance_optional_bucket_config_result, resolve_rebalance_save_task_result, resolve_rebalance_stats_update_result,
|
||||
resolve_rebalance_terminal_error, resolve_rebalance_worker_result, run_rebalance_listing_with_retry,
|
||||
send_rebalance_done_signal, should_cleanup_rebalance_source_entry, should_count_rebalance_version_complete,
|
||||
should_defer_rebalance_entry_failure, should_retry_rebalance_listing, should_skip_rebalance_delete_marker,
|
||||
wait_rebalance_entry_tasks, wait_rebalance_listing_retry, with_rebalance_entry_context,
|
||||
};
|
||||
use super::{
|
||||
DiskStat, GetObjectReader, ObjectInfo, ObjectOptions, RebalSaveOpt, RebalStatus, RebalanceBucketConfigs,
|
||||
RebalanceBucketOutcome, RebalanceCleanupWarnings, RebalanceDeferKind, RebalanceEntryOutcome, RebalanceInfo, RebalanceMeta,
|
||||
RebalanceStats, RebalanceStopPropagationRecord,
|
||||
};
|
||||
use super::{
|
||||
REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX, REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX, REBALANCE_SOURCE_CLEANUP_MAX_DEFERS,
|
||||
RebalanceBucketOutcome, RebalanceCleanupWarnings, RebalanceEntryOutcome, RebalanceInfo, RebalanceMeta, RebalanceStats,
|
||||
RebalanceStopPropagationRecord,
|
||||
};
|
||||
use super::{REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX, REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX};
|
||||
use crate::bucket::replication::{ReplicationState, ReplicationStatusType, replication_state_to_filemeta};
|
||||
use crate::data_movement;
|
||||
use crate::data_movement::SourceCleanupError;
|
||||
@@ -1777,7 +1775,7 @@ fn test_resolve_rebalance_entry_cleanup_delete_result_ignores_not_found() {
|
||||
|
||||
#[test]
|
||||
fn test_resolve_rebalance_entry_cleanup_delete_result_returns_warning_for_failures() {
|
||||
let result = resolve_rebalance_entry_cleanup_delete_result(Err(Error::FileAccessDenied.into()), "bucket-a", "obj.txt");
|
||||
let result = resolve_rebalance_entry_cleanup_delete_result(Err(Error::SlowDown.into()), "bucket-a", "obj.txt");
|
||||
assert!(matches!(
|
||||
result,
|
||||
RebalanceEntryCleanupResult::Completed { warning: Some(ref message) }
|
||||
@@ -1785,122 +1783,6 @@ fn test_resolve_rebalance_entry_cleanup_delete_result_returns_warning_for_failur
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_rebalance_entry_cleanup_delete_result_defers_transient_failures() {
|
||||
let cases = [
|
||||
(Error::SlowDown, "slow down"),
|
||||
(
|
||||
Error::Lock(rustfs_lock::LockError::timeout("bucket-a/obj.txt@latest", Duration::from_secs(5))),
|
||||
"object lock timeout",
|
||||
),
|
||||
(
|
||||
Error::Lock(rustfs_lock::LockError::network(
|
||||
"peer unavailable",
|
||||
std::io::Error::from(std::io::ErrorKind::ConnectionReset),
|
||||
)),
|
||||
"object lock network failure",
|
||||
),
|
||||
(Error::ErasureWriteQuorum, "write quorum"),
|
||||
(Error::Io(std::io::Error::from(std::io::ErrorKind::TimedOut)), "io timeout"),
|
||||
(
|
||||
Error::other("Lock error: Lock acquisition timeout for resource 'bucket-a/obj.txt@latest' after 5s"),
|
||||
"rendered lock timeout text",
|
||||
),
|
||||
];
|
||||
|
||||
for (err, label) in cases {
|
||||
match resolve_rebalance_entry_cleanup_delete_result(Err(err.into()), "bucket-a", "obj.txt") {
|
||||
RebalanceEntryCleanupResult::Deferred { last_error } => {
|
||||
assert!(
|
||||
last_error.starts_with(REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX),
|
||||
"{label}: {last_error}"
|
||||
);
|
||||
assert!(last_error.contains("bucket-a/obj.txt"), "{label}: {last_error}");
|
||||
}
|
||||
RebalanceEntryCleanupResult::Completed { warning } => {
|
||||
panic!("{label} must defer source cleanup instead of completing the entry with warning {warning:?}")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_rebalance_entry_cleanup_delete_result_defers_stage_wrapped_lock_timeout() {
|
||||
let err = data_movement::data_movement_stage_error_for_test(
|
||||
"rebalance",
|
||||
"delete_object",
|
||||
"bucket-a",
|
||||
"obj.txt",
|
||||
Error::Lock(rustfs_lock::LockError::timeout("bucket-a/obj.txt@latest", Duration::from_secs(5))),
|
||||
);
|
||||
|
||||
assert!(matches!(
|
||||
resolve_rebalance_entry_cleanup_delete_result(Err(err.into()), "bucket-a", "obj.txt"),
|
||||
RebalanceEntryCleanupResult::Deferred { .. }
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_rebalance_entry_cleanup_delete_result_ignores_stage_wrapped_not_found() {
|
||||
let err = data_movement::data_movement_stage_error_for_test(
|
||||
"rebalance",
|
||||
"delete_object",
|
||||
"bucket-a",
|
||||
"obj.txt",
|
||||
Error::ObjectNotFound("bucket-a".to_string(), "obj.txt".to_string()),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
resolve_rebalance_entry_cleanup_delete_result(Err(err.into()), "bucket-a", "obj.txt"),
|
||||
RebalanceEntryCleanupResult::Completed { warning: None }
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_rebalance_deferred_last_error_hides_retryable_cleanup_conflicts() {
|
||||
let entry_error = format!("{REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX} timeout");
|
||||
assert_eq!(
|
||||
resolve_rebalance_deferred_last_error(RebalanceDeferKind::Entry, None, entry_error.as_str()),
|
||||
Some(entry_error.clone()),
|
||||
"transient migration deferrals must stay visible to the completion guards"
|
||||
);
|
||||
|
||||
let cleanup_error = format!("{REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX} lock acquisition timeout");
|
||||
assert_eq!(
|
||||
resolve_rebalance_deferred_last_error(RebalanceDeferKind::SourceCleanup, None, cleanup_error.as_str()),
|
||||
None,
|
||||
"a retryable source cleanup conflict is progress, not a pool failure"
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
resolve_rebalance_deferred_last_error(
|
||||
RebalanceDeferKind::SourceCleanup,
|
||||
Some(entry_error.as_str()),
|
||||
cleanup_error.as_str()
|
||||
),
|
||||
Some(entry_error.clone()),
|
||||
"a cleanup deferral for one bucket must not erase an unresolved migration deferral of the same pool"
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_rebalance_deferred_last_error(
|
||||
RebalanceDeferKind::SourceCleanup,
|
||||
Some(cleanup_error.as_str()),
|
||||
cleanup_error.as_str()
|
||||
),
|
||||
None,
|
||||
"a stale retryable cleanup message must not survive as a pool failure"
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_rebalance_deferred_last_error(
|
||||
RebalanceDeferKind::SourceCleanup,
|
||||
Some(entry_error.as_str()),
|
||||
entry_error.as_str()
|
||||
),
|
||||
Some(entry_error),
|
||||
"repeated cleanup deferrals must keep the pending migration deferral visible"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_rebalance_entry_cleanup_delete_result_defers_source_change() {
|
||||
let result = resolve_rebalance_entry_cleanup_delete_result(Err(SourceCleanupError::SourceChanged), "bucket-a", "obj.txt");
|
||||
@@ -1935,89 +1817,6 @@ fn test_source_cleanup_defer_does_not_fail_repeated_bucket_retry() {
|
||||
assert_eq!(source_cleanup_defer_attempt(&mut source_attempts, "bucket-c"), 1);
|
||||
assert_eq!(source_cleanup_defer_attempt(&mut source_attempts, "bucket-c"), 2);
|
||||
assert_eq!(source_cleanup_defer_attempt(&mut source_attempts, "bucket-c"), 3);
|
||||
|
||||
let mut bounded_attempts = std::collections::HashMap::new();
|
||||
for expected in 1..REBALANCE_SOURCE_CLEANUP_MAX_DEFERS {
|
||||
assert_eq!(source_cleanup_defer_attempt(&mut bounded_attempts, "bucket-d"), expected);
|
||||
assert!(
|
||||
!reached_rebalance_source_cleanup_defer_limit(expected),
|
||||
"a retryable cleanup conflict must stay retryable at deferral {expected}"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
source_cleanup_defer_attempt(&mut bounded_attempts, "bucket-d"),
|
||||
REBALANCE_SOURCE_CLEANUP_MAX_DEFERS
|
||||
);
|
||||
assert!(
|
||||
reached_rebalance_source_cleanup_defer_limit(REBALANCE_SOURCE_CLEANUP_MAX_DEFERS),
|
||||
"an unreclaimable source replica must fail the bucket after the bounded deferral budget"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_defer_rebalance_bucket_keeps_pending_entry_defer_without_surfacing_cleanup_conflicts() {
|
||||
let id = "rebalance-defer-last-error";
|
||||
let meta = RebalanceMeta {
|
||||
id: id.to_string(),
|
||||
pool_stats: vec![RebalanceStats {
|
||||
participating: true,
|
||||
buckets: vec!["bucket-a".to_string(), "bucket-b".to_string()],
|
||||
info: RebalanceInfo {
|
||||
status: RebalStatus::Started,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
let store = test_store_with_rebalance_meta(meta);
|
||||
let cleanup_error = format!("{REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX} lock acquisition timeout");
|
||||
|
||||
store
|
||||
.defer_rebalance_bucket(0, "bucket-a".to_string(), cleanup_error.clone(), id, RebalanceDeferKind::SourceCleanup)
|
||||
.await
|
||||
.expect("a retryable cleanup conflict must be deferrable");
|
||||
{
|
||||
let meta = store.rebalance_meta.read().await;
|
||||
let pool_stat = &meta.as_ref().expect("rebalance metadata should exist").pool_stats[0];
|
||||
assert_eq!(
|
||||
pool_stat.info.last_error, None,
|
||||
"a retryable cleanup conflict is progress and must not surface as a pool failure"
|
||||
);
|
||||
assert_eq!(pool_stat.buckets, vec!["bucket-b".to_string(), "bucket-a".to_string()]);
|
||||
}
|
||||
|
||||
let entry_error = format!("{REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX} slow down");
|
||||
{
|
||||
let mut meta = store.rebalance_meta.write().await;
|
||||
meta.as_mut().expect("rebalance metadata should exist").pool_stats[0]
|
||||
.info
|
||||
.last_error = Some(entry_error.clone());
|
||||
}
|
||||
store
|
||||
.defer_rebalance_bucket(0, "bucket-b".to_string(), cleanup_error, id, RebalanceDeferKind::SourceCleanup)
|
||||
.await
|
||||
.expect("a cleanup deferral must be accepted while another bucket still defers an entry");
|
||||
{
|
||||
let meta = store.rebalance_meta.read().await;
|
||||
let pool_stat = &meta.as_ref().expect("rebalance metadata should exist").pool_stats[0];
|
||||
assert_eq!(
|
||||
pool_stat.info.last_error,
|
||||
Some(entry_error),
|
||||
"a cleanup deferral must not erase the pending migration deferral that blocks goal completion"
|
||||
);
|
||||
assert!(has_deferred_rebalance_error(pool_stat));
|
||||
assert_eq!(pool_stat.buckets, vec!["bucket-a".to_string(), "bucket-b".to_string()]);
|
||||
}
|
||||
|
||||
let migration_error = format!("{REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX} i/o timeout");
|
||||
store
|
||||
.defer_rebalance_bucket(0, "bucket-a".to_string(), migration_error.clone(), id, RebalanceDeferKind::Entry)
|
||||
.await
|
||||
.expect("migration deferrals must keep their last error");
|
||||
let meta = store.rebalance_meta.read().await;
|
||||
let pool_stat = &meta.as_ref().expect("rebalance metadata should exist").pool_stats[0];
|
||||
assert_eq!(pool_stat.info.last_error, Some(migration_error));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -7,14 +7,13 @@ use super::meta::{
|
||||
validate_start_rebalance_state,
|
||||
};
|
||||
use super::worker::{
|
||||
rebalance_max_attempts, resolve_rebalance_bucket_result, resolve_rebalance_meta_save_result,
|
||||
resolve_rebalance_save_task_result, resolve_rebalance_terminal_error, retry_rebalance_metadata_access,
|
||||
send_rebalance_done_signal,
|
||||
resolve_rebalance_bucket_result, resolve_rebalance_meta_save_result, resolve_rebalance_save_task_result,
|
||||
resolve_rebalance_terminal_error, send_rebalance_done_signal,
|
||||
};
|
||||
use super::{
|
||||
EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE,
|
||||
REBALANCE_LISTING_RETRY_BASE_DELAY, REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX, REBALANCE_SOURCE_CLEANUP_MAX_DEFERS,
|
||||
RebalSaveOpt, RebalStatus, RebalanceBucketOutcome, RebalanceDeferKind,
|
||||
REBALANCE_LISTING_RETRY_BASE_DELAY, REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX, RebalSaveOpt, RebalStatus,
|
||||
RebalanceBucketOutcome,
|
||||
};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
@@ -41,12 +40,6 @@ pub(super) fn source_cleanup_defer_attempt(deferred_attempts: &mut HashMap<Strin
|
||||
*attempts
|
||||
}
|
||||
|
||||
/// Retryable source cleanup conflicts are bounded per run, so a source replica that stays
|
||||
/// unreclaimable fails the bucket explicitly instead of deferring forever.
|
||||
pub(super) fn reached_rebalance_source_cleanup_defer_limit(attempt: usize) -> bool {
|
||||
attempt >= REBALANCE_SOURCE_CLEANUP_MAX_DEFERS
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(super) enum RebalanceLocalActivationOutcome {
|
||||
Started,
|
||||
@@ -450,14 +443,13 @@ impl ECStore {
|
||||
};
|
||||
|
||||
if terminal_state_present {
|
||||
if let Err(err) = retry_rebalance_metadata_access(None, rebalance_max_attempts(), || {
|
||||
store.save_rebalance_stats_inner(
|
||||
if let Err(err) = store
|
||||
.save_rebalance_stats_inner(
|
||||
pool_index,
|
||||
RebalSaveOpt::Stats,
|
||||
Some(save_rebalance_id.as_ref()),
|
||||
)
|
||||
})
|
||||
.await
|
||||
.await
|
||||
{
|
||||
let mut rebalance_meta = store.rebalance_meta.write().await;
|
||||
*rebalance_meta = previous_meta;
|
||||
@@ -477,10 +469,9 @@ impl ECStore {
|
||||
}
|
||||
|
||||
if !terminal_state_saved
|
||||
&& let Err(err) = retry_rebalance_metadata_access(None, rebalance_max_attempts(), || {
|
||||
store.save_rebalance_stats_for_id(pool_index, RebalSaveOpt::Stats, save_rebalance_id.as_ref())
|
||||
})
|
||||
.await
|
||||
&& let Err(err) = store
|
||||
.save_rebalance_stats_for_id(pool_index, RebalSaveOpt::Stats, save_rebalance_id.as_ref())
|
||||
.await
|
||||
{
|
||||
let wrapped = Error::other(format!("rebalance save_task stats save failed for pool {pool_index}: {err}"));
|
||||
error!("{} err: {:?}", msg, wrapped);
|
||||
@@ -632,11 +623,6 @@ impl ECStore {
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let defer_kind = if source_cleanup_deferred {
|
||||
RebalanceDeferKind::SourceCleanup
|
||||
} else {
|
||||
RebalanceDeferKind::Entry
|
||||
};
|
||||
warn!(
|
||||
event = EVENT_REBALANCE_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -648,7 +634,7 @@ impl ECStore {
|
||||
"Deferred rebalance bucket after transient object failures"
|
||||
);
|
||||
if let Err(err) = self
|
||||
.defer_rebalance_bucket(pool_index, bucket.clone(), last_error.clone(), rebalance_id.as_ref(), defer_kind)
|
||||
.defer_rebalance_bucket(pool_index, bucket.clone(), last_error.clone(), rebalance_id.as_ref())
|
||||
.await
|
||||
{
|
||||
error!(
|
||||
@@ -668,10 +654,10 @@ impl ECStore {
|
||||
break;
|
||||
}
|
||||
if source_cleanup_deferred {
|
||||
if reached_rebalance_source_cleanup_defer_limit(source_cleanup_attempt) {
|
||||
if source_cleanup_attempt >= super::REBALANCE_SOURCE_CLEANUP_MAX_DEFERS {
|
||||
let err = Error::other(format!(
|
||||
"rebalance bucket {bucket} source cleanup remained unstable after {} deferrals: {last_error}",
|
||||
REBALANCE_SOURCE_CLEANUP_MAX_DEFERS
|
||||
super::REBALANCE_SOURCE_CLEANUP_MAX_DEFERS
|
||||
));
|
||||
warn!(
|
||||
event = EVENT_REBALANCE_BUCKET,
|
||||
|
||||
@@ -51,18 +51,6 @@ pub(super) enum RebalanceEntryOutcome {
|
||||
Deferred { last_error: String },
|
||||
}
|
||||
|
||||
/// Why a rebalance bucket was put back at the end of the queue.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum RebalanceDeferKind {
|
||||
/// A transient object-migration failure. Persisting it as `lastError` keeps the pool
|
||||
/// from being completed at the free-space goal while the entry is still retried.
|
||||
Entry,
|
||||
/// A retryable source-cleanup conflict. The bucket stays queued and is retried, so a
|
||||
/// transient lock conflict must not be recorded as a permanent warning that would block
|
||||
/// pool completion for the rest of the run.
|
||||
SourceCleanup,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
|
||||
pub enum RebalStatus {
|
||||
#[default]
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
use super::migration::MigrationVersionResult;
|
||||
use super::{
|
||||
DEFAULT_REBALANCE_MAX_ATTEMPTS, EVENT_REBALANCE_LISTING, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE, REBAL_META_NAME,
|
||||
REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX, REBALANCE_LISTING_RETRY_BASE_DELAY, REBALANCE_MAX_ATTEMPTS_ENV,
|
||||
REBALANCE_MIGRATION_LOCK_RETRY_CAP, REBALANCE_MIGRATION_RETRY_BASE_DELAY, REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX,
|
||||
RebalanceBucketConfigs, RebalanceBucketOutcome, RebalanceDeferKind, RebalanceEntryOutcome, Result,
|
||||
REBALANCE_LISTING_RETRY_BASE_DELAY, REBALANCE_MAX_ATTEMPTS_ENV, REBALANCE_MIGRATION_LOCK_RETRY_CAP,
|
||||
REBALANCE_MIGRATION_RETRY_BASE_DELAY, REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX, RebalanceBucketConfigs,
|
||||
RebalanceBucketOutcome, RebalanceEntryOutcome, Result,
|
||||
};
|
||||
use crate::cache_value::metacache_set::{ListPathRawOptions, list_path_raw};
|
||||
use crate::core::pools::ListCallback;
|
||||
@@ -109,13 +109,7 @@ pub(super) fn resolve_rebalance_save_task_result(
|
||||
}
|
||||
|
||||
pub(super) fn resolve_rebalance_meta_save_result(result: Result<()>, stage: &str) -> Result<()> {
|
||||
// Keep the source error reachable: the metadata retry policy classifies
|
||||
// transient lock timeouts by inspecting the source chain, so collapsing the
|
||||
// failure into a plain string here would make that retry a no-op.
|
||||
result.map_err(|err| {
|
||||
let rendered = format!("rebalance meta save failed during {stage}: {err}");
|
||||
crate::data_movement::data_movement_context_error(rendered, err)
|
||||
})
|
||||
result.map_err(|err| Error::other(format!("rebalance meta save failed during {stage}: {err}")))
|
||||
}
|
||||
|
||||
pub(super) fn rebalance_meta_lock_error(err: rustfs_lock::LockError, mode: &'static str) -> Error {
|
||||
@@ -183,7 +177,7 @@ pub(super) fn resolve_rebalance_entry_cleanup_delete_result(
|
||||
) -> RebalanceEntryCleanupResult {
|
||||
match result {
|
||||
Ok(_) => RebalanceEntryCleanupResult::Completed { warning: None },
|
||||
Err(SourceCleanupError::Storage(err)) if is_source_cleanup_not_found(&err) => {
|
||||
Err(SourceCleanupError::Storage(err)) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {
|
||||
RebalanceEntryCleanupResult::Completed { warning: None }
|
||||
}
|
||||
Err(SourceCleanupError::SourceChanged) => RebalanceEntryCleanupResult::Deferred {
|
||||
@@ -191,25 +185,12 @@ pub(super) fn resolve_rebalance_entry_cleanup_delete_result(
|
||||
"{REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX} source changed during cleanup preflight for {bucket}/{object_name}"
|
||||
),
|
||||
},
|
||||
// A transient cleanup failure is not evidence that the source replica is gone, so the
|
||||
// entry stays incomplete and the bucket is retried instead of recording a permanent
|
||||
// cleanup warning that would block pool completion.
|
||||
Err(SourceCleanupError::Storage(err)) if is_transient_rebalance_error(&err) => RebalanceEntryCleanupResult::Deferred {
|
||||
last_error: format!(
|
||||
"{REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX} transient source cleanup failure for {bucket}/{object_name} will be retried: {err}"
|
||||
),
|
||||
},
|
||||
Err(SourceCleanupError::Storage(err)) => RebalanceEntryCleanupResult::Completed {
|
||||
warning: Some(format!("rebalance cleanup delete failed for {bucket}/{object_name}: {err}")),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn is_source_cleanup_not_found(err: &Error) -> bool {
|
||||
let err = rebalance_error_source(err);
|
||||
is_err_object_not_found(err) || is_err_version_not_found(err)
|
||||
}
|
||||
|
||||
pub(super) fn resolve_rebalance_migrate_result_error(
|
||||
err: Option<Error>,
|
||||
pool_idx: usize,
|
||||
@@ -229,23 +210,6 @@ pub(super) fn should_defer_rebalance_entry_failure(err: &Error) -> bool {
|
||||
is_transient_rebalance_error(err)
|
||||
}
|
||||
|
||||
pub(super) fn resolve_rebalance_deferred_last_error(
|
||||
kind: RebalanceDeferKind,
|
||||
pending_entry_defer: Option<&str>,
|
||||
last_error: &str,
|
||||
) -> Option<String> {
|
||||
match kind {
|
||||
RebalanceDeferKind::Entry => Some(last_error.to_string()),
|
||||
// A retryable cleanup conflict is progress, not a pool failure, so it must not surface as
|
||||
// `lastError`. It also must not erase an unresolved migration deferral: that marker is the
|
||||
// only signal keeping the pool from completing at the free-space goal while an entry is
|
||||
// still retried, and the two deferrals can be reported by different buckets of one pool.
|
||||
RebalanceDeferKind::SourceCleanup => pending_entry_defer
|
||||
.filter(|pending| pending.starts_with(REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX))
|
||||
.map(str::to_string),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn resolve_load_rebalance_stats_update_result(result: Result<()>) -> Result<()> {
|
||||
result.map_err(|err| Error::other(format!("rebalance metadata stats refresh failed after load: {err}")))
|
||||
}
|
||||
@@ -335,10 +299,7 @@ fn is_rebalance_transient_io_error(err: &std::io::Error) -> bool {
|
||||
|
||||
fn is_rebalance_transient_message(message: &str) -> bool {
|
||||
let message = message.to_ascii_lowercase();
|
||||
// `LockError::Timeout` renders "Lock acquisition timeout for resource ...", while the
|
||||
// namespace-lock layer renders "lock acquisition timed out on ..."; both are retryable.
|
||||
message.contains("lock acquisition timeout")
|
||||
|| message.contains("lock acquisition timed out")
|
||||
message.contains("lock acquisition timed out")
|
||||
|| message.contains("remote lock rpc timed out")
|
||||
|| message.contains("keepalivetimedout")
|
||||
|| message.contains("i/o timeout")
|
||||
@@ -419,8 +380,7 @@ fn is_rebalance_lock_or_rpc_timeout(err: &Error) -> bool {
|
||||
|
||||
fn is_rebalance_lock_or_rpc_timeout_message(message: &str) -> bool {
|
||||
let message = message.to_ascii_lowercase();
|
||||
message.contains("lock acquisition timeout")
|
||||
|| message.contains("lock acquisition timed out")
|
||||
message.contains("lock acquisition timed out")
|
||||
|| message.contains("remote lock rpc timed out")
|
||||
|| message.contains("keepalivetimedout")
|
||||
}
|
||||
@@ -745,29 +705,6 @@ mod error_source_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rebalance_metadata_retry_engages_for_wrapped_meta_save_lock_timeout() {
|
||||
let mut attempts = 0;
|
||||
let result = retry_rebalance_metadata_access(None, 3, || {
|
||||
attempts += 1;
|
||||
std::future::ready(resolve_rebalance_meta_save_result(
|
||||
Err(rebalance_meta_lock_error(
|
||||
rustfs_lock::LockError::timeout(".rustfs.sys/rebalance.bin@latest", Duration::from_secs(5)),
|
||||
"write",
|
||||
)),
|
||||
"save_rebalance_stats for pool 0 opt Stats",
|
||||
))
|
||||
})
|
||||
.await;
|
||||
|
||||
assert_eq!(attempts, 3, "a wrapped meta save lock timeout must stay retryable");
|
||||
let err = result.expect_err("persistent lock contention must not become success");
|
||||
assert!(matches!(
|
||||
rebalance_error_source(&err),
|
||||
Error::Lock(rustfs_lock::LockError::Timeout { .. })
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rebalance_metadata_retry_cancels_a_pending_attempt() {
|
||||
struct DropProbe(Arc<std::sync::atomic::AtomicUsize>);
|
||||
@@ -870,44 +807,4 @@ mod error_source_tests {
|
||||
}
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn rendered_lock_timeout_text_selects_the_lock_backoff() {
|
||||
// The lock backend renders a timeout as "Lock acquisition timeout for resource ...",
|
||||
// so the message matcher must recognize that text when the error arrives re-rendered
|
||||
// instead of as a typed `Error::Lock`.
|
||||
let rendered = rustfs_lock::LockError::timeout("bucket/object@latest", Duration::from_secs(5)).to_string();
|
||||
assert!(
|
||||
rendered.contains("Lock acquisition timeout for resource"),
|
||||
"unexpected lock timeout text: {rendered}"
|
||||
);
|
||||
|
||||
// The lock policy jitters the delay inside its own cap, so a far-out attempt identifies
|
||||
// the selected policy: the linear fallback would return `base * (attempt + 1)`.
|
||||
let far_attempt = 100;
|
||||
assert!(REBALANCE_MIGRATION_RETRY_BASE_DELAY * 101 > REBALANCE_MIGRATION_LOCK_RETRY_CAP);
|
||||
|
||||
let mut error = Error::other(format!("Lock error: {rendered}"));
|
||||
for depth in 0..=3 {
|
||||
assert!(
|
||||
is_transient_rebalance_error(&error),
|
||||
"rendered lock timeout lost retryability at depth {depth}: {error:?}"
|
||||
);
|
||||
assert!(
|
||||
is_rebalance_lock_or_rpc_timeout(&error),
|
||||
"rendered lock timeout lost the lock backoff at depth {depth}: {error:?}"
|
||||
);
|
||||
let delay = rebalance_migration_retry_delay(far_attempt, &error);
|
||||
assert!(
|
||||
delay <= REBALANCE_MIGRATION_LOCK_RETRY_CAP && delay >= Duration::from_millis(1),
|
||||
"rendered lock timeout must stay inside the lock backoff cap at depth {depth}: {delay:?}"
|
||||
);
|
||||
error = crate::data_movement::data_movement_stage_error_for_test(
|
||||
"rebalance_object",
|
||||
"put_object",
|
||||
"bucket",
|
||||
"baseline/00042.bin",
|
||||
error,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -37,7 +37,7 @@ use rustfs_s3_client::{
|
||||
api_s3_datatypes::ListVersionsResult,
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
provider_versions::validate_remote_version_id,
|
||||
transition_api::{BucketLookupType, ObjectInfo, Options, TransitionClient, TransitionCore},
|
||||
transition_api::{BucketLookupType, Options, TransitionClient, TransitionCore},
|
||||
transition_api::{ReadCloser, ReaderImpl},
|
||||
};
|
||||
use rustfs_utils::egress::validate_outbound_url;
|
||||
@@ -278,7 +278,19 @@ impl WarmBackendS3 {
|
||||
let mut stat_opts = GetObjectOptions::default();
|
||||
stat_opts.version_id.clone_from(&version.version_id);
|
||||
let info = self.client.stat_object(&self.bucket, &remote_object, &stat_opts).await?;
|
||||
let metadata = transition_candidate_metadata(info);
|
||||
let mut metadata = info.user_metadata;
|
||||
for (name, value) in &info.metadata {
|
||||
if (name
|
||||
.as_str()
|
||||
.starts_with(rustfs_utils::http::metadata_compat::RUSTFS_INTERNAL_PREFIX)
|
||||
|| name
|
||||
.as_str()
|
||||
.starts_with(rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX))
|
||||
&& let Ok(value) = value.to_str()
|
||||
{
|
||||
metadata.insert(name.as_str().to_string(), value.to_string());
|
||||
}
|
||||
}
|
||||
if transition_candidate_metadata_matches(&metadata, identity)? {
|
||||
if matched_version.is_some() {
|
||||
return Ok(TransitionCandidateProbe::Ambiguous);
|
||||
@@ -301,53 +313,6 @@ impl WarmBackendS3 {
|
||||
advance_version_markers(&mut key_marker, &mut version_id_marker, &versions)?;
|
||||
}
|
||||
}
|
||||
|
||||
async fn probe_unversioned_transition_candidate_identity(
|
||||
&self,
|
||||
object: &str,
|
||||
identity: TransitionCandidateIdentity,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
let remote_object = self.get_dest(object);
|
||||
match self
|
||||
.client
|
||||
.stat_object(&self.bucket, &remote_object, &GetObjectOptions::default())
|
||||
.await
|
||||
{
|
||||
Ok(info) => {
|
||||
let metadata = transition_candidate_metadata(info);
|
||||
if transition_candidate_metadata_matches(&metadata, identity)? {
|
||||
Ok(TransitionCandidateProbe::UnversionedPresent)
|
||||
} else {
|
||||
Ok(TransitionCandidateProbe::Unsupported)
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
let response = to_error_response(&err);
|
||||
if response.code == S3ErrorCode::NoSuchKey {
|
||||
Ok(TransitionCandidateProbe::Missing)
|
||||
} else {
|
||||
Err(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn transition_candidate_metadata(info: ObjectInfo) -> HashMap<String, String> {
|
||||
let mut metadata = info.user_metadata;
|
||||
for (name, value) in &info.metadata {
|
||||
if (name
|
||||
.as_str()
|
||||
.starts_with(rustfs_utils::http::metadata_compat::RUSTFS_INTERNAL_PREFIX)
|
||||
|| name
|
||||
.as_str()
|
||||
.starts_with(rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX))
|
||||
&& let Ok(value) = value.to_str()
|
||||
{
|
||||
metadata.insert(name.as_str().to_string(), value.to_string());
|
||||
}
|
||||
}
|
||||
metadata
|
||||
}
|
||||
|
||||
fn transition_candidate_metadata_matches(
|
||||
@@ -560,13 +525,6 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn scripted_probe_fixture(responses: Vec<String>) -> Option<(WarmBackendS3, tokio::task::JoinHandle<Vec<String>>)> {
|
||||
scripted_probe_fixture_for_tier(responses, "s3").await
|
||||
}
|
||||
|
||||
async fn scripted_probe_fixture_for_tier(
|
||||
responses: Vec<String>,
|
||||
tier_type: &str,
|
||||
) -> Option<(WarmBackendS3, tokio::task::JoinHandle<Vec<String>>)> {
|
||||
let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await {
|
||||
Ok(listener) => listener,
|
||||
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None,
|
||||
@@ -613,7 +571,7 @@ mod tests {
|
||||
max_retries: 1,
|
||||
..Default::default()
|
||||
},
|
||||
tier_type,
|
||||
"s3",
|
||||
)
|
||||
.await
|
||||
.expect("fixture client should build"),
|
||||
@@ -717,37 +675,6 @@ mod tests {
|
||||
assert!(requests[8].to_ascii_lowercase().contains("?versionid=historical-version"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unversioned_candidate_reconciler_uses_head_without_version_apis() {
|
||||
let identity = candidate_identity();
|
||||
let transaction_id = identity.transaction_id;
|
||||
let destination_id = rustfs_utils::crypto::hex(identity.destination_id);
|
||||
let response = format!(
|
||||
"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nx-amz-meta-x-rustfs-internal-transition-transaction-id: {transaction_id}\r\nx-amz-meta-x-rustfs-internal-transition-tier-destination-id: {destination_id}\r\nConnection: close\r\n\r\n"
|
||||
);
|
||||
let Some((backend, fixture)) = scripted_probe_fixture_for_tier(vec![response], "r2").await else {
|
||||
return;
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_transition_candidate_for(
|
||||
&backend,
|
||||
"archive/object",
|
||||
identity,
|
||||
)
|
||||
.await
|
||||
.expect("R2 candidate probe should use the current unversioned object"),
|
||||
TransitionCandidateProbe::UnversionedPresent
|
||||
);
|
||||
let requests = fixture.await.expect("R2 candidate fixture should finish");
|
||||
assert_eq!(requests.len(), 1);
|
||||
let request = requests[0].to_ascii_lowercase();
|
||||
assert!(request.starts_with("head /bucket/archive/object"));
|
||||
assert!(!request.contains("versioning"));
|
||||
assert!(!request.contains("versions"));
|
||||
assert!(!request.contains("versionid="));
|
||||
}
|
||||
|
||||
fn legacy_probe_xml_response(body: &str) -> String {
|
||||
format!(
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/xml\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
|
||||
@@ -1162,10 +1089,6 @@ impl TransitionCandidateReconciler for WarmBackendS3 {
|
||||
object: &str,
|
||||
identity: TransitionCandidateIdentity,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
let capabilities = self.client.provider_version_capabilities();
|
||||
if !capabilities.bucket_versioning_state && !capabilities.list_object_versions {
|
||||
return self.probe_unversioned_transition_candidate_identity(object, identity).await;
|
||||
}
|
||||
let bucket_versioning = self.remote_bucket_versioning().await?;
|
||||
self.probe_transition_candidate_identity(object, identity, bucket_versioning)
|
||||
.await
|
||||
|
||||
@@ -1163,10 +1163,13 @@ pub(in crate::set_disk) struct ReadRepairHealSubmission<'a> {
|
||||
pub(in crate::set_disk) set_index: usize,
|
||||
pub(in crate::set_disk) part_number: Option<usize>,
|
||||
pub(in crate::set_disk) reason: &'static str,
|
||||
/// Best-effort ingress for the durable MRF consumer. Its identity coalescer
|
||||
/// is independent of the heal reservation: an earlier metadata repair must
|
||||
/// not suppress newly observed payload damage, and rejected ingress must
|
||||
/// remain retryable. Channel admission does not prove checkpoint commit.
|
||||
/// Durable MRF journal intent to file alongside the read-repair request
|
||||
/// (backlog#1894 axis A): the intent kind plus its native `Uuid`
|
||||
/// version id (the submission's string form stays display-only). Bound
|
||||
/// to the reservation — the intent is only delivered when this sighting
|
||||
/// wins the dedup TTL, so a burst of reads failing on the same object
|
||||
/// books exactly one journal record instead of one per retry. `None`
|
||||
/// keeps the historical no-intent behavior.
|
||||
pub(in crate::set_disk) mrf_intent: Option<(rustfs_common::mrf_channel::MrfKind, Option<uuid::Uuid>)>,
|
||||
}
|
||||
|
||||
@@ -1221,27 +1224,6 @@ pub(in crate::set_disk) async fn submit_read_repair_heal_with_submitter(
|
||||
mrf_intent,
|
||||
} = submission;
|
||||
|
||||
if let Some((kind, version_uuid)) = mrf_intent
|
||||
&& let (Ok(pool_index), Ok(set_index)) = (u32::try_from(pool_index), u32::try_from(set_index))
|
||||
{
|
||||
let scope = rustfs_common::mrf_channel::MrfScope { pool_index, set_index };
|
||||
let ingress = rustfs_common::mrf_channel::try_send_mrf_intent_typed(kind, bucket, object, version_uuid, Some(scope));
|
||||
debug!(
|
||||
event = EVENT_SET_DISK_READ,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
state = "mrf_ingress",
|
||||
mrf_kind = ?kind,
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
pool_index,
|
||||
set_index,
|
||||
result = ?ingress,
|
||||
"Read-repair MRF ingress result"
|
||||
);
|
||||
}
|
||||
|
||||
let Some(dedup_key) = reserve_read_repair_heal(bucket, object, version_id, pool_index, set_index).await else {
|
||||
record_read_repair_dedup("duplicate");
|
||||
debug!(
|
||||
@@ -1251,6 +1233,15 @@ pub(in crate::set_disk) async fn submit_read_repair_heal_with_submitter(
|
||||
return;
|
||||
};
|
||||
|
||||
// Reservation won: this sighting owns the repair records for the object,
|
||||
// including the durable journal intent when the caller asked for one.
|
||||
if let Some((kind, version_uuid)) = mrf_intent
|
||||
&& let (Ok(pool_index), Ok(set_index)) = (u32::try_from(pool_index), u32::try_from(set_index))
|
||||
{
|
||||
let scope = rustfs_common::mrf_channel::MrfScope { pool_index, set_index };
|
||||
let _ = rustfs_common::mrf_channel::try_send_mrf_intent_typed(kind, bucket, object, version_uuid, Some(scope));
|
||||
}
|
||||
|
||||
let mut request = rustfs_heal_contracts::heal_channel::create_heal_request_with_options(
|
||||
bucket.to_string(),
|
||||
Some(object.to_string()),
|
||||
@@ -3503,44 +3494,19 @@ fn dangling_delete_grace() -> time::Duration {
|
||||
/// Result of scanning one disk's copy of a directory prefix while deciding
|
||||
/// whether an orphan (metadata-less) directory tree can be safely purged.
|
||||
enum OrphanDirScan {
|
||||
/// The prefix exists on this disk. `blocked` holds every directory that is
|
||||
/// itself, or an ancestor of, object metadata or uncommitted data (closed
|
||||
/// under taking parents, root included when anything under it is blocked);
|
||||
/// `dirs` is the pre-order list of every directory reached that is not
|
||||
/// blocked, and `committed_files` the erasure data and committed delete
|
||||
/// markers found in the unblocked UUID data dirs among them.
|
||||
Scanned {
|
||||
blocked: HashSet<String>,
|
||||
dirs: Vec<String>,
|
||||
/// The subtree holds object metadata or uncommitted data, so it must not be
|
||||
/// purged.
|
||||
HasData,
|
||||
/// The prefix contains only empty directories and/or UUID data directories
|
||||
/// carrying a committed delete marker.
|
||||
Purgeable {
|
||||
empty_dirs: Vec<String>,
|
||||
committed_files: Vec<String>,
|
||||
},
|
||||
/// A directory read failed for a reason other than absence, so nothing
|
||||
/// under the prefix can be classified on this disk.
|
||||
Unreadable,
|
||||
/// The prefix does not exist on this disk.
|
||||
Missing,
|
||||
}
|
||||
|
||||
/// How long an orphan prefix whose purge scan met unpurgeable data is left
|
||||
/// alone before an empty listing scans it again.
|
||||
const ORPHAN_PURGE_BACKOFF: Duration = Duration::from_secs(60);
|
||||
/// Upper bound on remembered backoff entries; the oldest is dropped first.
|
||||
const ORPHAN_PURGE_BACKOFF_MAX_ENTRIES: usize = 4096;
|
||||
|
||||
/// Mark `dir` and every ancestor up to and including `root` as blocked.
|
||||
fn block_orphan_dir_chain(blocked: &mut HashSet<String>, root: &str, dir: &str) {
|
||||
let mut current = dir;
|
||||
loop {
|
||||
if !blocked.insert(current.to_owned()) || current == root {
|
||||
return;
|
||||
}
|
||||
let Some((parent, _)) = current.rsplit_once(SLASH_SEPARATOR) else {
|
||||
return;
|
||||
};
|
||||
current = parent;
|
||||
}
|
||||
}
|
||||
|
||||
fn is_safe_orphan_dir_entry(entry: &str) -> bool {
|
||||
let component = entry.strip_suffix(SLASH_SEPARATOR).unwrap_or(entry);
|
||||
!component.is_empty()
|
||||
@@ -6345,12 +6311,9 @@ impl SetDisks {
|
||||
.await
|
||||
}
|
||||
|
||||
/// Scan a single disk's copy of `prefix` and classify every directory
|
||||
/// under it. Only empty directories and UUID data directories with valid
|
||||
/// committed delete markers are purgeable; anything else blocks its whole
|
||||
/// ancestor chain while sibling subtrees stay purgeable, so committed
|
||||
/// residue is still reclaimed when it shares an ancestor with residue from
|
||||
/// an older build that never wrote markers (#6898).
|
||||
/// Scan a single disk's copy of `prefix` and decide whether it is an orphan
|
||||
/// directory subtree. Only empty directories and UUID data directories with
|
||||
/// valid committed delete markers are purgeable; every child is still scanned.
|
||||
async fn scan_orphan_dir(disk: &DiskStore, bucket: &str, prefix: &str) -> OrphanDirScan {
|
||||
let root = prefix.trim_end_matches(SLASH_SEPARATOR).to_string();
|
||||
let mut stack = vec![root.clone()];
|
||||
@@ -6358,7 +6321,6 @@ impl SetDisks {
|
||||
// so reversing it yields a safe children-first removal order.
|
||||
let mut dirs: Vec<String> = Vec::new();
|
||||
let mut committed_files: Vec<String> = Vec::new();
|
||||
let mut blocked: HashSet<String> = HashSet::new();
|
||||
let mut existed = false;
|
||||
|
||||
while let Some(dir) = stack.pop() {
|
||||
@@ -6373,18 +6335,16 @@ impl SetDisks {
|
||||
}
|
||||
// Classification must fail closed: committed residue is safe to
|
||||
// remove only after every reachable child was inspected.
|
||||
Err(_) => return OrphanDirScan::Unreadable,
|
||||
Err(_) => return OrphanDirScan::HasData,
|
||||
};
|
||||
|
||||
existed = true;
|
||||
let mut child_dirs = Vec::new();
|
||||
let mut files = Vec::new();
|
||||
|
||||
let mut has_data = false;
|
||||
for entry in entries {
|
||||
if !is_safe_orphan_dir_entry(&entry) {
|
||||
has_data = true;
|
||||
break;
|
||||
return OrphanDirScan::HasData;
|
||||
}
|
||||
match entry.strip_suffix(SLASH_SEPARATOR) {
|
||||
Some(child) => child_dirs.push(format!("{dir}{SLASH_SEPARATOR}{child}")),
|
||||
@@ -6392,29 +6352,28 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
if !has_data && !files.is_empty() {
|
||||
if !files.is_empty() {
|
||||
let data_dir_name = dir.rsplit(SLASH_SEPARATOR).next().unwrap_or_default();
|
||||
let is_uuid_data_dir = Uuid::parse_str(data_dir_name).is_ok_and(|uuid| !uuid.is_nil());
|
||||
let has_committed_delete = files.iter().any(|entry| is_committed_delete_marker(entry));
|
||||
has_data = !is_uuid_data_dir || !has_committed_delete || files.iter().any(|entry| entry == STORAGE_FORMAT_FILE);
|
||||
}
|
||||
|
||||
if has_data {
|
||||
// Nothing below a blocked directory is ever removed, so its
|
||||
// children need no classification.
|
||||
block_orphan_dir_chain(&mut blocked, &root, &dir);
|
||||
if !is_uuid_data_dir || !has_committed_delete || files.iter().any(|entry| entry == STORAGE_FORMAT_FILE) {
|
||||
return OrphanDirScan::HasData;
|
||||
}
|
||||
|
||||
committed_files.extend(files.into_iter().map(|entry| path_join_buf(&[&dir, &entry])));
|
||||
dirs.push(dir);
|
||||
stack.extend(child_dirs);
|
||||
continue;
|
||||
}
|
||||
|
||||
committed_files.extend(files.into_iter().map(|entry| format!("{dir}{SLASH_SEPARATOR}{entry}")));
|
||||
dirs.push(dir);
|
||||
stack.extend(child_dirs);
|
||||
}
|
||||
|
||||
if existed {
|
||||
OrphanDirScan::Scanned {
|
||||
blocked,
|
||||
dirs,
|
||||
OrphanDirScan::Purgeable {
|
||||
empty_dirs: dirs,
|
||||
committed_files,
|
||||
}
|
||||
} else {
|
||||
@@ -6509,101 +6468,43 @@ impl SetDisks {
|
||||
/// of this set (the caller should surface the original NotFound), and `Err` on
|
||||
/// a hard disk failure.
|
||||
pub(crate) async fn purge_orphan_dir_object(&self, bucket: &str, object: &str) -> disk::error::Result<bool> {
|
||||
let backoff_key = format!("{bucket}{SLASH_SEPARATOR}{object}");
|
||||
if self.orphan_purge_in_backoff(&backoff_key) {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let disks = self.get_disks_internal().await;
|
||||
|
||||
// Phase 1: classify every online disk. A directory that holds object
|
||||
// data or uncommitted residue on ANY disk blocks itself and its
|
||||
// ancestors on every disk, so a degraded/healable object is never
|
||||
// destroyed; purgeable subtrees beside it are still reclaimed.
|
||||
let mut per_disk: Vec<(usize, Vec<String>, Vec<String>)> = Vec::new();
|
||||
let mut blocked: HashSet<String> = HashSet::new();
|
||||
// Phase 1: classify every online disk. Refuse to purge if ANY disk holds
|
||||
// object data under the prefix, so a degraded/healable object is never
|
||||
// destroyed.
|
||||
let mut per_disk_dirs: Vec<(usize, Vec<String>, Vec<String>)> = Vec::new();
|
||||
let mut existed = false;
|
||||
for (i, disk) in disks.iter().enumerate() {
|
||||
let Some(disk) = disk else { continue };
|
||||
match Self::scan_orphan_dir(disk, bucket, object).await {
|
||||
OrphanDirScan::Unreadable => return Ok(false),
|
||||
OrphanDirScan::Scanned {
|
||||
blocked: disk_blocked,
|
||||
dirs,
|
||||
OrphanDirScan::HasData => return Ok(false),
|
||||
OrphanDirScan::Purgeable {
|
||||
empty_dirs,
|
||||
committed_files,
|
||||
} => {
|
||||
blocked.extend(disk_blocked);
|
||||
per_disk.push((i, dirs, committed_files));
|
||||
existed = true;
|
||||
per_disk_dirs.push((i, empty_dirs, committed_files));
|
||||
}
|
||||
OrphanDirScan::Missing => {}
|
||||
}
|
||||
}
|
||||
if !blocked.is_empty() {
|
||||
// Whatever is purgeable goes now; what blocks the rest will still
|
||||
// block it on the next empty listing, so do not rescan for a while.
|
||||
self.record_orphan_purge_backoff(backoff_key);
|
||||
|
||||
if !existed {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
// Phase 2: remove only the files classified as committed residue in
|
||||
// unblocked data dirs, then remove unblocked directories
|
||||
// children-first. Every directory delete is non-recursive, so a
|
||||
// directory that concurrently gained an object fails with
|
||||
// DirectoryNotEmpty and is skipped — a racing PutObject is never
|
||||
// Phase 2: remove only the files classified as committed residue, then
|
||||
// remove directories children-first. Every directory delete is
|
||||
// non-recursive, so a directory that concurrently gained an object fails
|
||||
// with DirectoryNotEmpty and is skipped — a racing PutObject is never
|
||||
// clobbered.
|
||||
let mut purged = false;
|
||||
for (i, dirs, committed_files) in per_disk {
|
||||
for (i, empty_dirs, committed_files) in per_disk_dirs {
|
||||
let Some(disk) = disks[i].as_ref() else { continue };
|
||||
let dirs = dirs.into_iter().filter(|dir| !blocked.contains(dir)).collect::<Vec<_>>();
|
||||
let committed_files = committed_files
|
||||
.into_iter()
|
||||
.filter(|file| {
|
||||
file.rsplit_once(SLASH_SEPARATOR)
|
||||
.is_some_and(|(data_dir, _)| !blocked.contains(data_dir))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
if dirs.is_empty() && committed_files.is_empty() {
|
||||
continue;
|
||||
}
|
||||
purged = true;
|
||||
Self::delete_purgeable_orphan_entries(disk, bucket, object, dirs, committed_files).await;
|
||||
Self::delete_purgeable_orphan_entries(disk, bucket, object, empty_dirs, committed_files).await;
|
||||
}
|
||||
|
||||
Ok(purged)
|
||||
}
|
||||
|
||||
fn orphan_purge_in_backoff(&self, key: &str) -> bool {
|
||||
let backoff = self
|
||||
.orphan_purge_backoff
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
backoff
|
||||
.get(key)
|
||||
.is_some_and(|scanned_at| scanned_at.elapsed() < ORPHAN_PURGE_BACKOFF)
|
||||
}
|
||||
|
||||
fn record_orphan_purge_backoff(&self, key: String) {
|
||||
let mut backoff = self
|
||||
.orphan_purge_backoff
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
let now = Instant::now();
|
||||
backoff.retain(|_, scanned_at| now.duration_since(*scanned_at) < ORPHAN_PURGE_BACKOFF);
|
||||
if backoff.len() >= ORPHAN_PURGE_BACKOFF_MAX_ENTRIES
|
||||
&& let Some(oldest) = backoff
|
||||
.iter()
|
||||
.min_by_key(|(_, scanned_at)| **scanned_at)
|
||||
.map(|(key, _)| key.clone())
|
||||
{
|
||||
backoff.remove(&oldest);
|
||||
}
|
||||
backoff.insert(key, now);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn clear_orphan_purge_backoff(&self) {
|
||||
self.orphan_purge_backoff
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.clear();
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Reclaim orphaned physical data directories under `bucket/object` that no
|
||||
@@ -7720,15 +7621,13 @@ mod tests {
|
||||
.await
|
||||
.expect("committed delete marker should be written");
|
||||
|
||||
let OrphanDirScan::Scanned {
|
||||
blocked,
|
||||
dirs: empty_dirs,
|
||||
let OrphanDirScan::Purgeable {
|
||||
empty_dirs,
|
||||
committed_files,
|
||||
} = SetDisks::scan_orphan_dir(&disk, "bucket", "pfx/").await
|
||||
else {
|
||||
panic!("committed residue should be classified as purgeable");
|
||||
};
|
||||
assert!(blocked.is_empty(), "committed residue alone blocks nothing");
|
||||
|
||||
let nested_object = residue.join("nested");
|
||||
tokio::fs::create_dir_all(&nested_object)
|
||||
@@ -7770,15 +7669,13 @@ mod tests {
|
||||
.await
|
||||
.expect("committed delete marker should be written");
|
||||
|
||||
let OrphanDirScan::Scanned {
|
||||
blocked,
|
||||
dirs: empty_dirs,
|
||||
let OrphanDirScan::Purgeable {
|
||||
empty_dirs,
|
||||
committed_files,
|
||||
} = SetDisks::scan_orphan_dir(&disk, "bucket", "pfx/").await
|
||||
else {
|
||||
panic!("committed residue should be classified as purgeable");
|
||||
};
|
||||
assert!(blocked.is_empty(), "committed residue alone blocks nothing");
|
||||
tokio::fs::set_permissions(&residue, std::fs::Permissions::from_mode(0o555))
|
||||
.await
|
||||
.expect("residue directory should become read-only");
|
||||
@@ -12970,7 +12867,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn mrf_ingress_is_coalesced_independently_of_read_repair_reservations() {
|
||||
async fn mrf_intent_is_filed_once_per_read_repair_reservation() {
|
||||
// Serial: owns the process-global MRF channel for this test binary
|
||||
// (same key as the other channel-owning tests above).
|
||||
let bucket = format!("mrf-intent-bucket-{}", Uuid::new_v4());
|
||||
@@ -12991,13 +12888,8 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
// An earlier metadata/missing-shard admission has no payload MRF intent.
|
||||
let mut metadata_submission = intent_submission(&bucket, &object);
|
||||
metadata_submission.mrf_intent = None;
|
||||
submit_read_repair_heal_with_submitter(metadata_submission, accepted_read_repair_submitter).await;
|
||||
assert!(receiver.try_recv().is_err());
|
||||
|
||||
// Newly observed corruption must reach MRF despite that reservation.
|
||||
// First sighting wins the reservation: the journal intent is filed
|
||||
// synchronously before the admission task is spawned.
|
||||
submit_read_repair_heal_with_submitter(intent_submission(&bucket, &object), accepted_read_repair_submitter).await;
|
||||
let first = receiver.try_recv().expect("first sighting must file exactly one MRF intent");
|
||||
assert_eq!(*first.bucket, bucket);
|
||||
@@ -13007,20 +12899,6 @@ mod tests {
|
||||
// second journal record.
|
||||
submit_read_repair_heal_with_submitter(intent_submission(&bucket, &object), accepted_read_repair_submitter).await;
|
||||
assert!(receiver.try_recv().is_err(), "duplicate sighting must not file another MRF intent");
|
||||
|
||||
let retry_object = format!("retry-object-{}", Uuid::new_v4());
|
||||
rustfs_common::mrf_channel::set_mrf_delivery_enabled(false);
|
||||
submit_read_repair_heal_with_submitter(intent_submission(&bucket, &retry_object), accepted_read_repair_submitter).await;
|
||||
assert!(receiver.try_recv().is_err());
|
||||
rustfs_common::mrf_channel::set_mrf_delivery_enabled(true);
|
||||
submit_read_repair_heal_with_submitter(intent_submission(&bucket, &retry_object), accepted_read_repair_submitter).await;
|
||||
let retried = receiver
|
||||
.try_recv()
|
||||
.expect("a heal reservation cannot suppress a previously rejected MRF intent");
|
||||
assert_eq!(*retried.object, retry_object);
|
||||
rustfs_common::mrf_channel::release_mrf_intent(&first);
|
||||
rustfs_common::mrf_channel::release_mrf_intent(&retried);
|
||||
rustfs_common::mrf_channel::set_mrf_delivery_enabled(false);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -874,7 +874,6 @@ mod ctx;
|
||||
mod metadata;
|
||||
mod ops;
|
||||
pub(crate) use ops::bucket::BucketInfoQuorum;
|
||||
pub(crate) use ops::heal::AbsenceProofRequest;
|
||||
#[cfg(test)]
|
||||
pub(crate) use ops::heal::DanglingDeleteFailure;
|
||||
pub(crate) use ops::heal::HealedObjectAbsence;
|
||||
@@ -3877,9 +3876,6 @@ pub struct SetDisks {
|
||||
pub format: FormatV3,
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
disk_health_cache: Arc<RwLock<Vec<Option<DiskHealthEntry>>>>,
|
||||
/// Coalesce bounded GET probes of missing remote slots. Shared by set clones;
|
||||
/// the timestamp also limits repeated probes while a peer remains absent.
|
||||
read_reconnect: Arc<tokio::sync::Mutex<Option<tokio::time::Instant>>>,
|
||||
get_object_metadata_cache: moka::future::Cache<GetObjectMetadataCacheKey, Arc<GetObjectMetadataCacheEntry>>,
|
||||
get_object_metadata_cache_hash_builder: std::collections::hash_map::RandomState,
|
||||
get_object_metadata_cache_generations: Arc<[AtomicU64]>,
|
||||
@@ -3904,12 +3900,6 @@ pub struct SetDisks {
|
||||
/// writes skip the global registry mutex (backlog#1315). `Arc` so clones of
|
||||
/// a set share one generation marker.
|
||||
capacity_dirty_generation: Arc<AtomicU64>,
|
||||
/// Orphan prefixes whose last purge scan met data that can never be
|
||||
/// purged by listing (residue without a committed marker, an in-flight
|
||||
/// write), keyed by `bucket/prefix` with the time of that scan. Empty
|
||||
/// listings of such a prefix are frequent and the scan reads the whole
|
||||
/// subtree, so it is not repeated within `ORPHAN_PURGE_BACKOFF` (#6898).
|
||||
orphan_purge_backoff: Arc<std::sync::Mutex<HashMap<String, std::time::Instant>>>,
|
||||
#[cfg(test)]
|
||||
storage_class_config_override: Arc<std::sync::RwLock<Option<Arc<storageclass::Config>>>>,
|
||||
#[cfg(test)]
|
||||
@@ -3918,72 +3908,6 @@ pub struct SetDisks {
|
||||
>,
|
||||
}
|
||||
|
||||
fn marker_purge_receipt_path(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version: Uuid,
|
||||
purge: &rustfs_common::mrf_channel::MrfDeleteMarkerPurge,
|
||||
) -> String {
|
||||
let object_hash = rustfs_utils::crypto::hex(Sha256::digest(object.as_bytes()));
|
||||
let marker_hash = rustfs_utils::crypto::hex(purge.marker_identity);
|
||||
format!(
|
||||
"marker-purge-receipts/{}/{}/{}-{}-{}.json",
|
||||
bucket, purge.bucket_incarnation_id, version, object_hash, marker_hash
|
||||
)
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
/// Persist a quorum DELETE receipt outside the object marker itself. The
|
||||
/// receipt is written to the metadata bucket so it survives a node restart
|
||||
/// while an inaccessible member still holds the stale marker.
|
||||
pub(crate) async fn persist_marker_purge_receipt(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version: Uuid,
|
||||
purge: &rustfs_common::mrf_channel::MrfDeleteMarkerPurge,
|
||||
) -> bool {
|
||||
let path = marker_purge_receipt_path(bucket, object, version, purge);
|
||||
let api = Arc::new(self.clone());
|
||||
crate::config::com::save_config_with_opts(
|
||||
api,
|
||||
&path,
|
||||
b"rustfs-marker-purge-receipt-v1".to_vec(),
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.is_ok()
|
||||
}
|
||||
|
||||
pub(crate) async fn has_marker_purge_receipt(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version: Uuid,
|
||||
purge: &rustfs_common::mrf_channel::MrfDeleteMarkerPurge,
|
||||
) -> bool {
|
||||
let path = marker_purge_receipt_path(bucket, object, version, purge);
|
||||
crate::config::com::read_config_limited_preserve_empty(Arc::new(self.clone()), &path, 128)
|
||||
.await
|
||||
.is_ok()
|
||||
}
|
||||
|
||||
pub(crate) async fn consume_marker_purge_receipt(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version: Uuid,
|
||||
purge: &rustfs_common::mrf_channel::MrfDeleteMarkerPurge,
|
||||
) {
|
||||
let path = marker_purge_receipt_path(bucket, object, version, purge);
|
||||
let _ = crate::config::com::delete_config_no_lock(Arc::new(self.clone()), &path).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Read every physical copy before selecting a version quorum. A minority
|
||||
/// legacy record is still evidence and must not disappear behind a majority
|
||||
/// not-found result. Only an explicit file/volume absence produces `None`;
|
||||
@@ -4746,7 +4670,6 @@ impl SetDisks {
|
||||
format,
|
||||
set_endpoints,
|
||||
disk_health_cache: Arc::new(RwLock::new(Vec::new())),
|
||||
read_reconnect: Arc::new(tokio::sync::Mutex::new(None)),
|
||||
get_object_metadata_cache: moka::future::Cache::builder()
|
||||
.max_capacity(get_object_metadata_cache_max_entries() as u64)
|
||||
.time_to_live(GET_OBJECT_METADATA_CACHE_TTL)
|
||||
@@ -4767,7 +4690,6 @@ impl SetDisks {
|
||||
ctx,
|
||||
capacity_scope_cache: Arc::new(std::sync::RwLock::new(CapacityScopeCache::default())),
|
||||
capacity_dirty_generation: Arc::new(AtomicU64::new(u64::MAX)),
|
||||
orphan_purge_backoff: Arc::new(std::sync::Mutex::new(HashMap::new())),
|
||||
#[cfg(test)]
|
||||
storage_class_config_override: Arc::new(std::sync::RwLock::new(None)),
|
||||
#[cfg(test)]
|
||||
@@ -4781,12 +4703,6 @@ impl SetDisks {
|
||||
&self.ctx
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn set_instance_ctx_for_test(&mut self, ctx: Arc<InstanceContext>) {
|
||||
self.local_lock_manager = ctx.lock_manager();
|
||||
self.ctx = ctx;
|
||||
}
|
||||
|
||||
/// Read the persisted bucket identity through this set's metadata owner.
|
||||
/// Missing or non-authoritative legacy identities remain errors.
|
||||
pub async fn bucket_incarnation_id_from_disk(&self, bucket: &str) -> Result<Uuid> {
|
||||
@@ -5304,24 +5220,6 @@ fn known_put_object_storage_size(data_size: i64) -> i64 {
|
||||
}
|
||||
}
|
||||
|
||||
/// Shard size the inline admission check evaluates for a single PUT.
|
||||
///
|
||||
/// A compressed or encrypted stream reports `SIZE_PRESERVE_LAYER` as its
|
||||
/// stored size because the transformed length is only known after the write.
|
||||
/// MinIO's `putObject` sizes such objects for inline admission by their
|
||||
/// plaintext `ActualSize`; without that fallback every transformed object,
|
||||
/// however small, lands in `part.1` files. A stream with neither size known
|
||||
/// yields a negative shard size, which `should_inline` rejects.
|
||||
fn inline_admission_shard_size(erasure: &coding::Erasure, stored_size: i64, actual_size: i64) -> i64 {
|
||||
if stored_size >= 0 {
|
||||
return erasure.shard_file_size(stored_size);
|
||||
}
|
||||
if actual_size > 0 {
|
||||
return erasure.shard_file_size(actual_size);
|
||||
}
|
||||
HashReader::SIZE_PRESERVE_LAYER
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn build_inline_bitrot_readers(
|
||||
files: &[FileInfo],
|
||||
@@ -6477,28 +6375,6 @@ async fn verify_inline_part_bitrot(meta: &FileInfo) -> disk::error::Result<()> {
|
||||
.map_err(|_| DiskError::FileCorrupt)
|
||||
}
|
||||
|
||||
fn validate_deep_scan_results(results: &[usize], expected_parts: usize) -> disk::error::Result<()> {
|
||||
if results.len() != expected_parts {
|
||||
return Err(DiskError::other(format!(
|
||||
"incomplete deep scan result: expected {expected_parts} parts, received {}",
|
||||
results.len()
|
||||
)));
|
||||
}
|
||||
for (part, status) in results.iter().enumerate() {
|
||||
match *status {
|
||||
CHECK_PART_SUCCESS | CHECK_PART_FILE_NOT_FOUND | CHECK_PART_FILE_CORRUPT => {}
|
||||
CHECK_PART_DISK_NOT_FOUND => return Err(DiskError::DiskNotFound),
|
||||
crate::disk::CHECK_PART_VOLUME_NOT_FOUND => return Err(DiskError::VolumeNotFound),
|
||||
_ => {
|
||||
return Err(DiskError::other(format!(
|
||||
"incomplete deep scan result: part {part} has unverified status {status}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// disks_with_all_partsv2 is a corrected version based on Go implementation.
|
||||
/// It sets partsMetadata and onlineDisks when xl.meta is inexistant/corrupted or outdated.
|
||||
/// It also checks if the status of each part (corrupted, missing, ok) in each drive.
|
||||
@@ -6719,13 +6595,9 @@ async fn disks_with_all_parts(
|
||||
// it needs healing too.
|
||||
match disk.verify_file(bucket, object, meta).await {
|
||||
Ok(v) => {
|
||||
validate_deep_scan_results(&v.results, latest_meta.parts.len())?;
|
||||
verify_resp = v;
|
||||
}
|
||||
Err(err) => {
|
||||
if !matches!(err, DiskError::FileNotFound | DiskError::FileVersionNotFound | DiskError::FileCorrupt) {
|
||||
return Err(err);
|
||||
}
|
||||
debug!(
|
||||
event = EVENT_SET_DISK_HEAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -9441,175 +9313,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
async fn write_committed_residue(object_dir: &std::path::Path) -> std::path::PathBuf {
|
||||
let residue = object_dir.join(Uuid::new_v4().to_string());
|
||||
fs::create_dir_all(&residue)
|
||||
.await
|
||||
.expect("committed data directory should be created");
|
||||
fs::write(residue.join("part.1"), b"stale")
|
||||
.await
|
||||
.expect("stale part should be written");
|
||||
fs::write(
|
||||
residue.join(format!("{}{}", crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX, Uuid::new_v4())),
|
||||
[],
|
||||
)
|
||||
.await
|
||||
.expect("committed delete marker should be written");
|
||||
residue
|
||||
}
|
||||
|
||||
// #6898: residue from a build that never wrote delete markers blocks only
|
||||
// its own ancestor chain; a committed sibling subtree is still reclaimed
|
||||
// and the call reports the partial purge.
|
||||
#[tokio::test]
|
||||
async fn purge_orphan_dir_object_reclaims_committed_subtree_beside_blocked_subtree() {
|
||||
let (dir0, disk0) = make_single_local_disk().await;
|
||||
let (dir1, disk1) = make_single_local_disk().await;
|
||||
|
||||
let committed = write_committed_residue(&dir0.path().join("bucket/pfx/a/obj")).await;
|
||||
let unmarked = dir1.path().join("bucket/pfx/b/obj").join(Uuid::new_v4().to_string());
|
||||
fs::create_dir_all(&unmarked)
|
||||
.await
|
||||
.expect("unmarked residue should be created");
|
||||
fs::write(unmarked.join("part.1"), b"stale")
|
||||
.await
|
||||
.expect("stale part should be written");
|
||||
fs::create_dir_all(dir0.path().join("bucket/pfx/b/obj"))
|
||||
.await
|
||||
.expect("disk0 copy of the blocked chain should be created");
|
||||
|
||||
let set = make_set_disks_with(vec![Some(disk0), Some(disk1)]).await;
|
||||
let purged = set
|
||||
.purge_orphan_dir_object("bucket", "pfx/")
|
||||
.await
|
||||
.expect("scan should succeed");
|
||||
|
||||
assert!(purged, "the committed subtree must be reported as purged");
|
||||
assert!(!committed.exists(), "committed residue beside a blocked subtree must be reclaimed");
|
||||
assert!(!dir0.path().join("bucket/pfx/a").exists(), "the reclaimed subtree's directories must go");
|
||||
assert!(unmarked.join("part.1").exists(), "unmarked residue must survive");
|
||||
assert!(
|
||||
dir0.path().join("bucket/pfx/b/obj").exists(),
|
||||
"a directory blocked on another disk must be left alone on this one"
|
||||
);
|
||||
assert!(dir0.path().join("bucket/pfx").exists() && dir1.path().join("bucket/pfx").exists());
|
||||
}
|
||||
|
||||
// A data dir that is committed residue on one disk but still holds an
|
||||
// object below it on another disk is blocked on every disk.
|
||||
#[tokio::test]
|
||||
async fn purge_orphan_dir_object_blocks_committed_files_by_other_disks_data() {
|
||||
let (dir0, disk0) = make_single_local_disk().await;
|
||||
let (dir1, disk1) = make_single_local_disk().await;
|
||||
|
||||
let data_dir = Uuid::new_v4();
|
||||
let committed = dir0.path().join("bucket/pfx/x").join(data_dir.to_string());
|
||||
fs::create_dir_all(&committed)
|
||||
.await
|
||||
.expect("committed data directory should be created");
|
||||
fs::write(committed.join("part.1"), b"stale")
|
||||
.await
|
||||
.expect("stale part should be written");
|
||||
fs::write(
|
||||
committed.join(format!("{}{}", crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX, Uuid::new_v4())),
|
||||
[],
|
||||
)
|
||||
.await
|
||||
.expect("committed delete marker should be written");
|
||||
let nested = dir1.path().join("bucket/pfx/x").join(data_dir.to_string()).join("nested");
|
||||
fs::create_dir_all(&nested)
|
||||
.await
|
||||
.expect("nested object dir should be created");
|
||||
fs::write(nested.join(STORAGE_FORMAT_FILE), b"meta")
|
||||
.await
|
||||
.expect("nested object metadata should be written");
|
||||
|
||||
let set = make_set_disks_with(vec![Some(disk0), Some(disk1)]).await;
|
||||
let purged = set
|
||||
.purge_orphan_dir_object("bucket", "pfx/")
|
||||
.await
|
||||
.expect("scan should succeed");
|
||||
|
||||
assert!(!purged);
|
||||
assert!(
|
||||
committed.join("part.1").exists(),
|
||||
"committed files under a dir blocked elsewhere must remain"
|
||||
);
|
||||
assert!(nested.join(STORAGE_FORMAT_FILE).exists());
|
||||
}
|
||||
|
||||
// An unreadable directory anywhere under the prefix aborts the purge on
|
||||
// every disk before anything is deleted.
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn purge_orphan_dir_object_aborts_when_a_directory_is_unreadable() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
let (dir, disk) = make_single_local_disk().await;
|
||||
let committed = write_committed_residue(&dir.path().join("bucket/pfx/b/obj")).await;
|
||||
let sealed = dir.path().join("bucket/pfx/a");
|
||||
fs::create_dir_all(&sealed).await.expect("sealed directory should be created");
|
||||
fs::set_permissions(&sealed, std::fs::Permissions::from_mode(0o000))
|
||||
.await
|
||||
.expect("sealed directory should become unreadable");
|
||||
|
||||
let set = make_set_disks_with(vec![Some(disk)]).await;
|
||||
let purged = set.purge_orphan_dir_object("bucket", "pfx/").await;
|
||||
fs::set_permissions(&sealed, std::fs::Permissions::from_mode(0o755))
|
||||
.await
|
||||
.expect("sealed directory should be restored");
|
||||
|
||||
assert!(matches!(purged, Ok(false)), "an unreadable directory must abort the purge");
|
||||
assert!(committed.join("part.1").exists(), "nothing may be deleted once classification failed");
|
||||
}
|
||||
|
||||
// After a scan met unpurgeable data the prefix is not rescanned for a
|
||||
// while, even when new committed residue appears under it.
|
||||
#[tokio::test]
|
||||
async fn purge_orphan_dir_object_backs_off_after_blocked_scan() {
|
||||
let (dir, disk) = make_single_local_disk().await;
|
||||
let unmarked = dir.path().join("bucket/pfx/b/obj").join(Uuid::new_v4().to_string());
|
||||
fs::create_dir_all(&unmarked)
|
||||
.await
|
||||
.expect("unmarked residue should be created");
|
||||
fs::write(unmarked.join("part.1"), b"stale")
|
||||
.await
|
||||
.expect("stale part should be written");
|
||||
|
||||
let set = make_set_disks_with(vec![Some(disk)]).await;
|
||||
assert!(
|
||||
!set.purge_orphan_dir_object("bucket", "pfx/")
|
||||
.await
|
||||
.expect("scan should succeed")
|
||||
);
|
||||
|
||||
let committed = write_committed_residue(&dir.path().join("bucket/pfx/a/obj")).await;
|
||||
assert!(
|
||||
!set.purge_orphan_dir_object("bucket", "pfx/")
|
||||
.await
|
||||
.expect("scan should succeed"),
|
||||
"a prefix in backoff is not rescanned"
|
||||
);
|
||||
assert!(committed.exists());
|
||||
assert!(
|
||||
set.purge_orphan_dir_object("bucket", "pfx/a/")
|
||||
.await
|
||||
.expect("scan should succeed"),
|
||||
"backoff is per prefix, a narrower prefix still purges"
|
||||
);
|
||||
assert!(!committed.exists());
|
||||
|
||||
let committed = write_committed_residue(&dir.path().join("bucket/pfx/a/obj")).await;
|
||||
set.clear_orphan_purge_backoff();
|
||||
assert!(
|
||||
set.purge_orphan_dir_object("bucket", "pfx/")
|
||||
.await
|
||||
.expect("scan should succeed")
|
||||
);
|
||||
assert!(!committed.exists());
|
||||
assert!(unmarked.join("part.1").exists());
|
||||
}
|
||||
|
||||
// Build an `xl.meta` under `object_dir` whose versions reference `data_dirs`
|
||||
// (one Object version per data dir, each with its own version id).
|
||||
async fn write_object_meta_with_data_dirs(object_dir: &std::path::Path, bucket: &str, object: &str, data_dirs: &[Uuid]) {
|
||||
@@ -11139,50 +10842,6 @@ mod tests {
|
||||
assert_eq!(conv_part_err_to_int(&Some(other_err)), CHECK_PART_UNKNOWN); // Other errors should return UNKNOWN, not SUCCESS
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deep_scan_results_accept_only_complete_verified_or_repairable_parts() {
|
||||
for (results, expected_parts) in [
|
||||
(vec![], 0),
|
||||
(vec![CHECK_PART_SUCCESS], 1),
|
||||
(vec![CHECK_PART_FILE_NOT_FOUND], 1),
|
||||
(vec![CHECK_PART_FILE_CORRUPT], 1),
|
||||
(vec![CHECK_PART_SUCCESS, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_FILE_CORRUPT], 3),
|
||||
] {
|
||||
validate_deep_scan_results(&results, expected_parts)
|
||||
.expect("complete results must allow healthy or repairable parts");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deep_scan_results_reject_unknown_and_incomplete_observations() {
|
||||
for (results, expected_parts) in [
|
||||
(vec![CHECK_PART_UNKNOWN], 1),
|
||||
(vec![usize::MAX], 1),
|
||||
(vec![CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN], 2),
|
||||
(vec![], 1),
|
||||
(vec![CHECK_PART_SUCCESS], 2),
|
||||
(vec![CHECK_PART_SUCCESS, CHECK_PART_SUCCESS], 1),
|
||||
(vec![CHECK_PART_SUCCESS], 0),
|
||||
] {
|
||||
let error =
|
||||
validate_deep_scan_results(&results, expected_parts).expect_err("unverified parts must not become healthy");
|
||||
assert!(matches!(error, DiskError::Io(_)), "an incomplete observation is not proven corruption");
|
||||
assert!(error.to_string().contains("incomplete deep scan result"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deep_scan_results_preserve_disk_and_volume_failures() {
|
||||
for (status, expected_error) in [
|
||||
(CHECK_PART_DISK_NOT_FOUND, DiskError::DiskNotFound),
|
||||
(CHECK_PART_VOLUME_NOT_FOUND, DiskError::VolumeNotFound),
|
||||
] {
|
||||
let error = validate_deep_scan_results(&[CHECK_PART_SUCCESS, status], 2)
|
||||
.expect_err("an unavailable part cannot certify a healthy disk");
|
||||
assert_eq!(error, expected_error);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_has_part_err() {
|
||||
// Test checking for part errors
|
||||
@@ -13702,7 +13361,6 @@ mod tests {
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
GET_OBJECT_PATH_LEGACY_DUPLEX,
|
||||
GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART,
|
||||
metrics_size_bucket,
|
||||
@@ -13816,7 +13474,6 @@ mod tests {
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
GET_OBJECT_PATH_LEGACY_DUPLEX,
|
||||
GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART,
|
||||
metrics_size_bucket,
|
||||
@@ -13928,29 +13585,6 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inline_admission_falls_back_to_actual_size_for_transformed_streams() {
|
||||
let erasure = coding::Erasure::new(2, 2, 1024 * 1024);
|
||||
let unknown = HashReader::SIZE_PRESERVE_LAYER;
|
||||
|
||||
// A known stored size is authoritative, whatever the plaintext size says.
|
||||
assert_eq!(
|
||||
inline_admission_shard_size(&erasure, 16 * 1024, 4 * 1024 * 1024),
|
||||
erasure.shard_file_size(16 * 1024)
|
||||
);
|
||||
assert_eq!(inline_admission_shard_size(&erasure, 0, 4 * 1024), 0);
|
||||
|
||||
// A transformed stream is sized by its plaintext length (MinIO parity).
|
||||
assert_eq!(
|
||||
inline_admission_shard_size(&erasure, unknown, 16 * 1024),
|
||||
erasure.shard_file_size(16 * 1024)
|
||||
);
|
||||
|
||||
// Neither size known, or an empty transformed stream, cannot be admitted.
|
||||
assert!(inline_admission_shard_size(&erasure, unknown, 0) < 0);
|
||||
assert!(inline_admission_shard_size(&erasure, unknown, unknown) < 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_object_part_fast_path_selection_matches_single_block_non_inline_rules() {
|
||||
assert!(should_use_single_block_non_inline_fast_path(false, 4096, 4096));
|
||||
|
||||
@@ -141,7 +141,6 @@ pub(in crate::set_disk::ops) async fn verify_written_bitrot_shards(
|
||||
mod tests {
|
||||
use super::super::object::hermetic_set_disks_support::hermetic_set_disks_for_pool_with_default_parity;
|
||||
use super::*;
|
||||
use crate::storage_api_contracts::object::ObjectIO as _;
|
||||
|
||||
async fn encode_streaming_shard(data: &[u8], shard_size: usize) -> Bytes {
|
||||
let mut writer = coding::BitrotWriter::new(Cursor::new(Vec::new()), shard_size, HashAlgorithm::HighwayHash256S);
|
||||
@@ -296,19 +295,4 @@ mod tests {
|
||||
assert!(err.to_string().contains("trailing data"));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn no_parity_put_round_trip_large_stream() {
|
||||
let (_temp_dirs, disks, set_disks) = hermetic_set_disks_for_pool_with_default_parity(1, 0, 0).await;
|
||||
let bucket = "no-parity-put-round-trip";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
}
|
||||
let payload = vec![0x5a; 13 * 1024 * 1024 + 727_213];
|
||||
let mut reader = crate::set_disk::PutObjReader::from_vec(payload);
|
||||
set_disks
|
||||
.put_object(bucket, "object", &mut reader, &crate::set_disk::ObjectOptions::default())
|
||||
.await
|
||||
.expect("single-disk no-parity PUT should pass self-verify");
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -341,17 +341,14 @@ impl SetDisks {
|
||||
pub async fn renew_disk(&self, ep: &Endpoint) {
|
||||
debug!("renew_disk: start {:?}", ep);
|
||||
|
||||
let previous_disk = {
|
||||
let previous_health = {
|
||||
let disks = self.disks.read().await;
|
||||
disks
|
||||
.iter()
|
||||
.filter_map(|disk| disk.as_ref())
|
||||
.find(|disk| disk.endpoint() == *ep)
|
||||
.cloned()
|
||||
.and_then(|disk| disk.local_health_tracker_epoch_for_reconnect())
|
||||
};
|
||||
let previous_health = previous_disk
|
||||
.as_ref()
|
||||
.and_then(|disk| disk.local_health_tracker_epoch_for_reconnect());
|
||||
|
||||
let (new_disk, fm) = match Self::connect_endpoint(ep, previous_health).await {
|
||||
Ok(res) => res,
|
||||
@@ -409,25 +406,6 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
let _ = new_disk.set_disk_id(Some(fm.erasure.this)).await;
|
||||
|
||||
if !new_disk.is_local() {
|
||||
let mut disks = self.disks.write().await;
|
||||
let slot = &mut disks[disk_idx];
|
||||
let unchanged = match (slot.as_ref(), previous_disk.as_ref()) {
|
||||
(None, None) => true,
|
||||
(Some(current), Some(previous)) => Arc::ptr_eq(current, previous),
|
||||
_ => false,
|
||||
};
|
||||
// A GET probe or another reconnect may have filled this slot while
|
||||
// format I/O was pending. Do not replace its handle or start a
|
||||
// second health monitor for an unpublished handle.
|
||||
if unchanged {
|
||||
new_disk.enable_health_check();
|
||||
*slot = Some(new_disk);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
new_disk.enable_health_check();
|
||||
|
||||
if new_disk.is_local() {
|
||||
|
||||
@@ -2613,7 +2613,16 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
|
||||
fi.parts = Vec::with_capacity(uploaded_parts.len());
|
||||
|
||||
let quota_context = reservation::begin(&self.ctx, bucket, object, opts, self.pool_index, self.set_index).await?;
|
||||
let quota_context = reservation::begin(
|
||||
&self.ctx,
|
||||
bucket,
|
||||
object,
|
||||
opts.quota_admission,
|
||||
opts.data_movement,
|
||||
self.pool_index,
|
||||
self.set_index,
|
||||
)
|
||||
.await?;
|
||||
let quota_mutation_fence = quota_context.is_enforced() || opts.quota_admission.is_some();
|
||||
let preserve_replication_ciphertext = opts.replication_request
|
||||
&& contains_key_str(&fi.metadata, rustfs_utils::http::SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT);
|
||||
|
||||
@@ -49,16 +49,16 @@ use super::super::{
|
||||
disk, ensure_delete_commit_locks_held, error, explicit_delete_removed_marker, finish_set_disk_read_lock,
|
||||
get_codec_streaming_reader_gate_with_plan, get_object_body_cache_hook, get_raw_etag,
|
||||
get_small_object_direct_memory_decision_with_threshold_and_plan, get_small_object_direct_memory_threshold,
|
||||
get_stage_timer_if_enabled, get_str, get_transitioned_object_reader_with_tier_manager, inline_admission_shard_size,
|
||||
inline_erasure_shard_file_offset, inline_erasure_shard_size, insert_str, is_deadlock_detection_enabled,
|
||||
is_err_object_not_found, is_err_version_not_found, is_explicit_null_version, is_get_codec_streaming_base_enabled,
|
||||
is_get_small_object_direct_memory_enabled, is_lock_optimization_enabled, issue3031_diag_enabled, join_all,
|
||||
known_put_object_storage_size, path_join_buf, put_restore_opts, record_compression_total_memory,
|
||||
record_get_codec_streaming_gate_decision, record_get_direct_memory_decision, record_get_object_pipeline_failure,
|
||||
record_get_object_pipeline_failure_for_path, record_get_object_reader_path_observation, record_get_stage_duration_if_enabled,
|
||||
record_lock_acquire, reduce_write_quorum_errs, release_materialized_read_lock, replication_write_may_pass_worm_gate,
|
||||
require_restore_operation_id, resolve_delete_version_state, resolve_tiered_decommission_write_quorum_result,
|
||||
resolve_write_layout, restore_commit_operation_id_from_metadata, restore_operation_id_from_metadata, send_event,
|
||||
get_stage_timer_if_enabled, get_str, get_transitioned_object_reader_with_tier_manager, inline_erasure_shard_file_offset,
|
||||
inline_erasure_shard_size, insert_str, is_deadlock_detection_enabled, is_err_object_not_found, is_err_version_not_found,
|
||||
is_explicit_null_version, is_get_codec_streaming_base_enabled, is_get_small_object_direct_memory_enabled,
|
||||
is_lock_optimization_enabled, issue3031_diag_enabled, join_all, known_put_object_storage_size, path_join_buf,
|
||||
put_restore_opts, record_compression_total_memory, record_get_codec_streaming_gate_decision,
|
||||
record_get_direct_memory_decision, record_get_object_pipeline_failure, record_get_object_pipeline_failure_for_path,
|
||||
record_get_object_reader_path_observation, record_get_stage_duration_if_enabled, record_lock_acquire,
|
||||
reduce_write_quorum_errs, release_materialized_read_lock, replication_write_may_pass_worm_gate, require_restore_operation_id,
|
||||
resolve_delete_version_state, resolve_tiered_decommission_write_quorum_result, resolve_write_layout,
|
||||
restore_commit_operation_id_from_metadata, restore_operation_id_from_metadata, send_event,
|
||||
set_disk_delete_creates_delete_marker, should_force_delete_marker_for_missing_version,
|
||||
should_persist_encryption_original_size, should_preserve_delete_replication_state, should_use_inline_fast_path_with_plan,
|
||||
take_prepared_get_object_metadata, to_object_err, try_read_inline_data_shards_direct, warn,
|
||||
@@ -2776,7 +2776,6 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
self.set_index,
|
||||
self.pool_index,
|
||||
opts.skip_verify_bitrot,
|
||||
opts.suppress_read_repair,
|
||||
true,
|
||||
true,
|
||||
GET_OBJECT_PATH_LEGACY_DUPLEX,
|
||||
@@ -2806,7 +2805,6 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
self.set_index,
|
||||
self.pool_index,
|
||||
opts.skip_verify_bitrot,
|
||||
opts.suppress_read_repair,
|
||||
true,
|
||||
false,
|
||||
GET_OBJECT_PATH_LEGACY_DUPLEX,
|
||||
@@ -2895,7 +2893,6 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
self.set_index,
|
||||
self.pool_index,
|
||||
opts.skip_verify_bitrot,
|
||||
opts.suppress_read_repair,
|
||||
true,
|
||||
false,
|
||||
GET_OBJECT_PATH_DIRECT_MEMORY,
|
||||
@@ -2952,7 +2949,6 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
self.set_index,
|
||||
self.pool_index,
|
||||
opts.skip_verify_bitrot,
|
||||
opts.suppress_read_repair,
|
||||
object_class.as_str(),
|
||||
size_bucket,
|
||||
false,
|
||||
@@ -2999,7 +2995,6 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
self.set_index,
|
||||
self.pool_index,
|
||||
opts.skip_verify_bitrot,
|
||||
opts.suppress_read_repair,
|
||||
object_class.as_str(),
|
||||
size_bucket,
|
||||
codec_streaming_gate.prefer_data_blocks_first_reader_setup,
|
||||
@@ -3068,7 +3063,6 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
let set_index = self.set_index;
|
||||
let pool_index = self.pool_index;
|
||||
let skip_verify = opts.skip_verify_bitrot;
|
||||
let suppress_read_repair = opts.suppress_read_repair;
|
||||
// The producer runs in a separate Tokio task, so carry the caller's
|
||||
// read policy across the task boundary explicitly. Tokio task-local
|
||||
// values are not inherited by spawned tasks.
|
||||
@@ -3099,7 +3093,6 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
set_index,
|
||||
pool_index,
|
||||
skip_verify,
|
||||
suppress_read_repair,
|
||||
false,
|
||||
false,
|
||||
GET_OBJECT_PATH_LEGACY_DUPLEX,
|
||||
@@ -3434,8 +3427,8 @@ impl SetDisks {
|
||||
Ok(removed)
|
||||
}
|
||||
|
||||
async fn validate_bucket_incarnation(&self, bucket: &str, expected: Uuid, opts: &ObjectOptions) -> Result<()> {
|
||||
let current = metadata_sys::get_bucket_incarnation_id_for_options_in(&self.ctx, bucket, opts).await?;
|
||||
async fn validate_bucket_incarnation(&self, bucket: &str, expected: Uuid) -> Result<()> {
|
||||
let current = metadata_sys::get_bucket_incarnation_id_in(&self.ctx, bucket).await?;
|
||||
if current != expected {
|
||||
return Err(StorageError::BucketNotFound(bucket.to_string()));
|
||||
}
|
||||
@@ -3640,25 +3633,9 @@ impl SetDisks {
|
||||
|
||||
let put_object_size = known_put_object_storage_size(data.size());
|
||||
let shard_file_size_raw = erasure.shard_file_size(put_object_size);
|
||||
// Transformed streams (unknown stored size) are admitted by their
|
||||
// plaintext size and then take the streaming encode with inline
|
||||
// buffer writers, since the single-block fast path needs a known length.
|
||||
// Every inline candidate keeps the single-block bound on its admission
|
||||
// size: the inline buffer writer grows without a cap, so an explicit
|
||||
// INLINE_BLOCK budget above `block_size` must not admit multi-block
|
||||
// payloads into memory and xl.meta. Protected inline writes also need
|
||||
// a known stored length for their proof metadata, so they never fall
|
||||
// back to the plaintext size.
|
||||
let inline_admission_size = if put_object_size >= 0 || protect_write {
|
||||
put_object_size
|
||||
} else {
|
||||
data.actual_size()
|
||||
};
|
||||
let is_inline_buffer = storage_class_config.should_inline(
|
||||
inline_admission_shard_size(&erasure, put_object_size, data.actual_size()),
|
||||
erasure.data_shards,
|
||||
opts.versioned,
|
||||
) && usize::try_from(inline_admission_size).is_ok_and(|size| size <= erasure.block_size);
|
||||
let is_inline_buffer = storage_class_config.should_inline(shard_file_size_raw, erasure.data_shards, opts.versioned)
|
||||
&& put_object_size >= 0
|
||||
&& usize::try_from(put_object_size).is_ok_and(|size| size <= erasure.block_size);
|
||||
|
||||
let collect_stage_timing = rustfs_io_metrics::put_stage_metrics_enabled() || issue3031_diag_enabled();
|
||||
let shard_file_size = shard_file_size_raw;
|
||||
@@ -4250,7 +4227,16 @@ impl SetDisks {
|
||||
None
|
||||
};
|
||||
|
||||
let quota_context = reservation::begin(&self.ctx, bucket, object, opts, self.pool_index, self.set_index).await?;
|
||||
let quota_context = reservation::begin(
|
||||
&self.ctx,
|
||||
bucket,
|
||||
object,
|
||||
opts.quota_admission,
|
||||
opts.data_movement,
|
||||
self.pool_index,
|
||||
self.set_index,
|
||||
)
|
||||
.await?;
|
||||
let quota_mutation_fence = quota_context.is_enforced() || opts.quota_admission.is_some();
|
||||
let mut replication_quota_size = None;
|
||||
|
||||
@@ -6722,7 +6708,7 @@ fn remote_version_state_writer_enabled() -> bool {
|
||||
}
|
||||
|
||||
fn remote_version_state_writer_fleet_proof() -> Option<RemoteVersionStateFleetProofToken> {
|
||||
crate::services::notification_sys::acquire_remote_version_state_writer_fleet_proof()
|
||||
transaction_fencing_fleet_proof(remote_version_state_writer_requested())
|
||||
}
|
||||
|
||||
fn remote_version_state_writer_requested() -> bool {
|
||||
@@ -7578,11 +7564,7 @@ impl SetDisks {
|
||||
let fi = &transported;
|
||||
let disks = self.disk_inventory().await;
|
||||
let namespace_owner = (!is_meta_bucketname(bucket)).then(|| self.ctx.begin_namespace_commit());
|
||||
// Quorum is a property of the configured set, not of the current
|
||||
// online snapshot. A decommission/offline refresh may temporarily
|
||||
// shorten the snapshot; deriving quorum from it would turn a
|
||||
// quorum-minus-one delete into an apparent success.
|
||||
let write_quorum = self.set_drive_count / 2 + 1;
|
||||
let write_quorum = disks.len() / 2 + 1;
|
||||
let rollback_dir = Uuid::new_v4();
|
||||
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
@@ -7710,11 +7692,10 @@ impl SetDisks {
|
||||
drop(namespace_owner);
|
||||
if quorum_result.is_ok()
|
||||
&& errs.iter().any(Option::is_some)
|
||||
&& let Some(purge) = delete_marker_purge.as_ref()
|
||||
&& let Some(purge) = delete_marker_purge
|
||||
&& let Some(version) = fi.version_id.filter(|version| !version.is_nil())
|
||||
{
|
||||
let _ = self.persist_marker_purge_receipt(bucket, object, version, purge).await;
|
||||
let _ = self.persist_delete_marker_purge(bucket, object, version, purge.clone()).await;
|
||||
let _ = self.persist_delete_marker_purge(bucket, object, version, purge).await;
|
||||
}
|
||||
// An explicit purge can carry deleted=true for the existing marker.
|
||||
// It must not create a repair intent that could reintroduce that marker.
|
||||
@@ -8881,8 +8862,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
return Ok(ObjectInfo::default());
|
||||
}
|
||||
if let Some(expected_incarnation_id) = opts.expected_bucket_incarnation_id {
|
||||
self.validate_bucket_incarnation(bucket, expected_incarnation_id, &opts)
|
||||
.await?;
|
||||
self.validate_bucket_incarnation(bucket, expected_incarnation_id).await?;
|
||||
}
|
||||
ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?;
|
||||
begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?;
|
||||
@@ -9296,8 +9276,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
check_object_lock_retention_update(bucket, object, &obj_info, opts)?;
|
||||
|
||||
if let Some(expected_incarnation_id) = opts.expected_bucket_incarnation_id {
|
||||
self.validate_bucket_incarnation(bucket, expected_incarnation_id, opts)
|
||||
.await?;
|
||||
self.validate_bucket_incarnation(bucket, expected_incarnation_id).await?;
|
||||
}
|
||||
if _lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| opts
|
||||
@@ -9500,7 +9479,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
let set_index = self.set_index;
|
||||
let pool_index = self.pool_index;
|
||||
let skip_verify = opts.skip_verify_bitrot;
|
||||
let suppress_read_repair = opts.suppress_read_repair;
|
||||
let metrics_size_bucket = rustfs_io_metrics::get_object_size_bucket(cloned_fi.size);
|
||||
let erasure_cache = Arc::clone(&self.erasure_cache);
|
||||
let producer = async move {
|
||||
@@ -9518,7 +9496,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
set_index,
|
||||
pool_index,
|
||||
skip_verify,
|
||||
suppress_read_repair,
|
||||
false,
|
||||
false,
|
||||
GET_OBJECT_PATH_LEGACY_DUPLEX,
|
||||
@@ -9879,8 +9856,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
.await?
|
||||
.acquire_bucket_lifecycle_read_lock(bucket)
|
||||
.await?;
|
||||
self.validate_bucket_incarnation(bucket, expected_incarnation_id, opts)
|
||||
.await?;
|
||||
self.validate_bucket_incarnation(bucket, expected_incarnation_id).await?;
|
||||
Some(guard)
|
||||
} else {
|
||||
None
|
||||
@@ -11664,241 +11640,6 @@ mod inline_put_commit_path_tests {
|
||||
assert_eq!(restored, payload);
|
||||
}
|
||||
|
||||
/// Counts `part.*` files under every hermetic disk root.
|
||||
fn count_part_files(temp_dirs: &[tempfile::TempDir]) -> usize {
|
||||
fn walk(dir: &std::path::Path, hits: &mut usize) {
|
||||
let Ok(entries) = std::fs::read_dir(dir) else {
|
||||
return;
|
||||
};
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
if path.is_dir() {
|
||||
walk(&path, hits);
|
||||
} else if path
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.is_some_and(|name| name.starts_with("part."))
|
||||
{
|
||||
*hits += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut hits = 0;
|
||||
for dir in temp_dirs {
|
||||
walk(dir.path(), &mut hits);
|
||||
}
|
||||
hits
|
||||
}
|
||||
|
||||
/// The reader shape the app layer hands to `put_object` for a compressed
|
||||
/// single PUT: the stored size is unknown (`SIZE_PRESERVE_LAYER`), the
|
||||
/// plaintext size is known, and the metadata marks the object compressed so
|
||||
/// GET decompresses it. Encrypted PUTs present the same shape.
|
||||
fn transformed_put(plaintext: Vec<u8>) -> (PutObjReader, ObjectOptions) {
|
||||
let actual_size = plaintext.len() as i64;
|
||||
let plain = HashReader::from_stream(Cursor::new(plaintext), actual_size, actual_size, None, None, false)
|
||||
.expect("hash reader over plaintext");
|
||||
let compressed = crate::io_support::rio::compression_reader(plain, rustfs_utils::CompressionAlgorithm::default(), false);
|
||||
let stream = HashReader::from_reader(compressed, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false)
|
||||
.expect("hash reader over transformed stream");
|
||||
let mut user_defined = HashMap::new();
|
||||
insert_str(
|
||||
&mut user_defined,
|
||||
SUFFIX_COMPRESSION,
|
||||
crate::io_support::rio::compression_metadata_value(rustfs_utils::CompressionAlgorithm::default()),
|
||||
);
|
||||
insert_str(&mut user_defined, SUFFIX_ACTUAL_SIZE, actual_size.to_string());
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
user_defined,
|
||||
..Default::default()
|
||||
};
|
||||
(PutObjReader::new(stream), opts)
|
||||
}
|
||||
|
||||
fn compressible_payload(size: usize) -> Vec<u8> {
|
||||
b"inline transformed stream payload. "
|
||||
.iter()
|
||||
.copied()
|
||||
.cycle()
|
||||
.take(size)
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn read_back(set_disks: &Arc<SetDisks>, bucket: &str, object: &str) -> Vec<u8> {
|
||||
let mut object_reader = set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("committed object should be readable");
|
||||
let mut restored = Vec::new();
|
||||
object_reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("object should stream");
|
||||
restored
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn transformed_small_put_is_stored_inline_and_round_trips() {
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "inline-transformed-small";
|
||||
let object = "object.txt";
|
||||
// 16 KiB plaintext over EC 2+2 is 8 KiB per data shard, inside the
|
||||
// default 128 KiB inline budget; the stored size is unknown up front.
|
||||
let plaintext = compressible_payload(16 * 1024);
|
||||
make_bucket(&disk_stores, bucket).await;
|
||||
|
||||
let (mut reader, opts) = transformed_put(plaintext.clone());
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut reader, &opts)
|
||||
.await
|
||||
.expect("transformed PUT should commit");
|
||||
|
||||
let read_data = ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
};
|
||||
for (disk_index, disk) in disk_stores.iter().enumerate() {
|
||||
let file_info = disk
|
||||
.read_version("", bucket, object, "", &read_data)
|
||||
.await
|
||||
.unwrap_or_else(|err| panic!("disk {disk_index} should persist metadata: {err}"));
|
||||
assert!(file_info.inline_data(), "disk {disk_index} must mark the transformed shard inline");
|
||||
assert!(
|
||||
file_info.data.as_ref().is_some_and(|data| !data.is_empty()),
|
||||
"disk {disk_index} must embed the shard in xl.meta"
|
||||
);
|
||||
assert!(file_info.is_compressed(), "disk {disk_index} must keep the compression marker");
|
||||
}
|
||||
assert_eq!(count_part_files(&temp_dirs), 0, "an inline transformed object must not leave part files");
|
||||
|
||||
assert_eq!(read_back(&set_disks, bucket, object).await, plaintext);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn protected_puts_keep_unbounded_inline_candidates_external() {
|
||||
for transformed in [true, false] {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let storage_class =
|
||||
temp_env::with_var(INLINE_BLOCK_ENV, Some("16MiB"), || lookup_config_for_pools(&KVS::new(), &[4]))
|
||||
.expect("large inline budget should resolve");
|
||||
set_disks.set_test_storage_class_config(storage_class);
|
||||
let bucket = "protected-inline-candidates";
|
||||
let object = "object.bin";
|
||||
make_bucket(&disk_stores, bucket).await;
|
||||
let plaintext = compressible_payload(if transformed { 16 * 1024 } else { 1024 * 1024 + 17 });
|
||||
let (mut reader, mut opts) = if transformed {
|
||||
transformed_put(plaintext.clone())
|
||||
} else {
|
||||
(PutObjReader::from_vec(plaintext.clone()), ObjectOptions::default())
|
||||
};
|
||||
opts.shard_integrity_write_mode = Some(crate::object_api::ShardIntegrityWriteMode::Protected);
|
||||
temp_env::async_with_vars(
|
||||
[(
|
||||
crate::set_disk::core::io_primitives::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE,
|
||||
Some("false"),
|
||||
)],
|
||||
set_disks.put_object(bucket, object, &mut reader, &opts),
|
||||
)
|
||||
.await
|
||||
.expect("protected PUT should commit with an external integrity index");
|
||||
|
||||
for (disk_index, disk) in disk_stores.iter().enumerate() {
|
||||
let file_info = disk
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.unwrap_or_else(|err| panic!("disk {disk_index} should persist protected metadata: {err}"));
|
||||
assert!(!file_info.inline_data(), "unknown or multi-block protected shards must remain external");
|
||||
assert!(file_info.parts[0].integrity.is_some(), "protected PUT must retain its integrity proof");
|
||||
}
|
||||
assert_eq!(read_back(&set_disks, bucket, object).await, plaintext);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn unprotected_puts_keep_multi_block_inline_candidates_external() {
|
||||
for transformed in [true, false] {
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let storage_class =
|
||||
temp_env::with_var(INLINE_BLOCK_ENV, Some("16MiB"), || lookup_config_for_pools(&KVS::new(), &[4]))
|
||||
.expect("large inline budget should resolve");
|
||||
set_disks.set_test_storage_class_config(storage_class);
|
||||
let bucket = "unprotected-inline-candidates";
|
||||
let object = "object.bin";
|
||||
make_bucket(&disk_stores, bucket).await;
|
||||
// Above the single erasure block: an explicit INLINE_BLOCK budget must
|
||||
// not pull a multi-block payload into the unbounded inline buffer.
|
||||
let plaintext = compressible_payload(1024 * 1024 + 17);
|
||||
let (mut reader, opts) = if transformed {
|
||||
transformed_put(plaintext.clone())
|
||||
} else {
|
||||
(PutObjReader::from_vec(plaintext.clone()), ObjectOptions::default())
|
||||
};
|
||||
temp_env::async_with_vars(
|
||||
[(
|
||||
crate::set_disk::core::io_primitives::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE,
|
||||
Some("false"),
|
||||
)],
|
||||
set_disks.put_object(bucket, object, &mut reader, &opts),
|
||||
)
|
||||
.await
|
||||
.expect("multi-block PUT should commit with part files");
|
||||
|
||||
for (disk_index, disk) in disk_stores.iter().enumerate() {
|
||||
let file_info = disk
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.unwrap_or_else(|err| panic!("disk {disk_index} should persist metadata: {err}"));
|
||||
assert!(
|
||||
!file_info.inline_data(),
|
||||
"disk {disk_index}: a payload above block_size must stay external (transformed={transformed})"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
count_part_files(&temp_dirs) > 0,
|
||||
"a multi-block object must keep part files (transformed={transformed})"
|
||||
);
|
||||
assert_eq!(read_back(&set_disks, bucket, object).await, plaintext);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn transformed_put_above_inline_budget_keeps_part_files() {
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "inline-transformed-large";
|
||||
let object = "object.txt";
|
||||
// 512 KiB plaintext is 256 KiB per data shard, above the 128 KiB budget,
|
||||
// even though the compressed bytes would fit: the plaintext size is the
|
||||
// admission input, exactly like a known-size PUT of the same object.
|
||||
let plaintext = compressible_payload(512 * 1024);
|
||||
make_bucket(&disk_stores, bucket).await;
|
||||
|
||||
let (mut reader, opts) = transformed_put(plaintext.clone());
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut reader, &opts)
|
||||
.await
|
||||
.expect("transformed PUT should commit");
|
||||
|
||||
let read_data = ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
};
|
||||
for (disk_index, disk) in disk_stores.iter().enumerate() {
|
||||
let file_info = disk
|
||||
.read_version("", bucket, object, "", &read_data)
|
||||
.await
|
||||
.unwrap_or_else(|err| panic!("disk {disk_index} should persist metadata: {err}"));
|
||||
assert!(!file_info.inline_data(), "disk {disk_index} must keep the shard outside xl.meta");
|
||||
}
|
||||
assert_eq!(count_part_files(&temp_dirs), disk_stores.len(), "every disk must hold one part file");
|
||||
|
||||
assert_eq!(read_back(&set_disks, bucket, object).await, plaintext);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn get_object_reader_wires_mid_size_to_single_inflight() {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -18,7 +18,6 @@ use crate::diagnostics::get::{
|
||||
use crate::disk::error::Error;
|
||||
use crate::layout::disks_layout::MAX_ERASURE_SET_DRIVE_COUNT;
|
||||
use smallvec::SmallVec;
|
||||
use std::sync::atomic::{AtomicU8, Ordering};
|
||||
|
||||
/// Generic codec callers may exceed the production set limit; `SmallVec` then
|
||||
/// spills without changing slot semantics.
|
||||
@@ -26,35 +25,6 @@ pub(crate) const INLINE_SHARD_SLOTS: usize = MAX_ERASURE_SET_DRIVE_COUNT;
|
||||
pub(crate) type ShardBuffers = SmallVec<[Option<Vec<u8>>; INLINE_SHARD_SLOTS]>;
|
||||
pub(crate) type ShardErrors = SmallVec<[Option<Error>; INLINE_SHARD_SLOTS]>;
|
||||
|
||||
/// Repair evidence is independent of reconstruction quorum. Successful shards
|
||||
/// must not vote away a minority's observed corruption. Streaming decoders
|
||||
/// share this small accumulator with the reader that completes the response.
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct ShardReadRepair {
|
||||
observed: AtomicU8,
|
||||
}
|
||||
|
||||
impl ShardReadRepair {
|
||||
pub(crate) fn observe(&self, errors: &[Option<Error>]) {
|
||||
let flags = errors.iter().flatten().fold(0, |flags, error| {
|
||||
flags
|
||||
| match error {
|
||||
Error::FileCorrupt => 1,
|
||||
Error::FileNotFound => 2,
|
||||
_ => 0,
|
||||
}
|
||||
});
|
||||
if flags != 0 {
|
||||
self.observed.fetch_or(flags, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn needed(&self, allow_missing: bool) -> bool {
|
||||
let mask = if allow_missing { 3 } else { 1 };
|
||||
self.observed.load(Ordering::Relaxed) & mask != 0
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum ShardReadCost {
|
||||
Local,
|
||||
@@ -194,19 +164,6 @@ mod tests {
|
||||
use super::*;
|
||||
use std::mem::size_of;
|
||||
|
||||
#[test]
|
||||
fn repair_evidence_survives_healthy_stripes_and_excludes_unproven_failures() {
|
||||
let evidence = ShardReadRepair::default();
|
||||
evidence.observe(&[None, Some(Error::Timeout), Some(Error::DiskNotFound)]);
|
||||
assert!(!evidence.needed(true));
|
||||
evidence.observe(&[None, None, Some(Error::FileNotFound)]);
|
||||
assert!(!evidence.needed(false), "unattempted reader slots do not prove a missing shard");
|
||||
assert!(evidence.needed(true));
|
||||
evidence.observe(&[None, None, Some(Error::FileCorrupt)]);
|
||||
evidence.observe(&[None, None, None]);
|
||||
assert!(evidence.needed(false), "a later healthy stripe must not erase observed corruption");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stripe_scratch_capacity_matches_the_production_set_limit() {
|
||||
type OversizedShardBuffers = SmallVec<[Option<Vec<u8>>; 32]>;
|
||||
|
||||
@@ -645,17 +645,6 @@ impl ECStore {
|
||||
.as_ref()
|
||||
.and_then(|info| info.created)
|
||||
.unwrap_or(OffsetDateTime::UNIX_EPOCH);
|
||||
// A sidecar left by an interrupted legacy migration is the
|
||||
// bucket's published incarnation; keep it rather than minting
|
||||
// one that would invalidate fences taken on the stored value.
|
||||
// A retired incarnation is residue of a deleted bucket and
|
||||
// must not come back.
|
||||
let store = metadata_sys::object_store_in(&self.ctx).await?;
|
||||
if let Some(stored) = crate::bucket::metadata::load_bucket_incarnation(store.clone(), bucket).await?
|
||||
&& !crate::bucket::retirement::is_retired(store, bucket, stored).await?
|
||||
{
|
||||
metadata.bucket_incarnation_id = stored;
|
||||
}
|
||||
} else if !metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() {
|
||||
return Err(Error::other(format!(
|
||||
"bucket incarnation sidecar is missing for new-format metadata: {bucket}"
|
||||
@@ -2892,57 +2881,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// rustfs/rustfs#8003: a force-create (site replication replay, admin
|
||||
/// import) against a bucket whose legacy migration stopped after the
|
||||
/// sidecar write must persist `.metadata.bin` under the stored incarnation
|
||||
/// rather than fail closed or replace the published identity.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn issue_8003_force_create_adopts_existing_incarnation_sidecar() {
|
||||
let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
|
||||
let bucket = format!("bucket-8003-sidecar-{}", Uuid::new_v4().simple());
|
||||
|
||||
ecstore
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let stored = metadata_sys::get_in(&ecstore.ctx, &bucket)
|
||||
.await
|
||||
.expect("metadata should load")
|
||||
.bucket_incarnation_id;
|
||||
assert!(!stored.is_nil());
|
||||
|
||||
let metadata_path = format!("{BUCKET_META_PREFIX}/{bucket}/{}", crate::bucket::metadata::BUCKET_METADATA_FILE);
|
||||
crate::config::com::delete_config(ecstore.clone(), &metadata_path)
|
||||
.await
|
||||
.expect("simulate the interrupted migration by removing only .metadata.bin");
|
||||
assert!(!any_disk_path_exists(&disk_paths, format!("{RUSTFS_META_BUCKET}/{metadata_path}")).await);
|
||||
metadata_sys::remove_bucket_metadata_in(&ecstore.ctx, &bucket)
|
||||
.await
|
||||
.expect("drop the cached copy so the next read hits disk");
|
||||
|
||||
ecstore
|
||||
.make_bucket(
|
||||
&bucket,
|
||||
&MakeBucketOptions {
|
||||
force_create: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("force create must repair the sidecar-only bucket");
|
||||
|
||||
let (repaired, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&ecstore.ctx, &bucket)
|
||||
.await
|
||||
.expect("repaired metadata should load");
|
||||
assert!(persisted, "force create must persist .metadata.bin again");
|
||||
assert!(repaired.bucket_incarnation_sidecar);
|
||||
assert_eq!(
|
||||
repaired.bucket_incarnation_id, stored,
|
||||
"the repair must keep the incarnation the sidecar already published"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn make_bucket_seeds_new_bucket_durability_override() {
|
||||
|
||||
@@ -58,26 +58,6 @@ pub(crate) fn bucket_heal_scope(bucket: &str) -> Option<Arc<BucketHealScope>> {
|
||||
.flatten()
|
||||
}
|
||||
|
||||
/// Resolve only the two canonical bucket records; other internal paths carry no bucket authority.
|
||||
pub(super) fn bucket_metadata_owner(object: &str) -> Option<&str> {
|
||||
use crate::bucket::metadata::{BUCKET_INCARNATION_FILE, BUCKET_METADATA_FILE};
|
||||
let (prefix, rest) = object.split_once('/')?;
|
||||
let (bucket, file) = rest.split_once('/')?;
|
||||
(prefix == disk::BUCKET_META_PREFIX
|
||||
&& matches!(file, BUCKET_METADATA_FILE | BUCKET_INCARNATION_FILE)
|
||||
&& check_valid_bucket_name_strict(bucket).is_ok())
|
||||
.then_some(bucket)
|
||||
}
|
||||
|
||||
pub(crate) fn bucket_heal_scope_for_object(bucket: &str, object: &str) -> Option<Arc<BucketHealScope>> {
|
||||
let owner = if bucket == RUSTFS_META_BUCKET {
|
||||
bucket_metadata_owner(object)?
|
||||
} else {
|
||||
bucket
|
||||
};
|
||||
bucket_heal_scope(owner)
|
||||
}
|
||||
|
||||
/// Storage-owned proof for the exact version and every selected erasure location.
|
||||
/// This is an in-process result, never reconstructed from admin drive telemetry.
|
||||
#[derive(Debug)]
|
||||
@@ -316,24 +296,6 @@ impl ECStore {
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn heal_mrf_object_at_incarnation(
|
||||
self: &Arc<Self>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: &str,
|
||||
expected: uuid::Uuid,
|
||||
opts: &HealOpts,
|
||||
) -> Result<HealObjectStorageResult> {
|
||||
let object = object.to_owned();
|
||||
let version_id = version_id.to_owned();
|
||||
self.run_bucket_heal_at_incarnation(bucket, expected, opts, |store, bucket, opts| async move {
|
||||
store
|
||||
.heal_object_with_authoritative_absence_proof(&bucket, &object, &version_id, &opts)
|
||||
.await
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn acquire_heal_format_fence(
|
||||
&self,
|
||||
) -> Result<(
|
||||
@@ -442,97 +404,6 @@ impl ECStore {
|
||||
Ok(Arc::ptr_eq(&target_set, &pool.get_disks_by_key(POOL_META_NAME)))
|
||||
}
|
||||
|
||||
/// Restore the bucket configuration and incarnation before retiring a replacement marker.
|
||||
pub async fn heal_replacement_bucket_metadata(
|
||||
self: &Arc<Self>,
|
||||
bucket: &str,
|
||||
opts: &HealOpts,
|
||||
targets: &[String],
|
||||
) -> Result<()> {
|
||||
use crate::bucket::metadata::{BUCKET_INCARNATION_FILE, BUCKET_METADATA_FILE};
|
||||
use crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_read_lock_in;
|
||||
|
||||
let (Some(pool_index), Some(set_index)) = (opts.pool, opts.set) else {
|
||||
return Err(Error::PreconditionFailed);
|
||||
};
|
||||
if targets.is_empty() || opts.dry_run || opts.no_lock || check_valid_bucket_name_strict(bucket).is_err() {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
let pool = self
|
||||
.pools
|
||||
.get(pool_index)
|
||||
.ok_or_else(|| invalid_heal_pool_index(pool_index, self.pools.len()))?;
|
||||
let target_set = pool.get_disks_for_heal_object(BUCKET_METADATA_FILE, opts)?;
|
||||
if targets.iter().any(|target| {
|
||||
target_set
|
||||
.set_endpoints
|
||||
.iter()
|
||||
.filter(|endpoint| endpoint.to_string() == *target)
|
||||
.count()
|
||||
!= 1
|
||||
}) {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
let objects: Vec<_> = [BUCKET_METADATA_FILE, BUCKET_INCARNATION_FILE]
|
||||
.into_iter()
|
||||
.map(|file| format!("{}/{bucket}/{file}", disk::BUCKET_META_PREFIX))
|
||||
.filter(|object| Arc::ptr_eq(&target_set, &pool.get_disks_by_key(object)))
|
||||
.collect();
|
||||
if objects.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Read the persisted identity without lazy migration: a replacement
|
||||
// must never turn lost configuration into newly fabricated defaults.
|
||||
let incarnation = self.bucket_incarnation_id_from_disk(bucket).await?;
|
||||
let targets = targets.to_vec();
|
||||
self.run_bucket_heal_at_incarnation(bucket, incarnation, opts, move |store, bucket, opts| async move {
|
||||
// Match config writers: lifecycle, metadata transaction, object locks.
|
||||
// Keep the transaction stable through target readback, including when
|
||||
// the caller is cancelled and the storage owner finishes its write.
|
||||
let transaction = acquire_bucket_metadata_transaction_read_lock_in(&store.ctx, &bucket).await?;
|
||||
for object in objects {
|
||||
let scope = bucket_heal_scope(&bucket).ok_or(Error::PreconditionFailed)?;
|
||||
scope.check()?;
|
||||
if transaction.is_lock_lost() {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
// Config objects occupy one pool, unlike pool.bin. Require a
|
||||
// successful all-pool lookup before accepting another pool's ownership.
|
||||
if !store.single_pool() {
|
||||
let (owner, _) = store
|
||||
.get_pool_info_for_delete_marker(RUSTFS_META_BUCKET, &object, &ObjectOptions::default())
|
||||
.await?;
|
||||
if owner.index != pool_index {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
let metadata_opts = HealOpts { remove: false, ..opts };
|
||||
let (result, error) = store.heal_object(RUSTFS_META_BUCKET, &object, "", &metadata_opts).await?;
|
||||
if let Some(error) = error {
|
||||
return Err(error);
|
||||
}
|
||||
for target in &targets {
|
||||
let mut drives = result.after.drives.iter().filter(|drive| &drive.endpoint == target);
|
||||
if !drives.next().is_some_and(|drive| drive.state == "ok") || drives.next().is_some() {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
}
|
||||
if !store
|
||||
.replacement_targets_have_version(RUSTFS_META_BUCKET, &object, "", pool_index, set_index, &targets)
|
||||
.await?
|
||||
{
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
}
|
||||
if transaction.is_lock_lost() {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Return every live erasure set selected by an object-heal scope.
|
||||
pub async fn heal_erasure_set_scopes(&self, opts: &HealOpts) -> Result<Vec<(usize, usize)>> {
|
||||
let pools = self.get_pools_for_heal_object(opts)?;
|
||||
@@ -813,18 +684,8 @@ impl ECStore {
|
||||
version_id: &str,
|
||||
opts: &HealOpts,
|
||||
) -> Result<(HealResultItem, Option<Error>)> {
|
||||
self.handle_heal_object_with_absence(
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
opts,
|
||||
&mut None,
|
||||
crate::set_disk::AbsenceProofRequest {
|
||||
retirement: None,
|
||||
allow_unversioned: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
self.handle_heal_object_with_absence(bucket, object, version_id, opts, &mut None, None)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn heal_object_with_proof(
|
||||
@@ -834,34 +695,7 @@ impl ECStore {
|
||||
version_id: &str,
|
||||
opts: &HealOpts,
|
||||
) -> Result<HealObjectStorageResult> {
|
||||
self.heal_object_with_absence_proof(bucket, object, version_id, opts, false)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn heal_object_with_authoritative_absence_proof(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: &str,
|
||||
opts: &HealOpts,
|
||||
) -> Result<HealObjectStorageResult> {
|
||||
self.heal_object_with_absence_proof(bucket, object, version_id, opts, true)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn heal_object_with_absence_proof(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: &str,
|
||||
opts: &HealOpts,
|
||||
allow_unversioned_absence: bool,
|
||||
) -> Result<HealObjectStorageResult> {
|
||||
if opts.dry_run
|
||||
|| opts.no_lock
|
||||
|| (!allow_unversioned_absence && version_id.is_empty())
|
||||
|| super::utils::is_reserved_or_invalid_bucket(bucket, false)
|
||||
{
|
||||
if opts.dry_run || opts.no_lock || version_id.is_empty() || super::utils::is_reserved_or_invalid_bucket(bucket, false) {
|
||||
let (item, error) = self.handle_heal_object(bucket, object, version_id, opts).await?;
|
||||
return Ok(HealObjectStorageResult {
|
||||
item,
|
||||
@@ -904,17 +738,7 @@ impl ECStore {
|
||||
};
|
||||
let mut proofs = None;
|
||||
let (item, mut error) = self
|
||||
.handle_heal_object_with_absence(
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
opts,
|
||||
&mut proofs,
|
||||
crate::set_disk::AbsenceProofRequest {
|
||||
retirement: Some(&retirement),
|
||||
allow_unversioned: allow_unversioned_absence,
|
||||
},
|
||||
)
|
||||
.handle_heal_object_with_absence(bucket, object, version_id, opts, &mut proofs, Some(&retirement))
|
||||
.await?;
|
||||
// Read the authoritative incarnation only for an absence candidate.
|
||||
// The lifecycle guard has pinned it throughout the storage operation.
|
||||
@@ -956,7 +780,7 @@ impl ECStore {
|
||||
version_id: &str,
|
||||
opts: &HealOpts,
|
||||
absence: &mut Option<Vec<crate::set_disk::HealedObjectAbsence>>,
|
||||
proof: crate::set_disk::AbsenceProofRequest<'_>,
|
||||
retirement: Option<&crate::bucket::retirement::MarkerRetirementContext<'_>>,
|
||||
) -> Result<(HealResultItem, Option<Error>)> {
|
||||
trace!(
|
||||
event = EVENT_HEAL_OBJECT_STARTED,
|
||||
@@ -1047,7 +871,7 @@ impl ECStore {
|
||||
}
|
||||
#[cfg(test)]
|
||||
crate::core::pools::notify_decommission_external_heal_operation_started(store_id);
|
||||
pool.heal_object_with_absence(bucket, &pool_object, version_id, &opts, proof)
|
||||
pool.heal_object_with_absence(bucket, &pool_object, version_id, &opts, retirement)
|
||||
.await
|
||||
}
|
||||
});
|
||||
@@ -1071,7 +895,7 @@ impl ECStore {
|
||||
move |opts| async move {
|
||||
#[cfg(test)]
|
||||
crate::core::pools::notify_decommission_external_heal_operation_started(store_id);
|
||||
pool.heal_object_with_absence(bucket, &pool_object, version_id, &opts, proof)
|
||||
pool.heal_object_with_absence(bucket, &pool_object, version_id, &opts, retirement)
|
||||
.await
|
||||
},
|
||||
));
|
||||
@@ -1456,16 +1280,6 @@ mod tests {
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
for file in [".metadata.bin", ".bucket-incarnation"] {
|
||||
let metadata = format!("buckets/{bucket}/{file}");
|
||||
assert!(
|
||||
store
|
||||
.rename_local_data_at_incarnation(&disk_ref, (&bucket, object), &fi, (RUSTFS_META_BUCKET, &metadata), old)
|
||||
.await
|
||||
.is_err(),
|
||||
"stale remote heal must not replace the successor's {file}"
|
||||
);
|
||||
}
|
||||
assert!(store.heal_bucket_at_incarnation(&bucket, old, &opts).await.is_err());
|
||||
assert!(
|
||||
store
|
||||
@@ -1864,194 +1678,6 @@ mod tests {
|
||||
(temp_dir, store, shutdown)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_metadata_heal_owner_rejects_path_aliases_and_unrelated_records() {
|
||||
for file in [".metadata.bin", ".bucket-incarnation"] {
|
||||
assert_eq!(bucket_metadata_owner(&format!("buckets/example/{file}")), Some("example"));
|
||||
}
|
||||
for path in [
|
||||
"/buckets/example/.metadata.bin",
|
||||
"buckets/../.metadata.bin",
|
||||
"buckets/example/../.metadata.bin",
|
||||
"buckets/example/.metadata.bin/extra",
|
||||
"buckets/example//.metadata.bin",
|
||||
"buckets/example/.metadata.bin.old",
|
||||
"config/iam/example/.metadata.bin",
|
||||
"buckets/example/usage.json",
|
||||
] {
|
||||
assert_eq!(bucket_metadata_owner(path), None, "unexpected bucket authority for {path}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replacement_bucket_metadata_respects_pool_ownership_and_required_records() {
|
||||
let (root, store, shutdown) = multi_pool_heal_store().await;
|
||||
let bucket = "replacement-bucket-metadata";
|
||||
for invalid_bucket in ["MixedCase", "../example", RUSTFS_META_BUCKET] {
|
||||
assert!(matches!(
|
||||
store
|
||||
.heal_replacement_bucket_metadata(
|
||||
invalid_bucket,
|
||||
&HealOpts {
|
||||
pool: Some(0),
|
||||
set: Some(0),
|
||||
..Default::default()
|
||||
},
|
||||
&[root.path().join("pool0-disk0").to_string_lossy().into_owned()],
|
||||
)
|
||||
.await,
|
||||
Err(Error::PreconditionFailed)
|
||||
));
|
||||
}
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket and its persisted records");
|
||||
let mut owned = 0;
|
||||
for pool in 0..2 {
|
||||
let target = root.path().join(format!("pool{pool}-disk0"));
|
||||
let records: Vec<_> = [".metadata.bin", ".bucket-incarnation"]
|
||||
.into_iter()
|
||||
.map(|file| {
|
||||
let path = target.join(RUSTFS_META_BUCKET).join("buckets").join(bucket).join(file);
|
||||
let exists = path.join("xl.meta").exists();
|
||||
if exists {
|
||||
std::fs::remove_dir_all(&path).expect("remove replacement metadata shard");
|
||||
owned += 1;
|
||||
}
|
||||
(path, exists)
|
||||
})
|
||||
.collect();
|
||||
store
|
||||
.heal_replacement_bucket_metadata(
|
||||
bucket,
|
||||
&HealOpts {
|
||||
pool: Some(pool),
|
||||
set: Some(0),
|
||||
recreate: true,
|
||||
..Default::default()
|
||||
},
|
||||
&[target.to_string_lossy().into_owned()],
|
||||
)
|
||||
.await
|
||||
.expect("repair owned records and accept authoritative ownership in the other pool");
|
||||
for (path, existed) in records {
|
||||
assert_eq!(path.join("xl.meta").exists(), existed, "wrong placement for {}", path.display());
|
||||
}
|
||||
}
|
||||
assert_eq!(owned, 2, "each required record must have exactly one owning pool");
|
||||
let object = format!("buckets/{bucket}/.metadata.bin");
|
||||
let owner = store
|
||||
.get_pool_idx_existing_with_opts(RUSTFS_META_BUCKET, &object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("persisted metadata owner");
|
||||
let disk = store.pools[owner].disk_set[0].disks.read().await[0]
|
||||
.clone()
|
||||
.expect("metadata target disk");
|
||||
let fi = disk
|
||||
.read_version(
|
||||
"",
|
||||
RUSTFS_META_BUCKET,
|
||||
&object,
|
||||
"",
|
||||
&disk::ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("inline metadata to repair through target-side admission");
|
||||
assert!(fi.data.is_some(), "fixture must contain an inline metadata shard");
|
||||
let target_path = root.path().join(format!("pool{owner}-disk0"));
|
||||
std::fs::remove_dir_all(target_path.join(RUSTFS_META_BUCKET).join(&object)).expect("remove remote target shard");
|
||||
let incarnation = store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("current incarnation");
|
||||
store
|
||||
.rename_local_data_at_incarnation(
|
||||
&disk.endpoint().to_string(),
|
||||
(disk::RUSTFS_META_TMP_BUCKET, "metadata-heal-control"),
|
||||
&fi,
|
||||
(RUSTFS_META_BUCKET, &object),
|
||||
incarnation,
|
||||
)
|
||||
.await
|
||||
.expect("the target-side RPC admission must accept current bucket metadata repair");
|
||||
assert!(target_path.join(RUSTFS_META_BUCKET).join(&object).join("xl.meta").exists());
|
||||
let (configuration, _) = read_config_no_lock_preserve_empty_with_metadata(store.clone(), &object)
|
||||
.await
|
||||
.expect("read the current bucket configuration");
|
||||
save_config(store.clone(), &object, configuration)
|
||||
.await
|
||||
.expect("publish a newer config revision in the same bucket incarnation");
|
||||
let committed =
|
||||
std::fs::read(target_path.join(RUSTFS_META_BUCKET).join(&object).join("xl.meta")).expect("new target configuration");
|
||||
assert!(
|
||||
store
|
||||
.rename_local_data_at_incarnation(
|
||||
&disk.endpoint().to_string(),
|
||||
(disk::RUSTFS_META_TMP_BUCKET, "delayed-metadata-heal"),
|
||||
&fi,
|
||||
(RUSTFS_META_BUCKET, &object),
|
||||
incarnation,
|
||||
)
|
||||
.await
|
||||
.is_err(),
|
||||
"a delayed repair must not overwrite a newer config in the same incarnation"
|
||||
);
|
||||
assert_eq!(
|
||||
std::fs::read(target_path.join(RUSTFS_META_BUCKET).join(&object).join("xl.meta")).expect("retained configuration"),
|
||||
committed,
|
||||
"rejected repair must leave the newer configuration unchanged"
|
||||
);
|
||||
for pool in 0..2 {
|
||||
for disk in 0..4 {
|
||||
let path = root
|
||||
.path()
|
||||
.join(format!("pool{pool}-disk{disk}"))
|
||||
.join(RUSTFS_META_BUCKET)
|
||||
.join("buckets")
|
||||
.join(bucket)
|
||||
.join(".metadata.bin");
|
||||
if path.exists() {
|
||||
std::fs::remove_dir_all(path).expect("remove every copy of required bucket configuration");
|
||||
}
|
||||
}
|
||||
}
|
||||
metadata_sys::remove_bucket_metadata_in(&store.ctx, bucket)
|
||||
.await
|
||||
.expect("remove the cached configuration so repair cannot rely on it");
|
||||
assert!(
|
||||
store
|
||||
.heal_replacement_bucket_metadata(
|
||||
bucket,
|
||||
&HealOpts {
|
||||
pool: Some(0),
|
||||
set: Some(0),
|
||||
recreate: true,
|
||||
..Default::default()
|
||||
},
|
||||
&[root.path().join("pool0-disk0").to_string_lossy().into_owned()],
|
||||
)
|
||||
.await
|
||||
.is_err(),
|
||||
"total metadata loss must not be accepted as ownership in another pool"
|
||||
);
|
||||
for pool in 0..2 {
|
||||
assert!(
|
||||
!root
|
||||
.path()
|
||||
.join(format!("pool{pool}-disk0"))
|
||||
.join(RUSTFS_META_BUCKET)
|
||||
.join(&object)
|
||||
.exists(),
|
||||
"missing required configuration must not be recreated with defaults"
|
||||
);
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn absence_proof_requires_every_selected_pool() {
|
||||
@@ -3150,16 +2776,7 @@ mod tests {
|
||||
|
||||
let mut active_reader = PutObjReader::from_vec(b"active owner".to_vec());
|
||||
store.pools[0]
|
||||
.put_object(
|
||||
&bucket,
|
||||
active_object,
|
||||
&mut active_reader,
|
||||
&ObjectOptions {
|
||||
// Drain the rename tail before removing a physical shard.
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.put_object(&bucket, active_object, &mut active_reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("active owner object should be written");
|
||||
let active_disks = store.pools[0].disk_set[0].disks.read().await.clone();
|
||||
@@ -3195,7 +2812,6 @@ mod tests {
|
||||
&mut duplicate_reader,
|
||||
&ObjectOptions {
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(mod_time)),
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
|
||||
@@ -2905,17 +2905,8 @@ mod tests {
|
||||
if !version.is_empty() {
|
||||
insert_str(&mut metadata, SUFFIX_TRANSITIONED_VERSION_ID, version.to_string());
|
||||
}
|
||||
// The commit path acknowledges at write quorum and drains
|
||||
// the remaining rename fan-out in the background. This
|
||||
// fixture "crashes" the store right after the overwrite,
|
||||
// so it must wait for that tail: a disk left with the old
|
||||
// live transitioned source makes exact cleanup fail closed
|
||||
// (a minority live owner still references the remote
|
||||
// tuple) until heal repairs it, which this fixture never
|
||||
// runs (rustfs#7921).
|
||||
let options = ObjectOptions {
|
||||
version_suspended: suspended,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
};
|
||||
store
|
||||
@@ -2987,7 +2978,6 @@ mod tests {
|
||||
assert_eq!(free.len(), 1, "{state:?}, suspended={suspended}, copy={self_copy}");
|
||||
assert_eq!(free[0].transitioned_objname, remote);
|
||||
assert_eq!(free[0].transition_version_state, state);
|
||||
assert_every_disk_holds_only_the_cleanup_owner(&set, &bucket, object, &remote).await;
|
||||
assert!(backend.contains(&remote).await, "commit must not delete remote bytes before cleanup");
|
||||
let removed_before = backend.remove_count().await;
|
||||
|
||||
@@ -3159,120 +3149,6 @@ mod tests {
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
/// rustfs/rustfs#7674: a bucket carrying a legacy snapshot-protocol quota
|
||||
/// (written before durable reservations existed, so `quota.json` has no
|
||||
/// `reservation_protocol`) must accept a cross-key CopyObject whose
|
||||
/// destination options carry the handler's quota admission. The storage
|
||||
/// layer fails closed with `PartMissingOrCorrupt` when a quota-enforced
|
||||
/// bucket sees a write without admission, so dropping the admission while
|
||||
/// rebuilding the destination options turns every copy into a
|
||||
/// deterministic "part missing or corrupt" failure even though the source
|
||||
/// object is perfectly readable.
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn copy_object_forwards_quota_admission_on_legacy_snapshot_quota_bucket() {
|
||||
let temp_dir = tempfile::tempdir().expect("create legacy quota copy store dir");
|
||||
let (ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-quota-copy", &[1])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
|
||||
let bucket = format!("legacy-quota-copy-{}", Uuid::new_v4());
|
||||
let source_object = "docker/registry/v2/repositories/example/_uploads/upload-id/data";
|
||||
let target_object = "docker/registry/v2/blobs/sha256/a5/digest/data";
|
||||
let payload = vec![0x5A; 8178];
|
||||
let quota_limit = 1u64 << 30;
|
||||
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket for legacy quota copy");
|
||||
// Legacy quota shape: no `reservation_protocol`, so the storage layer
|
||||
// requires the handler-supplied snapshot admission on every write.
|
||||
let legacy_quota = format!(r#"{{"quota":{quota_limit},"quota_type":"Hard"}}"#);
|
||||
crate::bucket::metadata_sys::update_in(
|
||||
&ctx,
|
||||
&bucket,
|
||||
crate::bucket::metadata::BUCKET_QUOTA_CONFIG_FILE,
|
||||
legacy_quota.into_bytes(),
|
||||
)
|
||||
.await
|
||||
.expect("persist legacy snapshot quota");
|
||||
let (quota, _, _) = crate::bucket::metadata_sys::get_quota_config_and_incarnation_from_disk_in(&ctx, &bucket)
|
||||
.await
|
||||
.expect("legacy quota should load");
|
||||
let quota = quota.expect("legacy quota must be persisted");
|
||||
assert_eq!(quota.quota, Some(quota_limit));
|
||||
assert!(!quota.uses_durable_reservations(), "fixture must stay on the snapshot protocol");
|
||||
|
||||
let mut write_opts = ObjectOptions::default();
|
||||
assert!(write_opts.set_quota_admission(0, quota_limit));
|
||||
|
||||
let upload = store
|
||||
.new_multipart_upload(&bucket, source_object, &write_opts)
|
||||
.await
|
||||
.expect("create source multipart upload");
|
||||
let mut part_reader = PutObjReader::from_vec(payload.clone());
|
||||
let part = store
|
||||
.put_object_part(&bucket, source_object, &upload.upload_id, 1, &mut part_reader, &write_opts)
|
||||
.await
|
||||
.expect("stage multipart source part");
|
||||
store
|
||||
.clone()
|
||||
.complete_multipart_upload(
|
||||
&bucket,
|
||||
source_object,
|
||||
&upload.upload_id,
|
||||
vec![crate::storage_api_contracts::multipart::CompletePart {
|
||||
part_num: part.part_num,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
}],
|
||||
&write_opts,
|
||||
)
|
||||
.await
|
||||
.expect("complete the multipart source under the legacy quota");
|
||||
|
||||
let source_reader = store
|
||||
.get_object_reader(&bucket, source_object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("completed multipart source should be readable");
|
||||
let mut copy_info = source_reader.object_info.clone();
|
||||
let actual_size = copy_info.get_actual_size().expect("copy source logical size should resolve");
|
||||
assert_eq!(actual_size, payload.len() as i64);
|
||||
let copy_reader = rustfs_rio::HashReader::from_stream(source_reader.stream, actual_size, actual_size, None, None, false)
|
||||
.expect("copy source hash reader should build");
|
||||
copy_info.put_object_reader = Some(PutObjReader::new(copy_reader));
|
||||
|
||||
let mut dst_opts = ObjectOptions::default();
|
||||
assert!(dst_opts.set_quota_admission(payload.len() as u64, quota_limit));
|
||||
store
|
||||
.copy_object(
|
||||
&bucket,
|
||||
source_object,
|
||||
&bucket,
|
||||
target_object,
|
||||
&mut copy_info,
|
||||
&ObjectOptions::default(),
|
||||
&dst_opts,
|
||||
)
|
||||
.await
|
||||
.expect("CopyObject must forward the handler quota admission to the destination write");
|
||||
|
||||
let mut target_reader = store
|
||||
.get_object_reader(&bucket, target_object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("copied target should be readable");
|
||||
let mut target_body = Vec::new();
|
||||
target_reader
|
||||
.stream
|
||||
.read_to_end(&mut target_body)
|
||||
.await
|
||||
.expect("target body should stream");
|
||||
assert_eq!(target_body, payload);
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
@@ -4090,7 +3966,7 @@ mod tests {
|
||||
.collect::<Vec<_>>();
|
||||
for disk in &disks {
|
||||
disk.close().await.expect("fault injection should stop per-disk monitoring");
|
||||
disk.force_offline_for_test();
|
||||
disk.force_runtime_state_for_test(crate::disk::health_state::RuntimeDriveHealthState::Offline);
|
||||
}
|
||||
|
||||
// Sets has an independent endpoint monitor that renews missing slots.
|
||||
@@ -4127,7 +4003,7 @@ mod tests {
|
||||
.collect::<Vec<_>>();
|
||||
for disk in &disks {
|
||||
disk.close().await.expect("fault injection should stop per-disk monitoring");
|
||||
disk.force_offline_for_test();
|
||||
disk.force_runtime_state_for_test(crate::disk::health_state::RuntimeDriveHealthState::Offline);
|
||||
}
|
||||
set.connect_disks().await;
|
||||
for disk in &disks {
|
||||
@@ -9836,7 +9712,14 @@ mod tests {
|
||||
.await
|
||||
.expect("manual task receipt path should resolve");
|
||||
let target_task_set = store.pools[1].get_disks_by_key(&manual_task_receipt_path);
|
||||
let offline_target_task_disks = force_set_disk_range_offline_for_test(&target_task_set, 0..2).await;
|
||||
let original_target_task_disks = {
|
||||
let mut disks = target_task_set.disks.write().await;
|
||||
let original = disks.clone();
|
||||
for disk in disks.iter_mut().take(2) {
|
||||
*disk = None;
|
||||
}
|
||||
original
|
||||
};
|
||||
let receipt_quorum_error = store
|
||||
.verify_and_cleanup_decommissioned_durable_ilm_record_for_test(
|
||||
0,
|
||||
@@ -9845,7 +9728,7 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect_err("target read quorum without receipt write quorum must retain the source");
|
||||
drop(offline_target_task_disks);
|
||||
*target_task_set.disks.write().await = original_target_task_disks;
|
||||
let receipt_quorum_error = receipt_quorum_error.to_string();
|
||||
assert!(receipt_quorum_error.contains("receipt"));
|
||||
assert!(receipt_quorum_error.contains(&manual_task_path));
|
||||
@@ -10052,18 +9935,9 @@ mod tests {
|
||||
recovered_transition_version.clone(),
|
||||
)))
|
||||
.await;
|
||||
let transaction_stats = {
|
||||
let _proof = crate::services::notification_sys::install_current_remote_version_state_fleet_proof_for_test();
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_WRITE, Some("true")),
|
||||
(rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED, Some("true")),
|
||||
],
|
||||
recover_transition_transaction_records(store.clone(), 100, None),
|
||||
)
|
||||
let transaction_stats = recover_transition_transaction_records(store.clone(), 100, None)
|
||||
.await
|
||||
.expect("transition recovery should advance and consume the migrated transaction before completion")
|
||||
};
|
||||
.expect("transition recovery should advance and consume the migrated transaction before completion");
|
||||
backend.set_transition_candidate_probe_override(None).await;
|
||||
assert_eq!(
|
||||
(
|
||||
@@ -12774,45 +12648,10 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// Every physical copy must carry the replacement plus its tier
|
||||
/// free-version owner, and none may still hold the pre-overwrite live
|
||||
/// transitioned source. A stale minority copy is exactly what a lost
|
||||
/// early-ACK rename tail leaves behind, and exact cleanup refuses to
|
||||
/// delete remote bytes while such a live reference exists.
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn assert_every_disk_holds_only_the_cleanup_owner(
|
||||
set: &crate::set_disk::SetDisks,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
remote: &str,
|
||||
) {
|
||||
use crate::disk::DiskAPI as _;
|
||||
|
||||
let disk_object = rustfs_utils::path::encode_dir_object(object);
|
||||
for (index, disk) in set.disk_inventory().await.into_iter().enumerate() {
|
||||
let disk = disk.unwrap_or_else(|| panic!("disk{index} should be online"));
|
||||
let raw = disk
|
||||
.read_xl(bucket, &disk_object, false)
|
||||
.await
|
||||
.unwrap_or_else(|err| panic!("disk{index} xl.meta should be readable after the overwrite: {err:?}"));
|
||||
let versions = rustfs_filemeta::FileMeta::load(&raw.buf)
|
||||
.and_then(|meta| meta.get_all_file_info_versions(bucket, object, true))
|
||||
.unwrap_or_else(|err| panic!("disk{index} xl.meta should decode: {err:?}"));
|
||||
let all: Vec<_> = versions.versions.iter().chain(versions.free_versions.iter()).collect();
|
||||
let owners = all.iter().filter(|fi| fi.tier_free_version()).count();
|
||||
let live_sources = all
|
||||
.iter()
|
||||
.filter(|fi| !fi.tier_free_version() && fi.transitioned_objname == remote)
|
||||
.count();
|
||||
assert_eq!(owners, 1, "disk{index} must hold exactly one cleanup owner after the drained overwrite");
|
||||
assert_eq!(live_sources, 0, "disk{index} must not retain the pre-overwrite live transitioned source");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn wait_for_expiry_workers_idle(store: &crate::store::ECStore) {
|
||||
let expiry_state = store.ctx.expiry_state();
|
||||
let idle = tokio::time::timeout(Duration::from_secs(30), async {
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
let idle = {
|
||||
let state = expiry_state.read().await;
|
||||
@@ -12832,15 +12671,8 @@ mod tests {
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
if idle.is_err() {
|
||||
let state = expiry_state.read().await;
|
||||
panic!(
|
||||
"lifecycle expiry workers should become idle: pending={} active={}",
|
||||
state.pending_tasks(),
|
||||
state.active_tasks()
|
||||
);
|
||||
}
|
||||
.await
|
||||
.expect("lifecycle expiry workers should become idle");
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
@@ -21947,18 +21779,9 @@ mod tests {
|
||||
expected_removes.push((transaction.remote_object, put_version));
|
||||
}
|
||||
|
||||
let stats = {
|
||||
let _proof = crate::services::notification_sys::install_current_remote_version_state_fleet_proof_for_test();
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_WRITE, Some("true")),
|
||||
(rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED, Some("true")),
|
||||
],
|
||||
recover_transition_transaction_records(store.clone(), 100, None),
|
||||
)
|
||||
let stats = recover_transition_transaction_records(store.clone(), 100, None)
|
||||
.await
|
||||
.expect("transition transaction recovery should run")
|
||||
};
|
||||
.expect("transition transaction recovery should run");
|
||||
|
||||
assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (3, 3, 0, 0));
|
||||
assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
|
||||
|
||||
@@ -66,7 +66,7 @@ use std::future::Future;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, AtomicU64, Ordering},
|
||||
atomic::{AtomicU64, Ordering},
|
||||
};
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
use tokio::io::duplex;
|
||||
@@ -341,18 +341,6 @@ pub struct ListPathOptions {
|
||||
pub cursor_generation: Option<String>,
|
||||
pub walkdir_timeout: Option<Duration>,
|
||||
pub walkdir_stall_timeout: Option<Duration>,
|
||||
/// Shared by the collector and raw producers to preserve bounded-walk
|
||||
/// completion when filtering removes all entries from a batch.
|
||||
pub producer_limit_reached: Option<Arc<AtomicBool>>,
|
||||
}
|
||||
|
||||
fn ensure_producer_limit_state(options: &mut ListPathOptions) -> Arc<AtomicBool> {
|
||||
let state = options
|
||||
.producer_limit_reached
|
||||
.clone()
|
||||
.unwrap_or_else(|| Arc::new(AtomicBool::new(false)));
|
||||
options.producer_limit_reached = Some(state.clone());
|
||||
state
|
||||
}
|
||||
|
||||
async fn can_skip_hidden_prefix_check(options: &ListPathOptions) -> bool {
|
||||
@@ -4316,7 +4304,6 @@ impl ECStore {
|
||||
// cancel channel
|
||||
let cancel = CancellationToken::new();
|
||||
let _cancel_guard = cancel.clone().drop_guard();
|
||||
ensure_producer_limit_state(&mut o);
|
||||
|
||||
let (err_tx, mut err_rx) = broadcast::channel::<Arc<Error>>(1);
|
||||
|
||||
@@ -5060,15 +5047,6 @@ async fn gather_results(
|
||||
}
|
||||
}
|
||||
|
||||
// A producer can close its stream after exhausting its own scan budget
|
||||
// before this collector reaches its output limit. In that case the input
|
||||
// channel closing is not authoritative EOF: the caller must advertise a
|
||||
// continuation page even when every remaining entry was filtered out.
|
||||
let producer_limit_reached = opts
|
||||
.producer_limit_reached
|
||||
.as_ref()
|
||||
.is_some_and(|reached| reached.load(Ordering::Acquire));
|
||||
|
||||
// finish not full, return eof
|
||||
let filtered = scanned_entries.saturating_sub(candidate_entries);
|
||||
if let Some(started) = gather_started {
|
||||
@@ -5105,7 +5083,7 @@ async fn gather_results(
|
||||
o: MetaCacheEntries(entries),
|
||||
..Default::default()
|
||||
}),
|
||||
err: (!producer_limit_reached).then_some(rustfs_filemeta::Error::Unexpected),
|
||||
err: Some(rustfs_filemeta::Error::Unexpected),
|
||||
})
|
||||
.await
|
||||
.is_err()
|
||||
@@ -5756,7 +5734,6 @@ impl Sets {
|
||||
let (err_tx, mut err_rx) = broadcast::channel::<Arc<Error>>(1);
|
||||
let (sender, recv) = mpsc::channel(o.limit as usize);
|
||||
|
||||
ensure_producer_limit_state(&mut o);
|
||||
let sets = self.clone();
|
||||
let opts = o.clone();
|
||||
let cancel_rx1 = cancel.clone();
|
||||
@@ -6800,7 +6777,6 @@ impl SetDisks {
|
||||
let (err_tx, mut err_rx) = broadcast::channel::<Arc<Error>>(1);
|
||||
let (sender, recv) = mpsc::channel(o.limit as usize);
|
||||
|
||||
ensure_producer_limit_state(&mut o);
|
||||
let set = self.clone();
|
||||
let opts = o.clone();
|
||||
let cancel_rx1 = cancel.clone();
|
||||
@@ -7122,7 +7098,6 @@ impl SetDisks {
|
||||
forward_to: opts.marker,
|
||||
min_disks: raw_min_disks,
|
||||
per_disk_limit: limit,
|
||||
producer_limit_reached: opts.producer_limit_reached.clone(),
|
||||
// A foreground listing is bounded by lack of drive progress (the walk
|
||||
// stall timeout) and by the page limit, never by how long a healthy
|
||||
// walk takes — a large prefix on slow media is not a fault (#4644).
|
||||
@@ -7353,7 +7328,6 @@ mod test {
|
||||
};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
use tokio::sync::mpsc;
|
||||
@@ -8373,45 +8347,6 @@ mod test {
|
||||
assert!(!cancel.is_cancelled());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_gather_results_preserves_bounded_producer_after_filtering() {
|
||||
let (entry_tx, entry_rx) = mpsc::channel(4);
|
||||
let (result_tx, mut result_rx) = mpsc::channel(1);
|
||||
let cancel = CancellationToken::new();
|
||||
let producer_limit_reached = Arc::new(AtomicBool::new(true));
|
||||
|
||||
entry_tx
|
||||
.send(test_meta_entry("outside-prefix"))
|
||||
.await
|
||||
.expect("filtered test entry should be queued");
|
||||
drop(entry_tx);
|
||||
|
||||
let handle = tokio::spawn(gather_results(
|
||||
cancel,
|
||||
ListPathOptions {
|
||||
bucket: "bucket".to_owned(),
|
||||
prefix: "requested/".to_owned(),
|
||||
limit: 8,
|
||||
incl_deleted: true,
|
||||
producer_limit_reached: Some(producer_limit_reached),
|
||||
..Default::default()
|
||||
},
|
||||
entry_rx,
|
||||
result_tx,
|
||||
));
|
||||
|
||||
let result = result_rx.recv().await.expect("bounded producer result should be delivered");
|
||||
assert!(result.entries.expect("entries should be present").entries().is_empty());
|
||||
assert!(result.err.is_none(), "bounded producer must not be reported as EOF");
|
||||
assert_eq!(
|
||||
handle
|
||||
.await
|
||||
.expect("gather task should not panic")
|
||||
.expect("gather should succeed"),
|
||||
GatherResultsState::InputClosed
|
||||
);
|
||||
}
|
||||
|
||||
/// A-1 guard (rustfs/backlog#1306): pin that a *successful* send is never
|
||||
/// misclassified as `ConsumerGone`. With the receiver alive, gather_results
|
||||
/// must return the real drain state and deliver the payload carrying the
|
||||
@@ -9743,131 +9678,6 @@ mod test {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn empty_delimiter_listing_of_residue_ancestor_hides_and_purges_whole_tree() {
|
||||
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
let bucket = "listing-purge-ancestor-bucket";
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created with authoritative metadata");
|
||||
let data_dir = uuid::Uuid::new_v4();
|
||||
let transaction = uuid::Uuid::new_v4();
|
||||
for dir in &dirs {
|
||||
let residue = dir
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join("metrics/kubelet/2026/08/28/23/74992556388248657933757.parquet")
|
||||
.join(data_dir.to_string());
|
||||
tokio::fs::create_dir_all(&residue)
|
||||
.await
|
||||
.expect("committed delete residue should be created");
|
||||
tokio::fs::write(residue.join("part.1"), b"stale")
|
||||
.await
|
||||
.expect("stale part should be written");
|
||||
tokio::fs::write(
|
||||
residue.join(format!("{}{}", crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX, transaction)),
|
||||
[],
|
||||
)
|
||||
.await
|
||||
.expect("committed delete marker should be written");
|
||||
}
|
||||
|
||||
// Browsing an ancestor of the deleted key must not show the empty
|
||||
// date folders, and the empty result reclaims the whole residue tree
|
||||
// in one pass instead of one level per listing.
|
||||
let result = store
|
||||
.clone()
|
||||
.list_objects_generic(bucket, "metrics/", None, Some("/".to_owned()), 1000, false)
|
||||
.await
|
||||
.expect("delimiter listing should succeed");
|
||||
assert!(result.objects.is_empty());
|
||||
assert!(result.prefixes.is_empty(), "ancestors of delete residue must not surface as prefixes");
|
||||
for dir in &dirs {
|
||||
assert!(
|
||||
!dir.path().join(bucket).join("metrics").exists(),
|
||||
"the empty delimiter listing should reclaim the committed delete residue tree under it"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn empty_delimiter_listing_of_mixed_residue_ancestor_reclaims_only_committed_subtree() {
|
||||
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
let bucket = "listing-purge-mixed-bucket";
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created with authoritative metadata");
|
||||
let committed_dir = uuid::Uuid::new_v4();
|
||||
let unmarked_dir = uuid::Uuid::new_v4();
|
||||
let transaction = uuid::Uuid::new_v4();
|
||||
for dir in &dirs {
|
||||
let committed = dir
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join("metrics/cpu/2026/08/28/22/a.parquet")
|
||||
.join(committed_dir.to_string());
|
||||
tokio::fs::create_dir_all(&committed)
|
||||
.await
|
||||
.expect("committed delete residue should be created");
|
||||
tokio::fs::write(committed.join("part.1"), b"stale")
|
||||
.await
|
||||
.expect("stale part should be written");
|
||||
tokio::fs::write(
|
||||
committed.join(format!("{}{}", crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX, transaction)),
|
||||
[],
|
||||
)
|
||||
.await
|
||||
.expect("committed delete marker should be written");
|
||||
|
||||
// Residue left by a build that never wrote delete markers, or a
|
||||
// PUT still streaming its parts: indistinguishable, so never purged.
|
||||
let unmarked = dir
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join("metrics/kubelet/2026/08/28/23/74992556388248657933757.parquet")
|
||||
.join(unmarked_dir.to_string());
|
||||
tokio::fs::create_dir_all(&unmarked)
|
||||
.await
|
||||
.expect("unmarked residue should be created");
|
||||
tokio::fs::write(unmarked.join("part.1"), b"stale")
|
||||
.await
|
||||
.expect("stale part should be written");
|
||||
}
|
||||
|
||||
let result = store
|
||||
.clone()
|
||||
.list_objects_generic(bucket, "metrics/", None, Some("/".to_owned()), 1000, false)
|
||||
.await
|
||||
.expect("delimiter listing should succeed");
|
||||
assert!(result.objects.is_empty());
|
||||
assert!(result.prefixes.is_empty(), "neither residue subtree may surface as a prefix");
|
||||
for dir in &dirs {
|
||||
let metrics = dir.path().join(bucket).join("metrics");
|
||||
assert!(
|
||||
!metrics.join("cpu").exists(),
|
||||
"the committed subtree must be reclaimed even though a sibling subtree is blocked"
|
||||
);
|
||||
assert!(
|
||||
metrics
|
||||
.join("kubelet/2026/08/28/23/74992556388248657933757.parquet")
|
||||
.join(unmarked_dir.to_string())
|
||||
.join("part.1")
|
||||
.exists(),
|
||||
"residue without a committed marker must survive the purge"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_objects_index_provider_state_uses_lifecycle_active_generation() {
|
||||
let provider = ListObjectsIndexProviderState::walker_key_only();
|
||||
|
||||
@@ -395,19 +395,12 @@ async fn scan_metadata_less_residue_with_budget(
|
||||
Ok(scan)
|
||||
}
|
||||
|
||||
async fn enqueue_transition_after_write(
|
||||
store: &ECStore,
|
||||
result: Result<ObjectInfo>,
|
||||
src: LcEventSrc,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
async fn enqueue_transition_after_write(result: Result<ObjectInfo>, src: LcEventSrc) -> Result<ObjectInfo> {
|
||||
match result {
|
||||
Ok(oi) => {
|
||||
if should_enqueue_transition_immediately(&oi) {
|
||||
enqueue_transition_immediate(&oi, src.clone()).await;
|
||||
if let Ok(api) = metadata_sys::object_store_in(&store.ctx).await {
|
||||
enqueue_immediate_expiry(api, &oi, src, opts).await;
|
||||
}
|
||||
enqueue_immediate_expiry(&oi, src).await;
|
||||
}
|
||||
Ok(oi)
|
||||
}
|
||||
@@ -426,8 +419,8 @@ mod bucket_fence;
|
||||
pub(crate) use bucket::await_bucket_namespace_operation;
|
||||
pub use bucket_fence::BucketIncarnationFenceGuard;
|
||||
mod heal;
|
||||
pub(crate) use heal::bucket_heal_scope;
|
||||
pub use heal::{HealObjectAbsenceProof, HealObjectStorageResult};
|
||||
pub(crate) use heal::{bucket_heal_scope, bucket_heal_scope_for_object};
|
||||
mod heal_walk;
|
||||
pub use heal_walk::HealWalkVersion;
|
||||
mod init;
|
||||
@@ -1288,11 +1281,9 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(ObjectInfo, Option<crate::disk::OldCurrentSize>)> {
|
||||
let result = match self.handle_put_object(bucket, object, data, opts).await {
|
||||
Ok((object_info, old_current_size)) => {
|
||||
enqueue_transition_after_write(self, Ok(object_info), LcEventSrc::S3PutObject, opts)
|
||||
.await
|
||||
.map(|object_info| (object_info, old_current_size))
|
||||
}
|
||||
Ok((object_info, old_current_size)) => enqueue_transition_after_write(Ok(object_info), LcEventSrc::S3PutObject)
|
||||
.await
|
||||
.map(|object_info| (object_info, old_current_size)),
|
||||
Err(err) => Err(err),
|
||||
};
|
||||
if result.is_ok() {
|
||||
@@ -1365,11 +1356,9 @@ impl crate::storage_api_contracts::object::ObjectOperations for ECStore {
|
||||
dst_opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
let result = enqueue_transition_after_write(
|
||||
self,
|
||||
self.handle_copy_object(src_bucket, src_object, dst_bucket, dst_object, src_info, src_opts, dst_opts)
|
||||
.await,
|
||||
LcEventSrc::S3CopyObject,
|
||||
dst_opts,
|
||||
)
|
||||
.await;
|
||||
if result.is_ok() {
|
||||
@@ -1636,12 +1625,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
let result = enqueue_transition_after_write(
|
||||
self.as_ref(),
|
||||
self.clone()
|
||||
.handle_complete_multipart_upload(bucket, object, upload_id, uploaded_parts, opts)
|
||||
.await,
|
||||
LcEventSrc::S3CompleteMultipartUpload,
|
||||
opts,
|
||||
)
|
||||
.await;
|
||||
if result.is_ok() {
|
||||
|
||||
@@ -355,7 +355,7 @@ impl ECStore {
|
||||
if let Some(guard) = guard.as_ref() {
|
||||
opts.add_bucket_lifecycle_lock_guard(guard);
|
||||
}
|
||||
let current = crate::bucket::metadata_sys::get_bucket_incarnation_id_for_options_in(&self.ctx, bucket, &opts).await?;
|
||||
let current = crate::bucket::metadata_sys::get_bucket_incarnation_id_in(&self.ctx, bucket).await?;
|
||||
if opts.expected_bucket_incarnation_id != Some(current) {
|
||||
return Err(StorageError::BucketNotFound(bucket.to_string()));
|
||||
}
|
||||
@@ -530,11 +530,7 @@ impl ECStore {
|
||||
return Ok((result, 0, opts.expected_bucket_incarnation_id));
|
||||
}
|
||||
|
||||
let capacity_owner = DecommissionCapacityOwner::from_options(&opts);
|
||||
if opts.data_movement && (opts.version_id.is_some() || capacity_owner.is_some()) {
|
||||
// Capacity-owned decommission writes must remain on the target
|
||||
// selected by the durable reservation, including unversioned
|
||||
// objects whose ObjectOptions carry no version ID.
|
||||
if opts.data_movement && opts.version_id.is_some() {
|
||||
let idx = self.select_data_movement_pool_idx(bucket, object, -1, &opts, false).await?;
|
||||
if idx == opts.src_pool_idx {
|
||||
return Err(StorageError::DataMovementOverwriteErr(
|
||||
@@ -546,9 +542,12 @@ impl ECStore {
|
||||
self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence)
|
||||
.await;
|
||||
let res = self
|
||||
.run_decommission_capacity_temporary_mutation(idx, capacity_owner, None, || async {
|
||||
self.pools[idx].new_multipart_upload(bucket, object, &opts).await
|
||||
})
|
||||
.run_decommission_capacity_temporary_mutation(
|
||||
idx,
|
||||
DecommissionCapacityOwner::from_options(&opts),
|
||||
None,
|
||||
|| async { self.pools[idx].new_multipart_upload(bucket, object, &opts).await },
|
||||
)
|
||||
.await?;
|
||||
return Ok((res, idx, opts.expected_bucket_incarnation_id));
|
||||
}
|
||||
@@ -1047,7 +1046,6 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
publication_fence: Option<RemoteTuplePublicationFence>,
|
||||
) -> Result<ObjectInfo> {
|
||||
let request_opts = opts;
|
||||
let (target_pool_idx, mutation_fence) = target;
|
||||
check_complete_multipart_args(bucket, object, upload_id)?;
|
||||
if !opts.data_movement {
|
||||
@@ -1133,8 +1131,7 @@ impl ECStore {
|
||||
)
|
||||
.await;
|
||||
drop(publication_guard);
|
||||
let result =
|
||||
enqueue_transition_after_write(self.as_ref(), result, LcEventSrc::S3CompleteMultipartUpload, request_opts).await;
|
||||
let result = enqueue_transition_after_write(result, LcEventSrc::S3CompleteMultipartUpload).await;
|
||||
if result.is_ok() {
|
||||
list_objects::observe_list_objects_mutation(self.as_ref(), bucket).await;
|
||||
}
|
||||
|
||||
@@ -29,8 +29,8 @@ use crate::bucket::lifecycle::{
|
||||
},
|
||||
};
|
||||
use crate::bucket::metadata_sys::{
|
||||
acquire_bucket_metadata_transaction_read_lock_in, get_bucket_incarnation_id_for_options_in,
|
||||
get_cached_bucket_incarnation_id_in, get_object_lock_config_and_incarnation_from_disk_in,
|
||||
acquire_bucket_metadata_transaction_read_lock_in, get_bucket_incarnation_id_in, get_cached_bucket_incarnation_id_in,
|
||||
get_object_lock_config_and_incarnation_from_disk_in,
|
||||
};
|
||||
use crate::bucket::object_lock::objectlock_sys::{
|
||||
check_object_lock_for_deletion_with_state, ensure_recursive_force_delete_allowed_for_state,
|
||||
@@ -2837,7 +2837,7 @@ impl ECStore {
|
||||
config_revision,
|
||||
state,
|
||||
lifecycle_fence.clone(),
|
||||
Arc::new(metadata_guard),
|
||||
metadata_guard,
|
||||
)))
|
||||
}
|
||||
|
||||
@@ -4296,7 +4296,6 @@ impl ECStore {
|
||||
if !opts.data_movement {
|
||||
return Err(Error::other("data movement PUT requires data_movement options"));
|
||||
}
|
||||
let request_opts = opts;
|
||||
let (object, mut opts) = self.prepare_put_object(bucket, object, opts).await?;
|
||||
ensure_decommission_capacity_mutation_id(bucket, &object, &mut opts);
|
||||
let idx = self
|
||||
@@ -4339,7 +4338,7 @@ impl ECStore {
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let result = enqueue_transition_after_write(self, result, LcEventSrc::S3PutObject, request_opts).await;
|
||||
let result = enqueue_transition_after_write(result, LcEventSrc::S3PutObject).await;
|
||||
if result.is_ok() {
|
||||
list_objects::observe_list_objects_mutation(self, bucket).await;
|
||||
}
|
||||
@@ -4446,7 +4445,7 @@ impl ECStore {
|
||||
};
|
||||
let current_bucket_incarnation_id = if let Some(guard) = _bucket_lifecycle_guard.as_ref() {
|
||||
dst_opts.add_bucket_lifecycle_lock_guard(guard);
|
||||
let current_incarnation_id = get_bucket_incarnation_id_for_options_in(&self.ctx, dst_bucket, &dst_opts).await?;
|
||||
let current_incarnation_id = get_bucket_incarnation_id_in(&self.ctx, dst_bucket).await?;
|
||||
if dst_opts
|
||||
.expected_bucket_incarnation_id
|
||||
.is_some_and(|expected| expected != current_incarnation_id)
|
||||
@@ -4565,7 +4564,6 @@ impl ECStore {
|
||||
namespace_lock_fence: dst_opts.namespace_lock_fence.clone(),
|
||||
bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
|
||||
object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
|
||||
quota_admission: dst_opts.quota_admission,
|
||||
..Default::default()
|
||||
};
|
||||
if !self.single_pool() {
|
||||
@@ -4608,7 +4606,6 @@ impl ECStore {
|
||||
namespace_lock_fence: dst_opts.namespace_lock_fence.clone(),
|
||||
bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
|
||||
object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
|
||||
quota_admission: dst_opts.quota_admission,
|
||||
..Default::default()
|
||||
};
|
||||
if !self.single_pool() {
|
||||
@@ -4661,7 +4658,6 @@ impl ECStore {
|
||||
namespace_lock_fence: dst_opts.namespace_lock_fence.clone(),
|
||||
bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
|
||||
object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
|
||||
quota_admission: dst_opts.quota_admission,
|
||||
..Default::default()
|
||||
};
|
||||
if !self.single_pool() {
|
||||
@@ -4827,26 +4823,22 @@ impl ECStore {
|
||||
get_cached_bucket_incarnation_id_in(&self.ctx, bucket).await?;
|
||||
}
|
||||
let _object_lock_metadata_guard = if !is_meta_bucketname(bucket) {
|
||||
Some(Arc::new(acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?))
|
||||
Some(acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(guard) = _object_lock_metadata_guard.as_ref() {
|
||||
opts.add_namespace_lock_guard(guard);
|
||||
}
|
||||
let current_bucket_incarnation_id = if let Some(metadata_guard) = _object_lock_metadata_guard.as_ref() {
|
||||
let current_bucket_incarnation_id = if _object_lock_metadata_guard.is_some() {
|
||||
let (state, incarnation_id, config_revision) =
|
||||
get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await?;
|
||||
opts.object_lock_config_snapshot = Some(Arc::new(ObjectLockConfigSnapshot::for_store_bucket_under_lifecycle_fence(
|
||||
opts.object_lock_config_snapshot = Some(Arc::new(ObjectLockConfigSnapshot::for_store_bucket(
|
||||
self.id,
|
||||
bucket,
|
||||
incarnation_id,
|
||||
config_revision,
|
||||
state,
|
||||
opts.bucket_lifecycle_lock_fence
|
||||
.clone()
|
||||
.unwrap_or_else(NamespaceLockFence::new),
|
||||
Arc::clone(metadata_guard),
|
||||
)));
|
||||
Some(incarnation_id)
|
||||
} else {
|
||||
@@ -5265,32 +5257,26 @@ impl ECStore {
|
||||
let _object_lock_metadata_guard = if is_meta_bucketname(bucket) {
|
||||
None
|
||||
} else {
|
||||
Some(Arc::new(
|
||||
match acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
|
||||
},
|
||||
))
|
||||
Some(match acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
|
||||
})
|
||||
};
|
||||
if let Some(guard) = _object_lock_metadata_guard.as_ref() {
|
||||
opts.add_namespace_lock_guard(guard);
|
||||
}
|
||||
let current_bucket_incarnation_id = if let Some(metadata_guard) = _object_lock_metadata_guard.as_ref() {
|
||||
let current_bucket_incarnation_id = if _object_lock_metadata_guard.is_some() {
|
||||
let (state, incarnation_id, config_revision) =
|
||||
match get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await {
|
||||
Ok(snapshot) => snapshot,
|
||||
Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
|
||||
};
|
||||
opts.object_lock_config_snapshot = Some(Arc::new(ObjectLockConfigSnapshot::for_store_bucket_under_lifecycle_fence(
|
||||
opts.object_lock_config_snapshot = Some(Arc::new(ObjectLockConfigSnapshot::for_store_bucket(
|
||||
self.id,
|
||||
bucket,
|
||||
incarnation_id,
|
||||
config_revision,
|
||||
state,
|
||||
opts.bucket_lifecycle_lock_fence
|
||||
.clone()
|
||||
.unwrap_or_else(NamespaceLockFence::new),
|
||||
Arc::clone(metadata_guard),
|
||||
)));
|
||||
Some(incarnation_id)
|
||||
} else {
|
||||
@@ -5631,7 +5617,7 @@ impl ECStore {
|
||||
opts.add_bucket_lifecycle_lock_guard(guard);
|
||||
}
|
||||
if !is_meta_bucketname(bucket) {
|
||||
let current_incarnation_id = get_bucket_incarnation_id_for_options_in(&self.ctx, bucket, &opts).await?;
|
||||
let current_incarnation_id = get_bucket_incarnation_id_in(&self.ctx, bucket).await?;
|
||||
if opts.expected_bucket_incarnation_id != Some(current_incarnation_id) {
|
||||
return Err(StorageError::BucketNotFound(bucket.to_string()));
|
||||
}
|
||||
@@ -5704,7 +5690,7 @@ impl ECStore {
|
||||
None
|
||||
} else {
|
||||
let guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
|
||||
let current_incarnation_id = get_bucket_incarnation_id_for_options_in(&self.ctx, bucket, &opts).await?;
|
||||
let current_incarnation_id = get_bucket_incarnation_id_in(&self.ctx, bucket).await?;
|
||||
if opts
|
||||
.expected_bucket_incarnation_id
|
||||
.is_some_and(|expected| expected != current_incarnation_id)
|
||||
@@ -9124,168 +9110,6 @@ mod tests {
|
||||
assert_eq!(dirs.len(), 16);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn current_marker_delete_ack_removes_stale_rejoined_copy() {
|
||||
let bucket = "current-marker-stale-rejoin";
|
||||
let object = "marker.bin";
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let (_dirs, original) = make_local_set_disks_with_ctx(16, 4, ctx.clone()).await;
|
||||
let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], ctx).await);
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
store
|
||||
.handle_make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create versioned bucket");
|
||||
let mut history = Vec::new();
|
||||
for value in 1..=3_u8 {
|
||||
let payload = vec![value; 4097];
|
||||
let version = store
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(payload.clone()),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write the historical data versions")
|
||||
.version_id
|
||||
.expect("historical version must have an explicit identity");
|
||||
history.push((version, payload));
|
||||
}
|
||||
let marker = store
|
||||
.delete_object(
|
||||
bucket,
|
||||
object,
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create current marker")
|
||||
.version_id
|
||||
.expect("marker has explicit version");
|
||||
let current = store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("bucket incarnation");
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
let before = set.disks.read().await.clone();
|
||||
for slot in 12..16 {
|
||||
set.disks.write().await[slot] = None;
|
||||
}
|
||||
store
|
||||
.delete_object(
|
||||
bucket,
|
||||
object,
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(marker.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("acknowledged delete should remove the marker from the online quorum");
|
||||
*set.disks.write().await = before.clone();
|
||||
|
||||
for disk in before.iter().take(12).flatten() {
|
||||
assert!(matches!(
|
||||
disk.read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
|
||||
.await,
|
||||
Err(crate::disk::error::DiskError::FileVersionNotFound | crate::disk::error::DiskError::FileNotFound)
|
||||
));
|
||||
}
|
||||
for disk in before.iter().skip(12).flatten() {
|
||||
assert!(
|
||||
disk.read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
|
||||
.await
|
||||
.is_ok(),
|
||||
"offline member must retain the stale current marker"
|
||||
);
|
||||
}
|
||||
let healed = store
|
||||
.heal_object_at_incarnation(
|
||||
bucket,
|
||||
object,
|
||||
&marker.to_string(),
|
||||
current,
|
||||
&rustfs_heal_contracts::heal_channel::HealOpts {
|
||||
remove: true,
|
||||
scan_mode: rustfs_heal_contracts::heal_channel::HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("stale current marker heal should complete");
|
||||
assert!(healed.error.is_none(), "stale current marker heal: {:?}", healed.error);
|
||||
assert!(healed.absence.as_ref().is_some_and(|proof| proof.removed));
|
||||
for disk in before.iter().flatten() {
|
||||
assert!(matches!(
|
||||
disk.read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
|
||||
.await,
|
||||
Err(crate::disk::error::DiskError::FileVersionNotFound | crate::disk::error::DiskError::FileNotFound)
|
||||
));
|
||||
}
|
||||
let replay = store
|
||||
.heal_object_at_incarnation(
|
||||
bucket,
|
||||
object,
|
||||
&marker.to_string(),
|
||||
current,
|
||||
&rustfs_heal_contracts::heal_channel::HealOpts {
|
||||
remove: true,
|
||||
scan_mode: rustfs_heal_contracts::heal_channel::HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("replay the exact removed marker identity");
|
||||
assert!(replay.error.is_none(), "exact marker replay: {:?}", replay.error);
|
||||
let absence = replay.absence.expect("exact replay must prove authoritative absence");
|
||||
assert!(!absence.removed, "replay must not claim another physical repair");
|
||||
assert_eq!(absence.bucket_incarnation_id, current);
|
||||
assert_eq!(absence.version_id, marker.to_string());
|
||||
let listed = store
|
||||
.clone()
|
||||
.inner_list_object_versions(bucket, "", None, None, None, 100)
|
||||
.await
|
||||
.expect("list surviving versions after marker cleanup");
|
||||
assert_eq!(listed.objects.len(), history.len(), "an absent marker is not enumerable in a new walk");
|
||||
assert!(listed.objects.iter().all(|info| !info.delete_marker));
|
||||
for (version, expected) in history {
|
||||
assert!(listed.objects.iter().any(|info| info.version_id == Some(version)));
|
||||
let mut reader = store
|
||||
.handle_get_object_reader(
|
||||
bucket,
|
||||
object,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
version_id: Some(version.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("historical data must remain readable after cleanup and replay");
|
||||
let mut actual = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut actual)
|
||||
.await
|
||||
.expect("read all historical bytes");
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipool_delete_marker_stays_with_existing_versions() {
|
||||
let bucket = "multipool-marker-routing";
|
||||
|
||||
@@ -161,52 +161,14 @@ impl ECStore {
|
||||
"incarnation-bound rename requires namespace locking and a non-nil identity",
|
||||
));
|
||||
}
|
||||
let bucket = if destination.0 == RUSTFS_META_BUCKET {
|
||||
super::heal::bucket_metadata_owner(destination.1).ok_or(DiskError::FileAccessDenied)?
|
||||
} else {
|
||||
destination.0
|
||||
};
|
||||
let guard = Arc::new(
|
||||
self.acquire_bucket_incarnation_fence(bucket, expected)
|
||||
self.acquire_bucket_incarnation_fence(destination.0, expected)
|
||||
.await
|
||||
.map_err(|error| DiskError::other(error.to_string()))?,
|
||||
);
|
||||
if guard.is_lock_lost() {
|
||||
return Err(DiskError::other("bucket heal incarnation fence was lost"));
|
||||
}
|
||||
let transaction = if destination.0 == RUSTFS_META_BUCKET {
|
||||
// The receiver must retain the config fence after an RPC timeout.
|
||||
// A delayed request must also reject a newer config in the same
|
||||
// bucket incarnation before publishing its older repair payload.
|
||||
let transaction = crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket)
|
||||
.await
|
||||
.map_err(|error| DiskError::other(error.to_string()))?;
|
||||
let (current, _) = self
|
||||
.get_pool_info_for_delete_marker(
|
||||
destination.0,
|
||||
destination.1,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(|error| DiskError::other(error.to_string()))?;
|
||||
let current = current.object_info;
|
||||
if transaction.is_lock_lost()
|
||||
|| guard.is_lock_lost()
|
||||
|| fi.mod_time.is_none()
|
||||
|| current.mod_time != fi.mod_time
|
||||
|| current.data_dir != fi.data_dir
|
||||
|| current.size != fi.size
|
||||
|| current.etag != fi.get_etag()
|
||||
{
|
||||
return Err(DiskError::FileAccessDenied);
|
||||
}
|
||||
Some(transaction)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
rename_local_data_with_ctx(
|
||||
&self.ctx,
|
||||
disk_ref,
|
||||
@@ -214,7 +176,7 @@ impl ECStore {
|
||||
fi,
|
||||
destination,
|
||||
RenameDataGuards {
|
||||
external_guard: Some(Arc::new((guard, transaction))),
|
||||
external_guard: Some(guard),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
|
||||
@@ -1,288 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#![cfg(feature = "test-util")]
|
||||
|
||||
mod storage_api;
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use storage_api::metadata_lock::{
|
||||
BucketOperations, CompletePart, Error, MakeBucketOptions, MultipartOperations, NamespaceLocking, ObjectIO, ObjectOperations,
|
||||
ObjectOptions, PutObjReader, PutObjectCommitBarrier, PutObjectCommitPause, init_bucket_metadata_sys,
|
||||
isolated_store_over_temp_disks,
|
||||
};
|
||||
use tokio::io::AsyncReadExt;
|
||||
use tokio::time::timeout;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[tokio::test]
|
||||
async fn replica_put_completes_while_metadata_writer_waits_for_its_snapshot() {
|
||||
replica_write_with_waiting_metadata_writer(false).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replica_multipart_completes_while_metadata_writer_waits_for_its_snapshot() {
|
||||
replica_write_with_waiting_metadata_writer(true).await;
|
||||
}
|
||||
|
||||
async fn replica_write_with_waiting_metadata_writer(multipart: bool) {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "replica-metadata-read-reuse";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
store
|
||||
.update_bucket_metadata_config(
|
||||
bucket,
|
||||
"lifecycle.xml",
|
||||
br#"<LifecycleConfiguration><Rule><ID>expire-old-versions</ID><Status>Enabled</Status><Filter><Prefix></Prefix></Filter><NoncurrentVersionExpiration><NoncurrentDays>1</NoncurrentDays></NoncurrentVersionExpiration></Rule></LifecycleConfiguration>"#.to_vec(),
|
||||
)
|
||||
.await
|
||||
.expect("configure post-write lifecycle evaluation");
|
||||
let opts = ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(Uuid::new_v4().to_string()),
|
||||
replication_request: true,
|
||||
expected_bucket_incarnation_id: Some(store.bucket_incarnation_id(bucket).await.expect("load incarnation")),
|
||||
object_lock_config_snapshot: Some(
|
||||
store
|
||||
.object_lock_config_snapshot(bucket)
|
||||
.await
|
||||
.expect("capture Object Lock snapshot"),
|
||||
),
|
||||
..Default::default()
|
||||
};
|
||||
let upload = if multipart {
|
||||
let upload = store
|
||||
.new_multipart_upload(bucket, "object", &opts)
|
||||
.await
|
||||
.expect("stage upload");
|
||||
let mut data = PutObjReader::from_vec(b"replicated version".to_vec());
|
||||
let part = store
|
||||
.put_object_part(bucket, "object", &upload.upload_id, 1, &mut data, &opts)
|
||||
.await
|
||||
.expect("stage part");
|
||||
Some((
|
||||
upload.upload_id,
|
||||
vec![CompletePart {
|
||||
part_num: 1,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
}],
|
||||
))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let lock = store
|
||||
.new_ns_lock(".rustfs.sys", &format!("bucket-targets/{bucket}/transaction.lock"))
|
||||
.await
|
||||
.expect("create metadata writer");
|
||||
let writer = lock.get_write_lock(Duration::from_secs(5));
|
||||
tokio::pin!(writer);
|
||||
// Leave the writer registered while the replica continues. A second read
|
||||
// acquisition would queue behind this writer, which needs our first read.
|
||||
assert!(futures::poll!(&mut writer).is_pending());
|
||||
timeout(Duration::from_secs(2), async {
|
||||
if let Some((upload_id, parts)) = upload {
|
||||
store
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, "object", &upload_id, parts, &opts)
|
||||
.await
|
||||
} else {
|
||||
let mut data = PutObjReader::from_vec(b"replicated version".to_vec());
|
||||
store.put_object(bucket, "object", &mut data, &opts).await
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("replica must reuse its metadata read lock instead of waiting behind the writer")
|
||||
.expect("replica PUT should commit");
|
||||
assert!(futures::poll!(&mut writer).is_pending(), "the snapshot must still fence metadata updates");
|
||||
let read_opts = ObjectOptions {
|
||||
version_id: opts.version_id.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
drop(opts);
|
||||
timeout(Duration::from_secs(2), writer)
|
||||
.await
|
||||
.expect("metadata writer must proceed after the snapshot drops")
|
||||
.expect("acquire metadata writer");
|
||||
let mut reader = store
|
||||
.get_object_reader(bucket, "object", None, Default::default(), &read_opts)
|
||||
.await
|
||||
.expect("read committed replica version");
|
||||
let mut bytes = Vec::new();
|
||||
reader.stream.read_to_end(&mut bytes).await.expect("read replica body");
|
||||
assert_eq!(bytes, b"replicated version");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reused_metadata_lock_still_enforces_quota() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "metadata-read-reuse-quota";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
store
|
||||
.update_bucket_metadata_config(bucket, "quota.json", br#"{"quota":1}"#.to_vec())
|
||||
.await
|
||||
.expect("set quota");
|
||||
let mut opts = ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(Uuid::new_v4().to_string()),
|
||||
expected_bucket_incarnation_id: Some(store.bucket_incarnation_id(bucket).await.expect("load incarnation")),
|
||||
object_lock_config_snapshot: Some(store.object_lock_config_snapshot(bucket).await.expect("capture snapshot")),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(opts.set_quota_admission(0, 1));
|
||||
let lock = store
|
||||
.new_ns_lock(".rustfs.sys", &format!("bucket-targets/{bucket}/transaction.lock"))
|
||||
.await
|
||||
.expect("create writer");
|
||||
let writer = lock.get_write_lock(Duration::from_secs(5));
|
||||
tokio::pin!(writer);
|
||||
assert!(futures::poll!(&mut writer).is_pending());
|
||||
let mut data = PutObjReader::from_vec(b"too large".to_vec());
|
||||
let error = timeout(Duration::from_secs(2), store.put_object(bucket, "object", &mut data, &opts))
|
||||
.await
|
||||
.expect("quota admission must not reacquire the lock")
|
||||
.expect_err("quota must reject growth");
|
||||
assert!(matches!(error, Error::QuotaExceeded { limit: 1, .. }), "{error:?}");
|
||||
drop(opts);
|
||||
timeout(Duration::from_secs(2), writer)
|
||||
.await
|
||||
.expect("release metadata reader after denial")
|
||||
.expect("acquire writer");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_put_releases_shared_metadata_guard() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "metadata-read-reuse-cancel";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let opts = ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(Uuid::new_v4().to_string()),
|
||||
object_lock_config_snapshot: Some(store.object_lock_config_snapshot(bucket).await.expect("capture snapshot")),
|
||||
..Default::default()
|
||||
};
|
||||
let barrier = PutObjectCommitBarrier::install(bucket, "object", PutObjectCommitPause::AfterQuotaReservation);
|
||||
let put_store = Arc::clone(&store);
|
||||
let put = tokio::spawn(async move {
|
||||
let mut data = PutObjReader::from_vec(b"cancel before commit".to_vec());
|
||||
put_store.put_object(bucket, "object", &mut data, &opts).await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
let lock = store
|
||||
.new_ns_lock(".rustfs.sys", &format!("bucket-targets/{bucket}/transaction.lock"))
|
||||
.await
|
||||
.expect("create writer");
|
||||
let writer = lock.get_write_lock(Duration::from_secs(5));
|
||||
tokio::pin!(writer);
|
||||
assert!(futures::poll!(&mut writer).is_pending());
|
||||
put.abort();
|
||||
assert!(put.await.expect_err("PUT should be cancelled").is_cancelled());
|
||||
timeout(Duration::from_secs(2), writer)
|
||||
.await
|
||||
.expect("all metadata reader owners must drop after cancellation")
|
||||
.expect("acquire writer");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_rejects_metadata_snapshots_from_another_scope() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let (_other_dirs, other_store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&other_store), Vec::new()).await;
|
||||
let bucket = "metadata-read-reuse-scope";
|
||||
for target in [&store, &other_store] {
|
||||
target
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
}
|
||||
store
|
||||
.make_bucket("other-bucket", &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create other bucket");
|
||||
let other_store_incarnation = other_store
|
||||
.bucket_incarnation_id(bucket)
|
||||
.await
|
||||
.expect("load other store incarnation");
|
||||
let other_bucket_incarnation = store
|
||||
.bucket_incarnation_id("other-bucket")
|
||||
.await
|
||||
.expect("load other bucket incarnation");
|
||||
for (snapshot_store, snapshot_bucket, expected) in [
|
||||
(&other_store, bucket, other_store_incarnation),
|
||||
(&store, "other-bucket", other_bucket_incarnation),
|
||||
(&store, bucket, Uuid::new_v4()),
|
||||
] {
|
||||
let opts = ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(expected),
|
||||
object_lock_config_snapshot: Some(
|
||||
snapshot_store
|
||||
.object_lock_config_snapshot(snapshot_bucket)
|
||||
.await
|
||||
.expect("capture snapshot"),
|
||||
),
|
||||
..Default::default()
|
||||
};
|
||||
let error = store
|
||||
.new_multipart_upload(bucket, "object", &opts)
|
||||
.await
|
||||
.expect_err("foreign snapshot must be rejected");
|
||||
assert!(error.to_string().contains("valid metadata transaction fence"), "{error:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_prefix_reuses_its_metadata_snapshot_for_generation_validation() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "metadata-read-reuse-delete";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let mut data = PutObjReader::from_vec(b"remove prefix".to_vec());
|
||||
store
|
||||
.put_object(bucket, "prefix/object", &mut data, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("seed object");
|
||||
store
|
||||
.delete_object(
|
||||
bucket,
|
||||
"prefix/",
|
||||
ObjectOptions {
|
||||
delete_prefix: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("delete prefix with an internally captured metadata snapshot");
|
||||
let error = store
|
||||
.get_object_info(bucket, "prefix/object", &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("prefix object must be deleted");
|
||||
assert!(matches!(error, Error::ObjectNotFound(..)), "{error:?}");
|
||||
}
|
||||
@@ -6,17 +6,6 @@ pub(crate) use rustfs_ecstore::api::erasure::Erasure;
|
||||
pub(crate) use rustfs_ecstore::api::object::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
|
||||
pub(crate) use rustfs_ecstore::api::{error::Error, set_disk::SetDisks, storage::ECStore};
|
||||
use rustfs_storage_api as storage_contracts;
|
||||
#[cfg(feature = "test-util")]
|
||||
pub(crate) mod metadata_lock {
|
||||
pub(crate) use super::storage_contracts::{
|
||||
BucketOperations, CompletePart, MakeBucketOptions, MultipartOperations, NamespaceLocking, ObjectIO, ObjectOperations,
|
||||
};
|
||||
pub(crate) use super::{Error, ObjectOptions, PutObjReader};
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{
|
||||
init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::set_disk::test_util::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
}
|
||||
|
||||
pub(crate) mod contract_compat {
|
||||
pub(crate) use super::storage_contracts::{
|
||||
|
||||
@@ -994,15 +994,6 @@ impl FileMeta {
|
||||
Uuid::nil()
|
||||
}
|
||||
};
|
||||
// A nil selector denotes the null version slot, which legacy metadata
|
||||
// may encode as either an absent UUID or an explicit nil UUID.
|
||||
let matches_version = |actual: Option<Uuid>| {
|
||||
if vid.is_nil() {
|
||||
actual.is_none_or(|version_id| version_id.is_nil())
|
||||
} else {
|
||||
actual == Some(vid)
|
||||
}
|
||||
};
|
||||
|
||||
let mut is_latest = true;
|
||||
let mut succ_mod_time = None;
|
||||
@@ -1040,7 +1031,7 @@ impl FileMeta {
|
||||
}
|
||||
}
|
||||
|
||||
if !matches_version(header.version_id) {
|
||||
if header.version_id != Some(vid) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
@@ -1049,7 +1040,7 @@ impl FileMeta {
|
||||
continue;
|
||||
}
|
||||
|
||||
if !version_id.is_empty() && !matches_version(header.version_id) {
|
||||
if !version_id.is_empty() && header.version_id != Some(vid) {
|
||||
is_latest = false;
|
||||
succ_mod_time = header.mod_time;
|
||||
continue;
|
||||
@@ -2694,48 +2685,6 @@ mod test {
|
||||
);
|
||||
assert!(fi.uses_legacy_checksum);
|
||||
assert!(fi.is_latest);
|
||||
|
||||
let selected = fm
|
||||
.into_fileinfo(".rustfs.sys", "pool.bin", &Uuid::nil().to_string(), true, false, true)
|
||||
.expect("nil selector should match the legacy null version");
|
||||
assert_eq!(selected.version_id, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nil_version_selector_matches_only_the_null_slot() {
|
||||
let mod_time = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid test timestamp");
|
||||
let version_id = Uuid::new_v4();
|
||||
let versioned = version_for_ordering(VersionType::Object, version_id, mod_time + time::Duration::seconds(1), 2);
|
||||
let mut null = version_for_ordering(VersionType::Object, Uuid::nil(), mod_time, 1);
|
||||
null.object
|
||||
.as_mut()
|
||||
.expect("ordering helper should construct an object")
|
||||
.version_id = None;
|
||||
|
||||
let mut fm = FileMeta::new();
|
||||
fm.add_version_filemata(versioned.clone()).expect("add versioned object");
|
||||
fm.add_version_filemata(null).expect("add legacy null object");
|
||||
|
||||
let selected_null = fm
|
||||
.into_fileinfo("bucket", "object", &Uuid::nil().to_string(), false, false, true)
|
||||
.expect("nil selector should match only the legacy null slot");
|
||||
assert_eq!(selected_null.version_id, None);
|
||||
|
||||
let selected_versioned = fm
|
||||
.into_fileinfo("bucket", "object", &version_id.to_string(), false, false, true)
|
||||
.expect("non-nil selector should still match its exact version");
|
||||
assert_eq!(selected_versioned.version_id, Some(version_id));
|
||||
|
||||
let mut versioned_only = FileMeta::new();
|
||||
versioned_only
|
||||
.add_version_filemata(versioned)
|
||||
.expect("add versioned object without a null slot");
|
||||
assert!(
|
||||
versioned_only
|
||||
.into_fileinfo("bucket", "object", &Uuid::nil().to_string(), false, false, true)
|
||||
.is_err(),
|
||||
"nil selector must not fall back to a non-nil latest version"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -907,21 +907,6 @@ impl ErasureSetHealer {
|
||||
}
|
||||
|
||||
if failed_objects == 0 && skipped_objects == 0 && failed_buckets == 0 {
|
||||
let targets = if self.pool_metadata_target_endpoints.is_empty() {
|
||||
self.target_endpoints.as_ref()
|
||||
} else {
|
||||
self.pool_metadata_target_endpoints.as_ref()
|
||||
};
|
||||
if self.replacement_task_id.is_some() || (!self.heal_opts.dry_run && self.heal_opts.recreate && !targets.is_empty()) {
|
||||
self.verify_replacement_identity_fence("bucket metadata").await?;
|
||||
// Recheck even resumed buckets: their user-object cursor does not
|
||||
// prove that the replacement holds the internal bucket records.
|
||||
for bucket in buckets {
|
||||
self.storage
|
||||
.heal_replacement_bucket_metadata(bucket, &self.heal_opts, targets)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
self.heal_replacement_pool_metadata(
|
||||
set_disk_id,
|
||||
&mut ErasureSetPassCounters {
|
||||
@@ -2282,8 +2267,6 @@ mod resume_loop_tests {
|
||||
list_include_lifecycle_object_info: Mutex<Vec<bool>>,
|
||||
replacement_target_identity_sequences: Mutex<VecDeque<Vec<ReplacementTargetIdentity>>>,
|
||||
pool_metadata_placement: Mutex<Option<ReplacementCommitEvidence>>,
|
||||
bucket_metadata_calls: Mutex<Vec<String>>,
|
||||
bucket_metadata_failure: AtomicBool,
|
||||
fail_listing: AtomicBool,
|
||||
fail_listing_buckets: Mutex<HashSet<String>>,
|
||||
}
|
||||
@@ -2479,13 +2462,6 @@ mod resume_loop_tests {
|
||||
None => Ok(true),
|
||||
}
|
||||
}
|
||||
async fn heal_replacement_bucket_metadata(&self, bucket: &str, _opts: &HealOpts, _targets: &[String]) -> Result<()> {
|
||||
self.bucket_metadata_calls.lock().unwrap().push(bucket.to_owned());
|
||||
if self.bucket_metadata_failure.load(Ordering::SeqCst) {
|
||||
return Err(Error::Storage(EcstoreError::PreconditionFailed));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
async fn replacement_targets_have_version(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
@@ -3082,18 +3058,6 @@ mod resume_loop_tests {
|
||||
)
|
||||
.with_replacement_targets(vec!["replacement-a".to_string()], Some(replacement_task_id.clone()));
|
||||
|
||||
env.storage.bucket_metadata_failure.store(true, Ordering::SeqCst);
|
||||
assert!(healer.heal_erasure_set(&["b".to_string()], "pool_0_set_0").await.is_err());
|
||||
let incomplete = ResumeManager::load_replacement_intent(env.healer.disk.clone(), &replacement_task_id)
|
||||
.await
|
||||
.expect("failed metadata repair must retain replacement intent")
|
||||
.get_state()
|
||||
.await;
|
||||
assert!(!incomplete.completed);
|
||||
assert_ne!(incomplete.replacement_phase, crate::heal::resume::ReplacementPhase::Verified);
|
||||
assert!(env.storage.calls().is_empty(), "metadata failure must stop completion before pool.bin");
|
||||
env.storage.bucket_metadata_failure.store(false, Ordering::SeqCst);
|
||||
|
||||
let error = healer
|
||||
.heal_erasure_set(&["b".to_string()], "pool_0_set_0")
|
||||
.await
|
||||
@@ -3109,11 +3073,6 @@ mod resume_loop_tests {
|
||||
assert_eq!(state.replacement_phase, crate::heal::resume::ReplacementPhase::Intent);
|
||||
assert_eq!(state.retry_count, 1);
|
||||
assert_eq!(env.storage.calls(), vec![(POOL_META_NAME.to_string(), None)]);
|
||||
assert_eq!(
|
||||
*env.storage.bucket_metadata_calls.lock().unwrap(),
|
||||
vec!["b", "b"],
|
||||
"resuming a completed user scan must retry bucket metadata"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -72,41 +72,6 @@ const LEGACY_ROOT_HEAL_PATH: &str = ".";
|
||||
const MAX_RECOVERABLE_HEAL_RETRIES: u32 = 3;
|
||||
const MAX_RECOVERABLE_HEAL_RETRY_DELAY: Duration = Duration::from_secs(30);
|
||||
const RESUME_GC_INTERVAL: Duration = Duration::from_secs(60 * 60);
|
||||
const BEST_EFFORT_QUEUE_RESERVE_PERCENT: usize = 10;
|
||||
const BEST_EFFORT_SOURCE_COUNT: usize = 3;
|
||||
|
||||
fn is_best_effort_source(source: HealRequestSource) -> bool {
|
||||
matches!(
|
||||
source,
|
||||
HealRequestSource::Scanner | HealRequestSource::AutoHeal | HealRequestSource::ReadRepair
|
||||
)
|
||||
}
|
||||
|
||||
fn best_effort_source_quota_index(source: HealRequestSource) -> Option<usize> {
|
||||
match source {
|
||||
HealRequestSource::Scanner => Some(0),
|
||||
HealRequestSource::ReadRepair => Some(1),
|
||||
HealRequestSource::AutoHeal => Some(2),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn best_effort_queue_reserve(queue_capacity: usize) -> usize {
|
||||
match queue_capacity {
|
||||
0..=2 => 0,
|
||||
capacity => (capacity.saturating_mul(BEST_EFFORT_QUEUE_RESERVE_PERCENT) / 100).max(BEST_EFFORT_SOURCE_COUNT),
|
||||
}
|
||||
}
|
||||
|
||||
fn best_effort_source_reserves(queue_capacity: usize) -> [usize; BEST_EFFORT_SOURCE_COUNT] {
|
||||
let reserve = best_effort_queue_reserve(queue_capacity);
|
||||
if reserve == 0 {
|
||||
return [0; BEST_EFFORT_SOURCE_COUNT];
|
||||
}
|
||||
let base = reserve / BEST_EFFORT_SOURCE_COUNT;
|
||||
let remainder = reserve % BEST_EFFORT_SOURCE_COUNT;
|
||||
[base + usize::from(remainder > 0), base + usize::from(remainder > 1), base]
|
||||
}
|
||||
|
||||
// Admission/scheduler outcomes for per-object requests (Object/Metadata/
|
||||
// ECDecode) log via demote_to_debug_when! — MRF, autoheal, and scanner
|
||||
@@ -1000,7 +965,11 @@ struct HealQueueContext<'a> {
|
||||
|
||||
impl HealManager {
|
||||
fn classify_full_admission(request: &HealRequest, config: &HealConfig) -> HealAdmissionResult {
|
||||
if request.priority == HealPriority::Low && config.low_priority_drop_when_full {
|
||||
let best_effort_source = matches!(
|
||||
request.source,
|
||||
HealRequestSource::Scanner | HealRequestSource::AutoHeal | HealRequestSource::ReadRepair
|
||||
);
|
||||
if best_effort_source || (request.priority == HealPriority::Low && config.low_priority_drop_when_full) {
|
||||
HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull)
|
||||
} else {
|
||||
HealAdmissionResult::Full
|
||||
@@ -1011,6 +980,32 @@ impl HealManager {
|
||||
queue_len.saturating_mul(100).checked_div(queue_capacity).unwrap_or(0)
|
||||
}
|
||||
|
||||
fn classify_pressure_admission(
|
||||
request: &HealRequest,
|
||||
queue_len: usize,
|
||||
queue_capacity: usize,
|
||||
) -> Option<HealAdmissionResult> {
|
||||
if request.force_start || queue_capacity == 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let queue_usage_pct = Self::queue_usage_pct(queue_len, queue_capacity);
|
||||
if queue_usage_pct < 80 {
|
||||
return None;
|
||||
}
|
||||
|
||||
match request.source {
|
||||
HealRequestSource::ReadRepair => Some(HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped)),
|
||||
HealRequestSource::Scanner if request.priority == HealPriority::Low => {
|
||||
Some(HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped))
|
||||
}
|
||||
HealRequestSource::AutoHeal if queue_usage_pct >= 95 => {
|
||||
Some(HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped))
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn duplicate_admission_for_request(request: &HealRequest, config: &HealConfig) -> HealAdmissionResult {
|
||||
if request.priority == HealPriority::Low && !config.low_priority_merge_enable {
|
||||
HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped)
|
||||
@@ -1161,8 +1156,6 @@ impl HealManager {
|
||||
publish_heal_queue_length(queue);
|
||||
let queue_capacity = config.queue_size;
|
||||
let per_object_request = request.heal_type.is_per_object();
|
||||
let best_effort_reserve = best_effort_queue_reserve(queue_capacity);
|
||||
let best_effort_source_reserves = best_effort_source_reserves(queue_capacity);
|
||||
|
||||
if queue_len >= queue_capacity && !request.force_start {
|
||||
if Self::can_displace_queued_work(&request)
|
||||
@@ -1250,55 +1243,27 @@ impl HealManager {
|
||||
return HealAdmissionDecision::new(admission);
|
||||
}
|
||||
|
||||
if !is_best_effort_source(request.source)
|
||||
&& request.priority <= HealPriority::Normal
|
||||
&& !request.force_start
|
||||
&& best_effort_reserve > 0
|
||||
&& queue_len >= queue_capacity.saturating_sub(best_effort_reserve)
|
||||
{
|
||||
Self::record_admission_metric(request.source, HealAdmissionResult::Full, context);
|
||||
demote_to_debug_when!(per_object_request, warn, target: "rustfs::heal::manager", {
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
request_id = %request.id,
|
||||
priority = ?request.priority,
|
||||
source = request.source.as_str(),
|
||||
context,
|
||||
queue_len,
|
||||
queue_capacity,
|
||||
reserved_for_best_effort = best_effort_reserve,
|
||||
result = "rejected_for_reserved_capacity",
|
||||
"Normal heal request deferred to preserve best-effort capacity"
|
||||
});
|
||||
return HealAdmissionDecision::new(HealAdmissionResult::Full);
|
||||
}
|
||||
|
||||
if is_best_effort_source(request.source)
|
||||
&& request.priority <= HealPriority::Normal
|
||||
&& !request.force_start
|
||||
&& best_effort_reserve > 0
|
||||
&& queue_len >= queue_capacity.saturating_sub(best_effort_reserve)
|
||||
&& let Some(source_index) = best_effort_source_quota_index(request.source)
|
||||
&& queue.best_effort_source_count(request.source) >= best_effort_source_reserves[source_index]
|
||||
{
|
||||
Self::record_admission_metric(request.source, HealAdmissionResult::Full, context);
|
||||
demote_to_debug_when!(per_object_request, warn, target: "rustfs::heal::manager", {
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
request_id = %request.id,
|
||||
priority = ?request.priority,
|
||||
source = request.source.as_str(),
|
||||
context,
|
||||
queue_len,
|
||||
queue_capacity,
|
||||
reserved_for_best_effort = best_effort_reserve,
|
||||
source_quota = best_effort_source_reserves[source_index],
|
||||
result = "rejected_for_source_quota",
|
||||
"Best-effort source deferred to preserve per-source capacity"
|
||||
});
|
||||
return HealAdmissionDecision::new(HealAdmissionResult::Full);
|
||||
if let Some(admission) = Self::classify_pressure_admission(&request, queue_len, queue_capacity) {
|
||||
Self::record_admission_metric(request.source, admission, context);
|
||||
if let HealAdmissionResult::Dropped(reason) = admission {
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
request_id = %request.id,
|
||||
priority = ?request.priority,
|
||||
source = request.source.as_str(),
|
||||
context,
|
||||
queue_len,
|
||||
queue_capacity,
|
||||
queue_usage_pct = Self::queue_usage_pct(queue_len, queue_capacity),
|
||||
reason = reason.as_str(),
|
||||
result = "dropped_pressure",
|
||||
"Heal queue request dropped under pressure"
|
||||
);
|
||||
}
|
||||
return HealAdmissionDecision::new(admission);
|
||||
}
|
||||
|
||||
if queue_capacity > 0 {
|
||||
|
||||
@@ -34,33 +34,6 @@ pub(super) struct PriorityHealQueue {
|
||||
pub(super) sequence: u64,
|
||||
/// Deduplication index for queued requests
|
||||
pub(super) dedup_keys: HashMap<String, DedupKeyEntry>,
|
||||
pub(super) best_effort_queued: usize,
|
||||
pub(super) best_effort_queued_by_source: [usize; 3],
|
||||
/// Number of MRF/Normal tasks dispatched since a best-effort task ran.
|
||||
/// This is kept with the queue so event-driven scheduler wakeups cannot
|
||||
/// reset the fairness budget between cycles.
|
||||
pub(super) mrf_normal_dispatches_since_best_effort: usize,
|
||||
pub(super) last_best_effort_source: Option<HealRequestSource>,
|
||||
}
|
||||
|
||||
const MRF_NORMAL_DISPATCH_QUANTUM: usize = 4;
|
||||
|
||||
fn best_effort_source_index(source: HealRequestSource) -> Option<usize> {
|
||||
match source {
|
||||
HealRequestSource::Scanner => Some(0),
|
||||
HealRequestSource::ReadRepair => Some(1),
|
||||
HealRequestSource::AutoHeal => Some(2),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn best_effort_source_order(last: Option<HealRequestSource>) -> [usize; 3] {
|
||||
match last {
|
||||
Some(HealRequestSource::Scanner) => [1, 2, 0],
|
||||
Some(HealRequestSource::ReadRepair) => [2, 0, 1],
|
||||
Some(HealRequestSource::AutoHeal) => [0, 1, 2],
|
||||
_ => [0, 1, 2],
|
||||
}
|
||||
}
|
||||
|
||||
/// Wrapper for heap items to implement proper ordering
|
||||
@@ -291,10 +264,6 @@ impl PriorityHealQueue {
|
||||
heap: BinaryHeap::new(),
|
||||
sequence: 0,
|
||||
dedup_keys: HashMap::new(),
|
||||
best_effort_queued: 0,
|
||||
best_effort_queued_by_source: [0; 3],
|
||||
mrf_normal_dispatches_since_best_effort: 0,
|
||||
last_best_effort_source: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -302,11 +271,9 @@ impl PriorityHealQueue {
|
||||
self.heap.len()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn pop_next(&mut self) -> Option<HealRequest> {
|
||||
self.heap.pop().map(|item| {
|
||||
Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item.dedup_key);
|
||||
self.remove_best_effort_count(item.request.source);
|
||||
item.request
|
||||
})
|
||||
}
|
||||
@@ -334,19 +301,12 @@ impl PriorityHealQueue {
|
||||
})
|
||||
.refcount += 1;
|
||||
self.sequence += 1;
|
||||
let source = request.source;
|
||||
self.heap.push(PriorityQueueItem {
|
||||
priority: request.priority,
|
||||
sequence: self.sequence,
|
||||
dedup_key: key,
|
||||
request,
|
||||
});
|
||||
if is_best_effort_source(source) {
|
||||
self.best_effort_queued += 1;
|
||||
if let Some(index) = best_effort_source_index(source) {
|
||||
self.best_effort_queued_by_source[index] += 1;
|
||||
}
|
||||
}
|
||||
QueuePushOutcome::Accepted
|
||||
}
|
||||
|
||||
@@ -396,7 +356,6 @@ impl PriorityHealQueue {
|
||||
|
||||
let displaced = displaced.map(|item| {
|
||||
Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item.dedup_key);
|
||||
self.remove_best_effort_count(item.request.source);
|
||||
self.refresh_dedup_representative(&item.dedup_key);
|
||||
item.request
|
||||
});
|
||||
@@ -441,7 +400,6 @@ impl PriorityHealQueue {
|
||||
pub(super) fn pop(&mut self) -> Option<HealRequest> {
|
||||
self.heap.pop().map(|item| {
|
||||
Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item.dedup_key);
|
||||
self.remove_best_effort_count(item.request.source);
|
||||
item.request
|
||||
})
|
||||
}
|
||||
@@ -471,115 +429,12 @@ impl PriorityHealQueue {
|
||||
(
|
||||
selected.map(|item| {
|
||||
Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item.dedup_key);
|
||||
self.remove_best_effort_count(item.request.source);
|
||||
item.request
|
||||
}),
|
||||
skipped,
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn pop_runnable_with_fairness<F, G>(&mut self, can_run: F, skip_label: G) -> (Option<HealRequest>, Vec<String>)
|
||||
where
|
||||
F: Fn(&HealRequest) -> bool,
|
||||
G: Fn(&HealRequest) -> Option<String>,
|
||||
{
|
||||
if self.mrf_normal_dispatches_since_best_effort < MRF_NORMAL_DISPATCH_QUANTUM {
|
||||
let (chosen, skipped) = self.pop_runnable_with_skips(can_run, skip_label);
|
||||
if let Some(request) = chosen.as_ref() {
|
||||
self.record_dispatch(request);
|
||||
}
|
||||
return (chosen, skipped);
|
||||
}
|
||||
|
||||
if self.best_effort_queued == 0 {
|
||||
self.mrf_normal_dispatches_since_best_effort = 0;
|
||||
let (chosen, skipped) = self.pop_runnable_with_skips(can_run, skip_label);
|
||||
if let Some(request) = chosen.as_ref() {
|
||||
self.record_dispatch(request);
|
||||
}
|
||||
return (chosen, skipped);
|
||||
}
|
||||
|
||||
let mut items = std::mem::take(&mut self.heap).into_vec();
|
||||
let mut skipped = Vec::new();
|
||||
let mut first_runnable: Option<usize> = None;
|
||||
let mut best_effort_candidates: [Option<usize>; 3] = [None; 3];
|
||||
for (index, item) in items.iter().enumerate() {
|
||||
if !can_run(&item.request) {
|
||||
if let Some(label) = skip_label(&item.request) {
|
||||
skipped.push(label);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if first_runnable.is_none_or(|current| item.cmp(&items[current]).is_gt()) {
|
||||
first_runnable = Some(index);
|
||||
}
|
||||
let Some(source_index) = best_effort_source_index(item.request.source) else {
|
||||
continue;
|
||||
};
|
||||
if best_effort_candidates[source_index].is_none_or(|current| {
|
||||
item.priority > items[current].priority
|
||||
|| (item.priority == items[current].priority && item.sequence < items[current].sequence)
|
||||
}) {
|
||||
best_effort_candidates[source_index] = Some(index);
|
||||
}
|
||||
}
|
||||
|
||||
let chosen_index = first_runnable.filter(|index| {
|
||||
let item = &items[*index];
|
||||
item.priority == HealPriority::Normal
|
||||
&& matches!(item.request.source, HealRequestSource::Mrf | HealRequestSource::Internal)
|
||||
});
|
||||
let best_effort_priority = best_effort_candidates
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.map(|index| items[index].priority)
|
||||
.max();
|
||||
let chosen_index = chosen_index
|
||||
.and_then(|_| {
|
||||
let priority = best_effort_priority?;
|
||||
best_effort_source_order(self.last_best_effort_source)
|
||||
.into_iter()
|
||||
.find_map(|source| best_effort_candidates[source].filter(|index| items[*index].priority == priority))
|
||||
})
|
||||
.or(first_runnable);
|
||||
let chosen = chosen_index.map(|index| items.swap_remove(index));
|
||||
self.heap = BinaryHeap::from(items);
|
||||
|
||||
let chosen = chosen.map(|item| {
|
||||
let request = item.request;
|
||||
Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item.dedup_key);
|
||||
self.remove_best_effort_count(request.source);
|
||||
self.record_dispatch(&request);
|
||||
request
|
||||
});
|
||||
(chosen, skipped)
|
||||
}
|
||||
|
||||
fn record_dispatch(&mut self, request: &HealRequest) {
|
||||
if is_best_effort_source(request.source) {
|
||||
self.mrf_normal_dispatches_since_best_effort = 0;
|
||||
self.last_best_effort_source = Some(request.source);
|
||||
} else if matches!(request.source, HealRequestSource::Mrf | HealRequestSource::Internal)
|
||||
&& request.priority == HealPriority::Normal
|
||||
{
|
||||
self.mrf_normal_dispatches_since_best_effort = self.mrf_normal_dispatches_since_best_effort.saturating_add(1);
|
||||
}
|
||||
}
|
||||
|
||||
fn remove_best_effort_count(&mut self, source: HealRequestSource) {
|
||||
if is_best_effort_source(source) {
|
||||
self.best_effort_queued = self.best_effort_queued.saturating_sub(1);
|
||||
if let Some(index) = best_effort_source_index(source) {
|
||||
self.best_effort_queued_by_source[index] = self.best_effort_queued_by_source[index].saturating_sub(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn best_effort_source_count(&self, source: HealRequestSource) -> usize {
|
||||
best_effort_source_index(source).map_or(0, |index| self.best_effort_queued_by_source[index])
|
||||
}
|
||||
|
||||
fn restore_deferred_items(&mut self, deferred: Vec<PriorityQueueItem>) {
|
||||
if deferred.is_empty() {
|
||||
return;
|
||||
@@ -735,7 +590,6 @@ impl PriorityHealQueue {
|
||||
if removed.is_none() && item.request.id == request_id {
|
||||
let key = item.dedup_key.clone();
|
||||
Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &key);
|
||||
self.remove_best_effort_count(item.request.source);
|
||||
affected_key = Some(key);
|
||||
removed = Some(item.request);
|
||||
} else {
|
||||
@@ -761,7 +615,6 @@ impl PriorityHealQueue {
|
||||
while let Some(item) = self.heap.pop() {
|
||||
if should_remove(&item.request) {
|
||||
Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item.dedup_key);
|
||||
self.remove_best_effort_count(item.request.source);
|
||||
affected_keys.push(item.dedup_key);
|
||||
removed.push(item.request);
|
||||
} else {
|
||||
|
||||
@@ -150,7 +150,7 @@ impl HealManager {
|
||||
for _ in 0..available_slots {
|
||||
let selected_request = if config.set_bulkhead_enable || mainline_pressure.is_some() {
|
||||
let max_concurrent_per_set = config.max_concurrent_per_set;
|
||||
let (selected_request, skipped_sets) = queue.pop_runnable_with_fairness(
|
||||
let (selected_request, skipped_sets) = queue.pop_runnable_with_skips(
|
||||
|request| {
|
||||
let set_allowed = !config.set_bulkhead_enable
|
||||
|| can_schedule_request(request, &running_per_set, max_concurrent_per_set);
|
||||
@@ -164,7 +164,7 @@ impl HealManager {
|
||||
}
|
||||
selected_request
|
||||
} else {
|
||||
queue.pop_runnable_with_fairness(|_| true, |_| None).0
|
||||
queue.pop_next()
|
||||
};
|
||||
|
||||
if let Some(mut request) = selected_request {
|
||||
@@ -811,9 +811,6 @@ pub(super) fn mrf_verified_repair_event_for_target(
|
||||
let disposition = match outcome.disposition {
|
||||
HealObjectDisposition::Repaired => MrfVerifiedRepairDisposition::Repaired,
|
||||
HealObjectDisposition::VerifiedHealthy => MrfVerifiedRepairDisposition::VerifiedHealthy,
|
||||
HealObjectDisposition::MetadataHealthy if target.kind == MrfKind::MetadataCorruption => {
|
||||
MrfVerifiedRepairDisposition::VerifiedHealthy
|
||||
}
|
||||
HealObjectDisposition::AuthoritativelyAbsent => MrfVerifiedRepairDisposition::AuthoritativelyAbsent,
|
||||
_ => return None,
|
||||
};
|
||||
|
||||
@@ -1488,164 +1488,6 @@ fn test_priority_queue_ordering() {
|
||||
assert_eq!(queue.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_priority_queue_fairness_gives_best_effort_bounded_service() {
|
||||
let mut queue = PriorityHealQueue::new();
|
||||
for index in 0..5 {
|
||||
assert_eq!(
|
||||
queue.push(bucket_request(&format!("mrf-{index}"), HealPriority::Normal, HealRequestSource::Mrf)),
|
||||
QueuePushOutcome::Accepted
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
queue.push(bucket_request("scanner", HealPriority::Low, HealRequestSource::Scanner)),
|
||||
QueuePushOutcome::Accepted
|
||||
);
|
||||
|
||||
for _ in 0..4 {
|
||||
let (request, skipped) = queue.pop_runnable_with_fairness(|_| true, |_| None);
|
||||
assert!(skipped.is_empty());
|
||||
assert_eq!(
|
||||
request.expect("MRF request should run before the fairness quantum").source,
|
||||
HealRequestSource::Mrf
|
||||
);
|
||||
}
|
||||
|
||||
let (request, skipped) = queue.pop_runnable_with_fairness(|_| true, |_| None);
|
||||
assert!(skipped.is_empty());
|
||||
assert_eq!(
|
||||
request
|
||||
.expect("best-effort work must get a bounded service opportunity")
|
||||
.source,
|
||||
HealRequestSource::Scanner
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_priority_queue_fairness_covers_internal_normal_background_work() {
|
||||
let mut queue = PriorityHealQueue::new();
|
||||
for index in 0..5 {
|
||||
assert_eq!(
|
||||
queue.push(bucket_request(
|
||||
&format!("internal-{index}"),
|
||||
HealPriority::Normal,
|
||||
HealRequestSource::Internal
|
||||
)),
|
||||
QueuePushOutcome::Accepted
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
queue.push(bucket_request("scanner", HealPriority::Low, HealRequestSource::Scanner)),
|
||||
QueuePushOutcome::Accepted
|
||||
);
|
||||
|
||||
for _ in 0..4 {
|
||||
let (request, _) = queue.pop_runnable_with_fairness(|_| true, |_| None);
|
||||
assert_eq!(request.expect("internal background work should run").source, HealRequestSource::Internal);
|
||||
}
|
||||
let (request, _) = queue.pop_runnable_with_fairness(|_| true, |_| None);
|
||||
assert_eq!(
|
||||
request.expect("Scanner should get a service opportunity").source,
|
||||
HealRequestSource::Scanner
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_priority_queue_fairness_round_robins_best_effort_sources() {
|
||||
let mut queue = PriorityHealQueue::new();
|
||||
for index in 0..8 {
|
||||
assert_eq!(
|
||||
queue.push(bucket_request(&format!("mrf-{index}"), HealPriority::Normal, HealRequestSource::Mrf)),
|
||||
QueuePushOutcome::Accepted
|
||||
);
|
||||
}
|
||||
for (bucket, source) in [
|
||||
("scanner", HealRequestSource::Scanner),
|
||||
("read-repair", HealRequestSource::ReadRepair),
|
||||
("auto-heal", HealRequestSource::AutoHeal),
|
||||
] {
|
||||
assert_eq!(queue.push(bucket_request(bucket, HealPriority::Low, source)), QueuePushOutcome::Accepted);
|
||||
}
|
||||
|
||||
for _ in 0..4 {
|
||||
let (request, _) = queue.pop_runnable_with_fairness(|_| true, |_| None);
|
||||
assert_eq!(request.expect("first MRF quantum should run").source, HealRequestSource::Mrf);
|
||||
}
|
||||
let (request, _) = queue.pop_runnable_with_fairness(|_| true, |_| None);
|
||||
assert_eq!(
|
||||
request.expect("Scanner should get the first best-effort turn").source,
|
||||
HealRequestSource::Scanner
|
||||
);
|
||||
|
||||
for _ in 0..4 {
|
||||
let (request, _) = queue.pop_runnable_with_fairness(|_| true, |_| None);
|
||||
assert_eq!(request.expect("second MRF quantum should run").source, HealRequestSource::Mrf);
|
||||
}
|
||||
let (request, _) = queue.pop_runnable_with_fairness(|_| true, |_| None);
|
||||
assert_eq!(
|
||||
request.expect("ReadRepair should get the next best-effort turn").source,
|
||||
HealRequestSource::ReadRepair
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_priority_queue_fairness_keeps_best_effort_priority_order() {
|
||||
let mut queue = PriorityHealQueue::new();
|
||||
for index in 0..4 {
|
||||
assert_eq!(
|
||||
queue.push(bucket_request(&format!("mrf-{index}"), HealPriority::Normal, HealRequestSource::Mrf)),
|
||||
QueuePushOutcome::Accepted
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
queue.push(bucket_request("scanner", HealPriority::Low, HealRequestSource::Scanner)),
|
||||
QueuePushOutcome::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
queue.push(bucket_request("read-repair", HealPriority::Normal, HealRequestSource::ReadRepair)),
|
||||
QueuePushOutcome::Accepted
|
||||
);
|
||||
|
||||
for _ in 0..4 {
|
||||
let (request, _) = queue.pop_runnable_with_fairness(|_| true, |_| None);
|
||||
assert_eq!(request.expect("MRF request should run").source, HealRequestSource::Mrf);
|
||||
}
|
||||
let (request, _) = queue.pop_runnable_with_fairness(|_| true, |_| None);
|
||||
assert_eq!(
|
||||
request.expect("higher-priority best-effort work should run first").source,
|
||||
HealRequestSource::ReadRepair
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_priority_queue_fairness_skips_blocked_best_effort_source() {
|
||||
let mut queue = PriorityHealQueue::new();
|
||||
for index in 0..4 {
|
||||
assert_eq!(
|
||||
queue.push(bucket_request(&format!("mrf-{index}"), HealPriority::Normal, HealRequestSource::Mrf)),
|
||||
QueuePushOutcome::Accepted
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
queue.push(bucket_request("scanner", HealPriority::Low, HealRequestSource::Scanner)),
|
||||
QueuePushOutcome::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
queue.push(bucket_request("read-repair", HealPriority::Low, HealRequestSource::ReadRepair)),
|
||||
QueuePushOutcome::Accepted
|
||||
);
|
||||
|
||||
for _ in 0..4 {
|
||||
let (request, _) = queue.pop_runnable_with_fairness(|_| true, |_| None);
|
||||
assert_eq!(request.expect("MRF request should run").source, HealRequestSource::Mrf);
|
||||
}
|
||||
let (request, _) = queue.pop_runnable_with_fairness(|request| request.source != HealRequestSource::Scanner, |_| None);
|
||||
assert_eq!(
|
||||
request.expect("runnable ReadRepair should bypass blocked Scanner").source,
|
||||
HealRequestSource::ReadRepair
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_priority_queue_fifo_same_priority() {
|
||||
let mut queue = PriorityHealQueue::new();
|
||||
@@ -4658,7 +4500,7 @@ async fn test_displacing_registered_mrf_task_drops_notice_ownership() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_submit_heal_request_admits_read_repair_under_pressure() {
|
||||
async fn test_submit_heal_request_drops_read_repair_under_pressure() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
storage,
|
||||
@@ -4668,7 +4510,7 @@ async fn test_submit_heal_request_admits_read_repair_under_pressure() {
|
||||
}),
|
||||
);
|
||||
|
||||
for index in 0..7 {
|
||||
for index in 0..8 {
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request(
|
||||
@@ -4687,12 +4529,12 @@ async fn test_submit_heal_request_admits_read_repair_under_pressure() {
|
||||
.await
|
||||
.expect("read repair admission should return a result");
|
||||
|
||||
assert_eq!(admission, HealAdmissionResult::Accepted);
|
||||
assert_eq!(admission, HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped));
|
||||
assert_eq!(manager.get_queue_length().await, 8);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_submit_heal_request_admits_low_scanner_under_pressure() {
|
||||
async fn test_submit_heal_request_drops_low_scanner_under_pressure() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
storage,
|
||||
@@ -4702,7 +4544,7 @@ async fn test_submit_heal_request_admits_low_scanner_under_pressure() {
|
||||
}),
|
||||
);
|
||||
|
||||
for index in 0..7 {
|
||||
for index in 0..8 {
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request(
|
||||
@@ -4721,220 +4563,10 @@ async fn test_submit_heal_request_admits_low_scanner_under_pressure() {
|
||||
.await
|
||||
.expect("scanner admission should return a result");
|
||||
|
||||
assert_eq!(admission, HealAdmissionResult::Accepted);
|
||||
assert_eq!(admission, HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped));
|
||||
assert_eq!(manager.get_queue_length().await, 8);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mrf_admission_preserves_best_effort_queue_reserve() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
Arc::new(MockStorage),
|
||||
Some(HealConfig {
|
||||
queue_size: 10,
|
||||
..HealConfig::default()
|
||||
}),
|
||||
);
|
||||
|
||||
for index in 0..7 {
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request(&format!("mrf-{index}"), HealPriority::Normal, HealRequestSource::Mrf,))
|
||||
.await
|
||||
.expect("MRF request should be admitted before the reserve threshold"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("mrf-deferred", HealPriority::Normal, HealRequestSource::Mrf))
|
||||
.await
|
||||
.expect("MRF admission should return a typed result"),
|
||||
HealAdmissionResult::Full
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("internal-deferred", HealPriority::Normal, HealRequestSource::Internal))
|
||||
.await
|
||||
.expect("ordinary background admission should return a typed result"),
|
||||
HealAdmissionResult::Full
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("scanner-reserved", HealPriority::Low, HealRequestSource::Scanner))
|
||||
.await
|
||||
.expect("Scanner should consume the reserved slot"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("read-repair-reserved", HealPriority::Low, HealRequestSource::ReadRepair))
|
||||
.await
|
||||
.expect("ReadRepair should consume its reserved slot"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("auto-heal-reserved", HealPriority::Low, HealRequestSource::AutoHeal))
|
||||
.await
|
||||
.expect("AutoHeal should consume its reserved slot"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(manager.get_queue_length().await, 10);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_small_queue_keeps_per_source_best_effort_slots() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
Arc::new(MockStorage),
|
||||
Some(HealConfig {
|
||||
queue_size: 9,
|
||||
..HealConfig::default()
|
||||
}),
|
||||
);
|
||||
|
||||
for index in 0..6 {
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request(&format!("mrf-{index}"), HealPriority::Normal, HealRequestSource::Mrf))
|
||||
.await
|
||||
.expect("MRF request should be admitted before the per-source reserve"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("mrf-deferred", HealPriority::Normal, HealRequestSource::Mrf))
|
||||
.await
|
||||
.expect("MRF admission should return a typed result"),
|
||||
HealAdmissionResult::Full
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("scanner-reserved", HealPriority::Low, HealRequestSource::Scanner))
|
||||
.await
|
||||
.expect("Scanner should consume its reserved slot"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("read-repair-reserved", HealPriority::Low, HealRequestSource::ReadRepair))
|
||||
.await
|
||||
.expect("ReadRepair should consume its reserved slot"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("auto-heal-reserved", HealPriority::Low, HealRequestSource::AutoHeal))
|
||||
.await
|
||||
.expect("AutoHeal should consume its reserved slot"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_queue_size_four_keeps_mrf_and_each_best_effort_source() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
Arc::new(MockStorage),
|
||||
Some(HealConfig {
|
||||
queue_size: 4,
|
||||
..HealConfig::default()
|
||||
}),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("mrf", HealPriority::Normal, HealRequestSource::Mrf))
|
||||
.await
|
||||
.expect("MRF should retain one queue slot"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("scanner", HealPriority::Low, HealRequestSource::Scanner))
|
||||
.await
|
||||
.expect("Scanner should use its source slot"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("read-repair", HealPriority::Low, HealRequestSource::ReadRepair))
|
||||
.await
|
||||
.expect("ReadRepair should use its source slot"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("auto-heal", HealPriority::Low, HealRequestSource::AutoHeal))
|
||||
.await
|
||||
.expect("AutoHeal should use its source slot"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_high_priority_mrf_bypasses_best_effort_reserve() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
Arc::new(MockStorage),
|
||||
Some(HealConfig {
|
||||
queue_size: 10,
|
||||
..HealConfig::default()
|
||||
}),
|
||||
);
|
||||
|
||||
for index in 0..7 {
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request(&format!("mrf-{index}"), HealPriority::Normal, HealRequestSource::Mrf))
|
||||
.await
|
||||
.expect("normal MRF request should be admitted before the reserve"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("urgent-mrf", HealPriority::Urgent, HealRequestSource::Mrf))
|
||||
.await
|
||||
.expect("urgent MRF admission should return a typed result"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_high_priority_best_effort_bypasses_source_quota() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
Arc::new(MockStorage),
|
||||
Some(HealConfig {
|
||||
queue_size: 10,
|
||||
..HealConfig::default()
|
||||
}),
|
||||
);
|
||||
|
||||
for index in 0..7 {
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request(&format!("mrf-{index}"), HealPriority::Normal, HealRequestSource::Mrf))
|
||||
.await
|
||||
.expect("normal MRF request should be admitted before the reserve"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("scanner-low", HealPriority::Low, HealRequestSource::Scanner))
|
||||
.await
|
||||
.expect("low Scanner request should be admitted into its source slot"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("scanner-high", HealPriority::High, HealRequestSource::Scanner))
|
||||
.await
|
||||
.expect("high Scanner retry should bypass its low-priority quota"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_submit_heal_request_accepts_admin_high_under_pressure() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
@@ -4946,7 +4578,7 @@ async fn test_submit_heal_request_accepts_admin_high_under_pressure() {
|
||||
}),
|
||||
);
|
||||
|
||||
for index in 0..7 {
|
||||
for index in 0..8 {
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request(
|
||||
@@ -4966,56 +4598,7 @@ async fn test_submit_heal_request_accepts_admin_high_under_pressure() {
|
||||
.expect("admin admission should return a result");
|
||||
|
||||
assert_eq!(admission, HealAdmissionResult::Accepted);
|
||||
assert_eq!(manager.get_queue_length().await, 8);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_admin_normal_displaces_best_effort_when_queue_is_full() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
Arc::new(MockStorage),
|
||||
Some(HealConfig {
|
||||
queue_size: 10,
|
||||
..HealConfig::default()
|
||||
}),
|
||||
);
|
||||
|
||||
for index in 0..7 {
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request(&format!("mrf-{index}"), HealPriority::Normal, HealRequestSource::Mrf))
|
||||
.await
|
||||
.expect("MRF request should be queued"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("scanner", HealPriority::Low, HealRequestSource::Scanner))
|
||||
.await
|
||||
.expect("Scanner request should be queued"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("read-repair", HealPriority::Low, HealRequestSource::ReadRepair))
|
||||
.await
|
||||
.expect("ReadRepair request should be queued"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("auto-heal", HealPriority::Low, HealRequestSource::AutoHeal))
|
||||
.await
|
||||
.expect("AutoHeal request should be queued"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("admin", HealPriority::Normal, HealRequestSource::Admin))
|
||||
.await
|
||||
.expect("admin admission should return a typed result"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(manager.get_queue_length().await, 9);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -76,9 +76,6 @@ pub enum HealObjectDisposition {
|
||||
Unknown,
|
||||
Repaired,
|
||||
VerifiedHealthy,
|
||||
/// Authoritative metadata/presence proof only. This does not certify
|
||||
/// payload integrity.
|
||||
MetadataHealthy,
|
||||
AuthoritativelyAbsent,
|
||||
Deferred {
|
||||
reason: HealDeferredReason,
|
||||
@@ -105,17 +102,12 @@ pub struct HealObjectReceipt {
|
||||
|
||||
impl HealObjectReceipt {
|
||||
pub(crate) fn verified_for(&self, expected: &HealObjectIdentity) -> bool {
|
||||
let disposition_verifies = match self.disposition {
|
||||
matches!(
|
||||
self.disposition,
|
||||
HealObjectDisposition::Repaired
|
||||
| HealObjectDisposition::VerifiedHealthy
|
||||
| HealObjectDisposition::AuthoritativelyAbsent => true,
|
||||
// A metadata/presence proof never certifies payload bytes, so it
|
||||
// cannot discharge a request that exists to decode the payload.
|
||||
HealObjectDisposition::MetadataHealthy => expected.kind != HealObjectKind::Decode,
|
||||
_ => false,
|
||||
};
|
||||
disposition_verifies
|
||||
&& self.identity.kind == expected.kind
|
||||
| HealObjectDisposition::VerifiedHealthy
|
||||
| HealObjectDisposition::AuthoritativelyAbsent
|
||||
) && self.identity.kind == expected.kind
|
||||
&& self.identity.bucket == expected.bucket
|
||||
&& self.identity.object == expected.object
|
||||
&& self.identity.version_id == expected.version_id
|
||||
@@ -375,9 +367,7 @@ impl HealTaskOutcome {
|
||||
counters.overflowed |= !increment_counter(&mut counters.processed);
|
||||
let counter = match item.disposition {
|
||||
HealObjectDisposition::Repaired => &mut counters.healed,
|
||||
HealObjectDisposition::VerifiedHealthy
|
||||
| HealObjectDisposition::MetadataHealthy
|
||||
| HealObjectDisposition::AuthoritativelyAbsent => &mut counters.unchanged,
|
||||
HealObjectDisposition::VerifiedHealthy | HealObjectDisposition::AuthoritativelyAbsent => &mut counters.unchanged,
|
||||
HealObjectDisposition::Failed(_) => &mut counters.failed,
|
||||
HealObjectDisposition::Unknown => {
|
||||
counters.overflowed |= !increment_counter(&mut counters.unknown);
|
||||
@@ -534,32 +524,6 @@ mod canonical_outcome_tests {
|
||||
assert!(serde_json::from_value::<HealTaskOutcome>(value).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_health_receipt_never_discharges_a_payload_decode_request() {
|
||||
let incarnation = Uuid::new_v4();
|
||||
let mut expected = HealObjectIdentity {
|
||||
bucket_incarnation_id: Some(incarnation),
|
||||
..item(HealObjectDisposition::Unknown).identity
|
||||
};
|
||||
let mut receipt = HealObjectReceipt {
|
||||
identity: expected.clone(),
|
||||
disposition: HealObjectDisposition::MetadataHealthy,
|
||||
};
|
||||
assert!(
|
||||
receipt.verified_for(&expected),
|
||||
"a presence proof still settles a metadata-level object request"
|
||||
);
|
||||
|
||||
expected.kind = HealObjectKind::Decode;
|
||||
receipt.identity.kind = HealObjectKind::Decode;
|
||||
assert!(
|
||||
!receipt.verified_for(&expected),
|
||||
"a presence-only proof must not clear a payload decode responsibility"
|
||||
);
|
||||
receipt.disposition = HealObjectDisposition::VerifiedHealthy;
|
||||
assert!(receipt.verified_for(&expected));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_outcome_categories_have_one_terminal_count() {
|
||||
let mut outcome = HealTaskOutcome::default();
|
||||
@@ -567,7 +531,6 @@ mod canonical_outcome_tests {
|
||||
HealObjectDisposition::Unknown,
|
||||
HealObjectDisposition::Repaired,
|
||||
HealObjectDisposition::VerifiedHealthy,
|
||||
HealObjectDisposition::MetadataHealthy,
|
||||
HealObjectDisposition::AuthoritativelyAbsent,
|
||||
HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::DanglingDeleteGrace,
|
||||
@@ -580,7 +543,7 @@ mod canonical_outcome_tests {
|
||||
outcome.record(item(disposition));
|
||||
}
|
||||
let c = &outcome.counters;
|
||||
assert_eq!((c.processed, c.healed, c.unchanged, c.skipped, c.failed, c.unknown), (9, 1, 3, 4, 1, 1));
|
||||
assert_eq!((c.processed, c.healed, c.unchanged, c.skipped, c.failed, c.unknown), (8, 1, 2, 4, 1, 1));
|
||||
assert_eq!(c.processed, c.healed + c.unchanged + c.skipped + c.failed);
|
||||
}
|
||||
|
||||
|
||||
@@ -109,9 +109,7 @@ fn verified_object_receipt(
|
||||
item: &HealResultItem,
|
||||
bucket_incarnation_id: Uuid,
|
||||
) -> Option<HealObjectReceipt> {
|
||||
if opts.dry_run
|
||||
|| (!item.integrity_verified && !item.repair_verified && !item.metadata_verified && !item.metadata_repair_verified)
|
||||
{
|
||||
if opts.dry_run || (!item.integrity_verified && !item.repair_verified) {
|
||||
return None;
|
||||
}
|
||||
let resolved_version = Uuid::from_bytes(item.resolved_version_id?);
|
||||
@@ -122,7 +120,7 @@ fn verified_object_receipt(
|
||||
}
|
||||
item.drives_reported()?;
|
||||
let drives_healed = item.drives_healed()?;
|
||||
if (item.repair_verified || item.metadata_repair_verified) && drives_healed == 0 {
|
||||
if item.repair_verified && drives_healed == 0 {
|
||||
return None;
|
||||
}
|
||||
let ok_drive_state = DriveState::Ok.to_string();
|
||||
@@ -144,12 +142,10 @@ fn verified_object_receipt(
|
||||
pool_index: opts.pool,
|
||||
set_index: opts.set,
|
||||
},
|
||||
disposition: if drives_healed > 0 && (item.integrity_verified || item.repair_verified || item.metadata_repair_verified) {
|
||||
disposition: if drives_healed > 0 {
|
||||
HealObjectDisposition::Repaired
|
||||
} else if drives_healed == 0 && item.integrity_verified {
|
||||
} else if item.integrity_verified {
|
||||
HealObjectDisposition::VerifiedHealthy
|
||||
} else if drives_healed == 0 && item.metadata_verified {
|
||||
HealObjectDisposition::MetadataHealthy
|
||||
} else {
|
||||
return None;
|
||||
},
|
||||
@@ -532,19 +528,6 @@ pub trait HealStorageAPI: Send + Sync {
|
||||
Err(Error::other("storage does not support incarnation-bound object healing"))
|
||||
}
|
||||
|
||||
/// Durable MRF repair may request an authoritative unversioned absence proof.
|
||||
async fn heal_mrf_object_at_incarnation(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<&str>,
|
||||
expected: Uuid,
|
||||
opts: &HealOpts,
|
||||
) -> Result<HealStorageObjectResult> {
|
||||
self.heal_object_at_incarnation(bucket, object, version_id, expected, opts)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Heal object using ecstore
|
||||
async fn heal_object(
|
||||
&self,
|
||||
@@ -591,11 +574,6 @@ pub trait HealStorageAPI: Send + Sync {
|
||||
Err(Error::other("replacement pool metadata placement is unsupported"))
|
||||
}
|
||||
|
||||
/// Heal and physically verify bucket configuration on its replacement targets.
|
||||
async fn heal_replacement_bucket_metadata(&self, _bucket: &str, _opts: &HealOpts, _targets: &[String]) -> Result<()> {
|
||||
Err(Error::Storage(StorageError::PreconditionFailed))
|
||||
}
|
||||
|
||||
/// Read target-specific physical evidence for one replacement version.
|
||||
///
|
||||
/// This is only used by automatic replacement healing after the normal
|
||||
@@ -774,10 +752,7 @@ impl ECStoreHealStorage {
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else if error.is_none()
|
||||
&& !opts.dry_run
|
||||
&& (item.integrity_verified || item.repair_verified || item.metadata_verified || item.metadata_repair_verified)
|
||||
{
|
||||
} else if error.is_none() && !opts.dry_run && (item.integrity_verified || item.repair_verified) {
|
||||
let bucket_incarnation_id = match expected {
|
||||
Some(expected) => Some(expected),
|
||||
None => self.ecstore.bucket_incarnation_id(bucket).await.ok(),
|
||||
@@ -985,24 +960,6 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
.await)
|
||||
}
|
||||
|
||||
async fn heal_mrf_object_at_incarnation(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<&str>,
|
||||
expected: Uuid,
|
||||
opts: &HealOpts,
|
||||
) -> Result<HealStorageObjectResult> {
|
||||
let result = self
|
||||
.ecstore
|
||||
.heal_mrf_object_at_incarnation(bucket, object, version_id.unwrap_or_default(), expected, opts)
|
||||
.await
|
||||
.map_err(|error| incarnation_storage_error(bucket, expected, error))?;
|
||||
Ok(self
|
||||
.object_result_with_receipt(bucket, object, version_id, opts, result, Some(expected))
|
||||
.await)
|
||||
}
|
||||
|
||||
async fn get_object_meta(&self, bucket: &str, object: &str) -> Result<Option<HealObjectInfo>> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
@@ -1600,13 +1557,6 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
.map_err(Error::Storage)
|
||||
}
|
||||
|
||||
async fn heal_replacement_bucket_metadata(&self, bucket: &str, opts: &HealOpts, targets: &[String]) -> Result<()> {
|
||||
self.ecstore
|
||||
.heal_replacement_bucket_metadata(bucket, opts, targets)
|
||||
.await
|
||||
.map_err(Error::Storage)
|
||||
}
|
||||
|
||||
async fn replacement_targets_have_version(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -2132,51 +2082,6 @@ mod tests {
|
||||
assert!(verified_object_receipt("bucket", "object", None, &options, &item, incarnation).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_health_and_marker_repair_have_distinct_receipts() {
|
||||
use super::{HealObjectDisposition, HealOpts, HealResultItem, Uuid, verified_object_receipt};
|
||||
use rustfs_madmin::heal_commands::HealDriveInfo;
|
||||
|
||||
let incarnation = Uuid::new_v4();
|
||||
let version = Uuid::new_v4().to_string();
|
||||
let options = HealOpts::default();
|
||||
let healthy = HealDriveInfo {
|
||||
state: "ok".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let mut item = HealResultItem {
|
||||
metadata_verified: true,
|
||||
version_id: version.clone(),
|
||||
resolved_version_id: Some(*Uuid::parse_str(&version).expect("version UUID").as_bytes()),
|
||||
..Default::default()
|
||||
};
|
||||
item.before.drives.push(healthy.clone());
|
||||
item.after.drives.push(healthy);
|
||||
|
||||
let receipt = verified_object_receipt("bucket", "object", Some(&version), &options, &item, incarnation)
|
||||
.expect("metadata quorum should certify metadata health");
|
||||
assert_eq!(receipt.disposition, HealObjectDisposition::MetadataHealthy);
|
||||
|
||||
item.integrity_verified = true;
|
||||
let receipt = verified_object_receipt("bucket", "object", Some(&version), &options, &item, incarnation)
|
||||
.expect("stronger integrity proof should remain available");
|
||||
assert_eq!(receipt.disposition, HealObjectDisposition::VerifiedHealthy);
|
||||
|
||||
item.integrity_verified = false;
|
||||
item.metadata_verified = false;
|
||||
item.metadata_repair_verified = true;
|
||||
item.before.drives[0].state = "missing".to_string();
|
||||
let receipt = verified_object_receipt("bucket", "object", Some(&version), &options, &item, incarnation)
|
||||
.expect("a committed metadata repair should certify the marker/version");
|
||||
assert_eq!(receipt.disposition, HealObjectDisposition::Repaired);
|
||||
|
||||
item.before.drives[0].state = "ok".to_string();
|
||||
assert!(
|
||||
verified_object_receipt("bucket", "object", Some(&version), &options, &item, incarnation).is_none(),
|
||||
"repair proof without a repaired drive must fail closed"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn next_heal_listing_token_returns_none_for_complete_page() {
|
||||
assert_eq!(
|
||||
|
||||
@@ -298,18 +298,6 @@ impl HealTask {
|
||||
"Heal object stage entered"
|
||||
);
|
||||
let object_size = result.object_size as u64;
|
||||
let ok_drive_state = DriveState::Ok.to_string();
|
||||
if self.source == HealRequestSource::Admin
|
||||
&& !self.options.dry_run
|
||||
&& !result.after.drives.is_empty()
|
||||
&& result.after.drives.iter().any(|drive| drive.state != ok_drive_state)
|
||||
{
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!(
|
||||
"Heal left one or more drives unhealthy for {bucket}/{object}; retry after the missing drives are restored"
|
||||
),
|
||||
});
|
||||
}
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_RESULT,
|
||||
@@ -429,10 +417,6 @@ impl HealTask {
|
||||
|
||||
/// Recreate missing object (for EC decode scenarios)
|
||||
async fn recreate_missing_object(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> {
|
||||
if self.source == HealRequestSource::Mrf {
|
||||
return self.recreate_missing_mrf_object(bucket, object, version_id).await;
|
||||
}
|
||||
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_STAGE,
|
||||
@@ -533,65 +517,4 @@ impl HealTask {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Durable MRF responsibilities may complete only with an exact storage proof.
|
||||
async fn recreate_missing_mrf_object(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> {
|
||||
let heal_opts = HealOpts {
|
||||
recursive: false,
|
||||
dry_run: self.options.dry_run,
|
||||
remove: false,
|
||||
recreate: true,
|
||||
scan_mode: HealScanMode::Deep,
|
||||
update_parity: true,
|
||||
no_lock: self.options.no_lock,
|
||||
read_repair: false,
|
||||
pool: self.options.pool_index,
|
||||
set: self.options.set_index,
|
||||
};
|
||||
let mut expected = self.outcome_identity(bucket, object, version_id, self.options.pool_index, self.options.set_index);
|
||||
let bucket_incarnation_id = self
|
||||
.outcome_bucket_incarnation_id(bucket, self.options.dry_run)
|
||||
.await?
|
||||
.ok_or_else(|| Error::TaskExecutionFailed {
|
||||
message: format!("Missing bucket incarnation for durable MRF repair {bucket}/{object}"),
|
||||
})?;
|
||||
expected.bucket_incarnation_id = Some(bucket_incarnation_id);
|
||||
|
||||
let storage_result = self
|
||||
.await_with_control(self.storage.heal_mrf_object_at_incarnation(
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
bucket_incarnation_id,
|
||||
&heal_opts,
|
||||
))
|
||||
.await?;
|
||||
if let Some(error) = storage_result.error {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to recreate missing object {bucket}/{object}: {error}"),
|
||||
});
|
||||
}
|
||||
|
||||
let object_size = storage_result.item.object_size as u64;
|
||||
let authoritatively_absent = matches!(
|
||||
storage_result.receipt.as_ref().map(|receipt| &receipt.disposition),
|
||||
Some(HealObjectDisposition::AuthoritativelyAbsent)
|
||||
);
|
||||
if !self.record_verified_storage_receipt(expected, storage_result.receipt).await {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Missing exact storage proof for durable MRF repair {bucket}/{object}"),
|
||||
});
|
||||
}
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
if authoritatively_absent {
|
||||
progress.update_object_progress(1, 0, 0, 1, 0);
|
||||
} else {
|
||||
progress.update_object_progress(1, 1, 0, 0, object_size);
|
||||
}
|
||||
}
|
||||
self.record_result_item(storage_result.item).await;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user