gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-25 13:36:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into HEAD

Browse Source 
# Conflicts:
#	.config/e2e-full-selection.txt
This commit is contained in:
overtrue
2026-08-23 22:48:30 +08:00
parent 26307be408 0d15ce1865
commit 1ceacaecde
185 changed files with 28834 additions and 2499 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.config/coverage-baselines.toml
+20
View File
@@ -0,0 +1,20 @@
# Report-only calibration baseline from https://github.com/rustfs/rustfs/actions/runs/29394996173.
# Update counts only with a linked coverage run and a reviewed explanation.
phase = "report-only"
allowed_drop_percentage_points = 1.0
[crates."crates/iam"]
covered = 5149
count = 8131
[crates."crates/kms"]
covered = 2950
count = 4200
[crates."crates/policy"]
covered = 4636
count = 5464
[crates."crates/crypto"]
covered = 469
count = 494
.config/e2e-full-selection.txt
+2 -2
View File
@@ -1,2 +1,2 @@
sha256-darwin=f1bcf046b2f4137ca2e05381ea1264ef32cf5462629410c6d44851474ffee102
sha256-linux=03651c12c23914d61196a037ee9753afebaf57fcceff31cdae55aec3b21163ba
sha256-darwin=caf2e48fdd2e2d4780e798397517a0ddee74360c4cc83e6f80d31b18e235eed7
sha256-linux=ef3be856bd3257c2c369428f66a48ad073dad8187229fe95840a600a81edf22b
.config/make/tests.mak
+1
View File
@@ -36,6 +36,7 @@ script-tests: ## Run shell script tests
./scripts/test_manual_transition_runbooks.sh
./scripts/check_embedded_secrets.sh --self-test
python3 ./scripts/check_test_wiring.py --self-test
python3 ./scripts/check_security_coverage.py --self-test
python3 ./scripts/check_scheduled_validation_freshness.py --self-test
python3 ./scripts/s3-tests/test_report_compat.py
bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
.config/nextest.toml
+14 -4
View File
@@ -78,6 +78,12 @@ test-group = 'embedded-test-ports'
filter = 'package(rustfs-ecstore) & test(manual_transition_page_checkpoint_persists_durable_job_progress)'
test-group = 'ecstore-serial-flaky'
# The durable ILM decommission regressions build isolated multi-pool stores and
# deliberately take source or target disks offline while checking fencing.
[[profile.default.overrides]]
filter = 'package(rustfs-ecstore) & (test(decommission_migrates_and_verifies_registered_durable_ilm_records) | test(decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success) | test(decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup) | test(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page) | test(decommission_durable_ilm_recovery_keeps_multiple_active_sources))'
test-group = 'ecstore-serial-flaky'
# Serialize the bucket-incarnation / lifecycle-fence tests. They drive
# init_bucket_metadata_sys and bucket_metadata_sys_of, i.e. process-global
# OnceLock state that serial_test's #[serial] cannot protect across nextest's
@@ -190,6 +196,10 @@ test-group = 'embedded-test-ports'
filter = 'package(rustfs-ecstore) & test(manual_transition_page_checkpoint_persists_durable_job_progress)'
test-group = 'ecstore-serial-flaky'
[[profile.ci.overrides]]
filter = 'package(rustfs-ecstore) & (test(decommission_migrates_and_verifies_registered_durable_ilm_records) | test(decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success) | test(decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup) | test(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page) | test(decommission_durable_ilm_recovery_keeps_multiple_active_sources))'
test-group = 'ecstore-serial-flaky'
# Serialize the bucket-incarnation / lifecycle-fence tests under the ci profile
# too (see the matching default-profile override near the top). No retries.
[[profile.ci.overrides]]
@@ -305,7 +315,7 @@ slow-timeout = { period = "60s", terminate-after = 2, grace-period = "10s" }
# the target incl. multipart and the resync path, SSE-C and
# target-without-KMS stay fail-closed), and one guards event/history
# observers.
# * 12 `_real_dual_node` site-replication tests — each spawns TWO full rustfs
# * 13 `_real_dual_node` site-replication tests — each spawns TWO full rustfs
# servers and drives the cross-process site-replication control plane.
# * 1 `_real_three_node` site-replication test.
# * 1 `_real_single_node` service-account round-trip test.
@@ -325,8 +335,8 @@ slow-timeout = { period = "60s", terminate-after = 2, grace-period = "10s" }
#
# Wired by .github/workflows/e2e-replication-nightly.yml (schedule +
# workflow_dispatch), which builds the rustfs binary once, installs awscurl so
# the STS dual-node test actually exercises its path (it skips gracefully with
# a visible log line when awscurl is absent), and routes scheduled failures
# the STS dual-node test actually exercises its path (the test fails when
# awscurl is absent), and routes scheduled failures
# through .github/actions/schedule-failure-issue (ci-8). Explicit division of
# labor with e2e-full: these tests run only in the consolidated nightly
# workflow, not in the merge/main lane.
@@ -396,7 +406,7 @@ path = "junit.xml"
# object_lambda) — too heavy for the merge budget; they run in the
# e2e-nightly serial cluster-fault lane.
# * replication_extension_test — repl-1 already splits it into the PR
# `e2e-smoke` (20 fast) and `e2e-repl-nightly` (55 slow) lanes and reserves
# `e2e-smoke` (20 fast) and `e2e-repl-nightly` (56 slow) lanes and reserves
# it for those, so e2e-full does not double-run it.
# * #[ignore]d tests — nextest skips them by default (no --run-ignored); the
# manual-localhost:9000 reliant/policy tests are ci-13's migration.
.github/workflows/ci.yml
+27
View File
@@ -681,6 +681,19 @@ jobs:
cache-save-if: 'false'
install-build-packaging-tools: 'false'
- name: Set up Python
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: "3.12"
- name: Install awscurl
run: |
python3 -m pip install --user --upgrade pip "awscurl==0.44"
echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"
- name: Verify awscurl
run: test -x "$AWSCURL_PATH"
# Download after the cache restore so the freshly built binary from the
# build job always wins over anything restored into target/debug.
- name: Download debug binary
@@ -803,6 +816,20 @@ jobs:
- name: Verify awscurl
run: test -x "$AWSCURL_PATH"
- name: Install mc
env:
MC_VERSION: RELEASE.2025-08-13T08-35-41Z
MC_SHA256: 01f866e9c5f9b87c2b09116fa5d7c06695b106242d829a8bb32990c00312e891
run: |
MC_BINARY="mc.linux-amd64.${MC_VERSION}"
curl -fsSLo "$RUNNER_TEMP/mc" "https://github.com/minio/mc/releases/download/${MC_VERSION}/${MC_BINARY}"
echo "${MC_SHA256} $RUNNER_TEMP/mc" | sha256sum --check --status
chmod +x "$RUNNER_TEMP/mc"
echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
- name: Verify mc
run: mc --version
- name: Install Vault
run: |
VAULT_VERSION="1.17.6"
.github/workflows/coverage.yml
+29 -12
View File
@@ -12,14 +12,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Weekly workspace line-coverage baseline (backlog#1153 infra-5).
# Workspace line-coverage baseline and security-crate calibration
# (backlog#1153 infra-5/infra-6).
#
# NON-BLOCKING by design: this workflow only runs on schedule and manual
# dispatch, so it never attaches a status to a PR and must never be made a
# required check. It exists to give coverage a visible baseline and trend
# (per-crate table in the job summary, lcov artifact kept 90 days) — the
# per-crate ratchet for the security-critical crates builds on it later
# (backlog#1153 infra-6, report-only first per the ci-11 ladder).
# NON-BLOCKING by design: the weekly job gives coverage a visible baseline and
# trend, while relevant pull requests run a report-only security-crate
# comparison. Neither job is a required check during calibration.
#
# Measurement scope matches the PR test gate (ci.yml "Run tests"):
# `--workspace --exclude e2e_test` with the `ci` nextest profile. Doctests are
@@ -31,6 +29,17 @@
name: coverage
on:
pull_request:
branches: [main]
paths:
- "crates/iam/**"
- "crates/kms/**"
- "crates/policy/**"
- "crates/crypto/**"
- ".config/coverage-baselines.toml"
- "scripts/coverage_per_crate.py"
- "scripts/check_security_coverage.py"
- ".github/workflows/coverage.yml"
workflow_dispatch:
schedule:
# 07:00 UTC Sunday — staggered clear of the other Sunday crons: ci (00:00),
@@ -39,6 +48,10 @@ on:
# e2e-replication-nightly (04:00) and performance-ab (06:00) lanes.
- cron: "43 7 * * 0"
concurrency:
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name != 'schedule' }}
# Only alert-on-failure needs more than read access; it declares its own
# job-level `issues: write`.
permissions:
@@ -46,12 +59,14 @@ permissions:
jobs:
coverage:
name: Workspace coverage (weekly)
name: Workspace line coverage
runs-on: sm-standard-4
# The instrumented build cannot reuse the regular CI cache (different
# RUSTFLAGS), so a cold week rebuilds the workspace before running the
# full suite; give it double the test job's 60-minute budget.
timeout-minutes: 120
# RUSTFLAGS), so a cold run rebuilds the workspace before running the
# full suite. Two later exact-head runs exhausted 150 minutes before the
# report steps, so allow one additional 90-minute cold-run margin while
# keeping the calibration job bounded.
timeout-minutes: 240
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
# Match the PR gate's nextest semantics (ci.yml runs `--profile ci`):
@@ -91,7 +106,9 @@ jobs:
cargo llvm-cov report --json --output-path target/llvm-cov/coverage.json
- name: Write per-crate summary
run: python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"
run: |
python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"
python3 scripts/check_security_coverage.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"
- name: Upload coverage artifact
if: always()
.github/workflows/e2e-replication-nightly.yml
+7 -8
View File
@@ -75,11 +75,7 @@ jobs:
cache-save-if: ${{ github.ref == 'refs/heads/main' }}
install-build-packaging-tools: 'false'
# awscurl lets the STS dual-node test actually exercise its path. Without
# it the test skips gracefully with a visible log line
# (`awscurl_available()` in crates/e2e_test/src/common.rs), so the lane
# still passes — installing it just upgrades that one test from skip to
# real coverage.
# The STS dual-node test requires awscurl and fails if it is unavailable.
- name: Set up Python
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
@@ -87,7 +83,7 @@ jobs:
- name: Install awscurl
run: |
python3 -m pip install --user --upgrade pip awscurl
python3 -m pip install --user --upgrade pip "awscurl==0.44"
echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"
- name: Verify awscurl
@@ -196,8 +192,11 @@ jobs:
cache-save-if: 'false'
install-build-packaging-tools: 'false'
- name: Verify protocol socket oracle
run: ss -tn state CLOSE-WAIT >/dev/null
- name: Install and verify protocol socket oracle
run: |
sudo apt-get update -qq
sudo apt-get install -y -qq iproute2
ss -tn state CLOSE-WAIT >/dev/null
# The suite owns fixed protocol ports and serializes its internal cases.
- name: Verify protocol e2e membership
.github/workflows/e2e-s3tests.yml
+88 -1
View File
@@ -21,6 +21,9 @@
# suite and reports promotion candidates. Regressions, unclassified tests,
# incomplete execution, and infrastructure errors fail the job; classified
# failures for not-yet-implemented features remain informational.
# - Non-blocking upstream HEAD canary: collects current upstream node IDs and
# reports new, removed, duplicate, or overlapping classifications without
# making upstream drift a release gate.
# - Manual runs (workflow_dispatch): same, with configurable mode/scope.
#
# All test execution is delegated to scripts/s3-tests/run.sh (single source of
@@ -178,9 +181,14 @@ jobs:
- name: Install Python tools
run: |
python3 -m pip install --user --upgrade pip awscurl tox
python3 -m pip install --user --upgrade pip "awscurl==0.44" "tox==4.60.0"
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
- name: Verify Python tools
run: |
test "$(python3 -c 'import importlib.metadata as m; print(m.version("awscurl"))')" = "0.44"
test "$(python3 -c 'import importlib.metadata as m; print(m.version("tox"))')" = "4.60.0"
- name: Enable buildx
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
@@ -354,6 +362,85 @@ jobs:
name: s3tests-${{ env.TEST_MODE }}-shard-${{ matrix.shard-index }}
path: artifacts/**
upstream-head-canary:
name: Upstream HEAD classification canary
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
continue-on-error: true
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: "3.12"
- name: Install collection tool
run: |
python3 -m pip install --user "tox==4.60.0"
python3 - <<'PY'
from importlib.metadata import version
assert version("tox") == "4.60.0"
PY
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
- name: Compare upstream HEAD classifications
id: upstream-compare
run: |
ARTIFACT_DIR="artifacts/s3tests-upstream-head"
UPSTREAM_DIR="${RUNNER_TEMP}/s3-tests-upstream"
mkdir -p "${ARTIFACT_DIR}"
git clone --depth 1 https://github.com/ceph/s3-tests.git "${UPSTREAM_DIR}"
git -C "${UPSTREAM_DIR}" rev-parse HEAD > "${ARTIFACT_DIR}/upstream-sha.txt"
cp "${UPSTREAM_DIR}/s3tests.conf.SAMPLE" "${UPSTREAM_DIR}/s3tests.conf"
(
cd "${UPSTREAM_DIR}"
S3TEST_CONF="${UPSTREAM_DIR}/s3tests.conf" tox -- \
-q --collect-only s3tests/functional/test_s3.py \
-m "not rustfs_never_marker"
) 2>&1 | tee "${ARTIFACT_DIR}/collect.log"
grep -E '^s3tests/functional/test_s3\.py::' \
"${ARTIFACT_DIR}/collect.log" > "${ARTIFACT_DIR}/collected-nodeids.txt"
python3 scripts/s3-tests/report_compat.py \
--lists-dir scripts/s3-tests \
--collected-nodeids "${ARTIFACT_DIR}/collected-nodeids.txt" \
--check-classifications-only 2>&1 | tee "${ARTIFACT_DIR}/classification-drift.txt"
- name: Publish canary report
if: always()
env:
CANARY_OUTCOME: ${{ steps.upstream-compare.outcome }}
run: |
{
echo "## ceph/s3-tests upstream HEAD canary"
echo
if [ -f artifacts/s3tests-upstream-head/upstream-sha.txt ]; then
echo "Upstream HEAD: $(cat artifacts/s3tests-upstream-head/upstream-sha.txt)"
fi
echo
echo '```text'
if [ -s artifacts/s3tests-upstream-head/classification-drift.txt ]; then
cat artifacts/s3tests-upstream-head/classification-drift.txt
elif [ "${CANARY_OUTCOME}" != "success" ]; then
echo "Canary did not complete; inspect the collection log artifact."
else
echo "No classification drift detected."
fi
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
- name: Upload canary artifacts
if: always() && env.ACT != 'true'
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: s3tests-upstream-head
path: artifacts/s3tests-upstream-head/**
retention-days: 14
alert-on-failure:
name: Alert on scheduled failure
needs: [s3tests]
.github/workflows/performance-ab.yml
+8 -4
View File
@@ -121,14 +121,14 @@ jobs:
candidate_sha="$(git rev-parse HEAD)"
if [[ "${{ github.event_name }}" == "schedule" ]]; then
baseline_sha="${SCHEDULED_BASELINE_SHA:-$candidate_sha}"
if ! git merge-base --is-ancestor "$baseline_sha" "$candidate_sha"; then
echo "::error::scheduled baseline $baseline_sha is not an ancestor of candidate $candidate_sha" >&2
exit 1
fi
else
baseline_sha="$(git rev-parse origin/main)"
fi
git cat-file -e "${baseline_sha}^{commit}"
if ! git merge-base --is-ancestor "$baseline_sha" "$candidate_sha"; then
echo "::error::baseline $baseline_sha is not an ancestor of candidate $candidate_sha; update the selected ref before comparing" >&2
exit 1
fi
echo "baseline_sha=$baseline_sha" >> "$GITHUB_OUTPUT"
echo "candidate_sha=$candidate_sha" >> "$GITHUB_OUTPUT"
echo "baseline commit: $baseline_sha"
@@ -342,6 +342,10 @@ jobs:
if: always()
run: |
status="${{ steps.ab.outputs.status }}"
if [[ -z "$status" ]]; then
echo "::error::warp A/B setup failed before the rig ran. Check the first failed workflow step." >&2
exit 1
fi
if [[ "$status" != "0" ]]; then
echo "::error::warp A/B budget gate failed (exit $status). See the step summary / gate.md artifact." >&2
exit "$status"
Cargo.lock
Generated
+109 -53
View File
@@ -68,16 +68,16 @@ dependencies = [
[[package]]
name = "aes-gcm"
version = "0.11.0"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdf011db2e21ce0d575593d749db5554b47fed37aff429e4dc50bc91ac93a028"
checksum = "7f2b8006a0c83f52b62ba44a97b58bf76fe2f70a329e588f67f89691d93d498f"
dependencies = [
"aead",
"aes 0.9.2",
"cipher 0.5.2",
"ctr",
"ctutils",
"ghash",
"subtle",
"zeroize",
]
@@ -333,6 +333,12 @@ dependencies = [
"password-hash",
]
[[package]]
name = "array-init"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc"
[[package]]
name = "arrayvec"
version = "0.7.8"
@@ -813,11 +819,12 @@ dependencies = [
[[package]]
name = "async_zip"
version = "0.0.18"
version = "0.0.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d8c50d65ce1b0e0cb65a785ff615f78860d7754290647d3b983208daa4f85e6"
checksum = "fb7f5f40e1eb30949a266fc900d37fd3267c7baf50c3705ac09c5d8ced5def63"
dependencies = [
"async-compression",
"binrw",
"crc32fast",
"futures-lite",
"pin-project",
@@ -869,9 +876,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
[[package]]
name = "aws-config"
version = "1.10.1"
version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b180a3c8b55960db3426d8964b8745e652466a1a49fe1a2eda828046d30b5e4"
checksum = "a767267da9e2c2e189b2f9df8b5657e850ecf5352644734ba130d4a57095cf1b"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -964,9 +971,9 @@ dependencies = [
[[package]]
name = "aws-sdk-kms"
version = "1.115.0"
version = "1.116.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d5b034f8b7ceadb873d0bc607c30bb4b0be68e09a84c837174e7c2c6878ff882"
checksum = "484ecdbea2a1cfc0e6eea69ce0a665f93913671b303ba40b2361b1d826544e7e"
dependencies = [
"arc-swap",
"aws-credential-types",
@@ -990,9 +997,9 @@ dependencies = [
[[package]]
name = "aws-sdk-s3"
version = "1.142.0"
version = "1.143.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9e15a5c55e05f4b0b7e483160b3c85cccdf77cff02c95504f3e71d460855cd2"
checksum = "a0ade5433c9561daac7c0c6bc910f1240b4f8ec0d6148b0b463aac0691d747c9"
dependencies = [
"arc-swap",
"aws-credential-types",
@@ -1027,9 +1034,9 @@ dependencies = [
[[package]]
name = "aws-sdk-sso"
version = "1.106.0"
version = "1.107.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d0efcee834347b6705eca3eea2defd88242f43774f55d7326604222e3c86260"
checksum = "769b0abd0f89cfe11da5099986dd493e4f94347ce9a4562cb86ddecfe926b6c0"
dependencies = [
"arc-swap",
"aws-credential-types",
@@ -1053,9 +1060,9 @@ dependencies = [
[[package]]
name = "aws-sdk-ssooidc"
version = "1.108.0"
version = "1.109.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a59312a04cf19c962cfee32b64ecfee758f8786407ff6da5b30fff46ae96f201"
checksum = "f4075b8a2c8cda4076a3dcc43b9d6dabd93e0c2502abeaaf7e14aaead9bb312b"
dependencies = [
"arc-swap",
"aws-credential-types",
@@ -1079,9 +1086,9 @@ dependencies = [
[[package]]
name = "aws-sdk-sts"
version = "1.111.0"
version = "1.112.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "120e7eb63457a9e547f9986fe3b273f77c43679da4d04f46359fa881c5e19b6e"
checksum = "3f582002918346a3e685be1b391c7bea155073088cea6bd4e4b7663df9e43b6c"
dependencies = [
"arc-swap",
"aws-credential-types",
@@ -1544,6 +1551,30 @@ dependencies = [
"serde",
]
[[package]]
name = "binrw"
version = "0.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ad120d555272286c1017d25165ab8bd74806f13fc85b258484ec7e4ce75458f"
dependencies = [
"array-init",
"binrw_derive",
"bytemuck",
]
[[package]]
name = "binrw_derive"
version = "0.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6df92e0e9baae4dc82c7bad7715ca40c0a5c71539057bf2ea04a5c29c980410b"
dependencies = [
"either",
"owo-colors",
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]]
name = "bit-set"
version = "0.8.0"
@@ -1631,9 +1662,9 @@ dependencies = [
[[package]]
name = "blocking"
version = "1.6.2"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e83f8d02be6967315521be875afa792a316e28d57b5a2d401897e2a7921b7f21"
checksum = "a70e4329df6cb94385eed412ec92375c3cdd8a6e502493d1229b6414e4036dfa"
dependencies = [
"async-channel",
"async-task",
@@ -1722,6 +1753,12 @@ version = "3.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
[[package]]
name = "bytemuck"
version = "1.25.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797"
[[package]]
name = "byteorder"
version = "1.5.0"
@@ -2325,9 +2362,9 @@ dependencies = [
[[package]]
name = "crc32fast"
version = "1.5.0"
version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550"
dependencies = [
"cfg-if",
]
@@ -3934,9 +3971,9 @@ dependencies = [
[[package]]
name = "either"
version = "1.17.0"
version = "1.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d"
checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34"
[[package]]
name = "elliptic-curve"
@@ -4470,6 +4507,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5"
dependencies = [
"polyval",
"zeroize",
]
[[package]]
@@ -5052,9 +5090,9 @@ dependencies = [
[[package]]
name = "hotpath"
version = "0.23.3"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dce755d457a63bdd0c95e4c91511daad1b58b33209543b7f38027b676f387e5e"
checksum = "e2645642a23d4061ec15a4a6e74f851a3145c3125356846cfc7772ff9c6f2737"
dependencies = [
"arc-swap",
"async-channel",
@@ -5086,9 +5124,9 @@ dependencies = [
[[package]]
name = "hotpath-macros"
version = "0.23.3"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a903af89a8429cb07790c3818bc15270b394f80af1bc254e5ccf9c7de2961770"
checksum = "89a3d3cdf9b0d4d3d4f6d4a29798f3b9170401ba500eaa58dd8f890f926af0f1"
dependencies = [
"proc-macro2",
"quote",
@@ -5097,15 +5135,15 @@ dependencies = [
[[package]]
name = "hotpath-macros-meta"
version = "0.23.3"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bcc0ab94ffbb2ee77f4a897df02b5a137a10cf24d69bda936e59aff4dd456e61"
checksum = "e84cd2417fa60938241cf1cd6c03e09953f5c821122dc5da9b8f27975d136c5b"
[[package]]
name = "hotpath-meta"
version = "0.23.3"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "053481f6cec8f775a3276c7f6e2f21123111d28261e4edc15ea7421c445964bb"
checksum = "4d2c145b67b1a4e7bcefa918995e212c30a49a85f05cc5962fe1f717878d560b"
dependencies = [
"hotpath-macros-meta",
]
@@ -5372,9 +5410,9 @@ checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa"
[[package]]
name = "icu_provider"
version = "2.3.0"
version = "2.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92a7ed671a6aad807a8651a2e1782a6598fda9ce5185dd8158549e95a91c6428"
checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73"
dependencies = [
"displaydoc",
"icu_locale_core",
@@ -5761,9 +5799,9 @@ dependencies = [
[[package]]
name = "keccak"
version = "0.2.1"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ffd9697dc4a9a62e2da93389f34400b77a28f0287711263cabb203b3ccb9c0e4"
checksum = "d8f198d1db720e4940b5a493201d199d9f24f568f8f746bd13706243a2f71598"
dependencies = [
"cfg-if",
"cpufeatures 0.3.0",
@@ -6075,9 +6113,9 @@ dependencies = [
[[package]]
name = "log"
version = "0.4.33"
version = "0.4.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
[[package]]
name = "lru"
@@ -6267,7 +6305,7 @@ dependencies = [
"hashbrown 0.16.1",
"indexmap 2.14.0",
"metrics",
"ordered-float 5.3.0",
"ordered-float 5.5.0",
"quanta",
"radix_trie",
"rand 0.9.5",
@@ -7215,9 +7253,9 @@ dependencies = [
[[package]]
name = "ordered-float"
version = "5.3.0"
version = "5.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e"
checksum = "8c7c9e0d9b23589f26070720bac724174bfec1083e82f7854cdd0267518343c0"
dependencies = [
"num-traits",
]
@@ -7252,6 +7290,12 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e"
[[package]]
name = "owo-colors"
version = "4.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d"
[[package]]
name = "p12-keystore"
version = "0.2.1"
@@ -7794,6 +7838,7 @@ dependencies = [
"cpubits",
"cpufeatures 0.3.0",
"universal-hash",
"zeroize",
]
[[package]]
@@ -8264,6 +8309,16 @@ name = "quick-xml"
version = "0.41.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1"
dependencies = [
"memchr",
"serde",
]
[[package]]
name = "quick-xml"
version = "0.42.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b"
dependencies = [
"encoding_rs",
"memchr",
@@ -8928,9 +8983,9 @@ dependencies = [
[[package]]
name = "russh"
version = "0.62.7"
version = "0.63.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9decb68e4e44e1079700e54f17c8f23806ec53d7e0db73ab1c71d9dabc666812"
checksum = "00cf00190c315093734a8d405225bd8773a219bc86538a9b73bfc51145b33995"
dependencies = [
"aes 0.9.2",
"aws-lc-rs",
@@ -9152,7 +9207,7 @@ dependencies = [
"percent-encoding",
"pin-project-lite",
"proptest",
"quick-xml",
"quick-xml 0.42.0",
"rand 0.10.2",
"rcgen",
"regex",
@@ -9418,7 +9473,7 @@ dependencies = [
"path-absolutize",
"pin-project-lite",
"proptest",
"quick-xml",
"quick-xml 0.42.0",
"rand 0.10.2",
"ratelimit",
"rcgen",
@@ -9564,6 +9619,7 @@ dependencies = [
"serde",
"serde_json",
"serial_test",
"sha2 0.11.0",
"temp-env",
"tempfile",
"thiserror 2.0.20",
@@ -9884,7 +9940,7 @@ dependencies = [
"jiff",
"metrics",
"percent-encoding",
"quick-xml",
"quick-xml 0.42.0",
"rayon",
"rustc-hash",
"rustfs-config",
@@ -10657,9 +10713,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"
[[package]]
name = "rustls-webpki"
version = "0.103.14"
version = "0.103.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0527518605e68109d875e248ea259b6758801cf165e4b2c2733ae3b51f12535a"
checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2"
dependencies = [
"aws-lc-rs",
"ring",
@@ -10703,7 +10759,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
[[package]]
name = "s3s"
version = "0.15.0-alpha.1"
source = "git+https://github.com/rustfs/s3s.git?rev=ed70cb048cc4be168419d461cb9ac3c2c7fa6d5a#ed70cb048cc4be168419d461cb9ac3c2c7fa6d5a"
source = "git+https://github.com/rustfs/s3s.git?rev=e080e38c56a3b43acbacce55710d765a5ce9003d#e080e38c56a3b43acbacce55710d765a5ce9003d"
dependencies = [
"arc-swap",
"arrayvec",
@@ -10730,7 +10786,7 @@ dependencies = [
"nom 8.0.0",
"numeric_cast",
"pin-project-lite",
"quick-xml",
"quick-xml 0.41.0",
"regex",
"serde",
"serde_json",
@@ -12675,9 +12731,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]]
name = "uuid"
version = "1.24.1"
version = "1.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2cefc03fd367c0c6d4305de1b312cf00248c4114f4a0418ce6a6af769e3b0bd9"
checksum = "f053576934f05a761a402421fbbe3d425d9366f75f978806a037b3ca481abecc"
dependencies = [
"getrandom 0.4.3",
"js-sys",
@@ -13397,9 +13453,9 @@ dependencies = [
[[package]]
name = "zerovec-derive"
version = "0.11.5"
version = "0.11.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f212a141d820099d57ffafb9569be9617a6f27d3dc881fbee8fb56642f917a9"
checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da"
dependencies = [
"proc-macro2",
"quote",
Cargo.toml
+11 -11
View File
@@ -135,7 +135,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.3" }
# Async Runtime and Networking
async-channel = "2.5.0"
async_zip = { default-features = false, version = "0.0.18" }
async_zip = { default-features = false, version = "0.0.19" }
mysql_async = { default-features = false, version = "0.37" }
async-compression = { version = "0.4.43" }
async-recursion = "1.1.1"
@@ -178,7 +178,7 @@ byteorder = "1.5.0"
flatbuffers = "25.12.19"
form_urlencoded = "1.2.2"
prost = "0.14.4"
quick-xml = "0.41.0"
quick-xml = "0.42.0"
rmp = { version = "0.8.15" }
rmp-serde = { version = "1.3.1" }
serde = { version = "1.0.229" }
@@ -191,7 +191,7 @@ serde_urlencoded = "0.7.1"
# matching stable releases are not available yet, while previous stable lines
# have incompatible APIs. Keep them exact-pinned and monitor upstream for stable
# releases.
aes-gcm = { version = "=0.11.0" }
aes-gcm = { version = "=0.11.1" }
argon2 = { version = "=0.6.0-rc.8" }
blake2 = "=0.11.0-rc.6"
chacha20poly1305 = { version = "=0.11.0" }
@@ -227,11 +227,11 @@ arc-swap = "1.9.2"
astral-tokio-tar = "0.6.4"
atoi = "3.1.0"
atomic_enum = "0.3.0"
aws-config = { version = "1.10.1" }
aws-config = { version = "1.11.0" }
aws-credential-types = { version = "1.3.0" }
aws-sdk-kms = { default-features = false, version = "1.115.0" }
aws-sdk-s3 = { default-features = false, version = "1.142.0" }
aws-sdk-sts = { default-features = false, version = "1.111.0" }
aws-sdk-kms = { default-features = false, version = "1.116.0" }
aws-sdk-s3 = { default-features = false, version = "1.143.0" }
aws-sdk-sts = { default-features = false, version = "1.112.0" }
aws-smithy-http-client = { default-features = false, version = "1.4.0" }
aws-smithy-runtime-api = { version = "1.15.0" }
aws-smithy-types = { version = "1.6.2" }
@@ -291,7 +291,7 @@ rustify = { version = "0.7", default-features = false }
rustix = { version = "1.1.4" }
rust-embed = { version = "8.12.0" }
rustc-hash = { version = "2.1.3" }
s3s = { git = "https://github.com/rustfs/s3s.git", rev = "ed70cb048cc4be168419d461cb9ac3c2c7fa6d5a" }
s3s = { git = "https://github.com/rustfs/s3s.git", rev = "e080e38c56a3b43acbacce55710d765a5ce9003d" }
serial_test = "4.0.1"
shadow-rs = { default-features = false, version = "2.0.0" }
siphasher = "1.0.3"
@@ -314,7 +314,7 @@ tracing-subscriber = { version = "0.3.23" }
transform-stream = "0.3.1"
url = "2.5.8"
urlencoding = "2.1.3"
uuid = { version = "1.24.1" }
uuid = { version = "1.25.0" }
vaultrs = { version = "0.8.0" }
tar = "0.4.46"
walkdir = "2.5.0"
@@ -343,7 +343,7 @@ libunftp = { version = "0.23.0" }
unftp-core = "0.1.0"
suppaftp = { version = "10.0.2" }
rcgen = { version = "0.14.9", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
russh = { version = "0.62.7" }
russh = { version = "0.63.0" }
russh-sftp = "2.4.0"
# WebDAV
@@ -352,7 +352,7 @@ dav-server = "0.11.0"
# Performance Analysis and Memory Profiling
rustfs-mimalloc = { version = "0.5.0" }
rustfs-mimalloc-sys = { version = "0.5.0" }
hotpath = { version = "0.23.3", default-features = false }
hotpath = { version = "0.24.0", default-features = false }
# Snapshot testing for output format regression detection
insta = { version = "1.48" }
crates/common/src/metrics.rs
+104
View File
@@ -918,7 +918,15 @@ pub struct Metrics {
scanner_dirty_usage_last_cycle_dirty_buckets: AtomicU64,
scanner_dirty_usage_last_cycle_cleared_buckets: AtomicU64,
scanner_usage_last_save_unix_secs: AtomicU64,
scanner_usage_last_durable_success_unix_secs: AtomicU64,
scanner_usage_last_publication_unix_secs: AtomicU64,
scanner_usage_last_publication_state: Mutex<String>,
scanner_usage_last_publication_reason: Mutex<String>,
scanner_usage_last_save_result: AtomicU8,
scanner_usage_deferred_pending: AtomicBool,
scanner_usage_deferred_total: AtomicU64,
scanner_usage_last_deferred_unix_secs: AtomicU64,
scanner_usage_last_deferred_reason: Mutex<String>,
scanner_source_work: Vec<ScannerSourceWorkCounters>,
current_scan_cycle_source_work_start: Vec<ScannerSourceWorkCounters>,
last_scan_cycle_source_work: Vec<ScannerSourceWorkCounters>,
@@ -1216,6 +1224,22 @@ pub struct ScannerUsageFreshnessSnapshot {
pub last_usage_save_unix_secs: u64,
pub last_usage_save_result: String,
pub last_usage_save_result_code: u64,
#[serde(default)]
pub last_durable_success_unix_secs: u64,
#[serde(default)]
pub last_publication_unix_secs: u64,
#[serde(default)]
pub last_publication_state: String,
#[serde(default)]
pub last_publication_reason: String,
#[serde(default)]
pub deferred_pending: bool,
#[serde(default)]
pub deferred_total: u64,
#[serde(default)]
pub last_deferred_unix_secs: u64,
#[serde(default)]
pub last_deferred_reason: String,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
@@ -1945,7 +1969,15 @@ impl Metrics {
scanner_dirty_usage_last_cycle_dirty_buckets: AtomicU64::new(0),
scanner_dirty_usage_last_cycle_cleared_buckets: AtomicU64::new(0),
scanner_usage_last_save_unix_secs: AtomicU64::new(0),
scanner_usage_last_durable_success_unix_secs: AtomicU64::new(0),
scanner_usage_last_publication_unix_secs: AtomicU64::new(0),
scanner_usage_last_publication_state: Mutex::new(String::new()),
scanner_usage_last_publication_reason: Mutex::new(String::new()),
scanner_usage_last_save_result: AtomicU8::new(ScannerUsageSaveResult::Unknown as u8),
scanner_usage_deferred_pending: AtomicBool::new(false),
scanner_usage_deferred_total: AtomicU64::new(0),
scanner_usage_last_deferred_unix_secs: AtomicU64::new(0),
scanner_usage_last_deferred_reason: Mutex::new(String::new()),
scanner_source_work: ScannerWorkSource::all()
.iter()
.map(|_| ScannerSourceWorkCounters::default())
@@ -2270,6 +2302,44 @@ impl Metrics {
.store(unix_now_secs(), Ordering::Relaxed);
}
/// Record an intentional retryable usage publication deferral separately
/// from the last durable save result.
pub fn record_scanner_usage_deferred(&self, reason: impl Into<String>) {
let reason = reason.into();
self.record_scanner_usage_publication("deferred", reason.clone());
self.scanner_usage_deferred_pending.store(true, Ordering::Release);
self.scanner_usage_deferred_total.fetch_add(1, Ordering::Relaxed);
self.scanner_usage_last_deferred_unix_secs
.store(unix_now_secs(), Ordering::Relaxed);
let mut last_reason = match self.scanner_usage_last_deferred_reason.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
*last_reason = reason;
}
pub fn record_scanner_usage_durable_success(&self) {
self.record_scanner_usage_publication("success", "");
self.scanner_usage_last_durable_success_unix_secs
.store(unix_now_secs(), Ordering::Relaxed);
self.scanner_usage_deferred_pending.store(false, Ordering::Release);
}
pub fn record_scanner_usage_publication(&self, state: &str, reason: impl Into<String>) {
self.scanner_usage_last_publication_unix_secs
.store(unix_now_secs(), Ordering::Relaxed);
let mut publication_state = match self.scanner_usage_last_publication_state.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
*publication_state = state.to_string();
let mut publication_reason = match self.scanner_usage_last_publication_reason.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
*publication_reason = reason.into();
}
pub fn record_scanner_source_work(&self, source: ScannerWorkSource, work: ScannerSourceWorkUpdate) {
if let Some(counters) = self.scanner_source_work.get(source.index()) {
counters.add(work);
@@ -3292,6 +3362,23 @@ impl Metrics {
last_usage_save_unix_secs: self.scanner_usage_last_save_unix_secs.load(Ordering::Relaxed),
last_usage_save_result: usage_save_result.as_str().to_string(),
last_usage_save_result_code: usage_save_result as u8 as u64,
last_durable_success_unix_secs: self.scanner_usage_last_durable_success_unix_secs.load(Ordering::Relaxed),
last_publication_unix_secs: self.scanner_usage_last_publication_unix_secs.load(Ordering::Relaxed),
last_publication_state: match self.scanner_usage_last_publication_state.lock() {
Ok(state) => state.clone(),
Err(poisoned) => poisoned.into_inner().clone(),
},
last_publication_reason: match self.scanner_usage_last_publication_reason.lock() {
Ok(reason) => reason.clone(),
Err(poisoned) => poisoned.into_inner().clone(),
},
deferred_pending: self.scanner_usage_deferred_pending.load(Ordering::Acquire),
deferred_total: self.scanner_usage_deferred_total.load(Ordering::Relaxed),
last_deferred_unix_secs: self.scanner_usage_last_deferred_unix_secs.load(Ordering::Relaxed),
last_deferred_reason: match self.scanner_usage_last_deferred_reason.lock() {
Ok(reason) => reason.clone(),
Err(poisoned) => poisoned.into_inner().clone(),
},
};
m.throttle_idle_mode_enabled = self.scanner_throttle_idle_mode_enabled.load(Ordering::Relaxed);
m.throttle_sleep_factor = self.scanner_throttle_sleep_factor_micros.load(Ordering::Relaxed) as f64 / 1_000_000.0;
@@ -4663,6 +4750,7 @@ mod tests {
metrics.record_scanner_dirty_usage_cycle_snapshot(1);
metrics.record_scanner_dirty_usage_cycle_clear(1, 1);
metrics.record_scanner_usage_save_result(ScannerUsageSaveResult::Success);
metrics.record_scanner_usage_deferred("data_movement");
let report = metrics.report().await;
@@ -4674,6 +4762,22 @@ mod tests {
assert!(report.usage_freshness.last_usage_save_unix_secs > 0);
assert_eq!(report.usage_freshness.last_usage_save_result, "success");
assert_eq!(report.usage_freshness.last_usage_save_result_code, 1);
assert!(report.usage_freshness.deferred_pending);
assert_eq!(report.usage_freshness.deferred_total, 1);
assert!(report.usage_freshness.last_deferred_unix_secs > 0);
assert_eq!(report.usage_freshness.last_deferred_reason, "data_movement");
metrics.record_scanner_usage_durable_success();
let report = metrics.report().await;
assert!(!report.usage_freshness.deferred_pending);
assert_eq!(report.usage_freshness.deferred_total, 1);
assert!(report.usage_freshness.last_durable_success_unix_secs > 0);
assert_eq!(report.usage_freshness.last_publication_state, "success");
metrics.record_scanner_usage_publication("no_update", "no_update");
let report = metrics.report().await;
assert_eq!(report.usage_freshness.last_publication_state, "no_update");
assert_eq!(report.usage_freshness.last_publication_reason, "no_update");
}
#[tokio::test]
crates/common/src/mrf_channel.rs
+333 -10
View File
@@ -23,21 +23,32 @@
//! unconsumed intents is the consumer's job (see `rustfs-heal`
//! `heal::mrf_queue`), mirroring MinIO's `.heal/mrf/list.bin`.
use std::collections::HashMap;
use std::collections::hash_map::RandomState;
use std::hash::{BuildHasher, Hash};
use std::sync::atomic::AtomicU64;
use std::sync::atomic::AtomicUsize;
use std::sync::{
Arc, OnceLock,
Arc, Mutex, OnceLock,
atomic::{AtomicBool, Ordering},
};
use std::time::{Duration, Instant};
use tokio::sync::mpsc;
use uuid::Uuid;
/// Bounded capacity of the global MRF channel. Backpressure is resolved by
/// dropping (and counting) intents, never by blocking the producer.
const MRF_CHANNEL_CAPACITY: usize = 8192;
const MRF_COALESCER_SHARDS: usize = 16;
const MRF_COALESCER_MAX_KEYS: usize = 8192;
const MRF_COALESCER_MAX_BYTES: usize = 16 * 1024 * 1024;
const MRF_COALESCER_TTL: Duration = Duration::from_secs(60);
const MRF_MAX_IDENTITY_COMPONENT: usize = 1024;
/// Why an intent was produced. Drives the heal priority mapping on the
/// consumer side (DecodeFailure -> Urgent, MetadataCorruption -> High,
/// PartialWrite -> Normal).
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum MrfKind {
/// Erasure decode failed while serving a read (read path).
DecodeFailure,
@@ -67,12 +78,52 @@ pub struct MrfIntent {
/// Version the intent targets, as raw UUID bytes.
pub version_id: Option<[u8; 16]>,
pub kind: MrfKind,
/// Stable erasure-set scope when the producer has it. Kept optional so
/// metadata corruption and legacy producers do not invent a scope.
pub scope: Option<MrfScope>,
/// Generation of the node-local ingress lease. It is not persisted in
/// the journal; replayed records acquire a fresh lease when re-enqueued.
pub lease: Option<MrfIngressLease>,
pub enqueued_at_ms: u64,
/// Times this intent has already been offered to the heal manager.
/// Dropped by the consumer once it reaches `MRF_MAX_ATTEMPTS`.
pub attempts: u8,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct MrfScope {
pub pool_index: u32,
pub set_index: u32,
}
/// Opaque generation used to release exactly the admission that created an
/// ingress entry. A generation prevents a late terminal callback from
/// deleting a newer retry for the same identity (ABA).
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct MrfIngressLease(u64);
impl MrfIngressLease {
const fn new(value: u64) -> Self {
Self(value)
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum MrfDropReason {
Disabled,
Uninitialized,
Full,
OversizedIdentity,
CoalescerFull,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum MrfIngressResult {
Enqueued,
Coalesced,
Dropped(MrfDropReason),
}
/// Consumer-side retry ceiling before an intent is given up on.
pub const MRF_MAX_ATTEMPTS: u8 = 3;
@@ -87,6 +138,159 @@ impl MrfIntent {
static GLOBAL_MRF_SENDER: OnceLock<mpsc::Sender<MrfIntent>> = OnceLock::new();
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
struct MrfIdentityKey {
kind: MrfKind,
bucket: Arc<str>,
object: Arc<str>,
version_id: Option<[u8; 16]>,
scope: Option<MrfScope>,
}
#[derive(Debug)]
struct IngressEntry {
lease: MrfIngressLease,
expires_at: Instant,
bytes: usize,
}
type MrfCoalescerShard = Mutex<HashMap<MrfIdentityKey, IngressEntry>>;
type MrfCoalescer = Box<[MrfCoalescerShard]>;
static MRF_COALESCER: OnceLock<MrfCoalescer> = OnceLock::new();
static NEXT_MRF_LEASE: AtomicU64 = AtomicU64::new(1);
static MRF_COALESCER_COUNT: AtomicUsize = AtomicUsize::new(0);
static MRF_COALESCER_BYTES: AtomicUsize = AtomicUsize::new(0);
static MRF_HASH_STATE: OnceLock<RandomState> = OnceLock::new();
fn coalescer() -> &'static [MrfCoalescerShard] {
MRF_COALESCER.get_or_init(|| {
(0..MRF_COALESCER_SHARDS)
.map(|_| Mutex::new(HashMap::new()))
.collect::<Vec<_>>()
.into_boxed_slice()
})
}
fn key_shard(key: &MrfIdentityKey) -> usize {
let hash = MRF_HASH_STATE.get_or_init(RandomState::new).hash_one(key);
usize::try_from(hash).unwrap_or(0) % MRF_COALESCER_SHARDS
}
fn canonical_version(version_id: Option<Uuid>) -> Option<[u8; 16]> {
version_id
.filter(|version| !version.is_nil())
.map(|version| *version.as_bytes())
}
fn canonical_identity(
kind: MrfKind,
version_id: Option<[u8; 16]>,
scope: Option<MrfScope>,
) -> (Option<[u8; 16]>, Option<MrfScope>) {
let version_id = version_id.filter(|bytes| *bytes != [0; 16]);
match kind {
MrfKind::MetadataCorruption => (None, None),
MrfKind::DecodeFailure | MrfKind::PartialWrite => (version_id, scope),
}
}
fn identity_estimated_bytes(key: &MrfIdentityKey) -> usize {
64usize
.saturating_add(key.bucket.len())
.saturating_add(key.object.len())
.saturating_add(key.version_id.map_or(0, |_| 16))
.saturating_add(key.scope.map_or(0, |_| 8))
}
fn reserve(counter: &AtomicUsize, limit: usize, amount: usize) -> bool {
let mut current = counter.load(Ordering::Relaxed);
loop {
let Some(next) = current.checked_add(amount) else {
return false;
};
if next > limit {
return false;
}
match counter.compare_exchange_weak(current, next, Ordering::Relaxed, Ordering::Relaxed) {
Ok(_) => return true,
Err(observed) => current = observed,
}
}
}
fn coalescer_admit(key: MrfIdentityKey) -> Result<MrfIngressLease, MrfIngressResult> {
let shard = key_shard(&key);
let mut entries = coalescer()[shard]
.lock()
.map_err(|_| MrfIngressResult::Dropped(MrfDropReason::CoalescerFull))?;
let now = Instant::now();
let before = entries.len();
let mut expired_bytes = 0usize;
entries.retain(|_, entry| {
if entry.expires_at > now {
true
} else {
expired_bytes = expired_bytes.saturating_add(entry.bytes);
false
}
});
let evicted = before.saturating_sub(entries.len());
if evicted > 0 {
MRF_COALESCER_COUNT.fetch_sub(evicted, Ordering::Relaxed);
MRF_COALESCER_BYTES.fetch_sub(expired_bytes, Ordering::Relaxed);
let evicted = u64::try_from(evicted).unwrap_or(u64::MAX);
metrics::counter!("rustfs_heal_mrf_coalescer_expired_total").increment(evicted);
metrics::counter!("rustfs_heal_mrf_coalescer_evictions_total").increment(evicted);
}
if entries.contains_key(&key) {
metrics::counter!("rustfs_heal_mrf_coalesced_total").increment(1);
return Err(MrfIngressResult::Coalesced);
}
let bytes = identity_estimated_bytes(&key);
let count_reserved = reserve(&MRF_COALESCER_COUNT, MRF_COALESCER_MAX_KEYS, 1);
let bytes_reserved = count_reserved && reserve(&MRF_COALESCER_BYTES, MRF_COALESCER_MAX_BYTES, bytes);
if !count_reserved || !bytes_reserved {
if count_reserved {
MRF_COALESCER_COUNT.fetch_sub(1, Ordering::Relaxed);
}
metrics::counter!("rustfs_heal_mrf_dropped_total", "reason" => "coalescer_full").increment(1);
return Err(MrfIngressResult::Dropped(MrfDropReason::CoalescerFull));
}
let lease = MrfIngressLease::new(NEXT_MRF_LEASE.fetch_add(1, Ordering::Relaxed));
if entries
.insert(
key,
IngressEntry {
lease,
expires_at: now + MRF_COALESCER_TTL,
bytes,
},
)
.is_some()
{
MRF_COALESCER_COUNT.fetch_sub(1, Ordering::Relaxed);
MRF_COALESCER_BYTES.fetch_sub(bytes, Ordering::Relaxed);
metrics::counter!("rustfs_heal_mrf_coalesced_total").increment(1);
return Err(MrfIngressResult::Coalesced);
}
Ok(lease)
}
fn coalescer_release(key: &MrfIdentityKey, lease: Option<MrfIngressLease>) {
let Some(lease) = lease else {
return;
};
if let Ok(mut entries) = coalescer()[key_shard(key)].lock() {
let should_remove = entries.get(key).is_some_and(|entry| entry.lease == lease);
if should_remove {
let bytes = entries.remove(key).map(|entry| entry.bytes).unwrap_or(0);
MRF_COALESCER_COUNT.fetch_sub(1, Ordering::Relaxed);
MRF_COALESCER_BYTES.fetch_sub(bytes, Ordering::Relaxed);
}
}
}
/// Delivery kill-switch, set from `RUSTFS_HEAL_MRF_ENABLE`. Producers check
/// this before touching the channel so the disabled path stays allocation- and
/// sync-free.
@@ -122,21 +326,90 @@ pub fn init_mrf_channel() -> Result<mpsc::Receiver<MrfIntent>, &'static str> {
/// This runs on IO error paths, so it stays synchronous and cheap: one
/// bounded allocation for the two `Arc<str>` handles plus the channel slot.
pub fn try_send_mrf_intent(kind: MrfKind, bucket: &str, object: &str, version_id: Option<Uuid>) -> bool {
matches!(
try_send_mrf_intent_typed(kind, bucket, object, version_id, None),
MrfIngressResult::Enqueued
)
}
/// Typed ingress result. `Coalesced` means an equivalent in-flight channel
/// intent already exists; it is not a second executable or durable admission.
pub fn try_send_mrf_intent_typed(
kind: MrfKind,
bucket: &str,
object: &str,
version_id: Option<Uuid>,
scope: Option<MrfScope>,
) -> MrfIngressResult {
if !mrf_delivery_enabled() {
return false;
return MrfIngressResult::Dropped(MrfDropReason::Disabled);
}
let Some(sender) = GLOBAL_MRF_SENDER.get() else {
return false;
return MrfIngressResult::Dropped(MrfDropReason::Uninitialized);
};
let intent = MrfIntent {
if bucket.len() > MRF_MAX_IDENTITY_COMPONENT || object.len() > MRF_MAX_IDENTITY_COMPONENT {
return MrfIngressResult::Dropped(MrfDropReason::OversizedIdentity);
}
let (version_id, scope) = canonical_identity(kind, canonical_version(version_id), scope);
let key = MrfIdentityKey {
kind,
bucket: Arc::from(bucket),
object: Arc::from(object),
version_id: version_id.map(|vid| *vid.as_bytes()),
version_id,
scope,
};
let lease = match coalescer_admit(key.clone()) {
Ok(lease) => lease,
Err(result) => return result,
};
let intent = MrfIntent {
bucket: key.bucket.clone(),
object: key.object.clone(),
version_id: key.version_id,
kind,
scope,
lease: Some(lease),
enqueued_at_ms: unix_now_ms(),
attempts: 0,
};
sender.try_send(intent).is_ok()
match sender.try_send(intent) {
Ok(()) => MrfIngressResult::Enqueued,
Err(mpsc::error::TrySendError::Full(_)) => {
coalescer_release(&key, Some(lease));
metrics::counter!("rustfs_heal_mrf_dropped_total", "reason" => "channel_full").increment(1);
MrfIngressResult::Dropped(MrfDropReason::Full)
}
Err(mpsc::error::TrySendError::Closed(_)) => {
coalescer_release(&key, Some(lease));
MrfIngressResult::Dropped(MrfDropReason::Uninitialized)
}
}
}
/// Release the ingress key once the consumer owns the intent.
pub fn release_mrf_intent(intent: &MrfIntent) {
release_mrf_identity(intent.kind, &intent.bucket, &intent.object, intent.version_id, intent.scope, intent.lease);
}
pub fn release_mrf_identity(
kind: MrfKind,
bucket: &str,
object: &str,
version_id: Option<[u8; 16]>,
scope: Option<MrfScope>,
lease: Option<MrfIngressLease>,
) {
let (version_id, scope) = canonical_identity(kind, version_id, scope);
coalescer_release(
&MrfIdentityKey {
kind,
bucket: Arc::from(bucket),
object: Arc::from(object),
version_id,
scope,
},
lease,
);
}
fn unix_now_ms() -> u64 {
@@ -144,7 +417,8 @@ fn unix_now_ms() -> u64 {
// failure would be a bug rather than something to handle here.
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_millis() as u64)
.ok()
.and_then(|d| u64::try_from(d.as_millis()).ok())
.unwrap_or(0)
}
@@ -215,12 +489,60 @@ mod tests {
object: Arc::from("object"),
version_id: Some([0u8; 16]),
kind: MrfKind::DecodeFailure,
scope: None,
lease: None,
enqueued_at_ms: 0,
attempts: 0,
};
assert!(intent.estimated_bytes() >= intent.bucket.len() + intent.object.len());
}
#[test]
fn ingress_duplicate_identity_coalesces_and_releases_for_retry() {
let key = MrfIdentityKey {
kind: MrfKind::DecodeFailure,
bucket: Arc::from("ingress-test-bucket"),
object: Arc::from("ingress-test-object"),
version_id: Some([9; 16]),
scope: Some(MrfScope {
pool_index: 3,
set_index: 4,
}),
};
let lease = coalescer_admit(key.clone()).expect("first identity should be admitted");
for _ in 0..999 {
assert_eq!(coalescer_admit(key.clone()), Err(MrfIngressResult::Coalesced));
}
coalescer_release(&key, Some(lease));
let retry_lease = coalescer_admit(key.clone()).expect("released identity must admit a retry");
coalescer_release(&key, Some(retry_lease));
}
#[test]
fn ingress_identity_preserves_kind_scope_and_version_boundaries() {
let (nil_version, nil_scope) = canonical_identity(
MrfKind::DecodeFailure,
Some([0; 16]),
Some(MrfScope {
pool_index: 1,
set_index: 2,
}),
);
assert_eq!(nil_version, None, "nil UUID is the unversioned identity");
assert!(nil_scope.is_some());
let (metadata_version, metadata_scope) = canonical_identity(
MrfKind::MetadataCorruption,
Some([7; 16]),
Some(MrfScope {
pool_index: 1,
set_index: 2,
}),
);
assert_eq!(metadata_version, None);
assert_eq!(metadata_scope, None);
}
#[tokio::test]
async fn try_send_delivers_and_respects_capacity() {
let mut receiver = init_mrf_channel().expect("first initialization should succeed");
@@ -230,6 +552,7 @@ mod tests {
let intent = receiver.recv().await.expect("intent should arrive");
assert_eq!(intent.kind, MrfKind::DecodeFailure);
assert_eq!(intent.bucket.as_ref(), "b");
release_mrf_intent(&intent);
// Disable delivery: producers become no-ops.
set_mrf_delivery_enabled(false);
@@ -239,8 +562,8 @@ mod tests {
// Fill the bounded channel past capacity: excess intents are dropped,
// never blocking.
let mut accepted = 0;
for _ in 0..(MRF_CHANNEL_CAPACITY + 64) {
if try_send_mrf_intent(MrfKind::PartialWrite, "b", "o", None) {
for index in 0..(MRF_CHANNEL_CAPACITY + 64) {
if try_send_mrf_intent(MrfKind::PartialWrite, "b", &format!("o-{index}"), None) {
accepted += 1;
}
}
crates/config/src/constants/runtime.rs
+2 -2
View File
@@ -60,9 +60,9 @@ pub const DEFAULT_RNG_SEED: Option<u64> = None; // None means random
/// Dedicated blocking thread pool for fsync/fdatasync operations.
/// When > 1, fsync operations are isolated from the main blocking pool to
/// prevent device-bound fsync from starving read operations (pread/stat/open).
/// Default 0 means auto (no isolation, use main runtime).
/// Default 64 isolates fsync from the main blocking pool to prevent device-bound fsync from starving read I/O.
pub const ENV_FSYNC_BLOCKING_THREADS: &str = "RUSTFS_RUNTIME_FSYNC_BLOCKING_THREADS";
pub const DEFAULT_FSYNC_BLOCKING_THREADS: usize = 0;
pub const DEFAULT_FSYNC_BLOCKING_THREADS: usize = 64;
// Dial9 Tokio Telemetry Default values
pub const DEFAULT_RUNTIME_DIAL9_ENABLED: bool = false; // Disabled by default
crates/data-usage/src/data_usage.rs
+873 -5
View File
File diff suppressed because it is too large Load Diff
crates/e2e_test/README.md
+13 -10
View File
@@ -123,7 +123,7 @@ via `create_s3_client(idx)` / `create_all_clients()`. See
| `find_available_port` | Random free port (isolation primitive) |
| `rustfs_binary_path` / `_with_features` | Locate/build the binary; honors `RUSTFS_BUILD_FEATURES` |
| `requested_rustfs_build_features` / `rustfs_build_feature_enabled` | Feature-gate a test to what the binary was built with |
| `awscurl_available` + `execute_awscurl` / `awscurl_post` / `_get` / `_put` / `_delete` / `awscurl_post_sts_form_urlencoded` | Admin/STS API calls via `awscurl` (skip gracefully when absent) |
| `execute_awscurl` / `awscurl_post` / `_get` / `_put` / `_delete` / `awscurl_post_sts_form_urlencoded` | Admin/STS API calls via `awscurl`; missing binaries are test failures |
| `replication_fast_env` | Env vars that shrink replication timers (from repl-4); pass to `start_rustfs_server_with_env` |
| `local_http_client` / `init_logging` | Loopback HTTP client; idempotent tracing init |
| `RustFSTestClusterEnvironment` (`new`/`start`/`start_node`/`stop_node`/`create_all_clients`) | Multi-node harness |
@@ -189,7 +189,7 @@ cargo nextest run --profile e2e-smoke -p e2e_test
cargo nextest run --profile e2e-full -p e2e_test
# Cluster fault nightly lane
cargo nextest run --profile e2e-nightly -p e2e_test
# Replication nightly lane; install awscurl so STS paths do not skip
# Replication nightly lane; awscurl is required for STS paths
cargo nextest run --profile e2e-repl-nightly -p e2e_test
# Fixed-port protocol nightly lane
RUSTFS_BUILD_FEATURES=ftps,webdav,sftp \
@@ -221,9 +221,8 @@ The `s3s-e2e` CI job selects a random `RUSTFS_TEST_PORT` (see the `e2e-tests`
job) to dodge this; local single-node tests already use random ports, so a
lingering orphan is usually the cause of a spurious bind failure.
**`awscurl` not found.** `awscurl`-dependent tests skip gracefully with a
visible log line (`awscurl_available()`); install `awscurl` to actually run
them.
**`awscurl` not found.** `awscurl`-dependent tests fail closed with a process
spawn error. Install the pinned CI version before running their profiles.
## Related
@@ -258,10 +257,9 @@ A test module may join the smoke filter only if every test in it is:
2. **Single-node** — spawns its own server via
`RustFSTestEnvironment`/`start_rustfs_server` on a random port with an
isolated temp dir. No `RustFSTestClusterEnvironment`, no fixed ports.
3. **Dependency-free** — no pre-started server at `localhost:9000`, no Vault,
no fixed protocol ports. Tools that may be absent on the runner (e.g.
`awscurl`) are acceptable only when the test skips gracefully with a
visible log line (see `bucket_policy_check_test.rs`).
3. **Hermetic dependencies** — no pre-started server at `localhost:9000`, no
Vault, and no fixed protocol ports. Any required CLI must be pinned and
installed by the workflow; a missing CLI must fail the test.
4. **Not `#[ignore]`** — ignored tests are activation work (backlog#1149
ci-13 / backlog#1148 ilm-3), not smoke candidates.
@@ -278,4 +276,9 @@ listed by `cargo nextest list -p e2e_test`. Regenerate it when adding or
moving e2e tests so acceptance numbers in the test-strategy issues
(backlog#1147–#1155) stay auditable. When a profile membership change is
intentional, review its JSON listing before updating the matching
`.config/e2e-*-selection.txt` test-ID digest.
`.config/e2e-*-selection.txt` test-ID digest. Update only the platform that
produced the listing:
```bash
python3 scripts/check_test_wiring.py --update-profile e2e-full /path/to/listing.json linux
```
crates/e2e_test/src/bucket_policy_check_test.rs
-4
View File
@@ -52,10 +52,6 @@ fn create_user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key:
#[tokio::test]
async fn test_bucket_policy_authenticated_user() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if !crate::common::awscurl_available() {
info!("Skipping test_bucket_policy_authenticated_user because awscurl is not available");
return Ok(());
}
info!("Starting test_bucket_policy_authenticated_user...");
let mut env = RustFSTestEnvironment::new().await?;
crates/e2e_test/src/cluster_concurrency_test.rs
+77 -16
View File
@@ -15,16 +15,27 @@
use crate::common::RustFSTestClusterEnvironment;
use aws_sdk_s3::Client;
use aws_sdk_s3::error::SdkError;
use aws_sdk_s3::types::{CorsConfiguration, CorsRule};
use bytes::Bytes;
use std::sync::Arc;
use tokio::sync::Barrier;
use tracing::{info, warn};
const BUCKET: &str = "conditional-put-race-bucket";
const BUCKET_METADATA_RELOAD_BUCKET: &str = "bucket-metadata-reload-barrier";
async fn cleanup_object(client: &Client, key: &str) {
if let Err(e) = client.delete_object().bucket(BUCKET).key(key).send().await {
warn!("Failed to delete object '{}' from bucket '{}' during cleanup: {:?}", key, BUCKET, e);
async fn cleanup_object(client: &Client, key: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
client.delete_object().bucket(BUCKET).key(key).send().await?;
Ok(())
}
async fn assert_bucket_cors_missing(client: &Client) {
let result = client.get_bucket_cors().bucket(BUCKET_METADATA_RELOAD_BUCKET).send().await;
match result {
Err(SdkError::ServiceError(error)) => {
assert_eq!(error.err().meta().code(), Some("NoSuchCORSConfiguration"));
}
result => panic!("expected the peer to report a missing CORS configuration: {result:?}"),
}
}
@@ -71,14 +82,13 @@ async fn run_race_iteration(
test_key: &str,
iteration: usize,
) -> Result<usize, Box<dyn std::error::Error + Send + Sync>> {
cleanup_object(&clients[0], test_key).await;
cleanup_object(&clients[0], test_key).await?;
tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
let head_result = clients[0].head_object().bucket(BUCKET).key(test_key).send().await;
if head_result.is_ok() {
warn!("Warning: Object still exists after cleanup, skipping iteration {}", iteration);
return Ok(0);
match clients[0].head_object().bucket(BUCKET).key(test_key).send().await {
Ok(_) => return Err(format!("object still exists after cleanup in iteration {iteration}").into()),
Err(error) if error.as_service_error().is_some_and(|error| error.is_not_found()) => {}
Err(error) => return Err(format!("failed to verify cleanup in iteration {iteration}: {error:?}").into()),
}
info!("\n=== Iteration {} ===", iteration);
@@ -120,14 +130,16 @@ async fn run_race_iteration(
info!("Result: {} out of {} succeeded", success_count, clients.len());
if had_error {
return Err("one or more conditional PUTs failed unexpectedly".into());
}
if success_count > 1 {
info!(">>> RACE CONDITION DETECTED!");
} else if success_count == 1 {
info!(">>> Correct behavior: exactly 1 writer succeeded.");
} else if had_error {
return Err("all conditional PUTs failed (e.g. cluster/bucket not ready)".into());
} else {
info!(">>> Unexpected: no writers succeeded.");
return Err("no conditional PUT succeeded".into());
}
Ok(success_count)
@@ -167,7 +179,7 @@ async fn test_conditional_put_race_cluster() -> Result<(), Box<dyn std::error::E
}
}
cleanup_object(&clients[0], &test_key).await;
cleanup_object(&clients[0], &test_key).await?;
tokio::time::sleep(tokio::time::Duration::from_millis(50)).await;
}
@@ -177,7 +189,7 @@ async fn test_conditional_put_race_cluster() -> Result<(), Box<dyn std::error::E
info!("Total iterations: {}", iterations);
info!("Correct (1 winner): {}", correct_count);
info!("Race conditions: {}", races_detected);
info!("Errors (skipped): {}", error_count);
info!("Failed iterations: {}", error_count);
assert_eq!(races_detected, 0, "Race conditions detected: {}/{}", races_detected, iterations);
assert_eq!(
@@ -185,6 +197,10 @@ async fn test_conditional_put_race_cluster() -> Result<(), Box<dyn std::error::E
"{} iteration(s) failed due to errors (e.g. cluster not ready)",
error_count
);
assert_eq!(
correct_count, iterations,
"only {correct_count}/{iterations} iterations observed exactly one winner"
);
Ok(())
}
@@ -201,7 +217,7 @@ async fn test_conditional_put_basic_cluster() -> Result<(), Box<dyn std::error::
let client = cluster.create_s3_client(0)?;
let test_key = "basic-conditional-put";
cleanup_object(&client, test_key).await;
cleanup_object(&client, test_key).await?;
let result = client
.put_object()
@@ -233,6 +249,51 @@ async fn test_conditional_put_basic_cluster() -> Result<(), Box<dyn std::error::
assert_eq!(code, "PreconditionFailed");
}
cleanup_object(&client, test_key).await;
cleanup_object(&client, test_key).await?;
Ok(())
}
#[tokio::test]
async fn test_bucket_cors_write_is_visible_on_peer_before_response() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
crate::common::init_logging();
let mut cluster = RustFSTestClusterEnvironment::new(2).await?;
cluster.start().await?;
cluster.create_test_bucket(BUCKET_METADATA_RELOAD_BUCKET).await?;
let writer = cluster.create_s3_client(0)?;
let reader = cluster.create_s3_client(1)?;
assert_bucket_cors_missing(&reader).await;
let rule = CorsRule::builder()
.allowed_methods("GET")
.allowed_origins("https://example.com")
.build()?;
let configuration = CorsConfiguration::builder().cors_rules(rule).build()?;
writer
.put_bucket_cors()
.bucket(BUCKET_METADATA_RELOAD_BUCKET)
.cors_configuration(configuration)
.send()
.await?;
let response = reader.get_bucket_cors().bucket(BUCKET_METADATA_RELOAD_BUCKET).send().await?;
let rules = response.cors_rules();
assert_eq!(
rules.len(),
1,
"peer should observe the committed CORS rule before the write response returns"
);
assert_eq!(rules[0].allowed_methods(), ["GET"]);
assert_eq!(rules[0].allowed_origins(), ["https://example.com"]);
writer
.delete_bucket_cors()
.bucket(BUCKET_METADATA_RELOAD_BUCKET)
.send()
.await?;
assert_bucket_cors_missing(&reader).await;
writer.delete_bucket().bucket(BUCKET_METADATA_RELOAD_BUCKET).send().await?;
Ok(())
}
crates/e2e_test/src/common.rs
+28 -7
View File
@@ -499,15 +499,20 @@ fn awscurl_binary_path() -> PathBuf {
.unwrap_or_else(|| PathBuf::from("awscurl"))
}
pub fn awscurl_available() -> bool {
let path = awscurl_binary_path();
if path.components().count() > 1 || path.is_absolute() {
return path.is_file();
fn verify_awscurl_path(path: &Path) -> std::io::Result<()> {
let output = Command::new(path).arg("--help").output()?;
if output.status.success() {
return Ok(());
}
std::env::var_os("PATH")
.map(|paths| std::env::split_paths(&paths).any(|dir| dir.join(&path).is_file()))
.unwrap_or(false)
Err(std::io::Error::other(format!(
"awscurl prerequisite check failed: {}",
String::from_utf8_lossy(&output.stderr).trim()
)))
}
pub fn require_awscurl() -> std::io::Result<()> {
verify_awscurl_path(&awscurl_binary_path())
}
// Global initialization
@@ -1752,6 +1757,22 @@ mod tests {
assert_eq!(normalize_rustfs_build_features(" , "), None);
}
#[test]
fn missing_awscurl_is_a_prerequisite_failure() {
let missing = std::env::temp_dir().join(format!("missing-awscurl-{}", Uuid::new_v4()));
let error = verify_awscurl_path(&missing).expect_err("a missing awscurl binary must fail the test prerequisite");
assert_eq!(error.kind(), ErrorKind::NotFound);
}
#[test]
fn available_awscurl_client_passes_prerequisite_check() {
let executable = std::env::current_exe().expect("the test executable should have a path");
verify_awscurl_path(&executable).expect("an available client with a working help command should pass");
}
#[test]
fn capture_log_path_uses_temp_directory_basename() {
assert_eq!(
crates/e2e_test/src/existing_object_tag_policy_test.rs
+1 -23
View File
@@ -16,9 +16,7 @@
//! session policy** (`Policy` parameter) via `awscurl --service sts` with explicit
//! `Content-Type: application/x-www-form-urlencoded` on `POST /`.
use crate::common::{
RustFSTestEnvironment, awscurl_available, awscurl_delete, awscurl_post_sts_form_urlencoded, awscurl_put, init_logging,
};
use crate::common::{RustFSTestEnvironment, awscurl_delete, awscurl_post_sts_form_urlencoded, awscurl_put, init_logging};
use aws_sdk_s3::config::{Credentials, Region};
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{Delete, ObjectIdentifier, Tag, Tagging};
@@ -175,11 +173,6 @@ async fn cleanup_bucket_and_object(admin: &Client, bucket: &str, key: &str) {
#[tokio::test]
async fn test_e2e_iam_policy_existing_object_tag_get_object() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if !awscurl_available() {
info!("Skipping test_e2e_iam_policy_existing_object_tag_get_object: awscurl not available");
return Ok(());
}
let suffix = Uuid::new_v4();
let user = format!("e2eiamtag-{suffix}");
let user_secret = "longSecretKeyForTest123!";
@@ -233,11 +226,6 @@ async fn test_e2e_iam_policy_existing_object_tag_get_object() -> Result<(), Box<
#[tokio::test]
async fn test_e2e_bucket_policy_existing_object_tag_get_object() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if !awscurl_available() {
info!("Skipping test_e2e_bucket_policy_existing_object_tag_get_object: awscurl not available");
return Ok(());
}
let suffix = Uuid::new_v4();
let user = format!("e2ebptag-{suffix}");
let user_secret = "longSecretKeyForTest456!";
@@ -294,11 +282,6 @@ async fn test_e2e_bucket_policy_existing_object_tag_get_object() -> Result<(), B
#[tokio::test]
async fn test_e2e_sts_assume_role_session_policy_existing_object_tag() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if !awscurl_available() {
info!("Skipping test_e2e_sts_assume_role_session_policy_existing_object_tag: awscurl not available");
return Ok(());
}
let suffix = Uuid::new_v4();
let parent = format!("e2e-sts-par-{suffix}");
let parent_secret = "longSecretKeyForParentSts99!";
@@ -370,11 +353,6 @@ async fn test_e2e_sts_assume_role_session_policy_existing_object_tag() -> Result
#[tokio::test]
async fn test_e2e_sts_session_policy_delete_objects_object_prefix_only() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if !awscurl_available() {
info!("Skipping test_e2e_sts_session_policy_delete_objects_object_prefix_only: awscurl not available");
return Ok(());
}
let suffix = Uuid::new_v4();
let parent = format!("e2e-sts-del-par-{suffix}");
let parent_secret = "longSecretKeyForParentDelete99!";
crates/e2e_test/src/group_delete_test.rs
+118 -36
View File
@@ -14,7 +14,7 @@
//! E2E tests for group management (fixes #2028).
use crate::common::{RustFSTestEnvironment, admin_request, awscurl_delete, awscurl_get, awscurl_put, init_logging};
use crate::common::{RustFSTestEnvironment, admin_ok, admin_request, init_logging};
use aws_sdk_s3::config::{Credentials, Region};
use aws_sdk_s3::{Client, Config};
use tracing::info;
@@ -83,7 +83,6 @@ async fn update_group_members_rejects_invalid_new_group_names() -> Result<(), Bo
/// Test that deleting a group with members fails, and deleting an empty group succeeds.
#[tokio::test(flavor = "multi_thread")]
#[ignore = "requires awscurl and spawns a real RustFS server"]
async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
@@ -91,29 +90,58 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std
env.start_rustfs_server(vec![]).await?;
// 1. Create a user
let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey=testuser1", env.url);
let user_body = serde_json::json!({
"secretKey": "testuser1secret",
"status": "enabled"
});
awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
admin_ok(
&env,
http::Method::PUT,
"/rustfs/admin/v3/add-user?accessKey=testuser1",
Some(user_body.to_string()),
)
.await?;
info!("Created testuser1");
// 2. Create a group with testuser1 as a member
let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
let add_member_body = serde_json::json!({
"group": "testgroup",
"members": ["testuser1"],
"isRemove": false,
"groupStatus": "enabled"
});
awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
admin_ok(
&env,
http::Method::PUT,
"/rustfs/admin/v3/update-group-members",
Some(add_member_body.to_string()),
)
.await?;
info!("Added testuser1 to testgroup");
// 3. Attempt to delete the group while it still has members — should fail
let delete_group_url = format!("{}/rustfs/admin/v3/group/testgroup", env.url);
let delete_result = awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await;
assert!(delete_result.is_err(), "deleting a non-empty group should fail");
let (delete_status, delete_body) = admin_request(
&env.url,
http::Method::DELETE,
"/rustfs/admin/v3/group/testgroup",
None,
&env.access_key,
&env.secret_key,
)
.await?;
assert_eq!(
delete_status,
reqwest::StatusCode::BAD_REQUEST,
"deleting a non-empty group must return HTTP 400, body: {delete_body}"
);
assert!(
delete_body.contains("<Code>InvalidRequest</Code>"),
"deleting a non-empty group must return InvalidRequest, body: {delete_body}"
);
assert!(
delete_body.contains("<Message>group is not empty</Message>"),
"deleting a non-empty group returned an unexpected message: {delete_body}"
);
info!("Delete of non-empty group correctly rejected");
// 4. Remove the member from the group
@@ -123,17 +151,42 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std
"isRemove": true,
"groupStatus": "enabled"
});
awscurl_put(&update_members_url, &remove_member_body.to_string(), &env.access_key, &env.secret_key).await?;
admin_ok(
&env,
http::Method::PUT,
"/rustfs/admin/v3/update-group-members",
Some(remove_member_body.to_string()),
)
.await?;
info!("Removed testuser1 from testgroup");
// 5. Delete the now-empty group — should succeed
awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await?;
admin_ok(&env, http::Method::DELETE, "/rustfs/admin/v3/group/testgroup", None).await?;
info!("Deleted empty testgroup successfully");
// 6. Verify the group no longer exists
let get_group_url = format!("{}/rustfs/admin/v3/group?group=testgroup", env.url);
let get_result = awscurl_get(&get_group_url, &env.access_key, &env.secret_key).await;
assert!(get_result.is_err(), "group should no longer exist after deletion");
let (get_status, get_body) = admin_request(
&env.url,
http::Method::GET,
"/rustfs/admin/v3/group?group=testgroup",
None,
&env.access_key,
&env.secret_key,
)
.await?;
assert_eq!(
get_status,
reqwest::StatusCode::NOT_FOUND,
"a deleted group must return HTTP 404, body: {get_body}"
);
assert!(
get_body.contains("<Code>NoSuchResource</Code>"),
"a deleted group must return NoSuchResource, body: {get_body}"
);
assert!(
get_body.contains("<Message>group &apos;testgroup&apos; does not exist</Message>"),
"a deleted group returned an unexpected message: {get_body}"
);
info!("Confirmed testgroup no longer exists");
Ok(())
@@ -142,7 +195,6 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std
/// Test that a user with only group membership (no explicit user policy) gets group policies
/// and can perform actions allowed by the group (regression test for #2028.1).
#[tokio::test(flavor = "multi_thread")]
#[ignore = "requires awscurl and spawns a real RustFS server"]
async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
@@ -160,39 +212,56 @@ async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn s
"Statement": [{
"Effect": "Allow",
"Action": ["s3:ListAllMyBuckets"],
"Resource": ["*"]
"Resource": ["arn:aws:s3:::*"]
}]
});
let add_policy_url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", env.url, policy_name);
awscurl_put(&add_policy_url, &policy_doc.to_string(), &env.access_key, &env.secret_key).await?;
admin_ok(
&env,
http::Method::PUT,
&format!("/rustfs/admin/v3/add-canned-policy?name={policy_name}"),
Some(policy_doc.to_string()),
)
.await?;
info!("Created canned policy {}", policy_name);
// 2. Create user with no explicit policy
let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, user_name);
let user_body = serde_json::json!({
"secretKey": user_secret,
"status": "enabled"
});
awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
admin_ok(
&env,
http::Method::PUT,
&format!("/rustfs/admin/v3/add-user?accessKey={user_name}"),
Some(user_body.to_string()),
)
.await?;
info!("Created user {} with no explicit policy", user_name);
// 3. Add user to group (creates group with this member; user_group_memberships must be updated)
let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
let add_member_body = serde_json::json!({
"group": group_name,
"members": [user_name],
"isRemove": false,
"groupStatus": "enabled"
});
awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
admin_ok(
&env,
http::Method::PUT,
"/rustfs/admin/v3/update-group-members",
Some(add_member_body.to_string()),
)
.await?;
info!("Added {} to group {}", user_name, group_name);
// 4. Attach policy to group
let set_policy_url = format!(
"{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=true",
env.url, policy_name, group_name
);
awscurl_put(&set_policy_url, "", &env.access_key, &env.secret_key).await?;
admin_ok(
&env,
http::Method::PUT,
&format!("/rustfs/admin/v3/set-user-or-group-policy?policyName={policy_name}&userOrGroup={group_name}&isGroup=true"),
Some(String::new()),
)
.await?;
info!("Attached policy {} to group {}", policy_name, group_name);
// 5. User with only group (no user policy) should be able to list buckets
@@ -209,7 +278,6 @@ async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn s
/// Test that after deleting a user who was the only member of a group, the group can be deleted
/// (regression test for #2028.2: delete group uses backend membership, not stale cache).
#[tokio::test(flavor = "multi_thread")]
#[ignore = "requires awscurl and spawns a real RustFS server"]
async fn test_delete_group_after_deleting_user() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
@@ -221,33 +289,47 @@ async fn test_delete_group_after_deleting_user() -> Result<(), Box<dyn std::erro
let group_name = "soledeletegroup";
// 1. Create user
let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, user_name);
let user_body = serde_json::json!({
"secretKey": user_secret,
"status": "enabled"
});
awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
admin_ok(
&env,
http::Method::PUT,
&format!("/rustfs/admin/v3/add-user?accessKey={user_name}"),
Some(user_body.to_string()),
)
.await?;
info!("Created user {}", user_name);
// 2. Add user to group
let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
let add_member_body = serde_json::json!({
"group": group_name,
"members": [user_name],
"isRemove": false,
"groupStatus": "enabled"
});
awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
admin_ok(
&env,
http::Method::PUT,
"/rustfs/admin/v3/update-group-members",
Some(add_member_body.to_string()),
)
.await?;
info!("Added {} to group {}", user_name, group_name);
// 3. Delete the user (backend and cache update so group membership becomes empty)
let remove_user_url = format!("{}/rustfs/admin/v3/remove-user?accessKey={}", env.url, user_name);
awscurl_delete(&remove_user_url, &env.access_key, &env.secret_key).await?;
admin_ok(
&env,
http::Method::DELETE,
&format!("/rustfs/admin/v3/remove-user?accessKey={user_name}"),
None,
)
.await?;
info!("Deleted user {}", user_name);
// 4. Deleting the group should succeed (backend has empty members; no stale cache)
let delete_group_url = format!("{}/rustfs/admin/v3/group/{}", env.url, group_name);
awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await?;
admin_ok(&env, http::Method::DELETE, &format!("/rustfs/admin/v3/group/{group_name}"), None).await?;
info!("Deleted group {} after user was removed", group_name);
Ok(())
crates/e2e_test/src/kms/common.rs
+1 -16
View File
@@ -22,9 +22,7 @@
//! - KMS backend configuration (Local and Vault)
//! - SSE encryption testing utilities
use crate::common::{
RustFSTestEnvironment, awscurl_available, awscurl_get, awscurl_post, init_logging as common_init_logging, local_http_client,
};
use crate::common::{RustFSTestEnvironment, awscurl_get, awscurl_post, init_logging as common_init_logging, local_http_client};
use aws_sdk_s3::Client;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::ServerSideEncryption;
@@ -59,15 +57,6 @@ pub fn init_logging() {
// Additional KMS-specific logging configuration can be added here if needed
}
pub fn skip_if_kms_admin_tool_unavailable(test_name: &str) -> bool {
if awscurl_available() {
return false;
}
info!("Skipping {} because awscurl is not available in PATH", test_name);
true
}
pub fn sse_customer_key_md5_base64(key: &str) -> String {
let mut hasher = Md5::new();
hasher.update(key.as_bytes());
@@ -490,10 +479,6 @@ pub async fn test_kms_key_management(
access_key: &str,
secret_key: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
if skip_if_kms_admin_tool_unavailable("test_kms_key_management") {
return Ok(());
}
info!("Testing KMS key management APIs");
// Test CreateKey
crates/e2e_test/src/kms/kms_local_test.rs
+1 -5
View File
@@ -20,8 +20,7 @@
//! - Complete encryption/decryption lifecycle
use super::common::{
LocalKMSTestEnvironment, get_kms_status, skip_if_kms_admin_tool_unavailable, sse_customer_key_md5_base64,
test_kms_key_management, test_sse_c_encryption,
LocalKMSTestEnvironment, get_kms_status, sse_customer_key_md5_base64, test_kms_key_management, test_sse_c_encryption,
};
use crate::common::{TEST_BUCKET, init_logging};
use tracing::{error, info};
@@ -29,9 +28,6 @@ use tracing::{error, info};
#[tokio::test]
async fn test_local_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_if_kms_admin_tool_unavailable("test_local_kms_end_to_end") {
return Ok(());
}
info!("Starting Local KMS End-to-End Test");
// Create LocalKMS test environment
crates/e2e_test/src/kms/kms_vault_test.rs
+2 -17
View File
@@ -22,8 +22,8 @@ use crate::common::{TEST_BUCKET, init_logging};
use tracing::{error, info};
use super::common::{
VAULT_KEY_NAME, VaultTestEnvironment, get_kms_status, skip_if_kms_admin_tool_unavailable, sse_customer_key_md5_base64,
start_kms, test_all_multipart_encryption_types, test_error_scenarios, test_kms_key_management, test_sse_c_encryption,
VAULT_KEY_NAME, VaultTestEnvironment, get_kms_status, sse_customer_key_md5_base64, start_kms,
test_all_multipart_encryption_types, test_error_scenarios, test_kms_key_management, test_sse_c_encryption,
test_sse_kms_encryption, test_sse_s3_encryption,
};
@@ -62,9 +62,6 @@ impl VaultKmsTestContext {
#[tokio::test]
async fn test_vault_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_if_kms_admin_tool_unavailable("test_vault_kms_end_to_end") {
return Ok(());
}
info!("Starting Vault KMS End-to-End Test with default key {}", VAULT_KEY_NAME);
let context = VaultKmsTestContext::new().await?;
@@ -117,9 +114,6 @@ async fn test_vault_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + S
#[tokio::test]
async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_isolation") {
return Ok(());
}
info!("Starting Vault KMS SSE-C key isolation test");
let context = VaultKmsTestContext::new().await?;
@@ -203,9 +197,6 @@ async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error
#[tokio::test]
async fn test_vault_kms_large_file() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_if_kms_admin_tool_unavailable("test_vault_kms_large_file") {
return Ok(());
}
info!("Starting Vault KMS large file SSE-S3 test");
let context = VaultKmsTestContext::new().await?;
@@ -267,9 +258,6 @@ async fn test_vault_kms_large_file() -> Result<(), Box<dyn std::error::Error + S
#[tokio::test]
async fn test_vault_kms_multipart_upload() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_if_kms_admin_tool_unavailable("test_vault_kms_multipart_upload") {
return Ok(());
}
info!("Starting Vault KMS multipart upload encryption suite");
let context = VaultKmsTestContext::new().await?;
@@ -297,9 +285,6 @@ async fn test_vault_kms_multipart_upload() -> Result<(), Box<dyn std::error::Err
#[tokio::test]
async fn test_vault_kms_key_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_operations") {
return Ok(());
}
info!("Starting Vault KMS key operations test (CRUD)");
let context = VaultKmsTestContext::new().await?;
crates/e2e_test/src/lib.rs
+5
View File
@@ -61,6 +61,11 @@ mod get_codec_streaming_compat_test;
#[cfg(test)]
mod version_id_regression_test;
// Receiver-side replication LWW (rustfs/backlog#1953): stale inbound
// replication metadata must not overwrite a newer local category state.
#[cfg(test)]
mod replication_lww_receiver_test;
// Data usage regression tests
#[cfg(test)]
mod data_usage_test;
crates/e2e_test/src/mc_mirror_small_bucket_test.rs
+1 -11
View File
@@ -41,13 +41,6 @@ async fn create_issue_3107_fixture(root: &Path) -> TestResult {
Ok(())
}
fn mc_available() -> bool {
Command::new("mc")
.arg("--version")
.output()
.is_ok_and(|output| output.status.success())
}
fn run_mc(args: &[&str]) -> TestResult {
let output = Command::new("mc").args(args).output()?;
if !output.status.success() {
@@ -75,10 +68,7 @@ fn count_files(root: &Path) -> usize {
async fn test_mc_mirror_small_bucket_completes_without_list_timeout() -> TestResult {
crate::common::init_logging();
info!("Starting issue #3107 mc mirror regression test");
if !mc_available() {
info!("Skipping issue #3107 mc mirror regression test because mc is not installed");
return Ok(());
}
run_mc(&["--version"])?;
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
crates/e2e_test/src/multipart_auth_test.rs
-4
View File
@@ -4278,10 +4278,6 @@ async fn test_signed_put_object_extract_preserves_pax_metadata_and_version_id()
async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retention_conditions()
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if !crate::common::awscurl_available() {
return Ok(());
}
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
crates/e2e_test/src/protocols/sftp_helpers.rs
+2 -2
View File
@@ -26,7 +26,7 @@ use aws_sdk_s3::config::{Credentials, Region};
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
use russh::client::{self, Handle};
use russh::keys::ssh_key::LineEnding;
use russh::keys::{Algorithm, PrivateKey, PublicKey};
use russh::keys::{Algorithm, PrivateKey, PublicKeyOrCertificate};
use russh_sftp::client::SftpSession;
use russh_sftp::protocol::OpenFlags;
use std::path::Path;
@@ -46,7 +46,7 @@ pub struct AcceptAnyServerKey;
impl client::Handler for AcceptAnyServerKey {
type Error = anyhow::Error;
async fn check_server_key(&mut self, _server_public_key: &PublicKey) -> Result<bool, Self::Error> {
async fn check_server_key(&mut self, _server_public_key: &PublicKeyOrCertificate) -> Result<bool, Self::Error> {
Ok(true)
}
}
crates/e2e_test/src/quota_test.rs
-51
View File
@@ -18,15 +18,6 @@ use http::{Method, StatusCode};
use tokio::time::{Duration, sleep, timeout};
use tracing::{debug, info};
fn skip_without_awscurl() -> bool {
if crate::common::awscurl_available() {
return false;
}
info!("Skipping quota test because awscurl is not available");
true
}
/// Test environment setup for quota tests
pub struct QuotaTestEnv {
pub env: RustFSTestEnvironment,
@@ -276,9 +267,6 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_basic_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
// Create test bucket
@@ -320,9 +308,6 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_admission_aws_chunked_declared_encoding() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
@@ -371,9 +356,6 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_update_and_clear() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
@@ -406,9 +388,6 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_delete_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
@@ -442,9 +421,6 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_usage_tracking() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
@@ -480,9 +456,6 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_statistics() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
@@ -513,9 +486,6 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_check_api() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
@@ -553,9 +523,6 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_multiple_buckets() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
// Create two buckets in the same environment
@@ -593,9 +560,6 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_error_handling() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
@@ -628,9 +592,6 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_http_endpoints() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
@@ -689,9 +650,6 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_normal_user_permissions() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
@@ -744,9 +702,6 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_copy_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
@@ -789,9 +744,6 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_batch_delete() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
@@ -847,9 +799,6 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_multipart_upload() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
crates/e2e_test/src/reliant/node_interact_test.rs
+174 -153
View File
@@ -13,207 +13,228 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::common::workspace_root;
use crate::common::RustFSTestEnvironment;
use crate::storage_api::node_interact::{
TonicInterceptor, VolumeInfo, WalkDirOptions, gen_tonic_signature_interceptor, node_service_time_out_client,
};
use futures::future::join_all;
use aws_sdk_s3::primitives::ByteStream;
use rmp_serde::{Deserializer, Serializer};
use rustfs_filemeta::{MetaCacheEntry, MetacacheReader, MetacacheWriter};
use rustfs_filemeta::MetaCacheEntry;
use rustfs_protos::proto_gen::node_service::WalkDirRequest;
use rustfs_protos::{
models::{PingBody, PingBodyBuilder},
proto_gen::node_service::{
ListVolumesRequest, LocalStorageInfoRequest, MakeVolumeRequest, PingRequest, PingResponse, ReadAllRequest,
},
proto_gen::node_service::{ListVolumesRequest, LocalStorageInfoRequest, MakeVolumeRequest, PingRequest, ReadAllRequest},
};
use serde::{Deserialize, Serialize};
use std::error::Error;
use std::io::Cursor;
use std::path::PathBuf;
use tokio::spawn;
use tonic::Request;
use tonic::codegen::tokio_stream::StreamExt;
const CLUSTER_ADDR: &str = "http://localhost:9000";
type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
const TEST_RPC_SECRET: &str = "rustfs-internode-signature-e2e-secret";
fn signature_interceptor() -> TonicInterceptor {
TonicInterceptor::Signature(gen_tonic_signature_interceptor())
}
fn rpc_client_error(error: Box<dyn Error>) -> std::io::Error {
std::io::Error::other(error.to_string())
}
async fn start_server() -> Result<RustFSTestEnvironment, Box<dyn Error + Send + Sync>> {
let _ = rustfs_credentials::set_global_rpc_secret(TEST_RPC_SECRET.to_string());
let effective = rustfs_credentials::try_get_rpc_token().expect("RPC secret must resolve in the test process");
assert_eq!(effective, TEST_RPC_SECRET, "the test process uses an unexpected RPC secret");
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server_without_cleanup_with_env(&[
("RUSTFS_RPC_SECRET", TEST_RPC_SECRET),
("RUSTFS_INTERNODE_RPC_SIGNATURE_STRICT", "false"),
("RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT", "false"),
("RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT", "false"),
("RUST_LOG", "error"),
])
.await?;
Ok(env)
}
#[tokio::test]
#[ignore = "requires running RustFS server at localhost:9000"]
async fn ping() -> Result<(), Box<dyn Error>> {
async fn ping() -> TestResult {
let env = start_server().await?;
let mut fbb = flatbuffers::FlatBufferBuilder::new();
let payload = fbb.create_vector(b"hello world");
let mut builder = PingBodyBuilder::new(&mut fbb);
builder.add_payload(payload);
let root = builder.finish();
fbb.finish(root, None);
let finished_data = fbb.finished_data();
let decoded_payload = flatbuffers::root::<PingBody>(finished_data);
assert!(decoded_payload.is_ok());
// Create client
let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
// Construct PingRequest
let request = Request::new(PingRequest {
version: 1,
body: bytes::Bytes::copy_from_slice(finished_data),
});
// Send request and get response
let response: PingResponse = client.ping(request).await?.into_inner();
// Print response
let ping_response_body = flatbuffers::root::<PingBody>(&response.body);
if let Err(e) = ping_response_body {
eprintln!("{e}");
} else {
println!("ping_resp:body(flatbuffer): {ping_response_body:?}");
}
let mut client = node_service_time_out_client(&env.url, signature_interceptor())
.await
.map_err(rpc_client_error)?;
let response = client
.ping(Request::new(PingRequest {
version: 1,
body: bytes::Bytes::copy_from_slice(fbb.finished_data()),
}))
.await?
.into_inner();
assert_eq!(response.version, 1);
let body = flatbuffers::root::<PingBody>(&response.body)?;
assert_eq!(body.payload().expect("ping response must contain a payload").bytes(), b"hello, caller");
Ok(())
}
#[tokio::test]
#[ignore = "requires running RustFS server at localhost:9000"]
async fn make_volume() -> Result<(), Box<dyn Error>> {
let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
let request = Request::new(MakeVolumeRequest {
disk: "data".to_string(),
volume: "dandan".to_string(),
});
async fn make_volume() -> TestResult {
let env = start_server().await?;
let mut client = node_service_time_out_client(&env.url, signature_interceptor())
.await
.map_err(rpc_client_error)?;
let response = client
.make_volume(Request::new(MakeVolumeRequest {
disk: env.temp_dir.clone(),
volume: "node-rpc-volume".to_string(),
}))
.await?
.into_inner();
let response = client.make_volume(request).await?.into_inner();
if response.success {
println!("success");
} else {
println!("failed: {:?}", response.error);
}
assert!(response.success, "make_volume failed: {:?}", response.error);
assert!(std::path::Path::new(&env.temp_dir).join("node-rpc-volume").is_dir());
Ok(())
}
#[tokio::test]
#[ignore = "requires running RustFS server at localhost:9000"]
async fn list_volumes() -> Result<(), Box<dyn Error>> {
let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
let request = Request::new(ListVolumesRequest {
disk: "data".to_string(),
});
async fn list_volumes() -> TestResult {
let env = start_server().await?;
let mut client = node_service_time_out_client(&env.url, signature_interceptor())
.await
.map_err(rpc_client_error)?;
let created = client
.make_volume(Request::new(MakeVolumeRequest {
disk: env.temp_dir.clone(),
volume: "node-rpc-listed-volume".to_string(),
}))
.await?
.into_inner();
assert!(created.success, "make_volume failed: {:?}", created.error);
let response = client.list_volumes(request).await?.into_inner();
let volume_infos: Vec<VolumeInfo> = response
let response = client
.list_volumes(Request::new(ListVolumesRequest {
disk: env.temp_dir.clone(),
}))
.await?
.into_inner();
assert!(response.success, "list_volumes failed: {:?}", response.error);
let volumes = response
.volume_infos
.into_iter()
.filter_map(|json_str| serde_json::from_str::<VolumeInfo>(&json_str).ok())
.collect();
println!("{volume_infos:?}");
.iter()
.map(|json| serde_json::from_str::<VolumeInfo>(json))
.collect::<Result<Vec<_>, _>>()?;
assert!(volumes.iter().any(|volume| volume.name == "node-rpc-listed-volume"));
Ok(())
}
#[tokio::test]
#[ignore = "requires running RustFS server at localhost:9000"]
async fn walk_dir() -> Result<(), Box<dyn Error>> {
println!("walk_dir");
// TODO: use writer
async fn walk_dir() -> TestResult {
let env = start_server().await?;
let s3 = env.create_s3_client();
let bucket = "node-rpc-walk-bucket";
let key = "prefix/object.txt";
env.create_test_bucket(bucket).await?;
s3.put_object()
.bucket(bucket)
.key(key)
.body(ByteStream::from_static(b"walk payload"))
.send()
.await?;
let opts = WalkDirOptions {
bucket: "dandan".to_owned(),
base_dir: "".to_owned(),
bucket: bucket.to_string(),
recursive: true,
..Default::default()
};
let (rd, mut wr) = tokio::io::duplex(1024);
let mut buf = Vec::new();
opts.serialize(&mut Serializer::new(&mut buf))?;
let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
let disk_path = std::env::var_os("RUSTFS_DISK_PATH").map(PathBuf::from).unwrap_or_else(|| {
let mut path = workspace_root();
path.push("target");
path.push(if cfg!(debug_assertions) { "debug" } else { "release" });
path.push("data");
path
});
let request = Request::new(WalkDirRequest {
disk: disk_path.to_string_lossy().into_owned(),
walk_dir_options: buf.into(),
});
let mut response = client.walk_dir(request).await?.into_inner();
let mut encoded = Vec::new();
opts.serialize(&mut Serializer::new(&mut encoded))?;
let mut client = node_service_time_out_client(&env.url, signature_interceptor())
.await
.map_err(rpc_client_error)?;
let mut stream = client
.walk_dir(Request::new(WalkDirRequest {
disk: env.temp_dir.clone(),
walk_dir_options: encoded.into(),
}))
.await?
.into_inner();
let job1 = spawn(async move {
let mut out = MetacacheWriter::new(&mut wr);
loop {
match response.next().await {
Some(Ok(resp)) => {
if !resp.success {
println!("{}", resp.error_info.unwrap_or_else(|| "".to_string()));
}
let entry = serde_json::from_str::<MetaCacheEntry>(&resp.meta_cache_entry)
.map_err(|_e| std::io::Error::other(format!("Unexpected response: {response:?}")))
.unwrap();
out.write_obj(&entry).await.unwrap();
}
None => {
let _ = out.close().await;
break;
}
_ => {
println!("Unexpected response: {response:?}");
let _ = out.close().await;
break;
}
}
}
});
let job2 = spawn(async move {
let mut reader = MetacacheReader::new(rd);
while let Ok(Some(entry)) = reader.peek().await {
println!("{entry:?}");
}
});
join_all(vec![job1, job2]).await;
Ok(())
}
#[tokio::test]
#[ignore = "requires running RustFS server at localhost:9000"]
async fn read_all() -> Result<(), Box<dyn Error>> {
let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
let request = Request::new(ReadAllRequest {
disk: "data".to_string(),
volume: "ff".to_string(),
path: "format.json".to_string(),
});
let response = client.read_all(request).await?.into_inner();
let volume_infos = response.data;
println!("{}", response.success);
println!("{volume_infos:?}");
Ok(())
}
#[tokio::test]
#[ignore = "requires running RustFS server at localhost:9000"]
async fn storage_info() -> Result<(), Box<dyn Error>> {
let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
let request = Request::new(LocalStorageInfoRequest { metrics: true });
let response = client.local_storage_info(request).await?.into_inner();
if !response.success {
println!("{:?}", response.error_info);
return Ok(());
let mut entries = Vec::new();
while let Some(response) = stream.next().await {
let response = response?;
assert!(response.success, "walk_dir failed: {:?}", response.error_info);
entries.push(serde_json::from_str::<MetaCacheEntry>(&response.meta_cache_entry)?);
}
let info = response.storage_info;
let mut buf = Deserializer::new(Cursor::new(info));
let storage_info: rustfs_madmin::StorageInfo = Deserialize::deserialize(&mut buf).unwrap();
println!("{storage_info:?}");
assert!(
entries.iter().any(|entry| entry.name == key),
"walk_dir did not return {key}: {entries:?}"
);
Ok(())
}
#[tokio::test]
async fn read_all() -> TestResult {
let env = start_server().await?;
let mut client = node_service_time_out_client(&env.url, signature_interceptor())
.await
.map_err(rpc_client_error)?;
let volume = "node-rpc-read-volume";
let created = client
.make_volume(Request::new(MakeVolumeRequest {
disk: env.temp_dir.clone(),
volume: volume.to_string(),
}))
.await?
.into_inner();
assert!(created.success, "make_volume failed: {:?}", created.error);
tokio::fs::write(std::path::Path::new(&env.temp_dir).join(volume).join("payload.bin"), b"read payload").await?;
let response = client
.read_all(Request::new(ReadAllRequest {
disk: env.temp_dir.clone(),
volume: volume.to_string(),
path: "payload.bin".to_string(),
}))
.await?
.into_inner();
assert!(response.success, "read_all failed: {:?}", response.error);
assert_eq!(response.data.as_ref(), b"read payload");
Ok(())
}
#[tokio::test]
async fn storage_info() -> TestResult {
let env = start_server().await?;
let mut client = node_service_time_out_client(&env.url, signature_interceptor())
.await
.map_err(rpc_client_error)?;
let response = client
.local_storage_info(Request::new(LocalStorageInfoRequest { metrics: true }))
.await?
.into_inner();
assert!(response.success, "local_storage_info failed: {:?}", response.error_info);
let mut decoder = Deserializer::new(Cursor::new(response.storage_info));
let storage_info: rustfs_madmin::StorageInfo = Deserialize::deserialize(&mut decoder)?;
let expected_disk = std::fs::canonicalize(&env.temp_dir)?;
assert!(!storage_info.disks.is_empty(), "local_storage_info returned no disks");
assert!(
storage_info
.disks
.iter()
.any(|disk| std::path::Path::new(&disk.drive_path) == expected_disk),
"local_storage_info did not include the configured disk: {:?}",
storage_info.disks
);
Ok(())
}
crates/e2e_test/src/replication_extension_test.rs
+422 -10
View File
@@ -13,9 +13,8 @@
// limitations under the License.
use crate::common::{
RustFSTestEnvironment, admin_create_user, awscurl_available, awscurl_post_sts_form_urlencoded, init_logging,
local_http_client, replication_fast_env, rustfs_binary_path, signed_request, signed_request_with_client,
signed_request_with_session_token,
RustFSTestEnvironment, admin_create_user, awscurl_post_sts_form_urlencoded, init_logging, local_http_client,
replication_fast_env, rustfs_binary_path, signed_request, signed_request_with_client, signed_request_with_session_token,
};
use crate::fake_s3_target::{
FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, FaultAction as FakeTargetFault, Operation as FakeTargetOperation,
@@ -57,7 +56,7 @@ use rustfs_madmin::{
AddServiceAccountReq, ListServiceAccountsResp, PeerInfo, PeerSite, ReplicateAddStatus, ReplicateEditStatus,
ReplicateRemoveStatus, SRRemoveReq, SRResyncOpStatus, SRStatusInfo, SiteReplicationInfo, SyncStatus,
};
use s3s::header::X_AMZ_REPLICATION_STATUS;
use s3s::header::{X_AMZ_REPLICATION_STATUS, X_AMZ_TAGGING};
use sha2::{Digest, Sha256};
use std::collections::BTreeMap;
use std::convert::Infallible;
@@ -2023,6 +2022,7 @@ async fn forward_replication_proxy_request(
client: &reqwest::Client,
request_count: &AtomicU64,
mut replication_enabled: watch::Receiver<bool>,
mut held_tagging: watch::Receiver<Option<String>>,
) -> Response<Full<bytes::Bytes>> {
let (parts, body) = request.into_parts();
let is_replication = parts
@@ -2036,6 +2036,17 @@ async fn forward_replication_proxy_request(
return proxy_error_response("replication gate closed");
}
}
// Content-keyed hold: park only the replication request whose
// `x-amz-tagging` matches the held value, letting every other delivery
// through, so a test can make one specific (stale) delivery the last
// write the backend sees.
if let Some(tagging) = parts.headers.get(X_AMZ_TAGGING).and_then(|value| value.to_str().ok()) {
while held_tagging.borrow().as_deref() == Some(tagging) {
if held_tagging.changed().await.is_err() {
return proxy_error_response("replication tag hold closed");
}
}
}
}
let Some(path_and_query) = parts.uri.path_and_query() else {
@@ -2070,12 +2081,26 @@ async fn start_replication_counting_proxy(
backend_url: &str,
tasks: &mut JoinSet<()>,
) -> Result<(String, Arc<AtomicU64>, watch::Sender<bool>), Box<dyn Error + Send + Sync>> {
let (proxy_url, request_count, replication_enabled, _held_tagging) =
start_replication_counting_proxy_with_tag_hold(backend_url, tasks).await?;
Ok((proxy_url, request_count, replication_enabled))
}
/// [`start_replication_counting_proxy`] plus a content-keyed hold: while the
/// returned `watch::Sender<Option<String>>` holds `Some(tagging)`, replication
/// requests whose `x-amz-tagging` equals `tagging` are parked (and still
/// counted); all other traffic flows. Send `None` to release them.
async fn start_replication_counting_proxy_with_tag_hold(
backend_url: &str,
tasks: &mut JoinSet<()>,
) -> Result<(String, Arc<AtomicU64>, watch::Sender<bool>, watch::Sender<Option<String>>), Box<dyn Error + Send + Sync>> {
let listener = TcpListener::bind("127.0.0.1:0").await?;
let proxy_url = format!("http://{}", listener.local_addr()?);
let backend_url = backend_url.to_string();
let request_count = Arc::new(AtomicU64::new(0));
let task_request_count = request_count.clone();
let (replication_enabled, task_replication_enabled) = watch::channel(true);
let (held_tagging, task_held_tagging) = watch::channel(None);
tasks.spawn(async move {
let client = local_http_client();
let mut connections = JoinSet::new();
@@ -2087,12 +2112,14 @@ async fn start_replication_counting_proxy(
let client = client.clone();
let request_count = task_request_count.clone();
let replication_enabled = task_replication_enabled.clone();
let held_tagging = task_held_tagging.clone();
connections.spawn(async move {
let service = service_fn(move |request| {
let backend_url = backend_url.clone();
let client = client.clone();
let request_count = request_count.clone();
let replication_enabled = replication_enabled.clone();
let held_tagging = held_tagging.clone();
async move {
Ok::<_, Infallible>(
forward_replication_proxy_request(
@@ -2101,6 +2128,7 @@ async fn start_replication_counting_proxy(
&client,
&request_count,
replication_enabled,
held_tagging,
)
.await,
)
@@ -2113,7 +2141,7 @@ async fn start_replication_counting_proxy(
}
}
});
Ok((proxy_url, request_count, replication_enabled))
Ok((proxy_url, request_count, replication_enabled, held_tagging))
}
async fn site_replication_remove(
@@ -6949,6 +6977,395 @@ async fn test_site_replication_active_active_converges_without_loops_real_dual_n
}
}
/// Replication status a site reports for one object version via HEAD
/// (`x-amz-replication-status`), or `None` when the header is absent.
async fn head_replication_status(
client: &Client,
bucket: &str,
key: &str,
version_id: &str,
) -> Result<Option<String>, Box<dyn Error + Send + Sync>> {
let head = client
.head_object()
.bucket(bucket)
.key(key)
.version_id(version_id)
.send()
.await?;
Ok(head.replication_status().map(|status| status.as_str().to_string()))
}
/// Poll one site until the version's replication status is one of `expected`.
async fn wait_for_version_replication_status(
client: &Client,
bucket: &str,
key: &str,
version_id: &str,
expected: &[&str],
site: &str,
) -> Result<String, Box<dyn Error + Send + Sync>> {
let deadline = tokio::time::Instant::now() + Duration::from_secs(60);
loop {
let last = head_replication_status(client, bucket, key, version_id).await?;
if let Some(status) = last.as_deref()
&& expected.contains(&status)
{
return Ok(status.to_string());
}
if tokio::time::Instant::now() >= deadline {
return Err(format!(
"{site}: {bucket}/{key}?versionId={version_id} replication status {last:?} never reached {expected:?}"
)
.into());
}
sleep(Duration::from_millis(200)).await;
}
}
async fn put_single_tag(
client: &Client,
bucket: &str,
key: &str,
version_id: &str,
tag_key: &str,
tag_value: &str,
) -> Result<(), Box<dyn Error + Send + Sync>> {
client
.put_object_tagging()
.bucket(bucket)
.key(key)
.version_id(version_id)
.tagging(
aws_sdk_s3::types::Tagging::builder()
.tag_set(aws_sdk_s3::types::Tag::builder().key(tag_key).value(tag_value).build()?)
.build()?,
)
.send()
.await?;
Ok(())
}
async fn get_single_tag(
client: &Client,
bucket: &str,
key: &str,
version_id: &str,
tag_key: &str,
) -> Result<Option<String>, Box<dyn Error + Send + Sync>> {
let tagging = client
.get_object_tagging()
.bucket(bucket)
.key(key)
.version_id(version_id)
.send()
.await?;
Ok(tagging
.tag_set()
.iter()
.find(|tag| tag.key() == tag_key)
.map(|tag| tag.value().to_string()))
}
/// Poll one site until the version's `tag_key` equals `expected`.
async fn wait_for_single_tag(
client: &Client,
bucket: &str,
key: &str,
version_id: &str,
tag_key: &str,
expected: &str,
site: &str,
) -> Result<(), Box<dyn Error + Send + Sync>> {
let deadline = tokio::time::Instant::now() + Duration::from_secs(60);
loop {
let observed = get_single_tag(client, bucket, key, version_id, tag_key).await?;
if observed.as_deref() == Some(expected) {
return Ok(());
}
if tokio::time::Instant::now() >= deadline {
return Err(format!(
"{site}: {bucket}/{key}?versionId={version_id} tag {tag_key}={observed:?} never became {expected}"
)
.into());
}
sleep(Duration::from_millis(200)).await;
}
}
/// Tag key the dual-node LWW scenario edits on both sites.
const LWW_TAG_KEY: &str = "owner";
/// Assert the version's [`LWW_TAG_KEY`] stays `expected` on both sites for a
/// full quiet window (no late stale delivery flips it back).
async fn assert_tag_stable_on_both_sites(
site_a_client: &Client,
site_b_client: &Client,
bucket: &str,
key: &str,
version_id: &str,
expected: &str,
quiet: Duration,
) -> Result<(), Box<dyn Error + Send + Sync>> {
let deadline = tokio::time::Instant::now() + quiet;
loop {
let on_a = get_single_tag(site_a_client, bucket, key, version_id, LWW_TAG_KEY).await?;
let on_b = get_single_tag(site_b_client, bucket, key, version_id, LWW_TAG_KEY).await?;
assert_eq!(on_a.as_deref(), Some(expected), "site A tag {LWW_TAG_KEY} regressed from the LWW winner");
assert_eq!(on_b.as_deref(), Some(expected), "site B tag {LWW_TAG_KEY} regressed from the LWW winner");
if tokio::time::Instant::now() >= deadline {
return Ok(());
}
sleep(Duration::from_millis(250)).await;
}
}
/// Wait until the counting proxy in front of a site has admitted `expected`
/// replication requests in total (requests held by a closed gate still count).
async fn wait_for_proxy_replication_requests(
counter: &AtomicU64,
expected: u64,
site: &str,
) -> Result<(), Box<dyn Error + Send + Sync>> {
let deadline = tokio::time::Instant::now() + Duration::from_secs(60);
loop {
let observed = counter.load(Ordering::Relaxed);
if observed >= expected {
return Ok(());
}
if tokio::time::Instant::now() >= deadline {
return Err(format!("{site} proxy saw {observed} replication requests, expected at least {expected}").into());
}
sleep(Duration::from_millis(25)).await;
}
}
/// rustfs/backlog#1953 (audit A4/P1-6): receiver-side LWW for replicated
/// metadata categories, exercised end to end over the real dual-node
/// active-active site-replication control plane — sender, worker, status
/// bookkeeping and persisted failure recovery all participate (the single-server
/// `replication_lww_receiver_test` only injects authorized replication PUTs).
///
/// Scenario on one versioned object:
/// 1. reciprocal tag edits in real order (A then B) converge both sites on the
/// newer tag and leave the author COMPLETED / the receiver REPLICA;
/// 2. out-of-order delivery: A's edit is held at B's inbound proxy while B
/// authors a newer edit that reaches A first; releasing the stale delivery
/// must NOT roll B back — both sites settle on B's value and stay there
/// through a quiet window, with no FAILED/PENDING status left behind;
/// 3. persisted retry: B is stopped, A's delivery reaches FAILED, A restarts,
/// then B returns and the scanner-replayed edit converges both sites forward.
/// Durable metadata-MRF serialization/reconstruction is covered separately by
/// `metadata_mrf_roundtrip_preserves_tags_and_admitted_targets`.
#[tokio::test]
async fn test_site_replication_tagging_lww_converges_active_active_real_dual_node() -> TestResult {
init_logging();
match tokio::time::timeout(Duration::from_secs(420), async {
// The scanner is fast for the final persisted-failure recovery phase.
// Step 2 finishes and proves a quiet stable winner before that phase,
// so a later scanner pass cannot mask its stale-delivery assertion.
let mut site_env = replication_fast_env();
site_env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
site_env.extend_from_slice(FAST_SCANNER_ENV);
let mut site_a_env = RustFSTestEnvironment::new().await?;
site_a_env.start_rustfs_server_with_env(vec![], &site_env).await?;
let mut site_b_env = RustFSTestEnvironment::new().await?;
site_b_env.start_rustfs_server_with_env(vec![], &site_env).await?;
let mut proxy_tasks = JoinSet::new();
let (site_a_proxy, site_a_replication_requests, _site_a_replication_enabled, site_a_held_tagging) =
start_replication_counting_proxy_with_tag_hold(&site_a_env.url, &mut proxy_tasks).await?;
let (site_b_proxy, site_b_replication_requests, _site_b_replication_enabled, site_b_held_tagging) =
start_replication_counting_proxy_with_tag_hold(&site_b_env.url, &mut proxy_tasks).await?;
let site_a_client = site_a_env.create_s3_client();
let site_b_client = site_b_env.create_s3_client();
let bucket = "site-repl-tag-lww";
let key = "lww.txt";
let add_status = site_replication_add(
&site_a_env,
&[
PeerSite {
name: "lww-site-a".to_string(),
endpoint: site_a_env.url.clone(),
access_key: site_a_env.access_key.clone(),
secret_key: site_a_env.secret_key.clone(),
..Default::default()
},
PeerSite {
name: "lww-site-b".to_string(),
endpoint: site_b_env.url.clone(),
access_key: site_b_env.access_key.clone(),
secret_key: site_b_env.secret_key.clone(),
..Default::default()
},
],
)
.await?;
assert!(add_status.success, "unexpected site add result: {add_status:?}");
let site_info = wait_for_site_replication_enabled(&site_a_env, 2).await?;
wait_for_site_replication_enabled(&site_b_env, 2).await?;
// Route both directions through the counting proxies so inbound
// replication to B can be held (out-of-order delivery) and observed.
for (env_url, proxy_url, label) in [(&site_a_env.url, &site_a_proxy, "A"), (&site_b_env.url, &site_b_proxy, "B")] {
let mut peer = site_info
.sites
.iter()
.find(|peer| peer.endpoint == *env_url)
.ok_or_else(|| format!("site {label} peer missing from replication info"))?
.clone();
peer.endpoint = proxy_url.clone();
peer.sync_state = SyncStatus::Enable;
let edit = site_replication_edit(&site_a_env, "", &peer).await?;
assert!(edit.success, "unexpected site {label} endpoint edit: {edit:?}");
}
for env in [&site_a_env, &site_b_env] {
wait_for_site_replication_info(env, |info| {
info.sites.iter().any(|peer| peer.endpoint == site_a_proxy)
&& info.sites.iter().any(|peer| peer.endpoint == site_b_proxy)
})
.await?;
}
site_a_client.create_bucket().bucket(bucket).send().await?;
wait_for_bucket_on_target(&site_b_client, bucket).await?;
let version_id = site_a_client
.put_object()
.bucket(bucket)
.key(key)
.body(ByteStream::from_static(b"tag lww payload"))
.send()
.await?
.version_id()
.ok_or("site A PUT omitted version ID")?
.to_string();
wait_for_replicated_object(&site_b_client, bucket, key, "tag lww payload").await?;
wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["COMPLETED"], "site A").await?;
wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["REPLICA"], "site B").await?;
wait_for_proxy_replication_requests(&site_b_replication_requests, 1, "site B").await?;
// --- 1. reciprocal edits in real order: A then B ----------------------
put_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY, "a1").await?;
wait_for_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY, "a1", "site B").await?;
wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["COMPLETED"], "site A").await?;
wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["REPLICA"], "site B").await?;
wait_for_proxy_replication_requests(&site_b_replication_requests, 2, "site B").await?;
put_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY, "b1").await?;
wait_for_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY, "b1", "site A").await?;
wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["COMPLETED"], "site B").await?;
wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["REPLICA"], "site A").await?;
assert_tag_stable_on_both_sites(&site_a_client, &site_b_client, bucket, key, &version_id, "b1", Duration::from_secs(3))
.await?;
// --- 2. concurrent edits, stale delivery last ------------------------
// Both sites edit the same version while each other's delivery is
// parked at the peer's inbound proxy (content-keyed: only the
// `owner=a2` / `owner=b2` replication PUTs wait, everything else
// flows). B's edit is the newer one. Releasing A's stale `a2` first
// makes it the last write B sees while A itself still holds `a2`, so
// nothing A could re-deliver carries the winner: only receiver-side
// LWW on B can keep `b2`. Releasing `b2` afterwards converges A.
site_b_held_tagging.send(Some("owner=a2".to_string()))?;
site_a_held_tagging.send(Some("owner=b2".to_string()))?;
let a2_parked_at = site_b_replication_requests.load(Ordering::Relaxed) + 1;
let b2_parked_at = site_a_replication_requests.load(Ordering::Relaxed) + 1;
put_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY, "a2").await?;
wait_for_proxy_replication_requests(&site_b_replication_requests, a2_parked_at, "site B").await?;
sleep(Duration::from_millis(50)).await;
put_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY, "b2").await?;
wait_for_proxy_replication_requests(&site_a_replication_requests, b2_parked_at, "site A").await?;
assert_eq!(
get_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY)
.await?
.as_deref(),
Some("a2")
);
assert_eq!(
get_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY)
.await?
.as_deref(),
Some("b2")
);
// Release the stale a2 delivery onto B: the newer local b2 must
// survive, and the delivery itself must still succeed (A reaches
// COMPLETED instead of looping through MRF with the stale value).
site_b_held_tagging.send(None)?;
wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["COMPLETED"], "site A").await?;
let stale_deadline = tokio::time::Instant::now() + Duration::from_secs(3);
loop {
assert_eq!(
get_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY)
.await?
.as_deref(),
Some("b2"),
"a stale inbound delivery rolled back site B's newer tag (receiver-side LWW regression)"
);
if tokio::time::Instant::now() >= stale_deadline {
break;
}
sleep(Duration::from_millis(250)).await;
}
// Release b2 onto A: the newer edit wins there and both sites settle.
// B's own version may legitimately read REPLICA here: the stale inbound
// a2 write re-labelled it as a replica write (keeping B's tags); what
// must not remain is PENDING/FAILED.
site_a_held_tagging.send(None)?;
wait_for_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY, "b2", "site A").await?;
wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["COMPLETED", "REPLICA"], "site B")
.await?;
assert_tag_stable_on_both_sites(&site_a_client, &site_b_client, bucket, key, &version_id, "b2", Duration::from_secs(4))
.await?;
for (client, site) in [(&site_a_client, "site A"), (&site_b_client, "site B")] {
let status = head_replication_status(client, bucket, key, &version_id).await?;
assert!(
matches!(status.as_deref(), Some("COMPLETED" | "REPLICA")),
"{site} must not be left PENDING/FAILED after the concurrent edits: {status:?}"
);
}
// --- 3. persisted FAILED state survives a source restart ------------
site_b_env.stop_server();
put_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY, "a3").await?;
wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["FAILED"], "site A").await?;
site_a_env.restart_server_preserving_data(vec![], &site_env).await?;
wait_for_site_replication_enabled(&site_a_env, 2).await?;
site_b_env.restart_server_preserving_data(vec![], &site_env).await?;
wait_for_site_replication_enabled(&site_b_env, 2).await?;
wait_for_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY, "a3", "site B").await?;
wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["COMPLETED"], "site A").await?;
wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["REPLICA"], "site B").await?;
assert_tag_stable_on_both_sites(&site_a_client, &site_b_client, bucket, key, &version_id, "a3", Duration::from_secs(3))
.await?;
// The object itself never forked: one version on each side.
tokio::time::timeout(
Duration::from_secs(70),
assert_replication_converged(&site_a_client, bucket, &site_b_client, bucket),
)
.await??;
let state = list_replication_state(&site_a_client, bucket).await?;
assert_eq!(state.len(), 1, "tag edits must not create new object versions: {state:?}");
assert_eq!(state[0].version_id, version_id);
proxy_tasks.abort_all();
Ok(())
})
.await
{
Ok(result) => result,
Err(_) => Err("site replication tagging LWW test timed out".into()),
}
}
#[tokio::test]
async fn test_site_replication_replicates_policy_backed_user_access_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {
init_logging();
@@ -7281,11 +7698,6 @@ async fn test_site_replication_replicates_multiple_service_accounts_real_dual_no
async fn test_site_replication_replicates_service_accounts_created_from_sts_session_real_dual_node() -> TestResult {
init_logging();
if !awscurl_available() {
eprintln!("Skipping STS site replication service-account test because awscurl is unavailable");
return Ok(());
}
let mut source_env = RustFSTestEnvironment::new().await?;
source_env
.start_rustfs_server_with_env(vec![], LOOPBACK_REPLICATION_TARGET_ENV)
crates/e2e_test/src/replication_lww_receiver_test.rs
+155
View File
@@ -0,0 +1,155 @@
#![cfg(test)]
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Receiver-side replication LWW over the wire (rustfs/backlog#1953, audit
//! A4/P1-6).
//!
//! In an active-active topology both sites' metadata states arrive at the
//! peer as authorized replication PUTs carrying per-category source
//! timestamps (`x-rustfs-source-replication-tagging-timestamp` header
//! family). Before the fix the receiver applied them unconditionally, so a
//! stale delivery overwrote a newer local state and the two sites diverged
//! permanently while both reported COMPLETED. This test drives one live
//! `rustfs` server with simulated inbound replication PUTs for the same
//! object version and asserts the newer tagging state wins regardless of
//! delivery order, while a stale delivery still succeeds at the object level
//! (a failure would loop through MRF re-delivering the stale value).
//!
//! The real dual-site path (sender, worker, status bookkeeping, MRF replay)
//! is covered by
//! `replication_extension_test::test_site_replication_tagging_lww_converges_active_active_real_dual_node`;
//! this file stays as the fast, single-process receiver check.
use crate::common::{RustFSTestEnvironment, init_logging};
use aws_sdk_s3::Client;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
const HDR_SOURCE_REPLICATION_REQUEST: &str = "x-rustfs-source-replication-request";
const HDR_SOURCE_VERSION_ID: &str = "x-rustfs-source-version-id";
const HDR_SOURCE_MTIME: &str = "x-rustfs-source-mtime";
const HDR_SOURCE_TAGGING_TIMESTAMP: &str = "x-rustfs-source-replication-tagging-timestamp";
const SOURCE_MTIME: &str = "2026-01-01T00:00:00Z";
const T_STALE: &str = "2026-01-01T00:00:01Z";
const T_LOCAL: &str = "2026-02-01T00:00:00Z";
const T_NEWER: &str = "2026-03-01T00:00:00Z";
/// Simulated inbound authorized replication PUT: same object version, tags and
/// the source-authored tagging timestamp carried in transport headers.
async fn inbound_replication_put(
client: &Client,
bucket: &str,
key: &str,
version_id: &str,
tags: &str,
tagging_timestamp: &str,
) -> TestResult {
let version_id = version_id.to_string();
let tagging_timestamp = tagging_timestamp.to_string();
client
.put_object()
.bucket(bucket)
.key(key)
.body(ByteStream::from_static(b"lww-e2e-body"))
.tagging(tags)
.customize()
.mutate_request(move |req| {
req.headers_mut().insert(HDR_SOURCE_REPLICATION_REQUEST, "true");
req.headers_mut().insert(HDR_SOURCE_VERSION_ID, version_id.clone());
req.headers_mut().insert(HDR_SOURCE_MTIME, SOURCE_MTIME);
req.headers_mut()
.insert(HDR_SOURCE_TAGGING_TIMESTAMP, tagging_timestamp.clone());
})
.send()
.await?;
Ok(())
}
async fn tag_value(client: &Client, bucket: &str, key: &str, version_id: &str, tag_key: &str) -> Option<String> {
let tagging = client
.get_object_tagging()
.bucket(bucket)
.key(key)
.version_id(version_id)
.send()
.await
.expect("object tagging should be readable");
tagging
.tag_set()
.iter()
.find(|tag| tag.key() == tag_key)
.map(|tag| tag.value().to_string())
}
#[tokio::test(flavor = "multi_thread")]
async fn receiver_lww_keeps_newer_tags_across_delivery_orders() -> TestResult {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
let client = env.create_s3_client();
let bucket = "replication-lww-receiver";
let key = "object";
client.create_bucket().bucket(bucket).send().await?;
client
.put_bucket_versioning()
.bucket(bucket)
.versioning_configuration(
VersioningConfiguration::builder()
.status(BucketVersioningStatus::Enabled)
.build(),
)
.send()
.await?;
// First delivery establishes version V with tags stamped T_LOCAL.
let version_id = uuid::Uuid::new_v4().to_string();
inbound_replication_put(&client, bucket, key, &version_id, "site=local", T_LOCAL).await?;
assert_eq!(
tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
Some("local"),
"the first delivery must establish the tagged version"
);
// A stale delivery (older source timestamp) must succeed at the object
// level but must NOT overwrite the newer tags.
inbound_replication_put(&client, bucket, key, &version_id, "site=stale", T_STALE).await?;
assert_eq!(
tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
Some("local"),
"a stale inbound delivery must not overwrite newer tags (rustfs/backlog#1953)"
);
// A newer delivery still converges the version onto the newest state.
inbound_replication_put(&client, bucket, key, &version_id, "site=newer", T_NEWER).await?;
assert_eq!(
tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
Some("newer"),
"a newer inbound delivery must overwrite older tags"
);
client
.delete_object()
.bucket(bucket)
.key(key)
.version_id(&version_id)
.send()
.await?;
env.delete_test_bucket(bucket).await.ok();
Ok(())
}
crates/e2e_test/src/security_boundary_test.rs
+4 -9
View File
@@ -21,12 +21,11 @@
//! - SSRF prevention (internal/private endpoints rejected for tiering)
//! - Race condition handling (concurrent writes converge without corruption)
use crate::common::{RustFSTestEnvironment, awscurl_available, awscurl_put, init_logging};
use crate::common::{RustFSTestEnvironment, awscurl_put, init_logging, require_awscurl};
use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart, Tag, Tagging};
use std::error::Error;
use tracing::info;
/// Oversized tagging payloads must be rejected by the per-object tag limit.
///
@@ -225,16 +224,12 @@ async fn test_concurrent_object_operations() -> Result<(), Box<dyn Error + Send
/// outcome — the internal endpoint is not accepted — is asserted here.
///
/// The admin API is exercised via signed `awscurl` requests, matching the
/// pattern used by the other admin-API E2E tests in this crate; the test is
/// skipped when `awscurl` is not installed.
/// pattern used by the other admin-API E2E tests in this crate. The full E2E
/// lane installs and verifies the pinned `awscurl` prerequisite.
#[tokio::test]
async fn test_tiering_url_validation() -> Result<(), Box<dyn Error + Send + Sync>> {
init_logging();
if !awscurl_available() {
info!("Skipping tiering URL validation test because awscurl is not available");
return Ok(());
}
require_awscurl()?;
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
crates/ecstore/src/api/mod.rs
+20 -16
View File
@@ -187,22 +187,24 @@ pub mod bucket {
pub use crate::bucket::replication::{
BucketReplicationResyncStatus, BucketReplicationStat, BucketReplicationStats, BucketStats,
DeleteReplicationConfigSnapshot, DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool, InQueueMetric,
MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION,
REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, REPLICATE_INCOMING_DELETE,
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
ReplicateDecision, ReplicateObjectInfo, ReplicationBatchAdmission, ReplicationConfig,
MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts, OperatorRuleContract,
REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
REPLICATE_INCOMING_DELETE, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS,
REPLICATION_WRITABLE_FIELDS, ReplicateDecision, ReplicateObjectInfo, ReplicationBatchAdmission, ReplicationConfig,
ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationDeleteScheduleInput,
ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge, ReplicationObjectIO,
ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge,
ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage, ReplicationTargetValidationError,
ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, TargetReplicationResyncStatus,
VersionPurgeStatusType, XferStats, commit_force_delete_intent, complete_force_delete_intent,
delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool,
get_global_replication_stats, get_proxy_targets, init_background_replication,
invalid_replication_config_status_field, persist_force_delete_intent, read_durable_mrf_backlog,
replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map, replication_target_arns,
resync_start_conflict_id, should_remove_replication_target, should_schedule_delete_replication,
should_use_existing_delete_replication_info, should_use_existing_delete_replication_source,
VersionPurgeStatusType, XferStats, assign_site_replication_rule_priorities, commit_force_delete_intent,
complete_force_delete_intent, delete_replication_state_from_config, delete_replication_version_id,
get_global_replication_pool, get_global_replication_stats, get_proxy_targets, init_background_replication,
invalid_replication_config_status_field, is_site_replication_role, is_site_replication_rule,
merge_incoming_replication_config, merge_user_replication_config, persist_force_delete_intent,
read_durable_mrf_backlog, replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map,
replication_target_arn_deployment_id, replication_target_arns, resync_start_conflict_id,
should_remove_replication_target, should_schedule_delete_replication, should_use_existing_delete_replication_info,
should_use_existing_delete_replication_source, site_replication_rule_deployment_id,
unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
version_purge_status_to_filemeta,
};
@@ -456,11 +458,13 @@ pub mod rpc {
ScannerBucketListing, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, build_put_file_auth_trailer,
check_and_record_signed_rpc_nonce, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_put_file_capability,
sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
tonic_rpc_auth_failure_reason, verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature,
verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest,
verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, sign_ns_scanner_capability,
sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer,
verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
verify_tonic_rpc_signature_with_bootstrap,
};
}
crates/ecstore/src/bucket/bucket_target_sys.rs
+41 -2
View File
@@ -58,8 +58,8 @@ use rustfs_config::{DEFAULT_TRUST_LEAF_CERT_AS_CA, ENV_TRUST_LEAF_CERT_AS_CA, RU
use rustfs_utils::egress::{OutboundUrlError, validate_outbound_url};
use rustfs_utils::http::{
AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_MODE,
AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_STORAGE_CLASS, AMZ_WEBSITE_REDIRECT_LOCATION, is_amz_header, is_minio_header,
is_rustfs_header, is_standard_header, is_storageclass_header,
AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_OBJECT_TAGGING_LOWER, AMZ_STORAGE_CLASS, AMZ_WEBSITE_REDIRECT_LOCATION, is_amz_header,
is_minio_header, is_rustfs_header, is_standard_header, is_storageclass_header,
};
use rustfs_utils::http::{
SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_PROXY_REQUEST,
@@ -1774,6 +1774,22 @@ impl PutObjectOptions {
Self::insert_checked(&mut header, AMZ_BUCKET_REPLICATION_STATUS, self.internal.replication_status.as_str());
}
// MinIO PutObjectOptions.Header parity: object tags travel on the
// `x-amz-tagging` header (form-urlencoded). `replication_put_object_options`
// fills `user_tags` from the source version; without this header the
// whole-object transport delivered a tagless replica, so tag edits
// never reached the peer and the receiver-side LWW comparison
// (rustfs/backlog#1953) had nothing to judge.
if !self.user_tags.is_empty() {
let mut tags: Vec<(&String, &String)> = self.user_tags.iter().collect();
tags.sort();
let mut encoded = url::form_urlencoded::Serializer::new(String::new());
for (key, value) in tags {
encoded.append_pair(key, value);
}
Self::insert_checked(&mut header, AMZ_OBJECT_TAGGING_LOWER, &encoded.finish());
}
for (k, v) in &self.user_metadata {
let Ok(header_value) = HeaderValue::from_str(v) else {
warn!("skipping user metadata header with invalid value: {}", k);
@@ -3195,6 +3211,29 @@ mod tests {
}
}
#[test]
fn put_object_headers_carry_user_tags_on_x_amz_tagging() {
// rustfs/backlog#1953: tag edits replicate through the whole-object
// transport, so the source tags must travel on x-amz-tagging.
let mut opts = PutObjectOptions::default();
opts.user_tags.insert("owner".to_string(), "site a".to_string());
opts.user_tags.insert("env".to_string(), "prod".to_string());
let header = opts.header();
let tagging = header
.get(AMZ_OBJECT_TAGGING_LOWER)
.expect("user tags must be transported on x-amz-tagging")
.to_str()
.expect("tag header must be ASCII");
// Deterministic key order; values are form-urlencoded.
assert_eq!(tagging, "env=prod&owner=site+a");
assert!(
PutObjectOptions::default().header().get(AMZ_OBJECT_TAGGING_LOWER).is_none(),
"a tagless source must not send an empty x-amz-tagging header"
);
}
#[test]
fn put_object_headers_omit_unset_replication_timestamps() {
// UNIX_EPOCH means "never modified on the source"; sending it would
crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs
+70 -3
View File
@@ -3739,17 +3739,55 @@ impl ManualTransitionRunReport {
}
pub fn merge_scan_report_preserving_worker(&mut self, scan_report: &ManualTransitionRunReport) {
let previous = self.clone();
let resumed_after_checkpoint = previous.continuation_token.is_some() && scan_report.scanned < previous.scanned;
let mut tier_failure_by_reason = self.tier_failure_by_reason.clone();
for (reason, count) in &scan_report.tier_failure_by_reason {
let current = tier_failure_by_reason.get(reason).copied().unwrap_or_default();
tier_failure_by_reason.insert(*reason, current.max(*count));
let merged = if resumed_after_checkpoint {
current.saturating_add(*count)
} else {
current.max(*count)
};
tier_failure_by_reason.insert(*reason, merged);
}
let transition_completed = self.transition_completed;
let transition_failed = self.transition_failed;
*self = scan_report.clone();
if resumed_after_checkpoint {
self.scanned = previous.scanned.saturating_add(scan_report.scanned);
self.eligible = previous.eligible.saturating_add(scan_report.eligible);
self.enqueued = previous.enqueued.saturating_add(scan_report.enqueued);
self.dry_run_eligible = previous.dry_run_eligible.saturating_add(scan_report.dry_run_eligible);
self.skipped_not_transition = previous
.skipped_not_transition
.saturating_add(scan_report.skipped_not_transition);
self.skipped_tier = previous.skipped_tier.saturating_add(scan_report.skipped_tier);
self.skipped_delete_marker = previous
.skipped_delete_marker
.saturating_add(scan_report.skipped_delete_marker);
self.skipped_directory = previous.skipped_directory.saturating_add(scan_report.skipped_directory);
self.skipped_replication = previous.skipped_replication.saturating_add(scan_report.skipped_replication);
self.skipped_already_transitioned = previous
.skipped_already_transitioned
.saturating_add(scan_report.skipped_already_transitioned);
self.skipped_already_in_flight = previous
.skipped_already_in_flight
.saturating_add(scan_report.skipped_already_in_flight);
self.skipped_queue_full = previous.skipped_queue_full.saturating_add(scan_report.skipped_queue_full);
self.skipped_queue_closed = previous.skipped_queue_closed.saturating_add(scan_report.skipped_queue_closed);
self.skipped_queue_timeout = previous
.skipped_queue_timeout
.saturating_add(scan_report.skipped_queue_timeout);
self.tier_failure = previous.tier_failure.saturating_add(scan_report.tier_failure);
}
self.lifecycle_config_found = previous.lifecycle_config_found || scan_report.lifecycle_config_found;
self.truncated_by_limit = previous.truncated_by_limit || scan_report.truncated_by_limit;
self.truncated_by_duration = previous.truncated_by_duration || scan_report.truncated_by_duration;
self.cancelled = previous.cancelled || scan_report.cancelled;
self.transition_completed = transition_completed;
self.transition_failed = transition_failed;
self.tier_failure = scan_report.tier_failure.saturating_add(transition_failed);
self.tier_failure = self.tier_failure.saturating_add(transition_failed);
self.tier_failure_by_reason = tier_failure_by_reason;
}
@@ -3765,7 +3803,10 @@ struct ManualTransitionContinuationToken {
version_marker: Option<String>,
}
fn encode_manual_transition_continuation_token(marker: Option<String>, version_marker: Option<String>) -> Option<String> {
pub(super) fn encode_manual_transition_continuation_token(
marker: Option<String>,
version_marker: Option<String>,
) -> Option<String> {
if marker.is_none() && version_marker.is_none() {
return None;
}
@@ -9136,6 +9177,7 @@ mod tests {
assert_eq!(loaded.report.scanned, 37);
assert_eq!(loaded.report.eligible, 11);
assert_eq!(loaded.report.enqueued, 5);
assert_eq!(loaded.cursor_revision, Some(37));
assert!(loaded.lease_expires_at_unix_nanos > 0);
let token = loaded
.report
@@ -9154,6 +9196,30 @@ mod tests {
assert_eq!(admission.lease_id, loaded.lease_id);
assert_eq!(admission.lease_expires_at_unix_nanos, loaded.lease_expires_at_unix_nanos);
let mut same_marker_report = report.clone();
same_marker_report.scanned += 1;
persist_manual_transition_page_checkpoint(
&checkpoint_options,
&same_marker_report,
Some("logs/page-end".to_string()),
Some("opaque-next-version".to_string()),
)
.await
.expect("same-marker version checkpoint should persist through the durable progress sink");
let same_marker_checkpointed = load_manual_transition_job_record(ecstore.clone(), job_id)
.await
.expect("same-marker version checkpoint should reload");
assert_eq!(same_marker_checkpointed.cursor_revision, Some(38));
let (_, version_marker) = decode_manual_transition_continuation_token(
same_marker_checkpointed
.report
.continuation_token
.as_deref()
.expect("same-marker version checkpoint should persist a cursor"),
)
.expect("same-marker version cursor should decode");
assert_eq!(version_marker.as_deref(), Some("opaque-next-version"));
create_test_bucket(&ecstore, &bucket).await;
let lifecycle_xml = format!(
r#"<?xml version="1.0" encoding="UTF-8"?>
@@ -9210,6 +9276,7 @@ mod tests {
assert_eq!(checkpointed.report.scanned, 1000);
assert_eq!(checkpointed.report.eligible, 1000);
assert_eq!(checkpointed.report.dry_run_eligible, 1000);
assert_eq!(checkpointed.cursor_revision, Some(1000));
let token = checkpointed
.report
.continuation_token
crates/ecstore/src/bucket/lifecycle/durable_namespace.rs
+1161
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs
+217 -25
View File
@@ -20,10 +20,16 @@ use serde::{Deserialize, Serialize};
use time::OffsetDateTime;
use uuid::Uuid;
#[cfg(test)]
use crate::bucket::lifecycle::bucket_lifecycle_ops::encode_manual_transition_continuation_token;
use crate::bucket::lifecycle::bucket_lifecycle_ops::{
ManualTransitionQueueSnapshot, ManualTransitionRunOptions, ManualTransitionRunReport,
};
use crate::bucket::lifecycle::config_boundary;
use crate::bucket::lifecycle::durable_namespace::{
MANUAL_TRANSITION_JOB_NAMESPACE, MANUAL_TRANSITION_SCOPE_NAMESPACE, MANUAL_TRANSITION_TASK_NAMESPACE,
MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE,
};
use crate::disk::RUSTFS_META_BUCKET;
use crate::error::{Error, Result as EcstoreResult};
use crate::object_api::ObjectOptions;
@@ -34,10 +40,10 @@ use crate::store::ECStore;
pub const MANUAL_TRANSITION_JOB_SCHEMA: &str = "rustfs-manual-transition-job-v1";
pub const MANUAL_TRANSITION_TASK_SCHEMA: &str = "rustfs-manual-transition-task-v1";
pub const MANUAL_TRANSITION_WORKER_RESULT_SCHEMA: &str = "rustfs-manual-transition-worker-result-v1";
pub const MANUAL_TRANSITION_JOB_RECORD_PREFIX: &str = "ilm/manual-transition/jobs";
pub const MANUAL_TRANSITION_SCOPE_RECORD_PREFIX: &str = "ilm/manual-transition/scopes";
pub const MANUAL_TRANSITION_TASK_PREFIX: &str = "ilm/manual-transition/tasks";
pub const MANUAL_TRANSITION_WORKER_RESULT_PREFIX: &str = "ilm/manual-transition/results";
pub const MANUAL_TRANSITION_JOB_RECORD_PREFIX: &str = MANUAL_TRANSITION_JOB_NAMESPACE.prefix;
pub const MANUAL_TRANSITION_SCOPE_RECORD_PREFIX: &str = MANUAL_TRANSITION_SCOPE_NAMESPACE.prefix;
pub const MANUAL_TRANSITION_TASK_PREFIX: &str = MANUAL_TRANSITION_TASK_NAMESPACE.prefix;
pub const MANUAL_TRANSITION_WORKER_RESULT_PREFIX: &str = MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE.prefix;
pub const MAX_MANUAL_TRANSITION_JOB_RECORD_SIZE: usize = 64 * 1024;
pub const MAX_MANUAL_TRANSITION_TASK_RECORD_SIZE: usize = 16 * 1024;
pub const MAX_MANUAL_TRANSITION_WORKER_RESULT_RECORD_SIZE: usize = 8 * 1024;
@@ -195,6 +201,8 @@ pub struct ManualTransitionJobRecord {
pub updated_at_unix_nanos: i128,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub completed_at_unix_nanos: Option<i128>,
#[serde(default, skip_serializing)]
pub cursor_revision: Option<u64>,
pub report: ManualTransitionRunReport,
pub queue_snapshot: ManualTransitionQueueSnapshot,
#[serde(default, skip_serializing_if = "Option::is_none")]
@@ -224,6 +232,7 @@ impl ManualTransitionJobRecord {
created_at_unix_nanos: now,
updated_at_unix_nanos: now,
completed_at_unix_nanos: None,
cursor_revision: None,
report: ManualTransitionRunReport {
bucket: bucket.to_string(),
prefix: options.prefix.clone(),
@@ -238,7 +247,7 @@ impl ManualTransitionJobRecord {
pub fn complete(&mut self, report: ManualTransitionRunReport, queue_snapshot: ManualTransitionQueueSnapshot) {
self.scan_completed = true;
self.report.merge_scan_report_preserving_worker(&report);
self.merge_scan_report(&report);
self.queue_snapshot = queue_snapshot;
self.error = None;
self.mark_terminal_if_worker_drained();
@@ -316,7 +325,7 @@ impl ManualTransitionJobRecord {
}
}
self.queue_snapshot = queue_snapshot;
self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
self.advance_updated_at();
self.mark_terminal_if_worker_drained();
}
@@ -359,14 +368,14 @@ impl ManualTransitionJobRecord {
self.report.tier_failure = scan_tier_failure.saturating_add(transition_failed);
self.report.tier_failure_by_reason = scan_tier_failure_by_reason;
self.queue_snapshot = queue_snapshot;
self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
self.advance_updated_at();
self.mark_terminal_if_worker_drained();
true
}
pub fn mark_cancel_requested(&mut self) {
self.cancel_requested = true;
self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
self.advance_updated_at();
}
pub fn claim_recovery_lease(&mut self, owner_id: impl Into<String>, queue_snapshot: ManualTransitionQueueSnapshot) {
@@ -380,7 +389,7 @@ impl ManualTransitionJobRecord {
pub fn abandon_recovery_lease(&mut self, lease_id: Uuid) {
if self.state == ManualTransitionJobState::Running && self.lease_id == lease_id {
self.lease_expires_at_unix_nanos = 0;
self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
self.advance_updated_at();
}
}
@@ -398,7 +407,7 @@ impl ManualTransitionJobRecord {
pub fn renew_lease(&mut self, queue_snapshot: ManualTransitionQueueSnapshot) {
let now = OffsetDateTime::now_utc().unix_timestamp_nanos();
self.updated_at_unix_nanos = now;
self.updated_at_unix_nanos = self.updated_at_unix_nanos.saturating_add(1).max(now);
self.lease_expires_at_unix_nanos = manual_transition_job_lease_expires_at(now);
self.queue_snapshot = queue_snapshot;
}
@@ -438,11 +447,16 @@ impl ManualTransitionJobRecord {
pub fn update_running_progress(&mut self, report: ManualTransitionRunReport, queue_snapshot: ManualTransitionQueueSnapshot) {
if self.state == ManualTransitionJobState::Running {
self.report.merge_scan_report_preserving_worker(&report);
self.merge_scan_report(&report);
self.renew_lease(queue_snapshot);
}
}
fn merge_scan_report(&mut self, report: &ManualTransitionRunReport) {
self.report.merge_scan_report_preserving_worker(report);
self.cursor_revision = manual_transition_cursor_revision(&self.report);
}
pub fn mark_unknown_if_unowned(&mut self) {
if self.state == ManualTransitionJobState::Running {
self.state = ManualTransitionJobState::Unknown;
@@ -463,9 +477,13 @@ impl ManualTransitionJobRecord {
}
fn mark_updated_terminal(&mut self) {
self.advance_updated_at();
self.completed_at_unix_nanos = Some(self.updated_at_unix_nanos);
}
fn advance_updated_at(&mut self) {
let now = OffsetDateTime::now_utc().unix_timestamp_nanos();
self.updated_at_unix_nanos = now;
self.completed_at_unix_nanos = Some(now);
self.updated_at_unix_nanos = self.updated_at_unix_nanos.saturating_add(1).max(now);
}
fn mark_terminal_if_worker_drained(&mut self) {
@@ -543,6 +561,7 @@ impl ManualTransitionJobRecord {
if job.state == ManualTransitionJobState::Cancelled && job.cancel_requested {
job.report.cancelled = true;
}
job.cursor_revision = manual_transition_cursor_revision(&job.report);
job.validate()?;
Ok(job)
}
@@ -1109,7 +1128,8 @@ pub fn manual_transition_scope_record_object_name(scope_key: &str) -> Result<Str
pub async fn save_manual_transition_job_record(api: Arc<ECStore>, job: &ManualTransitionJobRecord) -> EcstoreResult<()> {
let object = manual_transition_job_record_object_name(job.job_id).map_err(manual_transition_job_store_error)?;
let data = job.encode().map_err(manual_transition_job_store_error)?;
config_boundary::save_config(api, &object, data).await
config_boundary::save_config(api.clone(), &object, data.clone()).await?;
api.record_durable_ilm_decommission_progress(&object, &data).await
}
pub async fn load_manual_transition_job_record(api: Arc<ECStore>, job_id: Uuid) -> EcstoreResult<ManualTransitionJobRecord> {
@@ -1142,9 +1162,9 @@ pub async fn save_manual_transition_job_record_if_current(
let object = manual_transition_job_record_object_name(job.job_id).map_err(manual_transition_job_store_error)?;
let data = job.encode().map_err(manual_transition_job_store_error)?;
config_boundary::save_config_with_opts_quiet(
api,
api.clone(),
&object,
data,
data.clone(),
&ObjectOptions {
max_parity: true,
http_preconditions: Some(HTTPPreconditions {
@@ -1154,7 +1174,8 @@ pub async fn save_manual_transition_job_record_if_current(
..Default::default()
},
)
.await
.await?;
api.record_durable_ilm_decommission_progress(&object, &data).await
}
/// Applies a job-record mutation with optimistic concurrency control.
@@ -1592,9 +1613,9 @@ pub async fn save_manual_transition_scope_admission_if_absent(
let object = manual_transition_scope_record_object_name(&admission.scope_key).map_err(manual_transition_job_store_error)?;
let data = serde_json::to_vec(admission).map_err(Error::other)?;
config_boundary::save_config_with_opts(
api,
api.clone(),
&object,
data,
data.clone(),
&ObjectOptions {
max_parity: true,
http_preconditions: Some(HTTPPreconditions {
@@ -1604,7 +1625,8 @@ pub async fn save_manual_transition_scope_admission_if_absent(
..Default::default()
},
)
.await
.await?;
api.record_durable_ilm_decommission_progress(&object, &data).await
}
pub async fn load_manual_transition_scope_admission(
@@ -1642,9 +1664,9 @@ pub async fn save_manual_transition_scope_admission_if_current(
let object = manual_transition_scope_record_object_name(&admission.scope_key).map_err(manual_transition_job_store_error)?;
let data = serde_json::to_vec(admission).map_err(Error::other)?;
match config_boundary::save_config_with_opts(
api,
api.clone(),
&object,
data,
data.clone(),
&ObjectOptions {
max_parity: true,
http_preconditions: Some(HTTPPreconditions {
@@ -1660,7 +1682,8 @@ pub async fn save_manual_transition_scope_admission_if_current(
Err(Error::PreconditionFailed)
}
result => result,
}
}?;
api.record_durable_ilm_decommission_progress(&object, &data).await
}
pub async fn claim_manual_transition_scope_admission(
@@ -1953,13 +1976,15 @@ pub async fn delete_manual_transition_scope_admission_if_current(
job_id: Uuid,
lease_id: Uuid,
) -> EcstoreResult<bool> {
let etag = match load_manual_transition_scope_admission_with_etag(api.clone(), scope_key).await {
Ok((admission, etag)) if admission.job_id == job_id && admission.lease_id == lease_id => etag,
let (admission, etag) = match load_manual_transition_scope_admission_with_etag(api.clone(), scope_key).await {
Ok((admission, etag)) if admission.job_id == job_id && admission.lease_id == lease_id => (admission, etag),
Ok(_) => return Ok(false),
Err(Error::ConfigNotFound) => return Ok(true),
Err(err) => return Err(err),
};
let object = manual_transition_scope_record_object_name(scope_key).map_err(manual_transition_job_store_error)?;
let data = serde_json::to_vec(&admission).map_err(Error::other)?;
api.record_durable_ilm_decommission_terminal(&object, &data).await?;
match config_boundary::delete_config_if_match(api, &object, &etag).await {
Ok(()) | Err(Error::ConfigNotFound) => Ok(true),
Err(Error::PreconditionFailed) => Ok(false),
@@ -1971,6 +1996,11 @@ fn manual_transition_job_store_error(err: ManualTransitionJobError) -> Error {
Error::other(err)
}
fn manual_transition_cursor_revision(report: &ManualTransitionRunReport) -> Option<u64> {
report.continuation_token.as_ref()?;
(report.scanned > 0).then_some(report.scanned)
}
pub fn manual_transition_scope_admission_lease_expired(admission: &ManualTransitionScopeAdmission) -> bool {
OffsetDateTime::now_utc().unix_timestamp_nanos() > admission.lease_expires_at_unix_nanos
}
@@ -2210,6 +2240,76 @@ mod tests {
);
}
#[test]
fn manual_transition_job_scan_progress_accumulates_resumed_checkpoint_counters() {
let options = ManualTransitionRunOptions::default();
let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER);
let first_token =
encode_manual_transition_continuation_token(Some("logs/page-a".to_string()), Some("version-a".to_string()));
record.update_running_progress(
ManualTransitionRunReport {
bucket: "bucket".to_string(),
lifecycle_config_found: true,
scanned: 1000,
eligible: 800,
enqueued: 50,
dry_run_eligible: 10,
skipped_not_transition: 2,
skipped_tier: 3,
skipped_delete_marker: 4,
skipped_directory: 5,
skipped_replication: 6,
skipped_already_transitioned: 7,
skipped_already_in_flight: 8,
skipped_queue_full: 9,
skipped_queue_closed: 10,
skipped_queue_timeout: 11,
tier_failure: 12,
truncated_by_duration: true,
continuation_token: first_token,
..Default::default()
},
ManualTransitionQueueSnapshot::default(),
);
let next_token =
encode_manual_transition_continuation_token(Some("logs/page-b".to_string()), Some("version-b".to_string()));
record.update_running_progress(
ManualTransitionRunReport {
bucket: "bucket".to_string(),
scanned: 3,
eligible: 2,
enqueued: 1,
dry_run_eligible: 1,
skipped_not_transition: 1,
tier_failure: 1,
continuation_token: next_token.clone(),
..Default::default()
},
ManualTransitionQueueSnapshot::default(),
);
assert_eq!(record.report.scanned, 1003);
assert_eq!(record.report.eligible, 802);
assert_eq!(record.report.enqueued, 51);
assert_eq!(record.report.dry_run_eligible, 11);
assert_eq!(record.report.skipped_not_transition, 3);
assert_eq!(record.report.skipped_tier, 3);
assert_eq!(record.report.skipped_delete_marker, 4);
assert_eq!(record.report.skipped_directory, 5);
assert_eq!(record.report.skipped_replication, 6);
assert_eq!(record.report.skipped_already_transitioned, 7);
assert_eq!(record.report.skipped_already_in_flight, 8);
assert_eq!(record.report.skipped_queue_full, 9);
assert_eq!(record.report.skipped_queue_closed, 10);
assert_eq!(record.report.skipped_queue_timeout, 11);
assert_eq!(record.report.tier_failure, 13);
assert!(record.report.lifecycle_config_found);
assert!(record.report.truncated_by_duration);
assert_eq!(record.report.continuation_token, next_token);
assert_eq!(record.cursor_revision, Some(1003));
}
#[test]
fn manual_transition_job_apply_worker_result_counts_preserves_existing_failure_reasons() {
let options = ManualTransitionRunOptions::default();
@@ -2577,6 +2677,98 @@ mod tests {
assert!(decoded.report.tier_failure_by_reason.is_empty());
}
#[test]
fn manual_transition_job_record_derives_revision_from_legacy_cursor() {
let options = ManualTransitionRunOptions::default();
let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER);
let continuation_token =
encode_manual_transition_continuation_token(Some("logs/page-a".to_string()), Some("version-a".to_string()));
record.update_running_progress(
ManualTransitionRunReport {
bucket: "bucket".to_string(),
scanned: 9,
continuation_token,
..Default::default()
},
ManualTransitionQueueSnapshot::default(),
);
let encoded = record.encode().expect("job record should encode");
let mut value: serde_json::Value = serde_json::from_slice(&encoded).expect("encoded job should be json");
value["job"]
.as_object_mut()
.expect("job should be object")
.remove("cursor_revision");
let record_bytes = serde_json::to_vec(&value["job"]).expect("legacy job should encode");
value["content_sha256"] = serde_json::Value::String(hex_sha256(&record_bytes, ToOwned::to_owned));
let legacy = serde_json::to_vec(&value).expect("legacy envelope should encode");
let decoded = ManualTransitionJobRecord::decode(record.job_id, &legacy).expect("legacy job should decode");
assert_eq!(decoded.cursor_revision, Some(9));
}
#[test]
fn manual_transition_job_record_omits_cursor_revision_for_old_readers() {
#[allow(dead_code)]
#[derive(serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct LegacyPersistedManualTransitionJobRecord {
schema: String,
content_sha256: String,
job: LegacyManualTransitionJobRecord,
}
#[allow(dead_code)]
#[derive(serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct LegacyManualTransitionJobRecord {
job_id: Uuid,
scope_key: String,
bucket: String,
prefix: String,
tier: Option<String>,
dry_run: bool,
max_objects: Option<u64>,
max_duration: Option<std::time::Duration>,
owner_id: String,
lease_id: Uuid,
lease_expires_at_unix_nanos: i128,
state: ManualTransitionJobState,
scan_completed: bool,
cancel_requested: bool,
created_at_unix_nanos: i128,
updated_at_unix_nanos: i128,
completed_at_unix_nanos: Option<i128>,
report: ManualTransitionRunReport,
queue_snapshot: ManualTransitionQueueSnapshot,
error: Option<String>,
}
let options = ManualTransitionRunOptions::default();
let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER);
let continuation_token =
encode_manual_transition_continuation_token(Some("logs/page-a".to_string()), Some("version-a".to_string()));
record.update_running_progress(
ManualTransitionRunReport {
bucket: "bucket".to_string(),
scanned: 7,
continuation_token: continuation_token.clone(),
..Default::default()
},
ManualTransitionQueueSnapshot::default(),
);
assert_eq!(record.cursor_revision, Some(7));
let encoded = record.encode().expect("job record should encode");
let value: serde_json::Value = serde_json::from_slice(&encoded).expect("encoded job should be json");
assert!(value["job"].get("cursor_revision").is_none());
let legacy: LegacyPersistedManualTransitionJobRecord =
serde_json::from_slice(&encoded).expect("old reader should accept new job record");
assert_eq!(legacy.job.job_id, record.job_id);
assert_eq!(legacy.job.report.continuation_token, continuation_token);
}
#[test]
fn manual_transition_job_record_rejects_unknown_report_fields() {
let options = ManualTransitionRunOptions::default();
crates/ecstore/src/bucket/lifecycle/mod.rs
+6
View File
@@ -16,6 +16,7 @@ pub mod bucket_lifecycle_audit;
pub mod bucket_lifecycle_ops;
mod config_boundary;
pub mod core;
mod durable_namespace;
pub mod evaluator;
pub mod manual_transition_job;
mod metadata_boundary;
@@ -31,3 +32,8 @@ pub mod tier_free_version_recovery;
pub mod tier_last_day_stats;
pub mod tier_sweeper;
pub mod transition_transaction;
pub(crate) use durable_namespace::{
DurableIlmRecordCheckpoint, ILM_META_PREFIX, ValidatedDurableIlmRecord, classify_durable_ilm_record,
validate_durable_ilm_record,
};
crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs
+17 -1
View File
@@ -20,6 +20,7 @@ use tokio_util::sync::CancellationToken;
use tracing::{debug, warn};
use crate::bucket::lifecycle::config_boundary;
use crate::bucket::lifecycle::durable_namespace::TIER_DELETE_JOURNAL_NAMESPACE;
use crate::bucket::lifecycle::runtime_boundary;
use crate::bucket::lifecycle::tier_sweeper::{
Jentry, TierDeleteJournalState, TierDeleteSourceIdentity,
@@ -49,7 +50,7 @@ const TIER_DELETE_JOURNAL_VERSION: u8 = 2;
const TIER_DELETE_JOURNAL_EXACT_VERSION: u8 = 3;
const TIER_DELETE_JOURNAL_STATE_VERSION: u8 = 4;
const TIER_DELETE_JOURNAL_TRANSACTION_VERSION: u8 = 5;
pub(crate) const TIER_DELETE_JOURNAL_PREFIX: &str = "ilm/tier-delete-journal/";
pub(crate) const TIER_DELETE_JOURNAL_PREFIX: &str = TIER_DELETE_JOURNAL_NAMESPACE.prefix;
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
@@ -432,6 +433,21 @@ async fn process_committed_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jen
)
.await?;
}
let path = tier_delete_journal_object_name(je);
let data = encode_tier_delete_journal_entry(je).map_err(std::io::Error::other)?;
let target_pool_indices = api
.record_durable_ilm_decommission_terminal_target_pools(&path, &data)
.await
.map_err(std::io::Error::other)?;
if let Some(target_pool_indices) = target_pool_indices {
for target_pool_idx in target_pool_indices {
match config_boundary::delete_config(api.pools[target_pool_idx].clone(), &path).await {
Ok(()) | Err(Error::ConfigNotFound) => {}
Err(err) => return Err(std::io::Error::other(err)),
}
}
return Ok(());
}
remove_tier_delete_journal_entry(api, je).await
}
crates/ecstore/src/bucket/lifecycle/transition_transaction.rs
+22 -14
View File
@@ -21,6 +21,7 @@ use tracing::{debug, warn};
use uuid::Uuid;
use crate::bucket::lifecycle::config_boundary;
use crate::bucket::lifecycle::durable_namespace::TRANSITION_TRANSACTION_NAMESPACE;
use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
use crate::bucket::lifecycle::tier_sweeper::{
delete_confirmed_transition_candidate_exact_with_lease_idempotent,
@@ -42,7 +43,7 @@ const TRANSITION_TRANSACTION_RECOVERY_INTERVAL: Duration = Duration::from_secs(6
const TRANSITION_TRANSACTION_RECOVERY_TIMEOUT: Duration = Duration::from_secs(300);
pub const TRANSITION_TRANSACTION_SCHEMA: &str = "rustfs-transition-transaction-v1";
pub const TRANSITION_TRANSACTION_PREFIX: &str = "ilm/transition-transactions";
pub const TRANSITION_TRANSACTION_RECORD_PREFIX: &str = "ilm/transition-transactions/records";
pub const TRANSITION_TRANSACTION_RECORD_PREFIX: &str = TRANSITION_TRANSACTION_NAMESPACE.prefix;
pub const MAX_TRANSITION_TRANSACTION_SIZE: usize = 64 * 1024;
pub type Result<T> = std::result::Result<T, TransitionTransactionError>;
@@ -584,7 +585,8 @@ pub(crate) async fn save_transition_transaction_record(
let object =
transition_transaction_record_object_name(transaction.transaction_id).map_err(transition_transaction_store_error)?;
let data = transaction.encode().map_err(transition_transaction_store_error)?;
config_boundary::save_config(api, &object, data).await
config_boundary::save_config(api.clone(), &object, data.clone()).await?;
api.record_durable_ilm_decommission_progress(&object, &data).await
}
pub(crate) async fn load_transition_transaction_record(
@@ -596,8 +598,14 @@ pub(crate) async fn load_transition_transaction_record(
TransitionTransaction::decode(transaction_id, &data).map_err(transition_transaction_store_error)
}
pub(crate) async fn delete_transition_transaction_record(api: Arc<ECStore>, transaction_id: Uuid) -> EcstoreResult<()> {
let object = transition_transaction_record_object_name(transaction_id).map_err(transition_transaction_store_error)?;
pub(crate) async fn delete_transition_transaction_record(
api: Arc<ECStore>,
transaction: &TransitionTransaction,
) -> EcstoreResult<()> {
let object =
transition_transaction_record_object_name(transaction.transaction_id).map_err(transition_transaction_store_error)?;
let data = transaction.encode().map_err(transition_transaction_store_error)?;
api.record_durable_ilm_decommission_terminal(&object, &data).await?;
match config_boundary::delete_config(api, &object).await {
Ok(()) | Err(Error::ConfigNotFound) => Ok(()),
Err(err) => Err(err),
@@ -813,7 +821,7 @@ pub async fn finalize_missing_transition_transaction_for_operator(
if probe != TransitionOperatorProbe::Missing {
return Err(TransitionOperatorError::CandidateNotMissing(probe));
}
delete_transition_transaction_record(api, transaction_id)
delete_transition_transaction_record(api, &transaction)
.await
.map_err(TransitionOperatorError::Store)
}
@@ -849,22 +857,22 @@ pub async fn process_transition_transaction_record(
match transaction.state {
TransitionTransactionState::Uploaded => {
delete_transition_remote_candidate(api.clone(), transaction).await?;
delete_transition_transaction_record(api, transaction.transaction_id).await?;
delete_transition_transaction_record(api, transaction).await?;
Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
}
TransitionTransactionState::CleanupPending => match local_commit_matches_transaction(api.clone(), transaction).await {
Ok(true) => {
delete_transition_transaction_record(api, transaction.transaction_id).await?;
delete_transition_transaction_record(api, transaction).await?;
Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
}
Ok(false) => {
delete_transition_remote_candidate(api.clone(), transaction).await?;
delete_transition_transaction_record(api, transaction.transaction_id).await?;
delete_transition_transaction_record(api, transaction).await?;
Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
}
Err(err) if transition_source_is_missing(&err) => {
delete_transition_remote_candidate(api.clone(), transaction).await?;
delete_transition_transaction_record(api, transaction.transaction_id).await?;
delete_transition_transaction_record(api, transaction).await?;
Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
}
Err(err) => Err(err),
@@ -872,7 +880,7 @@ pub async fn process_transition_transaction_record(
TransitionTransactionState::LocalCommitStarted => {
match local_commit_matches_transaction(api.clone(), transaction).await {
Ok(true) => {
delete_transition_transaction_record(api, transaction.transaction_id).await?;
delete_transition_transaction_record(api, transaction).await?;
Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
}
Ok(false) => Ok(TransitionTransactionRecoveryOutcome::Retained),
@@ -881,7 +889,7 @@ pub async fn process_transition_transaction_record(
}
}
TransitionTransactionState::AbortedNoRemote | TransitionTransactionState::Committed => {
delete_transition_transaction_record(api, transaction.transaction_id).await?;
delete_transition_transaction_record(api, transaction).await?;
Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
}
TransitionTransactionState::UploadOutcomeUnknown => recover_unknown_upload_outcome(api, transaction).await,
@@ -907,7 +915,7 @@ async fn recover_unknown_upload_outcome(
.map_err(Error::other)?
{
TransitionCandidateProbe::Missing => {
delete_transition_transaction_record(api, transaction.transaction_id).await?;
delete_transition_transaction_record(api, transaction).await?;
Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
}
TransitionCandidateProbe::UnversionedPresent => {
@@ -925,7 +933,7 @@ async fn recover_unknown_upload_outcome(
)
.await
.map_err(Error::other)?;
delete_transition_transaction_record(api, transaction.transaction_id).await?;
delete_transition_transaction_record(api, transaction).await?;
Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
}
TransitionCandidateProbe::VersionedPresent(version_id) => {
@@ -958,7 +966,7 @@ async fn cleanup_recovered_unknown_upload_candidate(
.map_err(transition_transaction_store_error)?;
save_transition_transaction_record(api.clone(), &cleanup).await?;
delete_transition_remote_candidate(api.clone(), &cleanup).await?;
delete_transition_transaction_record(api, cleanup.transaction_id).await?;
delete_transition_transaction_record(api, &cleanup).await?;
Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
}
crates/ecstore/src/bucket/replication/mod.rs
+8 -5
View File
@@ -44,11 +44,14 @@ mod replication_versioning_boundary;
mod runtime_boundary;
pub use replication_config_boundary::{
ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationTargetValidationError,
invalid_replication_config_status_field, replication_target_arns, should_remove_replication_target,
unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
ObjectOpts, OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS,
REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS,
REPLICATION_WRITABLE_FIELDS, ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationTargetValidationError,
assign_site_replication_rule_priorities, invalid_replication_config_status_field, is_site_replication_role,
is_site_replication_rule, merge_incoming_replication_config, merge_user_replication_config,
replication_target_arn_deployment_id, replication_target_arns, should_remove_replication_target,
site_replication_rule_deployment_id, unsupported_replication_config_field, validate_replication_config_structure,
validate_replication_config_target_arns,
};
pub(crate) use replication_filemeta_boundary::version_purge_statuses_map;
pub use replication_filemeta_boundary::{
crates/ecstore/src/bucket/replication/replication_config_boundary.rs
+8 -5
View File
@@ -13,9 +13,12 @@
// limitations under the License.
pub use rustfs_replication::{
ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationRuleExt, ReplicationTargetValidationError,
invalid_replication_config_status_field, replication_target_arns, should_remove_replication_target,
unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
ObjectOpts, OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS,
REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS,
REPLICATION_WRITABLE_FIELDS, ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationRuleExt,
ReplicationTargetValidationError, assign_site_replication_rule_priorities, invalid_replication_config_status_field,
is_site_replication_role, is_site_replication_rule, merge_incoming_replication_config, merge_user_replication_config,
replication_target_arn_deployment_id, replication_target_arns, should_remove_replication_target,
site_replication_rule_deployment_id, unsupported_replication_config_field, validate_replication_config_structure,
validate_replication_config_target_arns,
};
crates/ecstore/src/bucket/replication/replication_pool.rs
+24
View File
@@ -4194,6 +4194,30 @@ mod tests {
assert_eq!(ri.checksum, Some(checksum));
}
#[test]
fn metadata_mrf_roundtrip_preserves_tags_and_admitted_targets() {
let target = "arn:rustfs:replication:target-a";
let object = ObjectInfo {
bucket: "source".to_string(),
name: "object".to_string(),
version_id: Some(Uuid::new_v4()),
user_tags: Arc::new("owner=a3".to_string()),
..Default::default()
};
let live =
replicate_object_info_from_object_info(object.clone(), test_replicate_decision(&[target]), ReplicationType::Metadata);
let persisted = live.to_mrf_entry();
let encoded = encode_mrf_file(std::slice::from_ref(&persisted)).expect("metadata MRF entry should encode");
let decoded = decode_mrf_file(&encoded).expect("metadata MRF entry should decode");
assert_eq!(decoded[0].op, MrfOpKind::Metadata);
assert_eq!(decoded[0].target_arns, vec![target.to_string()]);
let replayed = admitted_mrf_replicate_object(object, &decoded[0], ReplicationType::Metadata);
assert_eq!(replayed.op_type, ReplicationType::Metadata);
assert_eq!(replayed.user_tags, "owner=a3");
assert_eq!(replayed.admitted_target_arns(), vec![target.to_string()]);
}
#[tokio::test]
async fn mrf_save_admission_waits_for_capacity_instead_of_dropping() {
let (tx, mut rx) = mpsc::channel(1);
crates/ecstore/src/bucket/replication/replication_resyncer.rs
+137 -3
View File
@@ -76,7 +76,8 @@ use metrics::counter;
use rmp_serde;
use rustfs_s3_types::EventName;
use rustfs_utils::http::{
AMZ_TAGGING_DIRECTIVE, SUFFIX_REPLICATION_RESET, SUFFIX_REPLICATION_STATUS, has_internal_suffix, insert_str,
AMZ_BUCKET_REPLICATION_STATUS, AMZ_TAGGING_DIRECTIVE, SUFFIX_REPLICATION_RESET, SUFFIX_REPLICATION_STATUS,
has_internal_suffix, insert_str,
};
use rustfs_utils::{DEFAULT_SIP_HASH_KEY, get_env_usize, sip_hash};
#[cfg(test)]
@@ -174,6 +175,14 @@ fn has_raw_status(err: &SdkError<HeadObjectError>, status: u16) -> bool {
err.raw_response().is_some_and(|r| r.status().as_u16() == status)
}
fn metadata_requires_existing_target(op_type: ReplicationType, object_info: &ObjectInfo) -> bool {
op_type == ReplicationType::Metadata
&& object_info
.user_defined
.get(AMZ_BUCKET_REPLICATION_STATUS)
.is_some_and(|status| status.eq_ignore_ascii_case(ReplicationStatusType::Replica.as_str()))
}
const METRIC_VERSION_IDENTITY_DRIFT_TOTAL: &str = "rustfs_replication_version_identity_drift_total";
/// Targets that already produced a version-identity-drift warning this
@@ -3494,6 +3503,7 @@ async fn resolve_replicate_all_action(
start_time,
ssec_audit_required,
} = ctx;
let require_existing_target = metadata_requires_existing_target(roi.op_type, &object_info);
let replication_action;
match head_object_for_worker(tgt_client.as_ref(), &tgt_client.bucket, object, roi.version_id.map(|v| v.to_string())).await {
Ok(oi) => {
@@ -3555,7 +3565,13 @@ async fn resolve_replicate_all_action(
// Version-ID format mismatch: retry without versionId and compare ETags.
match head_object_fallback(tgt_client, object).await {
Ok(Some(oi)) => {
replication_action = if replication_etags_match(object_info.etag.as_deref(), oi.e_tag.as_deref()) {
let etags_match = replication_etags_match(object_info.etag.as_deref(), oi.e_tag.as_deref());
if require_existing_target && !etags_match {
rinfo.error = Some("replica metadata target does not contain matching object data".to_string());
rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
return None;
}
replication_action = if etags_match {
if ssec_audit_required
&& !settle_ssec_passthrough_evidence(&oi, tgt_client, bucket, object, rinfo).await
{
@@ -3568,6 +3584,11 @@ async fn resolve_replicate_all_action(
};
}
Ok(None) => {
if require_existing_target {
rinfo.error = Some("replica metadata target does not contain this object version".to_string());
rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
return None;
}
replication_action = ReplicationAction::All;
}
Err(e2) => {
@@ -3593,7 +3614,12 @@ async fn resolve_replicate_all_action(
return None;
}
}
} else if e.as_service_error().is_some_and(|se| se.is_not_found()) {
} else if e.as_service_error().is_some_and(|se| se.is_not_found()) || has_raw_status(&e, 404) {
if require_existing_target {
rinfo.error = Some("replica metadata target does not contain this object version".to_string());
rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
return None;
}
replication_action = ReplicationAction::All;
} else {
rinfo.error = Some(e.to_string());
@@ -3868,6 +3894,7 @@ async fn replicate_object_with_multipart<S: ReplicationObjectIO>(ctx: MultipartR
actual_size,
object_info.etag.clone().unwrap_or_default(),
object_info.mod_time,
&put_opts.internal,
),
)
.await
@@ -3921,6 +3948,113 @@ mod tests {
})
}
fn spawn_head_status_server(status: u16) -> (String, std::thread::JoinHandle<()>) {
use std::io::{Read, Write};
let listener = std::net::TcpListener::bind(("127.0.0.1", 0)).expect("test HTTP listener should bind");
let endpoint = format!("http://{}", listener.local_addr().expect("test HTTP listener should have an address"));
let handle = std::thread::spawn(move || {
let (mut stream, _) = listener.accept().expect("test HTTP client should connect");
let mut request = [0_u8; 8192];
let bytes_read = stream.read(&mut request).expect("test HTTP request should be read");
assert!(bytes_read > 0, "test HTTP request should not be empty");
assert!(request[..bytes_read].starts_with(b"HEAD "), "replication comparison must use HEAD");
write!(stream, "HTTP/1.1 {status} Test\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
.expect("test HTTP response should be written");
});
(endpoint, handle)
}
#[tokio::test]
async fn replica_metadata_missing_target_stops_before_full_put() {
let (endpoint, server) = spawn_head_status_server(404);
let target = test_target_client(endpoint);
let roi = ReplicateObjectInfo {
bucket: "source".to_string(),
name: "object".to_string(),
version_id: Some(Uuid::new_v4()),
op_type: ReplicationType::Metadata,
// Normal metadata writes replace REPLICA with per-target PENDING
// before constructing the worker request.
replication_status: ReplicationStatusType::Pending,
..Default::default()
};
let object_info = ObjectInfo {
bucket: roi.bucket.clone(),
name: roi.name.clone(),
version_id: roi.version_id,
etag: Some("source-etag".to_string()),
user_defined: Arc::new(HashMap::from([(
AMZ_BUCKET_REPLICATION_STATUS.to_string(),
ReplicationStatusType::Replica.as_str().to_string(),
)])),
..Default::default()
};
let mut rinfo = replicate_all_target_info(&roi, &target);
let action = resolve_replicate_all_action(
ReplicateAllActionContext {
roi: &roi,
tgt_client: &target,
bucket: &roi.bucket,
object: &roi.name,
start_time: OffsetDateTime::now_utc(),
ssec_audit_required: false,
},
object_info,
&mut rinfo,
)
.await;
assert!(action.is_none(), "missing replica metadata targets must not reach the payload PUT path");
assert_eq!(rinfo.replication_status, ReplicationStatusType::Failed);
assert_eq!(
rinfo.error.as_deref(),
Some("replica metadata target does not contain this object version")
);
server.join().expect("test HTTP server should finish");
}
#[tokio::test]
async fn source_metadata_missing_target_rebuilds_object() {
let (endpoint, server) = spawn_head_status_server(404);
let target = test_target_client(endpoint);
let roi = ReplicateObjectInfo {
bucket: "source".to_string(),
name: "object".to_string(),
version_id: Some(Uuid::new_v4()),
op_type: ReplicationType::Metadata,
replication_status: ReplicationStatusType::Pending,
..Default::default()
};
let object_info = ObjectInfo {
bucket: roi.bucket.clone(),
name: roi.name.clone(),
version_id: roi.version_id,
etag: Some("source-etag".to_string()),
..Default::default()
};
let mut rinfo = replicate_all_target_info(&roi, &target);
let action = resolve_replicate_all_action(
ReplicateAllActionContext {
roi: &roi,
tgt_client: &target,
bucket: &roi.bucket,
object: &roi.name,
start_time: OffsetDateTime::now_utc(),
ssec_audit_required: false,
},
object_info,
&mut rinfo,
)
.await;
assert!(matches!(action, Some((ReplicationAction::All, _))));
assert!(rinfo.error.is_none());
server.join().expect("test HTTP server should finish");
}
async fn register_test_target(target: &Arc<TargetClient>) {
ReplicationTargetStore::register_test_target(target).await;
}
crates/ecstore/src/bucket/replication/replication_target_boundary.rs
+30 -2
View File
@@ -472,6 +472,7 @@ pub(crate) fn replication_complete_multipart_options(
actual_size: String,
source_etag: String,
source_mtime: Option<OffsetDateTime>,
source_internal: &AdvancedPutOptions,
) -> PutObjectOptions {
let mut user_metadata = HashMap::new();
insert_header_map(&mut user_metadata, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, actual_size);
@@ -484,6 +485,14 @@ pub(crate) fn replication_complete_multipart_options(
// mtime must degrade to epoch so header() suppresses the header
// instead of asserting the replication time as the object's mtime.
source_mtime: source_mtime.unwrap_or(OffsetDateTime::UNIX_EPOCH),
// Carry the per-category LWW timestamps on the complete request as
// well: the receiver's CompleteMultipartUpload options builder
// parses the same headers, so the multipart transport gets the
// same receiver-side LWW as the single-PUT transport
// (rustfs/backlog#1953). Epoch values keep the headers suppressed.
tagging_timestamp: source_internal.tagging_timestamp,
retention_timestamp: source_internal.retention_timestamp,
legalhold_timestamp: source_internal.legalhold_timestamp,
replication_status: ReplicationStatusType::Replica,
replication_request: true,
..Default::default()
@@ -663,20 +672,39 @@ mod tests {
#[test]
fn replication_complete_multipart_options_sets_actual_size() {
let source_mtime = OffsetDateTime::from_unix_timestamp(1_716_170_000).expect("valid test timestamp");
let source_internal = AdvancedPutOptions {
tagging_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_100).expect("valid test timestamp"),
retention_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_200).expect("valid test timestamp"),
legalhold_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_300).expect("valid test timestamp"),
..Default::default()
};
let options = replication_complete_multipart_options(
"1024".to_string(),
"0123456789abcdef0123456789abcdef-3".to_string(),
Some(source_mtime),
&source_internal,
);
assert_eq!(options.internal.source_etag, "0123456789abcdef0123456789abcdef-3");
assert_eq!(options.internal.source_mtime, source_mtime);
// The complete request must carry the same per-category LWW timestamps
// as the initiate request; the receiver reads them from the complete
// headers (rustfs/backlog#1953).
assert_eq!(options.internal.tagging_timestamp, source_internal.tagging_timestamp);
assert_eq!(options.internal.retention_timestamp, source_internal.retention_timestamp);
assert_eq!(options.internal.legalhold_timestamp, source_internal.legalhold_timestamp);
// Absent source mtime must degrade to epoch (header suppressed), not
// the AdvancedPutOptions default of now_utc() — that default would
// stamp the replication time as the replica's mtime and break the
// multipart HEAD convergence.
let options_no_mtime = replication_complete_multipart_options("1024".to_string(), String::new(), None);
// multipart HEAD convergence. Unset category timestamps stay epoch so
// header() keeps suppressing them.
let options_no_mtime =
replication_complete_multipart_options("1024".to_string(), String::new(), None, &AdvancedPutOptions::default());
assert_eq!(options_no_mtime.internal.source_mtime.unix_timestamp(), 0);
assert_eq!(options_no_mtime.internal.tagging_timestamp.unix_timestamp(), 0);
assert_eq!(options_no_mtime.internal.retention_timestamp.unix_timestamp(), 0);
assert_eq!(options_no_mtime.internal.legalhold_timestamp.unix_timestamp(), 0);
assert_eq!(
get_header_map(&options.user_metadata, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE).as_deref(),
crates/ecstore/src/bucket/target/bucket_target.rs
+89 -4
View File
@@ -32,6 +32,10 @@ pub struct Credentials {
pub access_key: String,
#[serde(rename = "secretKey")]
pub secret_key: String,
// The aliases accept madmin's JSON tags (MinIO-written bucket-targets
// metadata and mc request bodies) without changing the snake_case
// persisted/peer wire format this struct serializes to.
#[serde(alias = "sessionToken")]
pub session_token: Option<String>,
pub expiration: Option<Timestamp>,
}
@@ -202,12 +206,14 @@ pub struct BucketTarget {
#[serde(default)]
pub region: String,
#[serde(alias = "bandwidth", default)]
// madmin-go v3.0.109 tags this `bandwidthlimit`; `bandwidth` is a legacy
// alias kept for inputs written before the madmin tag was verified.
#[serde(alias = "bandwidthlimit", alias = "bandwidth", default)]
pub bandwidth_limit: i64,
#[serde(rename = "replicationSync", default)]
pub replication_sync: bool,
#[serde(default)]
#[serde(alias = "storageclass", default)]
pub storage_class: String,
#[serde(rename = "skipTlsVerify", default)]
pub skip_tls_verify: bool,
@@ -220,7 +226,7 @@ pub struct BucketTarget {
#[serde(rename = "resetBeforeDate", with = "time::serde::rfc3339::option", default)]
pub reset_before_date: Option<OffsetDateTime>,
#[serde(default)]
#[serde(alias = "resetID", default)]
pub reset_id: String,
#[serde(rename = "totalDowntime", with = "duration_seconds", default)]
pub total_downtime: Duration,
@@ -233,7 +239,7 @@ pub struct BucketTarget {
#[serde(default)]
pub latency: LatencyStat,
#[serde(default)]
#[serde(alias = "deploymentID", default)]
pub deployment_id: String,
#[serde(default)]
@@ -531,6 +537,85 @@ mod tests {
assert_eq!(value["totalDowntime"], 90);
}
#[test]
fn bucket_target_persisted_wire_keys_stay_snake_case() {
// bucket-targets.json (persisted via `serde_json::to_vec(&BucketTargets)`
// in the admin set/remove handlers) and the msgpack struct-map form
// (`BucketTargets::marshal_msg`) both come straight from this struct's
// serde field names. madmin naming is applied only in the admin
// response layer (`remote_target_admin_json`); renaming here would
// silently break every existing deployment's persisted metadata.
let targets = BucketTargets {
targets: vec![BucketTarget {
credentials: Some(Credentials {
access_key: "ak".to_string(),
secret_key: "sk".to_string(),
session_token: Some("token".to_string()),
expiration: None,
}),
bandwidth_limit: 5,
storage_class: "STANDARD".to_string(),
reset_id: "reset-1".to_string(),
deployment_id: "deploy-1".to_string(),
..Default::default()
}],
};
let json = serde_json::to_value(&targets).expect("targets should serialize to JSON");
let msgpack: serde_json::Value =
rmp_serde::from_slice(&targets.marshal_msg().expect("targets should marshal to msgpack"))
.expect("msgpack struct map should decode into a JSON value");
for (wire, entry) in [("JSON", &json["targets"][0]), ("msgpack", &msgpack["targets"][0])] {
assert_eq!(entry["bandwidth_limit"], 5, "{wire} key `bandwidth_limit` must stay");
assert_eq!(entry["storage_class"], "STANDARD", "{wire} key `storage_class` must stay");
assert_eq!(entry["reset_id"], "reset-1", "{wire} key `reset_id` must stay");
assert_eq!(entry["deployment_id"], "deploy-1", "{wire} key `deployment_id` must stay");
assert_eq!(entry["credentials"]["session_token"], "token", "{wire} key `session_token` must stay");
}
}
#[test]
fn minio_written_bucket_targets_json_populates_madmin_named_fields() {
// A MinIO-written bucket-targets.json carries madmin's JSON tags
// (`bandwidthlimit`, `storageclass`, `resetID`, `deploymentID`,
// `credentials.sessionToken` — madmin-go v3.0.109 bucket-targets.go).
// On migration these must land in the matching fields instead of
// silently defaulting (backlog#1951).
let targets: BucketTargets = serde_json::from_value(serde_json::json!({
"targets": [{
"sourcebucket": "src",
"endpoint": "minio.example:9000",
"credentials": {
"accessKey": "ak",
"secretKey": "sk",
"sessionToken": "minio-session-token"
},
"targetbucket": "dst",
"type": "replication",
"replicationSync": true,
"bandwidthlimit": 107374182400i64,
"storageclass": "STANDARD",
"resetID": "reset-789",
"deploymentID": "deploy-123"
}]
}))
.expect("MinIO-written bucket-targets.json must deserialize");
let target = &targets.targets[0];
assert_eq!(target.bandwidth_limit, 107374182400);
assert_eq!(target.storage_class, "STANDARD");
assert_eq!(target.reset_id, "reset-789");
assert_eq!(target.deployment_id, "deploy-123");
assert_eq!(
target
.credentials
.as_ref()
.and_then(|credentials| credentials.session_token.as_deref()),
Some("minio-session-token")
);
}
#[test]
fn test_bucket_target_debug_redacts_credentials() {
let target = BucketTarget {
crates/ecstore/src/cluster/rpc/http_auth.rs
+68 -13
View File
@@ -94,6 +94,7 @@ const REPLAY_CACHE_AUTO_MEMORY_PERCENT: u64 = 13;
const REPLAY_CACHE_AUTO_RPC_RPS_PER_CPU: usize = 4096;
const REPLAY_CACHE_AUTO_MAX_CAPACITY: usize = 33_554_432;
const NS_SCANNER_CAPABILITY_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-capability-v3";
const NS_SCANNER_TIER_REGISTRY_GENERATION_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-tier-registry-generation-v1";
pub const TONIC_RPC_PREFIX: &str = "/node_service.NodeService";
static INTERNODE_RPC_SIGNATURE_STRICT: LazyLock<bool> = LazyLock::new(|| {
get_env_bool(
@@ -636,40 +637,79 @@ pub fn verify_put_file_capability(challenge: Uuid, server_epoch: Uuid, version:
.map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid put_file capability proof"))
}
fn update_ns_scanner_capability_mac(mac: &mut HmacSha256, challenge: Uuid, server_epoch: Uuid) {
fn update_ns_scanner_capability_mac(
mac: &mut HmacSha256,
challenge: Uuid,
server_epoch: Uuid,
supports_tier_registry_generation: bool,
) {
mac.update(NS_SCANNER_CAPABILITY_AUTH_DOMAIN);
mac.update(&NS_SCANNER_PROTOCOL_VERSION.to_be_bytes());
mac.update(challenge.as_bytes());
mac.update(server_epoch.as_bytes());
if supports_tier_registry_generation {
// The optional response capability is part of the authenticated
// scope. A proxy cannot turn an old/unsupported peer into a worker
// that receives generation-fenced scanner work.
mac.update(NS_SCANNER_TIER_REGISTRY_GENERATION_AUTH_DOMAIN);
}
}
fn generate_ns_scanner_capability_proof(secret: &str, challenge: Uuid, server_epoch: Uuid) -> std::io::Result<Vec<u8>> {
fn generate_ns_scanner_capability_proof(
secret: &str,
challenge: Uuid,
server_epoch: Uuid,
supports_tier_registry_generation: bool,
) -> std::io::Result<Vec<u8>> {
if challenge.is_nil() || server_epoch.is_nil() {
return Err(std::io::Error::other("Invalid namespace scanner capability scope"));
}
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch);
update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch, supports_tier_registry_generation);
Ok(mac.finalize().into_bytes().to_vec())
}
fn verify_ns_scanner_capability_proof(secret: &str, challenge: Uuid, server_epoch: Uuid, proof: &[u8]) -> std::io::Result<()> {
fn verify_ns_scanner_capability_proof(
secret: &str,
challenge: Uuid,
server_epoch: Uuid,
proof: &[u8],
supports_tier_registry_generation: bool,
) -> std::io::Result<()> {
if challenge.is_nil() || server_epoch.is_nil() {
return Err(std::io::Error::other("Invalid namespace scanner capability scope"));
}
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch);
update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch, supports_tier_registry_generation);
mac.verify_slice(proof)
.map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid namespace scanner capability proof"))
}
pub fn sign_ns_scanner_capability(challenge: Uuid, server_epoch: Uuid) -> std::io::Result<Vec<u8>> {
generate_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch)
sign_ns_scanner_capability_with_tier_registry_generation(challenge, server_epoch, false)
}
pub fn verify_ns_scanner_capability(challenge: Uuid, server_epoch: Uuid, proof: &[u8]) -> std::io::Result<()> {
verify_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch, proof)
verify_ns_scanner_capability_with_tier_registry_generation(challenge, server_epoch, proof, false)
}
pub fn sign_ns_scanner_capability_with_tier_registry_generation(
challenge: Uuid,
server_epoch: Uuid,
supports_tier_registry_generation: bool,
) -> std::io::Result<Vec<u8>> {
generate_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch, supports_tier_registry_generation)
}
pub fn verify_ns_scanner_capability_with_tier_registry_generation(
challenge: Uuid,
server_epoch: Uuid,
proof: &[u8],
supports_tier_registry_generation: bool,
) -> std::io::Result<()> {
verify_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch, proof, supports_tier_registry_generation)
}
#[derive(Clone, Copy)]
@@ -1709,13 +1749,28 @@ mod tests {
let secret = "test-scanner-capability-secret";
let challenge = Uuid::new_v4();
let server_epoch = Uuid::new_v4();
let proof =
generate_ns_scanner_capability_proof(secret, challenge, server_epoch).expect("capability proof should be generated");
let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false)
.expect("capability proof should be generated");
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof).is_ok());
assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof).is_err());
assert!(verify_ns_scanner_capability_proof(secret, challenge, Uuid::new_v4(), &proof).is_err());
assert!(verify_ns_scanner_capability_proof("different-secret", challenge, server_epoch, &proof).is_err());
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false).is_ok());
assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof, false).is_err());
assert!(verify_ns_scanner_capability_proof(secret, challenge, Uuid::new_v4(), &proof, false).is_err());
assert!(verify_ns_scanner_capability_proof("different-secret", challenge, server_epoch, &proof, false).is_err());
}
#[test]
fn namespace_scanner_capability_proof_binds_tier_registry_generation_support() {
let secret = "test-scanner-capability-secret";
let challenge = Uuid::new_v4();
let server_epoch = Uuid::new_v4();
let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, true)
.expect("generation capability proof should be generated");
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, true).is_ok());
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false).is_err());
let legacy = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false)
.expect("legacy capability proof should be generated");
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &legacy, true).is_err());
}
/// Security regression for GHSA-r5qv-rc46-hv8q (internode RPC fail-closed,
crates/ecstore/src/cluster/rpc/internode_data_transport.rs
+202 -27
View File
@@ -13,17 +13,18 @@
// limitations under the License.
use crate::cluster::rpc::{
build_auth_headers, build_put_file_auth_trailer, verify_ns_scanner_capability, verify_put_file_capability,
build_auth_headers, build_put_file_auth_trailer, verify_ns_scanner_capability_with_tier_registry_generation,
verify_put_file_capability,
};
use crate::disk::error::{Error, Result};
use crate::disk::{FileReader, FileWriter};
use crate::storage_api_contracts::internode::{
NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY, NS_SCANNER_LEADER_EPOCH_QUERY,
NS_SCANNER_PROTOCOL_VERSION, NS_SCANNER_PROTOCOL_VERSION_QUERY, NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY,
NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, NsScannerCapabilityResponse, PUT_FILE_AUTH_QUERY,
PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_CHALLENGE_QUERY, PUT_FILE_CAPABILITY_QUERY, PUT_FILE_CAPABILITY_VERSION,
PUT_FILE_NONCE_QUERY, PUT_FILE_SERVER_EPOCH_QUERY, PutFileCapabilityResponse, WALK_DIR_BODY_SHA256_QUERY,
WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY,
NsScannerCapabilityResponse, PUT_FILE_AUTH_QUERY, PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_CHALLENGE_QUERY,
PUT_FILE_CAPABILITY_QUERY, PUT_FILE_CAPABILITY_VERSION, PUT_FILE_NONCE_QUERY, PUT_FILE_SERVER_EPOCH_QUERY,
PutFileCapabilityResponse, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
};
use async_trait::async_trait;
use http::{HeaderMap, HeaderValue, Method, header::CONTENT_TYPE};
@@ -137,6 +138,12 @@ fn put_file_capability_status_is_legacy(status: u16) -> bool {
status == 404
}
fn ns_scanner_capability_error_allows_legacy(error: &Error) -> bool {
[400, 404, 405, 426]
.into_iter()
.any(|status| error.is_internode_http_status(status))
}
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
#[allow(
dead_code,
@@ -220,6 +227,7 @@ pub struct NsScannerStreamRequest {
#[derive(Debug, Clone)]
pub struct NsScannerCapabilityRequest {
pub endpoint: String,
pub supports_tier_registry_generation: bool,
}
/// Data-plane stream opener used by `RemoteDisk`.
@@ -252,6 +260,15 @@ pub trait InternodeDataTransport: Send + Sync + std::fmt::Debug {
async fn probe_ns_scanner(&self, _request: NsScannerCapabilityRequest) -> Result<Uuid> {
Err(Error::MethodNotAllowed)
}
async fn probe_ns_scanner_capability(&self, request: NsScannerCapabilityRequest) -> Result<NsScannerCapabilityResponse> {
let server_epoch = self.probe_ns_scanner(request).await?;
Ok(NsScannerCapabilityResponse {
version: NS_SCANNER_PROTOCOL_VERSION,
server_epoch,
proof: Vec::new(),
supports_tier_registry_generation: None,
})
}
// Interface facet nobody calls yet: every transport implements both, but no
// caller negotiates on them. Kept for the internode transport split
// (backlog#1350); deleting them would delete the seam and six impls.
@@ -335,27 +352,44 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
}
async fn probe_ns_scanner(&self, request: NsScannerCapabilityRequest) -> Result<Uuid> {
let challenge = Uuid::new_v4();
let url = build_ns_scanner_capability_url(&request, challenge);
let mut headers = msgpack_headers();
build_auth_headers(&url, &Method::GET, &mut headers)?;
let reader = HttpReader::new(url, Method::GET, headers, None).await?;
let mut body = Vec::new();
reader
.take(u64::try_from(NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE + 1).unwrap_or(u64::MAX))
.read_to_end(&mut body)
.await?;
if body.is_empty() || body.len() > NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE {
return Err(Error::other("invalid remote namespace scanner capability response size"));
Ok(self.probe_ns_scanner_capability(request).await?.server_epoch)
}
async fn probe_ns_scanner_capability(&self, request: NsScannerCapabilityRequest) -> Result<NsScannerCapabilityResponse> {
if request.supports_tier_registry_generation {
return match self.probe_ns_scanner_capability_once(&request).await {
Ok(response) => Ok(response),
Err(marked_error) if ns_scanner_capability_error_allows_legacy(&marked_error) => {
// A v3 peer may reject the additive query marker, ignore
// it, or return its legacy proof. Retry once without the
// marker and only downgrade after that legacy response is
// authenticated; an unverified epoch is never trusted.
let legacy_request = NsScannerCapabilityRequest {
endpoint: request.endpoint.clone(),
supports_tier_registry_generation: false,
};
match self.probe_ns_scanner_capability_once(&legacy_request).await {
Ok(mut response) => {
response.supports_tier_registry_generation = None;
Ok(response)
}
Err(legacy_error) if ns_scanner_capability_error_allows_legacy(&legacy_error) => {
// Some old deployments expose only the legacy
// protocol response (or advertise 426). Treat
// the pair as an explicit unsupported result so
// the scanner can use its coordinator fallback.
Err(Error::MethodNotAllowed)
}
Err(_) => Err(marked_error),
}
}
// A server failure, network failure, or authentication error
// is not evidence of an old parser. Do not issue an
// unauthenticated legacy probe or silently downgrade.
Err(marked_error) => Err(marked_error),
};
}
let response: NsScannerCapabilityResponse =
rmp_serde::from_slice(&body).map_err(|_| Error::other("invalid remote namespace scanner capability response"))?;
if response.version != NS_SCANNER_PROTOCOL_VERSION || response.server_epoch.is_nil() {
return Err(Error::other("incompatible remote namespace scanner capability response"));
}
verify_ns_scanner_capability(challenge, response.server_epoch, &response.proof)
.map_err(|err| Error::other(format!("remote namespace scanner capability authentication failed: {err}")))?;
Ok(response.server_epoch)
self.probe_ns_scanner_capability_once(&request).await
}
fn name(&self) -> &'static str {
@@ -368,6 +402,53 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
}
impl TcpHttpInternodeDataTransport {
async fn probe_ns_scanner_capability_once(
&self,
request: &NsScannerCapabilityRequest,
) -> Result<NsScannerCapabilityResponse> {
let challenge = Uuid::new_v4();
let url = build_ns_scanner_capability_url(request, challenge);
let mut headers = msgpack_headers();
build_auth_headers(&url, &Method::GET, &mut headers)?;
let reader = HttpReader::new(url, Method::GET, headers, None).await?;
let mut body = Vec::new();
reader
.take(u64::try_from(NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE + 1).unwrap_or(u64::MAX))
.read_to_end(&mut body)
.await?;
if body.is_empty() || body.len() > NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE {
return Err(Error::other("invalid remote namespace scanner capability response size"));
}
let mut response: NsScannerCapabilityResponse =
rmp_serde::from_slice(&body).map_err(|_| Error::other("invalid remote namespace scanner capability response"))?;
if response.version != NS_SCANNER_PROTOCOL_VERSION || response.server_epoch.is_nil() {
return Err(Error::other("incompatible remote namespace scanner capability response"));
}
if let Err(err) = verify_ns_scanner_capability_with_tier_registry_generation(
challenge,
response.server_epoch,
&response.proof,
request.supports_tier_registry_generation,
) {
// A permissive older peer can ignore the additive marker and
// return a valid legacy-scope proof with HTTP 200. Accept that
// response only after independently authenticating the legacy
// scope; all other verification failures remain fail-closed.
if request.supports_tier_registry_generation && ns_scanner_capability_legacy_proof_is_valid(challenge, &response) {
response.supports_tier_registry_generation = None;
return Ok(response);
}
return Err(Error::other(format!("remote namespace scanner capability authentication failed: {err}")));
}
// The proof authenticates the requested capability scope, not the
// optional response field. Derive the client-facing bit from that
// verified scope so an intermediary cannot strip or rewrite the field
// and force a silent downgrade after a successful generation-bound
// handshake.
normalize_ns_scanner_capability_response(&mut response, request.supports_tier_registry_generation);
Ok(response)
}
async fn put_file_auth_capability(&self, endpoint: &str) -> Result<Option<Uuid>> {
resolve_put_file_auth_capability(endpoint, || async {
tokio::time::timeout(PUT_FILE_CAPABILITY_PROBE_TIMEOUT, self.probe_put_file_auth(endpoint))
@@ -649,6 +730,14 @@ fn build_walk_dir_url(request: &WalkDirStreamRequest) -> String {
)
}
fn normalize_ns_scanner_capability_response(response: &mut NsScannerCapabilityResponse, requested_generation_support: bool) {
response.supports_tier_registry_generation = requested_generation_support.then_some(true);
}
fn ns_scanner_capability_legacy_proof_is_valid(challenge: Uuid, response: &NsScannerCapabilityResponse) -> bool {
verify_ns_scanner_capability_with_tier_registry_generation(challenge, response.server_epoch, &response.proof, false).is_ok()
}
fn build_ns_scanner_url(request: &NsScannerStreamRequest) -> String {
let body_sha256 = hex_simd::encode_to_string(Sha256::digest(&request.body), hex_simd::AsciiCase::Lower);
format!(
@@ -675,13 +764,18 @@ fn build_ns_scanner_url(request: &NsScannerStreamRequest) -> String {
fn build_ns_scanner_capability_url(request: &NsScannerCapabilityRequest, challenge: Uuid) -> String {
format!(
"{}{}?{}={}&{}={}",
"{}{}?{}={}&{}={}{}",
request.endpoint,
NS_SCANNER_PATH,
NS_SCANNER_PROTOCOL_VERSION_QUERY,
NS_SCANNER_PROTOCOL_VERSION,
NS_SCANNER_CAPABILITY_CHALLENGE_QUERY,
challenge
challenge,
if request.supports_tier_registry_generation {
format!("&{}=true", NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY)
} else {
String::new()
}
)
}
@@ -794,6 +888,7 @@ mod tests {
let probe_err = transport
.probe_ns_scanner(NsScannerCapabilityRequest {
endpoint: "http://node1:9000".to_string(),
supports_tier_registry_generation: false,
})
.await
.expect_err("legacy transport should report namespace scanner as unsupported");
@@ -1387,6 +1482,7 @@ mod tests {
let url = build_ns_scanner_capability_url(
&NsScannerCapabilityRequest {
endpoint: "http://node1:9000".to_string(),
supports_tier_registry_generation: false,
},
challenge,
);
@@ -1399,6 +1495,85 @@ mod tests {
);
}
#[test]
fn ns_scanner_capability_url_marks_generation_support_only_when_requested() {
let challenge = Uuid::new_v4();
let url = build_ns_scanner_capability_url(
&NsScannerCapabilityRequest {
endpoint: "http://node1:9000".to_string(),
supports_tier_registry_generation: true,
},
challenge,
);
assert!(url.contains(&format!("&{}=true", NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY)));
}
#[test]
fn ns_scanner_capability_legacy_fallback_requires_explicit_compatibility_status() {
for status in [400, 404, 405, 426] {
let error = Error::from(rustfs_rio::new_test_internode_http_io_error(
rustfs_rio::InternodeHttpErrorKind::HttpStatus(http::StatusCode::from_u16(status).expect("test status")),
));
assert!(
ns_scanner_capability_error_allows_legacy(&error),
"status {status} should permit legacy retry"
);
}
let marked_server_error = Error::from(rustfs_rio::new_test_internode_http_io_error(
rustfs_rio::InternodeHttpErrorKind::HttpStatus(http::StatusCode::INTERNAL_SERVER_ERROR),
));
let network_error = Error::from(rustfs_rio::new_test_internode_http_io_error(
rustfs_rio::InternodeHttpErrorKind::ConnectionRefused,
));
let authentication_error = Error::other("remote namespace scanner capability authentication failed");
assert!(!ns_scanner_capability_error_allows_legacy(&marked_server_error));
assert!(!ns_scanner_capability_error_allows_legacy(&network_error));
assert!(!ns_scanner_capability_error_allows_legacy(&authentication_error));
}
#[test]
fn authenticated_ns_scanner_capability_ignores_unprotected_response_bit() {
let mut response = NsScannerCapabilityResponse {
version: NS_SCANNER_PROTOCOL_VERSION,
server_epoch: Uuid::new_v4(),
proof: Vec::new(),
supports_tier_registry_generation: None,
};
normalize_ns_scanner_capability_response(&mut response, true);
assert_eq!(response.supports_tier_registry_generation, Some(true));
response.supports_tier_registry_generation = Some(false);
normalize_ns_scanner_capability_response(&mut response, false);
assert_eq!(response.supports_tier_registry_generation, None);
}
#[test]
fn ns_scanner_capability_accepts_only_authenticated_legacy_scope_after_marker_mismatch() {
crate::runtime::sources::ensure_test_rpc_secret();
let challenge = Uuid::new_v4();
let response = NsScannerCapabilityResponse {
version: NS_SCANNER_PROTOCOL_VERSION,
server_epoch: Uuid::new_v4(),
proof: crate::cluster::rpc::sign_ns_scanner_capability(challenge, Uuid::new_v4())
.expect("placeholder proof should be generated"),
supports_tier_registry_generation: None,
};
// A proof bound to a different challenge cannot authorize the legacy
// fallback, even though the response has the expected shape.
assert!(!ns_scanner_capability_legacy_proof_is_valid(challenge, &response));
let server_epoch = response.server_epoch;
let valid_response = NsScannerCapabilityResponse {
proof: crate::cluster::rpc::sign_ns_scanner_capability(challenge, server_epoch)
.expect("legacy proof should be generated"),
..response
};
assert!(ns_scanner_capability_legacy_proof_is_valid(challenge, &valid_response));
}
#[test]
fn transport_config_defaults_to_tcp_http() {
let transport = build_internode_data_transport(None).unwrap();
crates/ecstore/src/cluster/rpc/mod.rs
+3 -2
View File
@@ -35,8 +35,9 @@ pub use http_auth::{
TONIC_RPC_PREFIX, build_auth_headers, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, gen_signature_headers,
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
set_tonic_mutation_body_digest, set_tonic_rolling_canonical_body_digest, set_tonic_rolling_mutation_body_digest,
sign_ns_scanner_capability, sign_put_file_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge,
tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_put_file_auth_trailer,
sign_ns_scanner_capability, sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability,
sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer,
verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
verify_tonic_rpc_signature_with_bootstrap,
crates/ecstore/src/cluster/rpc/peer_rest_client.rs
+34 -20
View File
@@ -85,6 +85,7 @@ const PEER_REST_RECOVERY_MAX_ATTEMPTS: u32 = 60;
const PEER_REST_RECOVERY_MAX_BACKOFF: Duration = Duration::from_secs(30);
const SCANNER_ACTIVITY_MAX_MESSAGE_SIZE: usize = 1024;
const REPLICATION_STATS_MAX_MESSAGE_SIZE: usize = 8 * 1024 * 1024;
const BUCKET_METADATA_RELOAD_TIMEOUT: Duration = Duration::from_secs(5);
/// Error for a peer that reported `success = false` without an `error_info` payload.
///
@@ -1089,7 +1090,9 @@ impl PeerRestClient {
.await?
.max_decoding_message_size(BACKGROUND_HEAL_STATUS_MAX_MESSAGE_SIZE);
let response = match client
.background_heal_status(Request::new(BackgroundHealStatusRequest::default()))
.background_heal_status(Request::new(BackgroundHealStatusRequest {
protocol_version: rustfs_protos::BACKGROUND_HEAL_STATUS_PROTOCOL_VERSION,
}))
.await
{
Ok(response) => response.into_inner(),
@@ -1328,27 +1331,38 @@ impl PeerRestClient {
}
pub async fn load_bucket_metadata(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
self.finalize_result(
async {
let mut client = self.get_client().await?;
let mut request = Request::new(LoadBucketMetadataRequest {
bucket: bucket.to_string(),
scanner_maintenance_change,
});
set_tonic_mutation_body_digest(&mut request)?;
let response = client.load_bucket_metadata(request).await?.into_inner();
if !response.success {
if let Some(msg) = response.error_info {
return Err(Error::other(msg));
}
return Err(peer_failure_without_details("load_bucket_metadata", Some(bucket)));
}
Ok(())
let result = tokio::time::timeout(BUCKET_METADATA_RELOAD_TIMEOUT, async {
let result = self.load_bucket_metadata_once(bucket, scanner_maintenance_change).await;
if let Err(err) = &result
&& Self::is_network_like_error(err)
{
self.prepare_retry().await;
return self.load_bucket_metadata_once(bucket, scanner_maintenance_change).await;
}
.await,
)
result
})
.await
.unwrap_or_else(|_| Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))));
self.finalize_result(result).await
}
async fn load_bucket_metadata_once(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
let mut client = self.get_client().await?;
let mut request = Request::new(LoadBucketMetadataRequest {
bucket: bucket.to_string(),
scanner_maintenance_change,
});
set_tonic_mutation_body_digest(&mut request)?;
request.set_timeout(BUCKET_METADATA_RELOAD_TIMEOUT);
let response = client.load_bucket_metadata(request).await?.into_inner();
if !response.success {
if let Some(msg) = response.error_info {
return Err(Error::other(msg));
}
return Err(peer_failure_without_details("load_bucket_metadata", Some(bucket)));
}
Ok(())
}
pub async fn delete_bucket_metadata(&self, bucket: &str) -> Result<()> {
crates/ecstore/src/cluster/rpc/remote_disk.rs
+58 -3
View File
@@ -781,14 +781,16 @@ impl RemoteDisk {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
let probe = self.data_transport.probe_ns_scanner(NsScannerCapabilityRequest {
let probe = self.data_transport.probe_ns_scanner_capability(NsScannerCapabilityRequest {
endpoint: self.endpoint.grid_host(),
supports_tier_registry_generation: true,
});
let result = timeout(NS_SCANNER_CAPABILITY_PROBE_TIMEOUT, probe)
.await
.map_err(|_| DiskError::other("remote namespace scanner capability probe timed out"))?;
match result {
Ok(server_epoch) => Ok(Some(server_epoch)),
Ok(response) if response.supports_tier_registry_generation == Some(true) => Ok(Some(response.server_epoch)),
Ok(_) => Ok(None),
// RUSTFS_COMPAT_TODO(ns-scanner-rpc-v3): old peers and legacy transports lack the authenticated startup-epoch handshake. Remove after every supported peer implements namespace scanner protocol v3.
Err(DiskError::MethodNotAllowed) => Ok(None),
Err(err)
@@ -4040,10 +4042,21 @@ mod tests {
NsScannerProbe(NsScannerCapabilityRequest),
}
#[derive(Debug, Clone, Default)]
#[derive(Debug, Clone)]
struct RecordingInternodeDataTransport {
calls: Arc<StdMutex<Vec<RecordedTransportCall>>>,
ns_scanner_probe_status: Arc<StdMutex<Option<u16>>>,
ns_scanner_generation_support: Arc<StdMutex<Option<bool>>>,
}
impl Default for RecordingInternodeDataTransport {
fn default() -> Self {
Self {
calls: Arc::default(),
ns_scanner_probe_status: Arc::default(),
ns_scanner_generation_support: Arc::new(StdMutex::new(Some(true))),
}
}
}
#[derive(Clone, Debug)]
@@ -4263,6 +4276,15 @@ mod tests {
Self {
calls: Arc::default(),
ns_scanner_probe_status: Arc::new(StdMutex::new(Some(status))),
ns_scanner_generation_support: Arc::new(StdMutex::new(Some(true))),
}
}
fn with_ns_scanner_generation_support(support: Option<bool>) -> Self {
Self {
calls: Arc::default(),
ns_scanner_probe_status: Arc::default(),
ns_scanner_generation_support: Arc::new(StdMutex::new(support)),
}
}
@@ -4945,6 +4967,23 @@ mod tests {
Ok(Uuid::from_u128(1))
}
async fn probe_ns_scanner_capability(
&self,
request: NsScannerCapabilityRequest,
) -> Result<crate::storage_api_contracts::internode::NsScannerCapabilityResponse> {
let server_epoch = self.probe_ns_scanner(request).await?;
let supports_tier_registry_generation = *self
.ns_scanner_generation_support
.lock()
.expect("namespace scanner generation support lock poisoned");
Ok(crate::storage_api_contracts::internode::NsScannerCapabilityResponse {
version: crate::storage_api_contracts::internode::NS_SCANNER_PROTOCOL_VERSION,
server_epoch,
proof: Vec::new(),
supports_tier_registry_generation,
})
}
fn name(&self) -> &'static str {
"recording"
}
@@ -6757,6 +6796,22 @@ mod tests {
}
}
#[tokio::test]
async fn test_remote_disk_namespace_scanner_capability_falls_back_without_generation_support() {
for support in [None, Some(false)] {
let transport = RecordingInternodeDataTransport::with_ns_scanner_generation_support(support);
let remote_disk = new_remote_disk_with_transport(Arc::new(transport)).await;
assert_eq!(
remote_disk
.ns_scanner_server_epoch()
.await
.expect("missing generation support should be classified as unsupported"),
None
);
}
}
#[tokio::test]
async fn test_remote_disk_namespace_scanner_capability_rejects_legacy_transport() {
let remote_disk = new_remote_disk_with_transport(Arc::new(RetryingOpenReadInternodeDataTransport::default())).await;
crates/ecstore/src/config/com.rs
+19
View File
@@ -406,6 +406,25 @@ where
Ok(data)
}
pub(crate) async fn read_config_limited_preserve_empty<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<Vec<u8>>
where
S: EcstoreObjectIO,
{
let (data, _obj) = read_config_limited_preserve_empty_with_metadata(api, file, max_bytes).await?;
Ok(data)
}
pub(crate) async fn read_config_limited_preserve_empty_with_metadata<S>(
api: Arc<S>,
file: &str,
max_bytes: usize,
) -> Result<(Vec<u8>, ObjectInfo)>
where
S: EcstoreObjectIO,
{
read_config_with_metadata_inner(api, file, &ObjectOptions::default(), true, Some(max_bytes)).await
}
/// Read an existing config object without treating an empty payload as absent.
/// Callers that validate their own payload format need to distinguish corruption
/// from `ConfigNotFound`.
crates/ecstore/src/core/pools.rs
+2478 -211
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/data_movement/mod.rs
+1 -1
View File
@@ -813,7 +813,7 @@ pub(crate) fn is_equivalent_data_movement_metadata(
.all(|(key, value)| source.user_defined.get(key) == Some(value))
}
fn is_equivalent_data_movement_object_identity(
pub(crate) fn is_equivalent_data_movement_object_identity(
source: &ObjectInfo,
target: &ObjectInfo,
compare_mod_time: bool,
crates/ecstore/src/data_usage/mod.rs
+369 -33
View File
@@ -73,6 +73,16 @@ struct CachedBucketUsage {
// mutation. A strictly later generation is required before the mutation
// evidence can be discarded.
pending_scanner_position: Option<(u64, u64)>,
// Deletes are visible to admin immediately, but quota admission keeps
// them pending until a complete scanner generation reconciles the set.
// This marker intentionally remains process-local: the delete request
// updates this overlay before the scanner writes a durable snapshot. If
// the process restarts first, loading the persisted complete snapshot
// restores the pre-reconciliation (larger) baseline, which is
// conservative for quota admission. A persisted post-delete snapshot is
// necessarily a complete scanner reconciliation and therefore creates a
// fresh cache entry with no pending hold.
pending_negative_delta: u64,
}
type UsageMemoryCache = Arc<RwLock<HashMap<String, CachedBucketUsage>>>;
@@ -378,8 +388,12 @@ pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store:
"nonconverged data usage observations cannot replace the quota-authoritative snapshot",
));
}
let Some(expected_publication_epoch) = store.scanner_data_usage_publication_epoch().await else {
return Err(Error::other("data usage publication is blocked by data movement"));
};
// Prevent older data from overwriting newer persisted stats
if let Ok((existing, source)) = load_data_usage_snapshot(store.clone()).await
let existing_snapshot = load_data_usage_snapshot(store.clone()).await;
if let Ok((existing, source)) = existing_snapshot
&& source.is_authoritative()
&& let Some(reason) = stale_data_usage_persist_reason_for_source(&data_usage_info, &existing, source, SystemTime::now())
{
@@ -390,19 +404,31 @@ pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store:
return Ok(());
}
save_data_usage_in_backend(data_usage_info, store).await
save_data_usage_in_backend(data_usage_info, store, expected_publication_epoch).await
}
async fn save_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<ECStore>) -> Result<(), Error> {
async fn save_data_usage_in_backend(
data_usage_info: DataUsageInfo,
store: Arc<ECStore>,
expected_publication_epoch: u64,
) -> Result<(), Error> {
let data =
serde_json::to_vec(&data_usage_info).map_err(|e| Error::other(format!("Failed to serialize data usage info: {e}")))?;
// Save to backend using the same mechanism as original code
let Some((publication_guard, publication_epoch)) = store.scanner_data_usage_publication_admission_guard().await else {
return Err(Error::other("data usage publication is blocked by data movement"));
};
if publication_epoch != expected_publication_epoch {
return Err(Error::other("data usage publication epoch changed before save"));
}
crate::config::com::save_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH, data)
.await
.map_err(Error::other)?;
drop(publication_guard);
cleanup_observed_data_usage_after_authoritative_save(store.as_ref(), &data_usage_info).await;
cleanup_observed_data_usage_after_authoritative_save_with_publication(store.as_ref(), &data_usage_info, Some(store.as_ref()))
.await;
// Invalidate the cached snapshot so readers observe the new save on their
// next request instead of waiting out the remaining TTL. The next cached
@@ -439,11 +465,24 @@ impl ObservedDataUsageSnapshotCleanup for ECStore {
}
}
async fn cleanup_observed_data_usage_after_authoritative_save<S>(store: &S, authoritative: &DataUsageInfo)
where
async fn cleanup_observed_data_usage_after_authoritative_save_with_publication<S>(
store: &S,
authoritative: &DataUsageInfo,
publication_store: Option<&ECStore>,
) where
S: EcstoreObjectIO + ObservedDataUsageSnapshotCleanup + ?Sized,
{
let (observed, revision) = match load_data_usage_for_bucket_removal(store, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await {
let observed_read_epoch = match publication_store {
Some(publication_store) => {
let Some(epoch) = publication_store.scanner_data_usage_publication_epoch().await else {
return;
};
Some(epoch)
}
None => None,
};
let observed_snapshot = load_data_usage_for_bucket_removal(store, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await;
let (observed, revision) = match observed_snapshot {
Ok(Some(snapshot)) => snapshot,
Ok(None) => return,
Err(err) => {
@@ -459,6 +498,19 @@ where
return;
}
let publication_guard = match publication_store {
Some(publication_store) => {
let Some((guard, publication_epoch)) = publication_store.scanner_data_usage_publication_admission_guard().await
else {
return;
};
if observed_read_epoch.is_some_and(|expected| expected != publication_epoch) {
return;
}
Some(guard)
}
None => None,
};
match store.delete_observed_data_usage_snapshot(&revision).await {
Ok(()) | Err(Error::ConfigNotFound | Error::FileNotFound | Error::ObjectNotFound(_, _) | Error::PreconditionFailed) => {}
Err(err) => {
@@ -469,6 +521,15 @@ where
);
}
}
drop(publication_guard);
}
#[cfg(test)]
async fn cleanup_observed_data_usage_after_authoritative_save<S>(store: &S, authoritative: &DataUsageInfo)
where
S: EcstoreObjectIO + ObservedDataUsageSnapshotCleanup + ?Sized,
{
cleanup_observed_data_usage_after_authoritative_save_with_publication(store, authoritative, None).await;
}
fn set_buckets_count_from_usage(data_usage_info: &mut DataUsageInfo) {
@@ -509,7 +570,7 @@ pub async fn remove_bucket_usage_from_backend(store: Arc<ECStore>, bucket: &str)
pub(crate) async fn remove_bucket_usage_for_namespace_change(store: &ECStore, bucket: &str) -> Result<(), Error> {
prepare_bucket_usage_for_namespace_change(bucket, None).await?;
remove_bucket_usage_from_backend_with_guard(store, bucket, None).await
remove_bucket_usage_from_backend_with_guard_fenced(store, bucket, None).await
}
pub(crate) async fn prepare_bucket_usage_for_namespace_change(
@@ -532,6 +593,7 @@ pub(crate) async fn prepare_bucket_usage_for_namespace_change(
Ok(())
}
#[cfg(test)]
pub(crate) async fn remove_bucket_usage_from_backend_with_guard<S>(
store: &S,
bucket: &str,
@@ -541,6 +603,24 @@ where
S: EcstoreObjectIO + ?Sized,
{
let result = remove_bucket_usage_from_backend_with_store_and_guard(store, bucket, guard).await;
invalidate_bucket_usage_snapshot_caches(guard, bucket).await?;
result
}
pub(crate) async fn remove_bucket_usage_from_backend_with_guard_fenced(
store: &ECStore,
bucket: &str,
guard: Option<&rustfs_lock::NamespaceLockGuard>,
) -> Result<(), Error> {
let result = remove_bucket_usage_from_backend_with_store_and_guard_and_publication(store, bucket, guard, Some(store)).await;
invalidate_bucket_usage_snapshot_caches(guard, bucket).await?;
result
}
async fn invalidate_bucket_usage_snapshot_caches(
guard: Option<&rustfs_lock::NamespaceLockGuard>,
bucket: &str,
) -> Result<(), Error> {
let mut snapshot_cache = data_usage_snapshot_cache().write().await;
ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cache invalidation")?;
clear_data_usage_snapshot_cache(&mut snapshot_cache);
@@ -548,7 +628,7 @@ where
let mut admin_snapshot_cache = admin_data_usage_snapshot_cache().write().await;
ensure_bucket_namespace_guard(guard, bucket, "admin data usage snapshot cache invalidation")?;
clear_admin_data_usage_snapshot_cache(&mut admin_snapshot_cache);
result
Ok(())
}
async fn load_data_usage_for_bucket_removal<S>(store: &S, object: &str) -> Result<Option<(DataUsageInfo, String)>, Error>
@@ -607,48 +687,83 @@ fn ensure_bucket_namespace_guard(
Ok(())
}
#[cfg(test)]
async fn remove_bucket_usage_from_backend_with_store_and_guard<S>(
store: &S,
bucket: &str,
guard: Option<&rustfs_lock::NamespaceLockGuard>,
) -> Result<(), Error>
where
S: EcstoreObjectIO + ?Sized,
{
remove_bucket_usage_from_backend_with_store_and_guard_and_publication(store, bucket, guard, None).await
}
async fn remove_bucket_usage_from_backend_with_store_and_guard_and_publication<S>(
store: &S,
bucket: &str,
guard: Option<&rustfs_lock::NamespaceLockGuard>,
publication_store: Option<&ECStore>,
) -> Result<(), Error>
where
S: EcstoreObjectIO + ?Sized,
{
ensure_bucket_namespace_guard(guard, bucket, "data usage primary cleanup")?;
let primary_seed_epoch = match publication_store {
Some(publication_store) => Some(
publication_store
.scanner_data_usage_publication_epoch()
.await
.ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?,
),
None => None,
};
let primary_seed = load_data_usage_seed_for_missing_primary(store).await?;
remove_bucket_usage_from_object_with_retries(
remove_bucket_usage_from_object_with_retries_and_publication(
store,
DATA_USAGE_OBJ_NAME_PATH.as_str(),
bucket,
DATA_USAGE_REMOVE_CAS_RETRIES,
Some(&primary_seed),
primary_seed.as_ref(),
guard,
publication_store.map(|store| (store, primary_seed_epoch)),
)
.await?;
ensure_bucket_namespace_guard(guard, bucket, "data usage backup cleanup")?;
let backup_seed_epoch = match publication_store {
Some(publication_store) => Some(
publication_store
.scanner_data_usage_publication_epoch()
.await
.ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?,
),
None => None,
};
let backup_seed = load_data_usage_for_bucket_removal(store, DATA_USAGE_OBJ_NAME_PATH.as_str())
.await?
.map_or(primary_seed, |(data_usage_info, _)| data_usage_info);
remove_bucket_usage_from_object_with_retries(
.map(|(data_usage_info, _)| data_usage_info)
.or_else(|| primary_seed.clone());
remove_bucket_usage_from_object_with_retries_and_publication(
store,
DATA_USAGE_OBJ_BACKUP_PATH.as_str(),
bucket,
DATA_USAGE_REMOVE_CAS_RETRIES,
Some(&backup_seed),
backup_seed.as_ref(),
guard,
publication_store.map(|store| (store, backup_seed_epoch)),
)
.await?;
ensure_bucket_namespace_guard(guard, bucket, "observed data usage cleanup")?;
if let Err(err) = remove_bucket_usage_from_object_with_retries(
if let Err(err) = remove_bucket_usage_from_object_with_retries_and_publication(
store,
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
bucket,
DATA_USAGE_REMOVE_CAS_RETRIES,
None,
guard,
publication_store.map(|store| (store, None)),
)
.await
{
@@ -662,12 +777,21 @@ where
LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(),
LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str(),
] {
remove_bucket_usage_from_object_with_retries(store, object, bucket, DATA_USAGE_REMOVE_CAS_RETRIES, None, guard).await?;
remove_bucket_usage_from_object_with_retries_and_publication(
store,
object,
bucket,
DATA_USAGE_REMOVE_CAS_RETRIES,
None,
guard,
publication_store.map(|store| (store, None)),
)
.await?;
}
Ok(())
}
async fn load_data_usage_seed_for_missing_primary<S>(store: &S) -> Result<DataUsageInfo, Error>
async fn load_data_usage_seed_for_missing_primary<S>(store: &S) -> Result<Option<DataUsageInfo>, Error>
where
S: EcstoreObjectIO + ?Sized,
{
@@ -680,12 +804,13 @@ where
if !authoritative {
data_usage_info.usage_snapshot_complete = false;
}
return Ok(data_usage_info);
return Ok(Some(data_usage_info));
}
}
Ok(DataUsageInfo::default())
Ok(None)
}
#[cfg(test)]
async fn remove_bucket_usage_from_object_with_retries<S>(
store: &S,
object: &str,
@@ -694,12 +819,42 @@ async fn remove_bucket_usage_from_object_with_retries<S>(
missing_seed: Option<&DataUsageInfo>,
guard: Option<&rustfs_lock::NamespaceLockGuard>,
) -> Result<(), Error>
where
S: EcstoreObjectIO + ?Sized,
{
remove_bucket_usage_from_object_with_retries_and_publication(store, object, bucket, cas_retries, missing_seed, guard, None)
.await
}
async fn remove_bucket_usage_from_object_with_retries_and_publication<S>(
store: &S,
object: &str,
bucket: &str,
cas_retries: usize,
missing_seed: Option<&DataUsageInfo>,
guard: Option<&rustfs_lock::NamespaceLockGuard>,
publication: Option<(&ECStore, Option<u64>)>,
) -> Result<(), Error>
where
S: EcstoreObjectIO + ?Sized,
{
for attempt in 0..=cas_retries {
ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cleanup")?;
let (mut data_usage_info, revision) = match load_data_usage_for_bucket_removal(store, object).await? {
let read_epoch = match publication {
Some((publication_store, expected_publication_epoch)) => {
let epoch = publication_store
.scanner_data_usage_publication_epoch()
.await
.ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?;
if expected_publication_epoch.is_some_and(|expected| expected != epoch) {
return Err(Error::other("data usage publication epoch changed before snapshot read"));
}
Some(epoch)
}
None => None,
};
let loaded_snapshot = load_data_usage_for_bucket_removal(store, object).await?;
let (mut data_usage_info, revision) = match loaded_snapshot {
Some((data_usage_info, revision)) => (data_usage_info, Some(revision)),
None => match missing_seed {
Some(data_usage_info) => (data_usage_info.clone(), None),
@@ -723,6 +878,22 @@ where
},
};
ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot commit")?;
let publication_guard = match publication {
Some((publication_store, expected_publication_epoch)) => {
let Some((guard, publication_epoch)) = publication_store.scanner_data_usage_publication_admission_guard().await
else {
return Err(Error::other("data usage publication is blocked by data movement"));
};
if expected_publication_epoch
.or(read_epoch)
.is_some_and(|expected| expected != publication_epoch)
{
return Err(Error::other("data usage publication epoch changed before snapshot commit"));
}
Some(guard)
}
None => None,
};
let save_result = store
.put_object(
RUSTFS_META_BUCKET,
@@ -735,6 +906,7 @@ where
},
)
.await;
drop(publication_guard);
match save_result {
Ok(_) => return Ok(()),
Err(err) => {
@@ -948,7 +1120,12 @@ async fn load_observed_data_usage_snapshot(store: Arc<ECStore>) -> Option<DataUs
};
match parse_usage_snapshot(&data) {
Ok(info) if info.usage_snapshot_converged == Some(false) && info.is_complete_bucket_usage_snapshot() => Some(info),
Ok(info)
if info.usage_snapshot_converged == Some(false)
&& (info.is_complete_bucket_usage_snapshot() || info.is_valid_partial_snapshot()) =>
{
Some(info)
}
Ok(_) => {
error!(
event = "data_usage_snapshot_load_failed",
@@ -993,7 +1170,7 @@ async fn load_admin_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataU
}
fn discard_incomplete_bucket_usage(data_usage_info: &mut DataUsageInfo) {
if !data_usage_info.is_complete_bucket_usage_snapshot() {
if !data_usage_info.is_complete_bucket_usage_snapshot() && !data_usage_info.usage_snapshot_partial {
data_usage_info.usage_snapshot_complete = false;
data_usage_info.buckets_usage.clear();
data_usage_info.bucket_sizes.clear();
@@ -1643,6 +1820,7 @@ fn cached_bucket_usage_from_backend(usage: BucketUsageInfo, updated_at: SystemTi
dirty: false,
stale_snapshot_pending: false,
pending_scanner_position: None,
pending_negative_delta: 0,
}
}
@@ -1656,6 +1834,7 @@ fn cached_bucket_usage_now(usage: BucketUsageInfo) -> CachedBucketUsage {
dirty: false,
stale_snapshot_pending: false,
pending_scanner_position: None,
pending_negative_delta: 0,
}
}
@@ -1808,6 +1987,7 @@ pub async fn record_bucket_object_delete_memory(bucket: &str, deleted_size: u64,
.or_insert_with(|| cached_bucket_usage_now(BucketUsageInfo::default()));
entry.usage.size = entry.usage.size.saturating_sub(deleted_size);
entry.pending_negative_delta = entry.pending_negative_delta.saturating_add(deleted_size);
if removed_current_object {
entry.usage.objects_count = entry.usage.objects_count.saturating_sub(1);
entry.usage.versions_count = entry.usage.versions_count.saturating_sub(1);
@@ -1863,7 +2043,7 @@ pub async fn get_bucket_usage_memory(bucket: &str) -> Option<u64> {
cache
.get(bucket)
.filter(|cached| cached.authoritative)
.map(|cached| cached.usage.size)
.map(|cached| cached.usage.size.saturating_add(cached.pending_negative_delta))
}
async fn update_usage_cache_if_needed() {
@@ -2268,6 +2448,8 @@ pub async fn init_compression_total_memory_from_backend(store: Arc<ECStore>) {
#[cfg(test)]
mod tests {
use super::*;
use crate::layout::endpoints::EndpointServerPools;
use crate::runtime::instance::InstanceContext;
use crate::storage_api_contracts::object::ObjectIO as _;
use rustfs_data_usage::BucketUsageInfo;
use rustfs_lock::{LocalClient, LockRequest, LockType, NamespaceLock, ObjectKey};
@@ -2290,6 +2472,7 @@ mod tests {
error_after_commit_put: Option<usize>,
advance_time_on_put: Option<Duration>,
advance_time_after_get: Option<(UsageObjectSlot, Duration)>,
advance_publication_epoch_after_get: Option<(UsageObjectSlot, Arc<InstanceContext>)>,
advance_time_before_put: Option<(usize, Duration)>,
advance_time_after_put: Option<(usize, Duration)>,
put_count: usize,
@@ -2367,10 +2550,21 @@ mod tests {
}
_ => None,
};
let advance_publication_epoch = match state.advance_publication_epoch_after_get {
Some((expected_slot, ref ctx)) if expected_slot == slot => {
let ctx = Arc::clone(ctx);
state.advance_publication_epoch_after_get = None;
Some(ctx)
}
_ => None,
};
drop(state);
if let Some(duration) = advance {
tokio::time::advance(duration).await;
}
if let Some(ctx) = advance_publication_epoch {
ctx.advance_data_movement_operation_epoch();
}
Ok(crate::object_api::GetObjectReader {
stream: Box::new(Cursor::new(data)),
object_info: ObjectInfo {
@@ -2571,6 +2765,23 @@ mod tests {
.to_string()
}
fn build_publication_store(ctx: Arc<InstanceContext>) -> Arc<ECStore> {
let endpoint_pools = EndpointServerPools::default();
Arc::new(ECStore {
id: uuid::Uuid::new_v4(),
disk_map: HashMap::new(),
pools: Vec::new(),
peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoint_pools, ctx.clone()),
pool_meta: RwLock::new(crate::core::pools::PoolMeta::default()),
rebalance_meta: RwLock::new(None),
decommission_cancelers: RwLock::new(Vec::new()),
start_gate: TokioMutex::new(()),
pool_meta_save_gate: TokioMutex::new(()),
ctx,
bucket_fence_registry: Arc::default(),
})
}
#[test]
fn data_usage_cache_absence_covers_the_variants_that_actually_arrive() {
// `to_object_err` rewrites the raw storage variants before they reach
@@ -2943,6 +3154,45 @@ mod tests {
assert_eq!(selected.usage_snapshot_converged, Some(true));
}
#[test]
fn persisted_authoritative_stalls_but_memory_overlay_remains_visible() {
let authoritative = DataUsageInfo {
last_update: Some(SystemTime::UNIX_EPOCH),
scanner_epoch: Some(4),
scanner_cycle: Some(10),
usage_snapshot_complete: true,
..Default::default()
};
let mut partial = authoritative.clone();
partial.last_update = Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1));
partial.scanner_cycle = Some(11);
partial.usage_snapshot_complete = false;
partial.usage_snapshot_partial = true;
partial.usage_snapshot_converged = Some(false);
partial.usage_snapshot_authoritative_baseline = Some(authoritative.snapshot_identity());
partial.usage_snapshot_set_states = vec![rustfs_data_usage::DataUsageSnapshotSetState {
pool_index: 0,
set_index: 0,
scanner_cycle: Some(10),
scanner_epoch: Some(4),
scan_plan_digest: Some([1; 32]),
complete: false,
tombstone: false,
}];
partial.buckets_usage.insert(
"bucket".to_string(),
BucketUsageInfo {
size: 100,
..Default::default()
},
);
partial.buckets_count = 1;
let (selected, _) = select_admin_data_usage_snapshot(authoritative, true, Some(partial));
assert!(selected.usage_snapshot_partial);
assert_eq!(selected.buckets_usage.get("bucket").map(|usage| usage.size), Some(100));
}
#[tokio::test]
async fn authoritative_save_cleanup_removes_observed_snapshot_best_effort() {
let store = UsageCasStore::default();
@@ -4276,7 +4526,15 @@ mod tests {
.expect("namespace lock acquisition should not fail")
.expect("namespace lock should be acquired"),
);
let store = Arc::new(UsageCasStore::default());
let snapshot = data_usage_info_for_test(BUCKET, 2, 84, SystemTime::now());
let encoded = serde_json::to_vec(&snapshot).expect("usage snapshot should encode");
let store = Arc::new(UsageCasStore {
state: Mutex::new(UsageCasState {
object: Some((encoded.clone(), 1)),
backup_object: Some((encoded, 1)),
..Default::default()
}),
});
let successor = data_usage_info_for_test(BUCKET, 7, 294, SystemTime::now());
let mut snapshot_cache = data_usage_snapshot_cache().write().await;
*snapshot_cache = Some(CachedDataUsageSnapshot {
@@ -4375,21 +4633,17 @@ mod tests {
}
#[tokio::test]
async fn remove_bucket_usage_creates_primary_and_backup_fences_when_missing() {
async fn remove_bucket_usage_does_not_synthesize_authoritative_snapshot_when_all_missing() {
let store = Arc::new(UsageCasStore::default());
remove_bucket_usage_from_backend_with_store(store.as_ref(), "bucket-a")
.await
.expect("bucket removal should create both usage fences");
.expect("bucket removal should remain a no-op without a usage baseline");
let state = store.state.lock().await;
assert_eq!(state.put_count, 2);
for (data, revision) in [state.object.as_ref(), state.backup_object.as_ref()].into_iter().flatten() {
let saved = serde_json::from_slice::<DataUsageInfo>(data).expect("saved usage snapshot should decode");
assert_eq!(*revision, 1);
assert!(saved.last_update.is_some());
assert!(!data_usage_contains_bucket(&saved, "bucket-a"));
}
assert_eq!(state.put_count, 0);
assert!(state.object.is_none());
assert!(state.backup_object.is_none());
}
#[tokio::test]
@@ -4541,6 +4795,39 @@ mod tests {
assert_eq!(backup_err, Error::PreconditionFailed);
}
#[tokio::test]
async fn remove_bucket_usage_rejects_movement_epoch_flip_between_read_and_commit() {
let ctx = Arc::new(InstanceContext::new());
let publication_store = build_publication_store(ctx.clone());
let snapshot = data_usage_info_for_test("bucket-a", 2, 84, SystemTime::now());
let store = Arc::new(UsageCasStore {
state: Mutex::new(UsageCasState {
object: Some((serde_json::to_vec(&snapshot).expect("usage snapshot should encode"), 1)),
advance_publication_epoch_after_get: Some((UsageObjectSlot::Primary, ctx)),
..Default::default()
}),
});
let expected_epoch = publication_store
.scanner_data_usage_publication_epoch()
.await
.expect("idle publication store should admit the initial read");
let err = remove_bucket_usage_from_object_with_retries_and_publication(
store.as_ref(),
DATA_USAGE_OBJ_NAME_PATH.as_str(),
"bucket-a",
0,
None,
None,
Some((publication_store.as_ref(), Some(expected_epoch))),
)
.await
.expect_err("a movement epoch flip during the read must fence the commit");
assert!(err.to_string().contains("epoch changed"));
assert_eq!(store.state.lock().await.put_count, 0);
}
#[tokio::test]
async fn remove_bucket_usage_confirms_ambiguous_committed_final_attempt() {
let initial = data_usage_info_for_test("bucket-a", 2, 84, SystemTime::now());
@@ -4665,6 +4952,55 @@ mod tests {
);
}
#[tokio::test]
#[serial]
async fn partial_usage_is_observational_not_authoritative_for_quota() {
clear_usage_memory_cache_for_test().await;
let mut partial = data_usage_info_for_test("bucket-a", 10, 100, SystemTime::now());
partial.usage_snapshot_complete = false;
partial.usage_snapshot_partial = true;
replace_bucket_usage_memory_from_info(&partial).await;
assert_eq!(get_bucket_usage_memory("bucket-a").await, None);
}
#[tokio::test]
#[serial]
async fn stale_quota_uses_complete_baseline_plus_positive_deltas() {
clear_usage_memory_cache_for_test().await;
let baseline = data_usage_info_for_test("bucket-a", 1, 100, SystemTime::now());
replace_bucket_usage_memory_from_info(&baseline).await;
record_bucket_object_write_memory("bucket-a", None, 25).await;
assert_eq!(get_bucket_usage_memory("bucket-a").await, Some(125));
}
#[tokio::test]
#[serial]
async fn negative_delta_waits_for_set_reconciliation() {
clear_usage_memory_cache_for_test().await;
let baseline = data_usage_info_for_test("bucket-a", 1, 100, SystemTime::UNIX_EPOCH + Duration::from_secs(100));
replace_bucket_usage_memory_from_info(&baseline).await;
record_bucket_object_delete_memory("bucket-a", 25, true).await;
assert_eq!(get_bucket_usage_memory("bucket-a").await, Some(100));
// Simulate a process restart: the request-path overlay is gone, but
// the persisted authoritative snapshot is still the pre-reconciliation
// baseline. Quota must remain conservative until a complete scanner
// result proves the delete.
clear_usage_memory_cache_for_test().await;
replace_bucket_usage_memory_from_info(&baseline).await;
assert_eq!(get_bucket_usage_memory("bucket-a").await, Some(100));
let reconciled = data_usage_info_for_test("bucket-a", 0, 75, SystemTime::UNIX_EPOCH + Duration::from_secs(101));
replace_bucket_usage_memory_from_info(&reconciled).await;
assert_eq!(get_bucket_usage_memory("bucket-a").await, Some(75));
}
#[tokio::test]
#[serial]
async fn memory_overlay_counts_versioned_overwrite_as_new_version() {
crates/ecstore/src/disk/local.rs
+77 -6
View File
@@ -8000,10 +8000,15 @@ impl DiskAPI for LocalDisk {
use std::io::Write as _;
let file_path = self.io_get_object_path(volume, path)?;
let lock_path = file_path.with_extension("rustfs-cas.lock");
let path = path.to_string();
let sync_metadata = effective_durability(volume).syncs_commit_metadata();
return Ok(tokio::task::spawn_blocking(move || {
// A persistent directory lock bounds metadata growth. Removing
// per-target lock files can split flock ownership across inodes.
let lock_path = file_path
.parent()
.ok_or_else(|| std::io::Error::new(ErrorKind::InvalidInput, "conditional file has no parent"))?
.join(".rustfs-cas.lock");
let lock = std::fs::OpenOptions::new()
.create(true)
.truncate(false)
@@ -8068,7 +8073,25 @@ impl DiskAPI for LocalDisk {
.map_err(DiskError::from)??);
}
#[cfg(not(unix))]
#[cfg(windows)]
{
let file_path = self.io_get_object_path(volume, path)?;
let sync_metadata = effective_durability(volume).syncs_commit_metadata();
let publication_root = self.publication_root.clone();
return Ok(tokio::task::spawn_blocking(move || {
os::compare_and_update_control_file(
&file_path,
expected.as_deref(),
replacement.as_deref(),
sync_metadata,
&publication_root,
)
})
.await
.map_err(DiskError::from)??);
}
#[cfg(not(any(unix, windows)))]
{
let _ = (volume, path, expected, replacement);
Err(DiskError::MethodNotAllowed)
@@ -8940,6 +8963,7 @@ impl DiskAPI for LocalDisk {
)
.await?;
out.close().await?;
Ok(())
}
@@ -21823,9 +21847,9 @@ mod test {
assert!(matches!(results[1].as_ref().unwrap_err(), DiskError::Io(_)));
}
#[cfg(unix)]
#[cfg(any(unix, windows))]
#[tokio::test]
async fn conditional_file_update_never_deletes_a_new_owner() {
async fn windows_and_unix_conditional_file_update_never_deletes_a_new_owner() {
use tempfile::tempdir;
let dir = tempdir().expect("temp dir should be created");
@@ -21856,8 +21880,18 @@ mod test {
disk.read_all(RUSTFS_META_BUCKET, HEALING_MARKER_PATH)
.await
.expect("new owner marker should remain"),
owner_b
owner_b.clone()
);
assert_eq!(
disk.compare_and_update_file(RUSTFS_META_BUCKET, HEALING_MARKER_PATH, Some(owner_b), None)
.await
.expect("current owner should remove marker"),
ConditionalFileUpdate::Updated
);
assert!(matches!(
disk.read_all(RUSTFS_META_BUCKET, HEALING_MARKER_PATH).await,
Err(DiskError::FileNotFound)
));
}
#[cfg(unix)]
@@ -21872,7 +21906,10 @@ mod test {
let marker_path = disk
.get_object_path(RUSTFS_META_BUCKET, HEALING_MARKER_PATH)
.expect("marker path should resolve");
let lock_path = marker_path.with_extension("rustfs-cas.lock");
let lock_path = marker_path
.parent()
.expect("marker path should have a parent")
.join(".rustfs-cas.lock");
let lock = std::fs::OpenOptions::new()
.create(true)
.truncate(false)
@@ -21893,6 +21930,40 @@ mod test {
assert!(matches!(err, DiskError::Io(ref err) if err.kind() == ErrorKind::WouldBlock));
}
#[cfg(windows)]
#[tokio::test]
async fn windows_conditional_file_update_returns_would_block_when_marker_lock_is_contended() {
let dir = tempfile::tempdir().expect("temp dir should be created");
let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
ensure_test_volume(&disk, RUSTFS_META_BUCKET).await;
let marker_path = disk
.get_object_path(RUSTFS_META_BUCKET, HEALING_MARKER_PATH)
.expect("marker path should resolve");
let lock_path = marker_path
.parent()
.expect("marker path should have a parent")
.join(".rustfs-cas.lock");
let lock = std::fs::OpenOptions::new()
.create(true)
.truncate(false)
.read(true)
.write(true)
.open(lock_path)
.expect("marker lock should open");
lock.try_lock().expect("marker lock should be held");
let err = tokio::time::timeout(
Duration::from_secs(1),
disk.compare_and_update_file(RUSTFS_META_BUCKET, HEALING_MARKER_PATH, None, Some(Bytes::from_static(b"owner"))),
)
.await
.expect("contended conditional update must not block")
.expect_err("contended conditional update must retry");
assert!(matches!(err, DiskError::Io(ref err) if err.kind() == ErrorKind::WouldBlock));
}
#[cfg(target_os = "linux")]
#[tokio::test]
async fn replacement_io_paths_stay_under_the_mount_lease() {
crates/ecstore/src/disk/os.rs
+85
View File
@@ -12,6 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#[cfg(windows)]
use crate::disk::ConditionalFileUpdate;
use crate::disk::error::DiskError;
use crate::disk::error::Result;
use crate::disk::error_conv::to_file_error;
@@ -3458,6 +3460,89 @@ fn read_windows_relative_file(file_path: &Path, parent_guard: &ExistingBaseDirec
Ok(Some(data))
}
#[cfg(windows)]
pub(crate) fn compare_and_update_control_file(
file_path: &Path,
expected: Option<&[u8]>,
replacement: Option<&[u8]>,
sync_metadata: bool,
publication_root: &PublicationRoot,
) -> io::Result<ConditionalFileUpdate> {
use windows_sys::{
Wdk::Storage::FileSystem::{
FILE_NON_DIRECTORY_FILE, FILE_OPEN, FILE_OPEN_IF, FILE_OPEN_REPARSE_POINT, FILE_SYNCHRONOUS_IO_NONALERT,
},
Win32::Storage::FileSystem::{
DELETE, FILE_ATTRIBUTE_NORMAL, FILE_READ_ATTRIBUTES, FILE_SHARE_READ, FILE_SHARE_WRITE, FILE_WRITE_DATA, SYNCHRONIZE,
},
};
let parent = file_path
.parent()
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "conditional file has no parent"))?;
let parent_guard = lock_windows_directory_tree(parent, Some(parent), publication_root)?;
let lock = open_windows_relative(
parent_guard.last_handle()?,
std::ffi::OsStr::new(".rustfs-cas.lock"),
SYNCHRONIZE | FILE_READ_ATTRIBUTES | FILE_WRITE_DATA,
FILE_SHARE_READ | FILE_SHARE_WRITE,
FILE_OPEN_IF,
FILE_NON_DIRECTORY_FILE | FILE_OPEN_REPARSE_POINT | FILE_SYNCHRONOUS_IO_NONALERT,
FILE_ATTRIBUTE_NORMAL,
true,
)?;
validate_windows_owned_file(&lock)?;
match lock.as_file().try_lock() {
Ok(()) => {}
Err(std::fs::TryLockError::WouldBlock) => return Err(io::Error::from(io::ErrorKind::WouldBlock)),
Err(std::fs::TryLockError::Error(err)) => return Err(err),
}
let current = read_windows_relative_file(file_path, &parent_guard)?;
let matches = match (&current, expected) {
(None, None) => true,
(Some(current), Some(expected)) => current.as_slice() == expected,
_ => false,
};
if !matches {
return Ok(match current {
None => ConditionalFileUpdate::Missing,
Some(_) => ConditionalFileUpdate::Mismatch,
});
}
match replacement {
Some(replacement) => RenameDestinationPathGuard {
directory: parent.to_path_buf(),
_directory_guard: parent_guard,
}
.write_file_for_path_access(file_path, replacement, sync_metadata, sync_metadata)?,
None => {
let file_name = file_path
.file_name()
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "conditional file must have a name"))?;
let file = open_windows_relative(
parent_guard.last_handle()?,
file_name,
DELETE | SYNCHRONIZE | FILE_READ_ATTRIBUTES,
FILE_SHARE_READ,
FILE_OPEN,
FILE_NON_DIRECTORY_FILE | FILE_OPEN_REPARSE_POINT | FILE_SYNCHRONOUS_IO_NONALERT,
0,
true,
)?;
validate_windows_owned_file(&file)?;
set_windows_file_delete_on_close(&file, true)?;
drop(file);
if sync_metadata {
fsync_dir_std(parent)?;
}
}
}
Ok(ConditionalFileUpdate::Updated)
}
#[cfg(windows)]
fn open_windows_directory_component(
parent: &WindowsDirectoryHandle,
crates/ecstore/src/runtime/instance.rs
+76 -8
View File
@@ -52,11 +52,18 @@ use crate::services::tier::tier::TierConfigMgr;
use rustfs_lock::{GlobalLockManager, get_global_lock_manager};
use s3s::region::Region;
use std::collections::{HashMap, HashSet};
use std::sync::{Arc, OnceLock};
use std::sync::{
Arc, OnceLock,
atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering},
};
use tokio::sync::{OnceCell, RwLock};
use tokio_util::sync::CancellationToken;
use uuid::Uuid;
const SCANNER_PUBLICATION_STATE_UNKNOWN: u8 = 0;
const SCANNER_PUBLICATION_STATE_ALLOWED: u8 = 1;
const SCANNER_PUBLICATION_STATE_BLOCKED: u8 = 2;
/// Runtime state owned by a single `ECStore` instance.
///
/// This is intentionally minimal in the first migration slice; subsequent
@@ -160,10 +167,22 @@ pub struct InstanceContext {
/// workers (scanner/heal/tier/lifecycle) without touching another instance.
/// Replaces the process-global cancel-token static.
background_cancel_token: OnceLock<CancellationToken>,
/// Serializes decommission data-movement operations with cancellation and
/// a subsequent restart. Readers are held across one object side effect;
/// the transition path takes the writer after cancelling the routine.
decommission_operation_gate: Arc<RwLock<()>>,
/// Serializes data-movement transitions with scanner publication commits.
/// Readers are held across one publication commit; movement transitions
/// take the writer at their durable state commit boundary.
data_movement_operation_gate: Arc<RwLock<()>>,
/// Monotonic admission epoch paired with the operation gate. A
/// publication admitted before a movement transition must never be
/// mistaken for one admitted after the transition.
data_movement_operation_epoch: AtomicU64,
/// Once the admission epoch reaches its reserved terminal value, no new
/// publication may be admitted. Keeping this state separate from the
/// saturating counter prevents an unchanged `u64::MAX` value from being
/// mistaken for a fresh epoch after overflow.
data_movement_operation_epoch_exhausted: AtomicBool,
/// Last storage-owned movement snapshot observed under the operation
/// gate. SetDisks cache writers fail closed until ECStore refreshes it.
scanner_publication_state: AtomicU8,
/// Resolves object-encryption material at the application boundary.
object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
@@ -204,7 +223,10 @@ impl InstanceContext {
local_disk_set_drives: Arc::new(RwLock::new(Vec::new())),
bucket_metadata_sys: std::sync::Mutex::new(None),
background_cancel_token: OnceLock::new(),
decommission_operation_gate: Arc::new(RwLock::new(())),
data_movement_operation_gate: Arc::new(RwLock::new(())),
data_movement_operation_epoch: AtomicU64::new(0),
data_movement_operation_epoch_exhausted: AtomicBool::new(false),
scanner_publication_state: AtomicU8::new(SCANNER_PUBLICATION_STATE_UNKNOWN),
object_encryption_resolver: OnceLock::new(),
tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()),
@@ -223,8 +245,54 @@ impl InstanceContext {
self.lock_manager.clone()
}
pub(crate) fn decommission_operation_gate(&self) -> Arc<RwLock<()>> {
Arc::clone(&self.decommission_operation_gate)
pub(crate) fn data_movement_operation_gate(&self) -> Arc<RwLock<()>> {
Arc::clone(&self.data_movement_operation_gate)
}
pub(crate) fn data_movement_operation_epoch(&self) -> u64 {
self.data_movement_operation_epoch.load(Ordering::Acquire)
}
pub(crate) fn data_movement_operation_epoch_exhausted(&self) -> bool {
self.data_movement_operation_epoch_exhausted.load(Ordering::Acquire)
}
pub(crate) fn scanner_publication_state_allowed(&self) -> bool {
!self.data_movement_operation_epoch_exhausted()
&& self.scanner_publication_state.load(Ordering::Acquire) == SCANNER_PUBLICATION_STATE_ALLOWED
}
pub(crate) fn set_scanner_publication_state(&self, blocked: bool) {
self.scanner_publication_state.store(
if blocked {
SCANNER_PUBLICATION_STATE_BLOCKED
} else {
SCANNER_PUBLICATION_STATE_ALLOWED
},
Ordering::Release,
);
}
pub(crate) fn advance_data_movement_operation_epoch(&self) -> u64 {
self.scanner_publication_state
.store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release);
let _ = self
.data_movement_operation_epoch
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |epoch| Some(epoch.saturating_add(1)));
let result = self.data_movement_operation_epoch.load(Ordering::Acquire);
if result == u64::MAX {
self.data_movement_operation_epoch_exhausted.store(true, Ordering::Release);
}
result
}
#[cfg(test)]
pub(crate) fn set_data_movement_operation_epoch_for_test(&self, epoch: u64) {
self.data_movement_operation_epoch.store(epoch, Ordering::Release);
self.data_movement_operation_epoch_exhausted
.store(epoch == u64::MAX, Ordering::Release);
self.scanner_publication_state
.store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release);
}
/// Install the application-owned object-encryption resolver once.
crates/ecstore/src/services/metrics_realtime.rs
+24
View File
@@ -220,6 +220,14 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo
last_usage_save_unix_secs: metrics.usage_freshness.last_usage_save_unix_secs,
last_usage_save_result: metrics.usage_freshness.last_usage_save_result,
last_usage_save_result_code: metrics.usage_freshness.last_usage_save_result_code,
last_durable_success_unix_secs: metrics.usage_freshness.last_durable_success_unix_secs,
last_publication_unix_secs: metrics.usage_freshness.last_publication_unix_secs,
last_publication_state: metrics.usage_freshness.last_publication_state,
last_publication_reason: metrics.usage_freshness.last_publication_reason,
deferred_pending: metrics.usage_freshness.deferred_pending,
deferred_total: metrics.usage_freshness.deferred_total,
last_deferred_unix_secs: metrics.usage_freshness.last_deferred_unix_secs,
last_deferred_reason: metrics.usage_freshness.last_deferred_reason,
},
maintenance_control: MadminScannerMaintenanceControlSnapshot {
primary_control: metrics.maintenance_control.primary_control,
@@ -816,6 +824,14 @@ mod test {
last_usage_save_unix_secs: 12,
last_usage_save_result: "success".to_string(),
last_usage_save_result_code: 1,
last_durable_success_unix_secs: 13,
last_publication_unix_secs: 14,
last_publication_state: "published".to_string(),
last_publication_reason: "complete".to_string(),
deferred_pending: true,
deferred_total: 15,
last_deferred_unix_secs: 16,
last_deferred_reason: "data_movement".to_string(),
},
..Default::default()
});
@@ -828,6 +844,14 @@ mod test {
assert_eq!(scanner.usage_freshness.last_usage_save_unix_secs, 12);
assert_eq!(scanner.usage_freshness.last_usage_save_result, "success");
assert_eq!(scanner.usage_freshness.last_usage_save_result_code, 1);
assert_eq!(scanner.usage_freshness.last_durable_success_unix_secs, 13);
assert_eq!(scanner.usage_freshness.last_publication_unix_secs, 14);
assert_eq!(scanner.usage_freshness.last_publication_state, "published");
assert_eq!(scanner.usage_freshness.last_publication_reason, "complete");
assert!(scanner.usage_freshness.deferred_pending);
assert_eq!(scanner.usage_freshness.deferred_total, 15);
assert_eq!(scanner.usage_freshness.last_deferred_unix_secs, 16);
assert_eq!(scanner.usage_freshness.last_deferred_reason, "data_movement");
}
#[test]
crates/ecstore/src/services/rebalance/control.rs
+76 -11
View File
@@ -48,12 +48,12 @@ fn pool_rebalance_status_from_meta(meta: Option<&RebalanceMeta>, pool_index: usi
.unwrap_or_default()
}
fn merge_rebalance_status_refresh(current: &mut Option<RebalanceMeta>, persisted: RebalanceMeta) {
fn merge_rebalance_status_refresh(current: &mut Option<RebalanceMeta>, persisted: RebalanceMeta) -> bool {
if persisted.id.is_empty() && persisted.pool_stats.is_empty() {
clear_rebalance_status_refresh(current);
return;
return clear_rebalance_status_refresh(current);
}
let before = current.clone();
match current.as_mut() {
Some(current_meta) => {
if merge_rebalance_meta(current_meta, &persisted) == RebalanceMetaMergeOutcome::RejectedActiveConflict
@@ -66,14 +66,41 @@ fn merge_rebalance_status_refresh(current: &mut Option<RebalanceMeta>, persisted
*current = Some(persisted);
}
}
match (before.as_ref(), current.as_ref()) {
(None, None) => false,
(None, Some(_)) | (Some(_), None) => true,
(Some(before), Some(after)) => rebalance_movement_snapshot_changed(Some(before), after),
}
}
fn clear_rebalance_status_refresh(current: &mut Option<RebalanceMeta>) {
fn clear_rebalance_status_refresh(current: &mut Option<RebalanceMeta>) -> bool {
if current.as_ref().is_none_or(|meta| !is_rebalance_actively_running(meta)) {
*current = None;
current.take().is_some()
} else {
false
}
}
fn rebalance_movement_snapshot_changed(current: Option<&RebalanceMeta>, persisted: &RebalanceMeta) -> bool {
let Some(current) = current else {
return true;
};
current.id != persisted.id
|| current.stopped_at != persisted.stopped_at
|| current.pool_stats.len() != persisted.pool_stats.len()
|| current
.pool_stats
.iter()
.zip(persisted.pool_stats.iter())
.any(|(current, persisted)| {
current.participating != persisted.participating
|| current.info.status != persisted.info.status
|| current.info.stopping != persisted.info.stopping
})
}
impl ECStore {
pub(super) async fn save_rebalance_meta_with_merge<S>(
&self,
@@ -121,7 +148,10 @@ impl ECStore {
"Loading rebalance metadata"
);
let pool = clone_first_arc(&self.pools, "rebalanceMeta: no pools available")?;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
if resolve_rebalance_meta_load_result(meta.load(pool).await)? {
let movement_changed = rebalance_movement_snapshot_changed(self.rebalance_meta.read().await.as_ref(), &meta);
{
let mut rebalance_meta = self.rebalance_meta.write().await;
@@ -130,6 +160,10 @@ impl ECStore {
drop(rebalance_meta);
}
if movement_changed {
self.ctx.advance_data_movement_operation_epoch();
}
drop(_movement_guard);
resolve_load_rebalance_stats_update_result(self.update_rebalance_stats().await)?;
debug!(
event = EVENT_REBALANCE_STATE,
@@ -139,10 +173,15 @@ impl ECStore {
"Loaded rebalance metadata"
);
} else {
let movement_changed = self.rebalance_meta.read().await.is_some();
{
let mut rebalance_meta = self.rebalance_meta.write().await;
*rebalance_meta = None;
}
if movement_changed {
self.ctx.advance_data_movement_operation_epoch();
}
drop(_movement_guard);
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
@@ -160,14 +199,20 @@ impl ECStore {
pub async fn refresh_rebalance_status_meta(&self) -> Result<()> {
let pool = clone_first_arc(&self.pools, "refresh_rebalance_status_meta: no pools available")?;
let mut persisted = RebalanceMeta::new();
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
match persisted.load(pool).await {
Ok(()) => {
let mut rebalance_meta = self.rebalance_meta.write().await;
merge_rebalance_status_refresh(&mut rebalance_meta, persisted);
if merge_rebalance_status_refresh(&mut rebalance_meta, persisted) {
self.ctx.advance_data_movement_operation_epoch();
}
}
Err(Error::ConfigNotFound) => {
let mut rebalance_meta = self.rebalance_meta.write().await;
clear_rebalance_status_refresh(&mut rebalance_meta);
if clear_rebalance_status_refresh(&mut rebalance_meta) {
self.ctx.advance_data_movement_operation_epoch();
}
}
Err(err) => {
return Err(Error::other(format!("rebalance metadata refresh failed during pool status: {err}")));
@@ -349,6 +394,8 @@ impl ECStore {
#[tracing::instrument(skip(self, bucktes))]
pub async fn init_rebalance_start(self: &Arc<Self>, bucktes: Vec<String>) -> Result<String> {
let _start_guard = self.start_gate.lock().await;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
let decommission_running = self.is_decommission_running().await;
{
@@ -356,12 +403,16 @@ impl ECStore {
validate_init_rebalance_state(decommission_running, rebalance_meta.as_ref())?;
}
self.init_rebalance_meta(bucktes).await
let id = self.init_rebalance_meta(bucktes).await?;
self.ctx.advance_data_movement_operation_epoch();
Ok(id)
}
#[tracing::instrument(skip(self))]
pub async fn start_rebalance_for_id(self: &Arc<Self>, expected_id: &str) -> Result<()> {
let _start_guard = self.start_gate.lock().await;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
{
let rebalance_meta = self.rebalance_meta.read().await;
@@ -379,7 +430,10 @@ impl ECStore {
}
}
self.start_rebalance().await
if self.start_rebalance_inner().await? {
self.ctx.advance_data_movement_operation_epoch();
}
Ok(())
}
pub async fn rollback_rebalance_start_for_id(self: &Arc<Self>, expected_id: Option<&str>, start_error: String) -> Result<()> {
@@ -548,6 +602,9 @@ impl ECStore {
#[tracing::instrument(skip(self))]
pub async fn stop_rebalance_for_id(self: &Arc<Self>, expected_id: Option<&str>) -> Result<()> {
let _start_guard = self.start_gate.lock().await;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
let meta_to_save = {
let mut rebalance_meta = self.rebalance_meta.write().await;
stop_rebalance_meta_snapshot_for_id(rebalance_meta.as_mut(), OffsetDateTime::now_utc(), expected_id)
@@ -560,6 +617,7 @@ impl ECStore {
.await,
"stop_rebalance",
)?;
self.ctx.advance_data_movement_operation_epoch();
}
Ok(())
@@ -570,6 +628,9 @@ impl ECStore {
expected_id: Option<&str>,
start_error: String,
) -> Result<()> {
let _start_guard = self.start_gate.lock().await;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
let meta_to_save = {
let mut rebalance_meta = self.rebalance_meta.write().await;
rollback_rebalance_start_meta_snapshot_for_id(
@@ -587,6 +648,7 @@ impl ECStore {
.await,
"rollback_rebalance_start",
)?;
self.ctx.advance_data_movement_operation_epoch();
}
Ok(())
@@ -597,6 +659,8 @@ impl ECStore {
return Ok(());
}
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
let encoded_error = encode_rebalance_stop_propagation_record(&record);
let meta_to_save = {
let mut rebalance_meta = self.rebalance_meta.write().await;
@@ -610,6 +674,7 @@ impl ECStore {
.await,
"record_rebalance_stop_propagation",
)?;
self.ctx.advance_data_movement_operation_epoch();
}
Ok(())
@@ -682,7 +747,7 @@ mod tests {
..Default::default()
};
merge_rebalance_status_refresh(&mut current, persisted);
assert!(merge_rebalance_status_refresh(&mut current, persisted));
let refreshed = current.as_ref().expect("refresh should keep rebalance metadata");
assert_eq!(refreshed.pool_stats[0].info.status, RebalStatus::Completed);
@@ -721,7 +786,7 @@ mod tests {
..Default::default()
};
merge_rebalance_status_refresh(&mut current, persisted);
assert!(!merge_rebalance_status_refresh(&mut current, persisted));
assert!(
current.as_ref().and_then(|meta| meta.cancel.as_ref()).is_some(),
crates/ecstore/src/services/rebalance/runtime.rs
+87 -47
View File
@@ -42,6 +42,15 @@ pub(super) fn source_cleanup_defer_attempt(deferred_attempts: &mut HashMap<Strin
impl ECStore {
#[tracing::instrument(skip_all)]
pub async fn start_rebalance(self: &Arc<Self>) -> Result<()> {
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
if self.start_rebalance_inner().await? {
self.ctx.advance_data_movement_operation_epoch();
}
Ok(())
}
pub(super) async fn start_rebalance_inner(self: &Arc<Self>) -> Result<bool> {
info!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
@@ -55,6 +64,7 @@ impl ECStore {
let cancel_tx = CancellationToken::new();
let rx = cancel_tx.clone();
let mut meta_to_save = None;
let mut movement_changed = false;
{
let mut rebalance_meta = self.rebalance_meta.write().await;
@@ -72,14 +82,16 @@ impl ECStore {
reason = "already_in_progress",
"Skipped duplicate rebalance start"
);
return Ok(());
return Ok(false);
}
let now = OffsetDateTime::now_utc();
if complete_rebalance_pools_at_goal(meta, now) {
meta_to_save = Some(meta.clone());
movement_changed = true;
}
if complete_rebalance_pools_with_empty_queue(meta, now) {
meta_to_save = Some(meta.clone());
movement_changed = true;
}
meta.cancel = Some(cancel_tx);
@@ -118,7 +130,7 @@ impl ECStore {
reason = "no_participants",
"Skipped rebalance start because no pools are participating"
);
return Ok(());
return Ok(movement_changed);
}
let mut workers_started = 0usize;
@@ -186,7 +198,7 @@ impl ECStore {
reason = "no_local_participants",
"Skipped rebalance start because no local pools are participating"
);
return Ok(());
return Ok(movement_changed);
}
info!(
@@ -197,7 +209,7 @@ impl ECStore {
worker_count = workers_started,
"Rebalance started"
);
Ok(())
Ok(true)
}
#[tracing::instrument(skip(self, rx))]
@@ -214,53 +226,77 @@ impl ECStore {
let mut quit = false;
loop {
let mut terminal_state_saved = false;
tokio::select! {
result = done_rx.recv() => {
quit = true;
let now = OffsetDateTime::now_utc();
let terminal_event = classify_rebalance_terminal_event(result, now);
msg = terminal_event.message().to_string();
let mut rebalance_meta = store.rebalance_meta.write().await;
if let Some(meta) = rebalance_meta.as_mut() {
let meta_stopped = meta.stopped_at.is_some();
if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) {
if matches!(&terminal_event, super::meta::RebalanceTerminalEvent::Completed { .. })
&& has_rebalance_cleanup_warnings(pool_stat)
{
pool_stat.info.stopping = false;
pool_stat.info.status = RebalStatus::Failed;
pool_stat.info.end_time = Some(now);
pool_stat.info.last_error = Some(
pool_stat
.cleanup_warnings
.last_message
.clone()
.unwrap_or_else(|| "rebalance source cleanup warnings prevented completion".to_string()),
);
} else if should_preserve_rebalance_stopped_state(
meta_stopped,
pool_stat.info.status,
&terminal_event,
) {
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
state = "stopped_preserved",
"Preserved stopped rebalance status"
);
let movement_gate = store.ctx.data_movement_operation_gate();
let movement_guard = movement_gate.write().await;
let previous_meta = store.rebalance_meta.read().await.clone();
let terminal_state_present = {
let mut rebalance_meta = store.rebalance_meta.write().await;
if let Some(meta) = rebalance_meta.as_mut() {
let meta_stopped = meta.stopped_at.is_some();
if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) {
if matches!(&terminal_event, super::meta::RebalanceTerminalEvent::Completed { .. })
&& has_rebalance_cleanup_warnings(pool_stat)
{
pool_stat.info.stopping = false;
pool_stat.info.status = RebalStatus::Failed;
pool_stat.info.end_time = Some(now);
pool_stat.info.last_error = Some(
pool_stat
.cleanup_warnings
.last_message
.clone()
.unwrap_or_else(|| "rebalance source cleanup warnings prevented completion".to_string()),
);
} else if should_preserve_rebalance_stopped_state(
meta_stopped,
pool_stat.info.status,
&terminal_event,
) {
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
state = "stopped_preserved",
"Preserved stopped rebalance status"
);
} else {
pool_stat.info.stopping = false;
apply_rebalance_terminal_event(
&mut pool_stat.info.status,
&mut pool_stat.info.end_time,
&mut pool_stat.info.last_error,
terminal_event,
now,
);
}
true
} else {
pool_stat.info.stopping = false;
apply_rebalance_terminal_event(
&mut pool_stat.info.status,
&mut pool_stat.info.end_time,
&mut pool_stat.info.last_error,
terminal_event,
now,
);
false
}
} else {
false
}
};
if terminal_state_present {
if let Err(err) = store.save_rebalance_stats_inner(pool_index, RebalSaveOpt::Stats).await {
let mut rebalance_meta = store.rebalance_meta.write().await;
*rebalance_meta = previous_meta;
drop(movement_guard);
return Err(Error::other(format!(
"rebalance terminal state save failed for pool {pool_index}: {err}"
)));
}
store.ctx.advance_data_movement_operation_epoch();
terminal_state_saved = true;
}
}
_ = timer.tick() => {
@@ -269,7 +305,7 @@ impl ECStore {
}
}
if let Err(err) = store.save_rebalance_stats(pool_index, RebalSaveOpt::Stats).await {
if !terminal_state_saved && let Err(err) = store.save_rebalance_stats(pool_index, RebalSaveOpt::Stats).await {
let wrapped = Error::other(format!("rebalance save_task stats save failed for pool {pool_index}: {err}"));
error!("{} err: {:?}", msg, wrapped);
if quit {
@@ -590,16 +626,14 @@ impl ECStore {
meta.percent_free_goal,
)
{
pool_stat.info.status = RebalStatus::Completed;
pool_stat.info.end_time = Some(OffsetDateTime::now_utc());
info!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
state = "completed",
state = "completion_ready",
percent_free = pfi,
"Marked rebalance pool completed"
"Rebalance pool reached completion goal"
);
return true;
}
@@ -612,6 +646,12 @@ impl ECStore {
impl ECStore {
#[tracing::instrument(skip(self))]
pub async fn save_rebalance_stats(&self, pool_idx: usize, opt: RebalSaveOpt) -> Result<()> {
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
self.save_rebalance_stats_inner(pool_idx, opt).await
}
pub(super) async fn save_rebalance_stats_inner(&self, pool_idx: usize, opt: RebalSaveOpt) -> Result<()> {
let meta_to_save = {
let mut rebalance_meta = self.rebalance_meta.write().await;
let Some(meta) = rebalance_meta.as_mut() else {
crates/ecstore/src/set_disk/core/io_primitives.rs
+105 -49
View File
@@ -71,6 +71,9 @@ use crate::set_disk::shard_source::ShardReadCost;
use futures::FutureExt as _;
use futures::stream::{FuturesUnordered, StreamExt};
use metrics::counter;
use rustfs_io_metrics::internode_metrics::{
INTERNODE_STAGE_BATCH_READ_VERSION_COALESCER_WAIT, INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MAP,
};
use std::{
collections::{HashMap, HashSet, VecDeque},
future::Future,
@@ -140,6 +143,21 @@ struct CoalescedReadVersionRequest {
tx: oneshot::Sender<disk::error::Result<FileInfo>>,
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct ExpectedBatchReadVersionItem {
path: String,
version_id: String,
}
impl From<&BatchReadVersionItem> for ExpectedBatchReadVersionItem {
fn from(item: &BatchReadVersionItem) -> Self {
Self {
path: item.path.clone(),
version_id: item.version_id.clone(),
}
}
}
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
struct ReadVersionCoalescerKey {
disk: usize,
@@ -178,6 +196,16 @@ fn record_read_version_coalescer_event(event: &'static str, item_count: usize) {
.increment(1);
}
fn batch_read_version_stage_timer() -> Option<Instant> {
rustfs_io_metrics::get_stage_metrics_enabled().then(Instant::now)
}
fn record_batch_read_version_stage(stage: &'static str, started_at: Option<Instant>) {
if let Some(started_at) = started_at {
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_stage(stage, started_at.elapsed());
}
}
async fn read_version_via_coalescer(
disk: DiskStore,
org_bucket: &str,
@@ -227,8 +255,12 @@ async fn read_version_via_coalescer(
flush_read_version_coalescer_pending(lane_key, disk, *opts, pending).await;
}
rx.await
.unwrap_or_else(|_| Err(DiskError::other("coalesced read_version response channel closed")))
let wait_started = batch_read_version_stage_timer();
let response = rx
.await
.unwrap_or_else(|_| Err(DiskError::other("coalesced read_version response channel closed")));
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_COALESCER_WAIT, wait_started);
response
}
async fn flush_read_version_coalescer_lane(lane_key: ReadVersionCoalescerKey, disk: DiskStore, opts: ReadOptions) {
@@ -266,7 +298,7 @@ async fn flush_read_version_coalescer_pending(
items.push(request.item);
}
let expected_items = items.clone();
let expected_items = items.iter().map(ExpectedBatchReadVersionItem::from).collect::<Vec<_>>();
record_read_version_coalescer_event("attempted_batch", items.len());
let result =
match tokio::time::timeout(get_drive_metadata_timeout(), disk.batch_read_version(BatchReadVersionReq { items, opts }))
@@ -277,7 +309,9 @@ async fn flush_read_version_coalescer_pending(
};
match result {
Ok(responses) => {
let map_started = batch_read_version_stage_timer();
let results = map_batch_read_version_responses(&expected_items, responses);
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MAP, map_started);
for (tx, result) in senders.into_iter().zip(results) {
let _ = tx.send(result);
}
@@ -292,7 +326,7 @@ async fn flush_read_version_coalescer_pending(
}
fn map_batch_read_version_responses(
expected_items: &[BatchReadVersionItem],
expected_items: &[ExpectedBatchReadVersionItem],
responses: Vec<BatchReadVersionResp>,
) -> Vec<crate::disk::error::Result<FileInfo>> {
let mut results = (0..expected_items.len())
@@ -1412,8 +1446,11 @@ pub(in crate::set_disk) async fn submit_read_repair_heal_with_submitter(
// Reservation won: this sighting owns the repair records for the object,
// including the durable journal intent when the caller asked for one.
if let Some((kind, version_uuid)) = mrf_intent {
rustfs_common::mrf_channel::try_send_mrf_intent(kind, bucket, object, version_uuid);
if let Some((kind, version_uuid)) = mrf_intent
&& let (Ok(pool_index), Ok(set_index)) = (u32::try_from(pool_index), u32::try_from(set_index))
{
let scope = rustfs_common::mrf_channel::MrfScope { pool_index, set_index };
let _ = rustfs_common::mrf_channel::try_send_mrf_intent_typed(kind, bucket, object, version_uuid, Some(scope));
}
let mut request = rustfs_common::heal_channel::create_heal_request_with_options(
@@ -3472,7 +3509,7 @@ type RenameDataLegacyTuple = (
);
fn put_rename_early_ack_enabled() -> bool {
rustfs_utils::get_env_bool(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, false)
rustfs_utils::get_env_bool(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, true)
}
impl RenameDataCommit {
@@ -6878,6 +6915,7 @@ mod tests {
},
];
let expected_items = expected_batch_read_version_items(&expected_items);
let mut results = map_batch_read_version_responses(&expected_items, responses).into_iter();
let first = results
.next()
@@ -6918,6 +6956,7 @@ mod tests {
version_id: "v-b".to_string(),
},
];
let expected_items = expected_batch_read_version_items(&expected_items);
let results = map_batch_read_version_responses(
&expected_items,
vec![
@@ -6957,6 +6996,7 @@ mod tests {
path: "object-a".to_string(),
version_id: "v-a".to_string(),
}];
let expected_items = expected_batch_read_version_items(&expected_items);
let mismatched = map_batch_read_version_responses(
&expected_items,
vec![BatchReadVersionResp {
@@ -7018,6 +7058,10 @@ mod tests {
);
}
fn expected_batch_read_version_items(items: &[BatchReadVersionItem]) -> Vec<ExpectedBatchReadVersionItem> {
items.iter().map(ExpectedBatchReadVersionItem::from).collect()
}
/// Isolation guard: unobserved objects record nothing (so parallel tests do
/// not inflate one another), and a scope clears its own counts on drop.
#[tokio::test]
@@ -8631,17 +8675,21 @@ mod tests {
inline_fi.mod_time = Some(OffsetDateTime::now_utc());
std::fs::create_dir_all(disk_root.join(RUSTFS_META_TMP_BUCKET).join("tmp-inline"))
.expect("inline staging dir should be created");
SetDisks::rename_data(
// Use rename_data_owned so we can await the tail_drain for cleanup.
let commit = SetDisks::rename_data_owned(
std::slice::from_ref(&online_disk),
RUSTFS_META_TMP_BUCKET,
"tmp-inline",
std::slice::from_ref(&inline_fi),
vec![inline_fi],
bucket,
object,
1,
)
.await
.expect("inline version should commit");
if let Some(td) = commit.tail_drain {
td.await.expect("inline commit tail drain must succeed");
}
// Overwrite the same (nil) version with a non-inline one.
let new_data_dir = Uuid::new_v4();
@@ -8654,17 +8702,20 @@ mod tests {
.join(new_data_dir.to_string());
std::fs::create_dir_all(&staged_data_dir).expect("streaming staging dir should be created");
std::fs::write(staged_data_dir.join("part.1"), b"streamed-body").expect("staged part should be written");
SetDisks::rename_data(
let commit = SetDisks::rename_data_owned(
std::slice::from_ref(&online_disk),
RUSTFS_META_TMP_BUCKET,
"tmp-streaming",
std::slice::from_ref(&streaming_fi),
vec![streaming_fi],
bucket,
object,
1,
)
.await
.expect("non-inline overwrite should commit");
if let Some(td) = commit.tail_drain {
td.await.expect("non-inline overwrite tail drain must succeed");
}
let mut leftovers: Vec<String> = std::fs::read_dir(disk_root.join(bucket).join(object))
.expect("committed object dir should be readable")
@@ -8825,49 +8876,54 @@ mod tests {
#[tokio::test]
#[serial_test::serial(rename_quorum_ack)]
async fn rename_data_waits_for_tail_disk_after_write_quorum() {
const DISKS: usize = 4;
let bucket = "rename-tail-success-bucket";
let object = "rename-tail-success-object";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
prepare_rename_source_dirs(&dirs, &disks, "source").await;
let file_infos = rename_commit_fileinfos(object, DISKS, "tail-success-etag");
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
// Explicitly test the serial (join_all) path: early ack is now the
// default, so disable it to verify the legacy behaviour still works.
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("false"))], async {
const DISKS: usize = 4;
let bucket = "rename-tail-success-bucket";
let object = "rename-tail-success-object";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
prepare_rename_source_dirs(&dirs, &disks, "source").await;
let file_infos = rename_commit_fileinfos(object, DISKS, "tail-success-etag");
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let rename = SetDisks::rename_data(&disks, RUSTFS_META_TMP_BUCKET, "source", &file_infos, bucket, object, 3);
tokio::pin!(rename);
tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
tokio::select! {
() = barrier.wait_until_paused() => {}
result = &mut rename => panic!("rename_data returned before the armed fan-out barrier: {result:?}"),
}
})
.await
.expect("paused disk must reach the armed rename barrier");
assert!(
tokio::time::timeout(Duration::from_millis(50), &mut rename).await.is_err(),
"current rename_data waits for the paused fan-out disk even after the other three disks can reach write quorum"
);
barrier.release();
rename
let rename = SetDisks::rename_data(&disks, RUSTFS_META_TMP_BUCKET, "source", &file_infos, bucket, object, 3);
tokio::pin!(rename);
tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
tokio::select! {
() = barrier.wait_until_paused() => {}
result = &mut rename => panic!("rename_data returned before the armed fan-out barrier: {result:?}"),
}
})
.await
.expect("tail success must complete the rename after the barrier is released");
.expect("paused disk must reach the armed rename barrier");
for (idx, dir) in dirs.iter().enumerate() {
let reopened = reopen_local_disk(dir).await;
let stored = reopened
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.unwrap_or_else(|err| panic!("disk {idx} must contain the tail-success commit after reopen: {err:?}"));
assert_eq!(
stored.metadata.get("etag").map(String::as_str),
Some("tail-success-etag"),
"disk {idx} must expose the same committed metadata after tail success and reopen"
assert!(
tokio::time::timeout(Duration::from_millis(50), &mut rename).await.is_err(),
"serial rename_data waits for the paused fan-out disk even after write quorum"
);
}
drop(dirs);
barrier.release();
rename
.await
.expect("tail success must complete the rename after the barrier is released");
for (idx, dir) in dirs.iter().enumerate() {
let reopened = reopen_local_disk(dir).await;
let stored = reopened
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.unwrap_or_else(|err| panic!("disk {idx} must contain the tail-success commit after reopen: {err:?}"));
assert_eq!(
stored.metadata.get("etag").map(String::as_str),
Some("tail-success-etag"),
"disk {idx} must expose the same committed metadata after tail success and reopen"
);
}
drop(dirs);
})
.await;
}
#[tokio::test]
crates/ecstore/src/set_disk/mod.rs
+29 -1
View File
@@ -3614,6 +3614,26 @@ impl SetDisks {
&self.ctx
}
/// Admit one short scanner cache publication under this set's instance
/// movement fence. The caller must hold the returned guard through its
/// final conditional cache write; no scan-round work belongs under it.
pub async fn scanner_data_usage_publication_admission_guard(&self) -> Option<(tokio::sync::OwnedRwLockReadGuard<()>, u64)> {
let operation_gate = self.ctx.data_movement_operation_gate();
let operation_guard = operation_gate.read_owned().await;
if self.ctx.scanner_publication_state_allowed() {
let epoch = self.ctx.data_movement_operation_epoch();
return Some((operation_guard, epoch));
}
// The owner deliberately marks the cached state UNKNOWN after every
// movement epoch advance. Do not strand remote scanner writers in that
// state: release this guard before asking the storage owner to refresh
// its durable movement snapshot, since the owner uses the same gate.
drop(operation_guard);
let owner = runtime_sources::object_store_handle().filter(|owner| Arc::ptr_eq(&owner.ctx, &self.ctx))?;
owner.scanner_data_usage_publication_admission_guard().await
}
/// Whether both sets' namespace-lock implementations cover the same object key.
pub(crate) async fn shares_namespace_lock_domain(&self, other: &Self) -> bool {
match (self.ctx.is_dist_erasure().await, other.ctx.is_dist_erasure().await) {
@@ -4736,7 +4756,15 @@ impl SetDisks {
achieved: 0,
});
}
let parts_metadata = vec![fi.clone(); disks.len()];
// Rebuilt tiered metadata starts with index zero, but shuffling validates
// each source slot before assigning the shuffled index below.
let parts_metadata: Vec<FileInfo> = (0..disks.len())
.map(|disk_index| {
let mut part = fi.clone();
part.erasure.index = fi.erasure.distribution[disk_index];
part
})
.collect();
let (shuffle_disks, parts_metadata) = Self::shuffle_disks_and_parts_metadata(&disks, &parts_metadata, &fi);
let mut errs = Vec::with_capacity(shuffle_disks.len());
crates/ecstore/src/set_disk/ops/multipart.rs
+177
View File
@@ -2318,6 +2318,43 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
fi.set_data_moved();
}
// Receiver-side LWW (rustfs/backlog#1953): the multipart replication
// transport carries the category values at CreateMultipartUpload (in
// the staged upload metadata) and the source category timestamps on
// the complete request. Read the destination version under the held
// object write lock and keep any category this site modified more
// recently. Only an absent version has no local state to compare;
// other read failures must leave the upload retryable rather than
// committing inbound metadata without the LWW check.
if crate::set_disk::ops::object::replication_lww_applicable(opts)
&& let Some(version_id) = fi.version_id
{
match self
.get_object_info(
bucket,
object,
&ObjectOptions {
version_id: Some(version_id.to_string()),
no_lock: true,
metadata_cache_safe: false,
versioned: opts.versioned,
version_suspended: opts.version_suspended,
..Default::default()
},
)
.await
{
Ok(existing) => {
let stored = crate::set_disk::ops::object::stored_replication_category_metadata(&existing);
crate::set_disk::ops::object::merge_replication_metadata_lww(&mut fi.metadata, &stored, opts);
}
// Version absent: first replication of this version, nothing
// local to compare — the normal path, not a degraded one.
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
Err(err) => return Err(err),
}
}
for meta in parts_metadatas.iter_mut() {
if meta.has_valid_erasure_geometry() {
meta.size = fi.size;
@@ -7055,6 +7092,146 @@ mod tests {
.await
}
/// Receiver-side LWW on the multipart replication transport
/// (rustfs/backlog#1953): a metadata-only replication of a multipart
/// source object rides CreateMultipartUpload (category values in the
/// upload metadata) + CompleteMultipartUpload (category timestamps in
/// the complete options). A stale inbound tagging timestamp must not
/// overwrite a newer locally-tagged destination version.
#[tokio::test]
#[serial]
async fn complete_multipart_upload_stale_replication_tags_keep_local() {
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
use rustfs_utils::http::{SUFFIX_TAGGING_TIMESTAMP, get_str};
use time::format_description::well_known::Rfc3339;
const T_OLD: &str = "2026-01-01T00:00:00Z";
const T_LOCAL: &str = "2026-02-01T00:00:00Z";
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "multipart-replication-lww-bucket";
let object = "object";
make_bucket_on_all(&disk_stores, bucket).await;
// Local destination version with newer tags.
let version_id = Uuid::new_v4();
let mut local_metadata = HashMap::new();
local_metadata.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
rustfs_utils::http::insert_str(&mut local_metadata, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
let mut local_reader = PutObjReader::from_vec(b"local body".to_vec());
set_disks
.put_object(
bucket,
object,
&mut local_reader,
&ObjectOptions {
versioned: true,
version_id: Some(version_id.to_string()),
user_defined: local_metadata,
// Explicit-version PUTs require the bucket Object Lock snapshot.
object_lock_config_snapshot: Some(Arc::new(crate::set_disk::ObjectLockConfigSnapshot::new(
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
))),
..Default::default()
},
)
.await
.expect("local versioned put should commit");
// Inbound replication upload carrying older tags for the same version.
let mut inbound_metadata = HashMap::new();
inbound_metadata.insert(AMZ_OBJECT_TAGGING.to_string(), "site=remote".to_string());
rustfs_utils::http::insert_str(&mut inbound_metadata, SUFFIX_TAGGING_TIMESTAMP, T_OLD.to_string());
let create_opts = ObjectOptions {
versioned: true,
user_defined: inbound_metadata,
..Default::default()
};
let (upload_id, parts) =
stage_upload_with_create_opts(&set_disks, bucket, object, &payload(0x5a), &create_opts).await;
rewrite_staged_upload_version_id(&set_disks, bucket, object, &upload_id, Some(version_id)).await;
let complete_opts = ObjectOptions {
versioned: true,
replication_request: true,
replication_tagging_timestamp: Some(OffsetDateTime::parse(T_OLD, &Rfc3339).expect("test timestamp should parse")),
..Default::default()
};
// Make the destination version unreadable on quorum while the
// staged upload remains intact. The commit barrier lets the old
// fail-open path move past the LWW read; restoring the metadata
// there proves it would otherwise commit the stale tags.
let mut damaged_metadata = Vec::new();
for temp_dir in temp_dirs.iter().take(3) {
let path = temp_dir.path().join(bucket).join(object).join(STORAGE_FORMAT_FILE);
let original = tokio::fs::read(&path).await.expect("destination xl.meta should be readable");
tokio::fs::write(&path, b"not an xl.meta")
.await
.expect("destination xl.meta should be corruptible");
damaged_metadata.push((path, original));
}
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::BeforeLockLost);
let first_set = set_disks.clone();
let first_upload_id = upload_id.clone();
let first_parts = parts.clone();
let first_opts = complete_opts.clone();
let mut first_completion = tokio::spawn(async move {
first_set
.complete_multipart_upload(bucket, object, &first_upload_id, first_parts, &first_opts)
.await
});
let first_result = tokio::select! {
result = &mut first_completion => result.expect("first completion task should finish"),
() = barrier.wait_until_paused() => {
for (path, original) in &damaged_metadata {
tokio::fs::write(path, original).await.expect("destination xl.meta should be restorable");
}
barrier.release();
first_completion.await.expect("released completion task should finish")
}
};
for (path, original) in &damaged_metadata {
tokio::fs::write(path, original)
.await
.expect("destination xl.meta should be restored");
}
drop(barrier);
let first_error = first_result.expect_err("unreadable destination metadata must fail before multipart commit");
assert!(
!(is_err_object_not_found(&first_error) || is_err_version_not_found(&first_error)),
"corrupt destination metadata must not be treated as an absent version: {first_error}"
);
set_disks
.clone()
.complete_multipart_upload(bucket, object, &upload_id, parts, &complete_opts)
.await
.expect("replication multipart completion should succeed even when a category keeps local values");
let info = set_disks
.get_object_info(
bucket,
object,
&ObjectOptions {
versioned: true,
version_id: Some(version_id.to_string()),
..Default::default()
},
)
.await
.expect("completed version should be readable");
assert_eq!(
info.user_tags.as_str(),
"site=local",
"older inbound multipart tags must not overwrite newer local tags"
);
assert_eq!(get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(), Some(T_LOCAL));
}
#[tokio::test]
#[serial]
async fn complete_multipart_upload_assigns_completion_version_id() {
crates/ecstore/src/set_disk/ops/object.rs
+533 -8
View File
@@ -37,7 +37,7 @@ use crate::bucket::lifecycle::{
transition_transaction::{
TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction,
TransitionTransactionInit, TransitionTransactionState, delete_transition_transaction_record,
save_transition_transaction_record,
load_transition_transaction_record, save_transition_transaction_record,
},
};
use crate::bucket::quota::reservation;
@@ -1881,6 +1881,110 @@ fn delete_file_info_with_replication_transport_metadata(fi: &FileInfo) -> FileIn
transported
}
/// True when an authorized replication write carries at least one per-category
/// source timestamp, i.e. receiver-side LWW has something to judge.
pub(in crate::set_disk) fn replication_lww_applicable(opts: &ObjectOptions) -> bool {
opts.replication_request
&& (opts.replication_tagging_timestamp.is_some()
|| opts.replication_retention_timestamp.is_some()
|| opts.replication_legalhold_timestamp.is_some())
}
/// The stored per-category state of a destination version, as compared by
/// [`merge_replication_metadata_lww`]. `ObjectInfo::from_file_info`
/// externalizes tags into `user_tags` (stripping the metadata key), so the
/// tag value is folded back into map form here.
pub(in crate::set_disk) fn stored_replication_category_metadata(existing: &ObjectInfo) -> HashMap<String, String> {
let mut stored = (*existing.user_defined).clone();
if !existing.user_tags.is_empty() {
stored.insert(rustfs_utils::http::headers::AMZ_OBJECT_TAGGING.to_string(), (*existing.user_tags).clone());
}
stored
}
/// Receiver-side last-writer-wins for authorized replication writes
/// (rustfs/backlog#1953, audit A4/P1-6). Metadata-only replication reuses the
/// whole-object transports, so in active-active topologies an inbound write
/// carries the source's tags / retention / legal hold verbatim and would
/// otherwise overwrite a category the destination modified more recently —
/// both sites end up permanently diverged while reporting COMPLETED.
///
/// Judged per category, only when the inbound request carries that category's
/// source timestamp (`ObjectOptions::replication_*_timestamp`):
/// - stored timestamp newer than inbound: the local category values and
/// timestamp are kept; the rest of the write proceeds per the inbound
/// metadata and the object-level result stays successful (failing the write
/// instead would loop through MRF, re-delivering the stale value forever);
/// - otherwise the inbound category wins and its internal timestamp key is
/// pinned to the source-authored time — the PUT path re-stamps the
/// object-lock timestamps with the receiver's clock
/// (`parse_object_lock_retention` / `parse_object_lock_legal_hold` insert
/// `now()` via `eval_metadata`), which would make the replica's clock the
/// LWW authority and wedge later convergence;
/// - no stored timestamp (pre-P1-6 data) or no inbound timestamp: the current
/// overwrite behavior is preserved.
///
/// Returns whether `inbound` was modified. Callers must hold the object write
/// lock so the stored values compared here are the ones being replaced.
pub(in crate::set_disk) fn merge_replication_metadata_lww(
inbound: &mut HashMap<String, String>,
existing: &HashMap<String, String>,
opts: &ObjectOptions,
) -> bool {
use rustfs_utils::http::headers::{
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, AMZ_OBJECT_TAGGING,
};
use rustfs_utils::http::metadata_compat::{
SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP, get_str,
remove_str,
};
use time::format_description::well_known::Rfc3339;
let categories: [(Option<OffsetDateTime>, &str, &[&str]); 3] = [
(opts.replication_tagging_timestamp, SUFFIX_TAGGING_TIMESTAMP, &[AMZ_OBJECT_TAGGING]),
(
opts.replication_retention_timestamp,
SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP,
&[AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER],
),
(
opts.replication_legalhold_timestamp,
SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP,
&[AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER],
),
];
let mut changed = false;
for (inbound_timestamp, timestamp_suffix, value_keys) in categories {
let Some(inbound_timestamp) = inbound_timestamp else { continue };
let is_category_value_key = |key: &str| value_keys.iter().any(|value_key| key.eq_ignore_ascii_case(value_key));
let stored_timestamp = get_str(existing, timestamp_suffix).and_then(|value| OffsetDateTime::parse(&value, &Rfc3339).ok());
if stored_timestamp.is_some_and(|stored| stored > inbound_timestamp) {
inbound.retain(|key, _| !is_category_value_key(key));
remove_str(inbound, timestamp_suffix);
for (key, value) in existing {
if is_category_value_key(key) {
inbound.insert(key.clone(), value.clone());
}
}
// Restore the winning timestamp via insert_str, not a verbatim key
// copy: a MinIO-written version may carry only the
// x-minio-internal- key, and the dual-key invariant requires every
// write to produce both keys.
if let Some(stored_value) = get_str(existing, timestamp_suffix) {
rustfs_utils::http::insert_str(inbound, timestamp_suffix, stored_value);
}
changed = true;
} else if let Ok(source_authored) = inbound_timestamp.format(&Rfc3339)
&& get_str(inbound, timestamp_suffix).as_deref() != Some(source_authored.as_str())
{
rustfs_utils::http::insert_str(inbound, timestamp_suffix, source_authored);
changed = true;
}
}
changed
}
impl SetDisks {
pub(in crate::set_disk) async fn persist_old_data_cleanup_receipts(
&self,
@@ -2073,6 +2177,14 @@ impl SetDisks {
user_defined.insert(key.clone(), value.clone());
}
}
if replication_lww_applicable(opts) {
// Object Lock evaluation stamps category timestamps with this
// receiver's clock. Pin them back to the source-authored times
// before the first copy of a version is committed; the existing-
// version branch below may still replace them with newer local
// state.
merge_replication_metadata_lww(&mut user_defined, &HashMap::new(), opts);
}
if expected_restore_operation_id.is_some() {
rustfs_utils::http::metadata_compat::remove_str(&mut user_defined, SUFFIX_RESTORE_OPERATION_ID);
}
@@ -2562,6 +2674,22 @@ impl SetDisks {
if check_object_lock_for_deletion_with_state(object_lock_config.state(), &existing, false)?.is_some() {
return Err(StorageError::PrefixAccessDenied(bucket.to_string(), object.to_string()));
}
// Receiver-side LWW (rustfs/backlog#1953): reuse this
// commit-lock read of the destination version so a
// category (tags / retention / legal hold) modified
// more recently on this site is kept instead of being
// overwritten by the inbound replication metadata.
if replication_lww_applicable(opts) {
let stored = stored_replication_category_metadata(&existing);
let mut merged = parts_metadatas[response_metadata_slot].metadata.clone();
if merge_replication_metadata_lww(&mut merged, &stored, opts) {
for (pfi, disk) in parts_metadatas.iter_mut().zip(shuffle_disks.iter()) {
if disk.is_some() {
pfi.metadata = merged.clone();
}
}
}
}
}
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
Err(err) => return Err(err),
@@ -4248,7 +4376,12 @@ fn record_transition_uploaded_save_attempt(transaction: &TransitionTransaction,
async fn delete_transition_transaction_if_available(api: Option<&Arc<ECStore>>, transaction_id: Uuid) -> Result<()> {
if let Some(api) = api {
return delete_transition_transaction_record(api.clone(), transaction_id).await;
let transaction = match load_transition_transaction_record(api.clone(), transaction_id).await {
Ok(transaction) => transaction,
Err(Error::ConfigNotFound) => return Ok(()),
Err(err) => return Err(err),
};
return delete_transition_transaction_record(api.clone(), &transaction).await;
}
Ok(())
}
@@ -6637,12 +6770,23 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> {
// MRF journal intent: partial-write recovery must survive a restart
// (HS-01); the heal request below remains the in-memory fast path.
rustfs_common::mrf_channel::try_send_mrf_intent(
rustfs_common::mrf_channel::MrfKind::PartialWrite,
bucket,
object,
uuid::Uuid::try_parse(version_id).ok(),
);
let version_uuid = if version_id.is_empty() {
Some(None)
} else {
uuid::Uuid::try_parse(version_id).ok().map(Some)
};
if let Some(version_uuid) = version_uuid
&& let (Ok(pool_index), Ok(set_index)) = (u32::try_from(self.pool_index), u32::try_from(self.set_index))
{
let scope = rustfs_common::mrf_channel::MrfScope { pool_index, set_index };
let _ = rustfs_common::mrf_channel::try_send_mrf_intent_typed(
rustfs_common::mrf_channel::MrfKind::PartialWrite,
bucket,
object,
version_uuid,
Some(scope),
);
}
let mut request = rustfs_common::heal_channel::create_heal_request_with_options(
bucket.to_string(),
Some(object.to_string()),
@@ -8066,6 +8210,387 @@ mod replication_quota_safety_tests {
}
}
#[cfg(test)]
mod replication_lww_tests {
//! Receiver-side LWW for authorized replication writes (rustfs/backlog#1953,
//! audit A4/P1-6): an inbound replication PUT whose per-category timestamp
//! (tags / retention / legal hold) is older than the destination version's
//! stored timestamp must keep the local category values instead of
//! overwriting them; categories are judged independently and the write
//! itself still succeeds.
use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
use super::*;
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
use rustfs_utils::http::headers::{
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, AMZ_OBJECT_TAGGING,
};
use rustfs_utils::http::{
SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP, get_str,
insert_str,
};
use time::format_description::well_known::Rfc3339;
const T_OLD: &str = "2026-01-01T00:00:00Z";
const T_LOCAL: &str = "2026-02-01T00:00:00Z";
const T_NEW: &str = "2026-03-01T00:00:00Z";
fn parse_ts(value: &str) -> OffsetDateTime {
OffsetDateTime::parse(value, &Rfc3339).expect("test timestamp should parse")
}
async fn make_bucket(disks: &[DiskStore], bucket: &str) {
for disk in disks {
disk.make_volume(bucket).await.expect("bucket volume should be created");
}
}
async fn put_version(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, version_id: &str, opts: &ObjectOptions) {
let mut reader = PutObjReader::from_vec(b"lww-body".to_vec());
set_disks
.put_object(bucket, object, &mut reader, opts)
.await
.expect("versioned put should commit");
assert_eq!(opts.version_id.as_deref(), Some(version_id));
}
fn versioned_opts(version_id: &str, user_defined: HashMap<String, String>) -> ObjectOptions {
ObjectOptions {
versioned: true,
version_id: Some(version_id.to_string()),
user_defined,
// Explicit-version PUTs require the bucket Object Lock snapshot.
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
))),
..Default::default()
}
}
/// Local state: version `version_id` with tags "site=local" stamped `T_LOCAL`.
async fn seed_local_tagged_version(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, version_id: &str) {
let mut user_defined = HashMap::new();
user_defined.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
insert_str(&mut user_defined, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
put_version(set_disks, bucket, object, version_id, &versioned_opts(version_id, user_defined)).await;
}
fn inbound_tagging_opts(version_id: &str, tags: &str, timestamp: &str) -> ObjectOptions {
let mut user_defined = HashMap::new();
user_defined.insert(AMZ_OBJECT_TAGGING.to_string(), tags.to_string());
insert_str(&mut user_defined, SUFFIX_TAGGING_TIMESTAMP, timestamp.to_string());
ObjectOptions {
replication_request: true,
replication_tagging_timestamp: Some(parse_ts(timestamp)),
..versioned_opts(version_id, user_defined)
}
}
async fn version_info(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, version_id: &str) -> ObjectInfo {
set_disks
.get_object_info(bucket, object, &versioned_opts(version_id, HashMap::new()))
.await
.expect("version should be readable")
}
#[tokio::test]
async fn inbound_stale_tagging_keeps_newer_local_tags() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "lww-tagging-stale";
let object = "object";
let version_id = Uuid::new_v4().to_string();
make_bucket(&disk_stores, bucket).await;
seed_local_tagged_version(&set_disks, bucket, object, &version_id).await;
put_version(
&set_disks,
bucket,
object,
&version_id,
&inbound_tagging_opts(&version_id, "site=remote", T_OLD),
)
.await;
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(
info.user_tags.as_str(),
"site=local",
"older inbound tags must not overwrite newer local tags"
);
assert_eq!(
get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(),
Some(T_LOCAL),
"the winning local tagging timestamp must be preserved"
);
}
#[tokio::test]
async fn inbound_newer_tagging_overwrites_local_tags() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "lww-tagging-newer";
let object = "object";
let version_id = Uuid::new_v4().to_string();
make_bucket(&disk_stores, bucket).await;
seed_local_tagged_version(&set_disks, bucket, object, &version_id).await;
put_version(
&set_disks,
bucket,
object,
&version_id,
&inbound_tagging_opts(&version_id, "site=remote", T_NEW),
)
.await;
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(
info.user_tags.as_str(),
"site=remote",
"newer inbound tags must overwrite older local tags"
);
assert_eq!(get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(), Some(T_NEW));
}
#[tokio::test]
async fn inbound_wins_when_local_has_no_tagging_timestamp() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "lww-tagging-no-local-ts";
let object = "object";
let version_id = Uuid::new_v4().to_string();
make_bucket(&disk_stores, bucket).await;
// Pre-P1-6 data: local tags without a stored tagging timestamp.
let mut user_defined = HashMap::new();
user_defined.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, user_defined)).await;
put_version(
&set_disks,
bucket,
object,
&version_id,
&inbound_tagging_opts(&version_id, "site=remote", T_OLD),
)
.await;
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(
info.user_tags.as_str(),
"site=remote",
"without a local timestamp the inbound category must win (pre-LWW data compatibility)"
);
}
#[tokio::test]
async fn categories_are_judged_independently() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "lww-category-independent";
let object = "object";
let version_id = Uuid::new_v4().to_string();
make_bucket(&disk_stores, bucket).await;
// Local: newer tags (T_LOCAL), older *cleared* retention (T_OLD) —
// timestamp key only, the shape a replicated retention clear stores.
// (An active local retention would already block the overwrite at the
// WORM gate; the LWW-reachable retention states are cleared/expired.)
let mut local = HashMap::new();
local.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
insert_str(&mut local, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
insert_str(&mut local, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_OLD.to_string());
put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
// Inbound: older tags (T_OLD), newer retention (T_NEW).
let mut inbound = HashMap::new();
inbound.insert(AMZ_OBJECT_TAGGING.to_string(), "site=remote".to_string());
insert_str(&mut inbound, SUFFIX_TAGGING_TIMESTAMP, T_OLD.to_string());
inbound.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "COMPLIANCE".to_string());
inbound.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2028-01-01T00:00:00Z".to_string());
insert_str(&mut inbound, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_NEW.to_string());
let opts = ObjectOptions {
replication_request: true,
replication_tagging_timestamp: Some(parse_ts(T_OLD)),
replication_retention_timestamp: Some(parse_ts(T_NEW)),
..versioned_opts(&version_id, inbound)
};
put_version(&set_disks, bucket, object, &version_id, &opts).await;
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(info.user_tags.as_str(), "site=local", "the stale tagging category must keep local values");
assert_eq!(
info.user_defined.get(AMZ_OBJECT_LOCK_MODE_LOWER).map(String::as_str),
Some("COMPLIANCE"),
"the newer retention category must be applied in the same write"
);
assert_eq!(get_str(&info.user_defined, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP).as_deref(), Some(T_NEW));
}
#[tokio::test]
async fn inbound_stale_legal_hold_keeps_local_value() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "lww-legalhold-stale";
let object = "object";
let version_id = Uuid::new_v4().to_string();
make_bucket(&disk_stores, bucket).await;
// Local: legal hold released (OFF) at T_LOCAL. (A local hold that is
// still ON already blocks the overwrite at the WORM gate; the
// LWW-reachable divergence is a stale inbound ON resurrecting a hold
// that was released more recently on this site.)
let mut local = HashMap::new();
local.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "OFF".to_string());
insert_str(&mut local, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, T_LOCAL.to_string());
put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
let mut inbound = HashMap::new();
inbound.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "ON".to_string());
insert_str(&mut inbound, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, T_OLD.to_string());
let opts = ObjectOptions {
replication_request: true,
replication_legalhold_timestamp: Some(parse_ts(T_OLD)),
..versioned_opts(&version_id, inbound)
};
put_version(&set_disks, bucket, object, &version_id, &opts).await;
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(
info.user_defined.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).map(String::as_str),
Some("OFF"),
"a stale inbound legal hold must not resurrect a hold released more recently"
);
assert_eq!(
get_str(&info.user_defined, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP).as_deref(),
Some(T_LOCAL)
);
}
/// Dual-key invariant under LWW: a MinIO-written destination version may
/// carry only the x-minio-internal timestamp key; when the local category
/// wins, the restored map must still hold BOTH compatibility keys.
#[test]
fn local_win_restores_both_internal_timestamp_keys_for_minio_only_metadata() {
let mut inbound = HashMap::new();
inbound.insert(AMZ_OBJECT_TAGGING.to_string(), "site=remote".to_string());
insert_str(&mut inbound, SUFFIX_TAGGING_TIMESTAMP, T_OLD.to_string());
let existing = HashMap::from([
(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string()),
("X-Minio-Internal-Tagging-Timestamp".to_string(), T_LOCAL.to_string()),
]);
let opts = ObjectOptions {
replication_request: true,
replication_tagging_timestamp: Some(parse_ts(T_OLD)),
..Default::default()
};
assert!(merge_replication_metadata_lww(&mut inbound, &existing, &opts));
assert_eq!(inbound.get(AMZ_OBJECT_TAGGING).map(String::as_str), Some("site=local"));
assert_eq!(
inbound.get("x-rustfs-internal-tagging-timestamp").map(String::as_str),
Some(T_LOCAL),
"the RustFS twin key must be materialized even when the source version only had the MinIO key"
);
assert_eq!(inbound.get("x-minio-internal-tagging-timestamp").map(String::as_str), Some(T_LOCAL));
}
/// When the inbound category wins, the stored timestamp must be the
/// source-authored one: the PUT path's eval_metadata stamps the
/// object-lock timestamps with the receiver's clock
/// (`parse_object_lock_retention`), which would otherwise make this
/// replica's clock the LWW authority and wedge later convergence.
#[tokio::test]
async fn inbound_win_pins_stored_timestamp_to_source_authored_value() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "lww-retention-ts-pinned";
let object = "object";
let version_id = Uuid::new_v4().to_string();
make_bucket(&disk_stores, bucket).await;
// Local cleared retention at T_OLD.
let mut local = HashMap::new();
insert_str(&mut local, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_OLD.to_string());
put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
// Inbound newer retention: the source authored T_LOCAL, but the PUT
// path's eval_metadata stomped the metadata key with receiver-now
// (simulated by T_NEW here).
let mut inbound = HashMap::new();
inbound.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "GOVERNANCE".to_string());
inbound.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2028-01-01T00:00:00Z".to_string());
insert_str(&mut inbound, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_NEW.to_string());
let opts = ObjectOptions {
replication_request: true,
replication_retention_timestamp: Some(parse_ts(T_LOCAL)),
..versioned_opts(&version_id, inbound)
};
put_version(&set_disks, bucket, object, &version_id, &opts).await;
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(
get_str(&info.user_defined, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP).as_deref(),
Some(T_LOCAL),
"the stored category timestamp must be the source-authored time, not the receiver's clock"
);
assert_eq!(info.user_defined.get(AMZ_OBJECT_LOCK_MODE_LOWER).map(String::as_str), Some("GOVERNANCE"));
}
#[tokio::test]
async fn first_inbound_version_pins_source_authored_timestamp() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "lww-first-version-ts-pinned";
let object = "object";
let version_id = Uuid::new_v4().to_string();
make_bucket(&disk_stores, bucket).await;
let mut inbound = HashMap::new();
inbound.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "OFF".to_string());
insert_str(&mut inbound, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, T_OLD.to_string());
let mut evaluated = inbound.clone();
insert_str(&mut evaluated, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, T_NEW.to_string());
let opts = ObjectOptions {
replication_request: true,
replication_legalhold_timestamp: Some(parse_ts(T_OLD)),
eval_metadata: Some(evaluated),
..versioned_opts(&version_id, inbound)
};
put_version(&set_disks, bucket, object, &version_id, &opts).await;
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(
get_str(&info.user_defined, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP).as_deref(),
Some(T_OLD),
"the first copy must store the source timestamp, not the receiver evaluation time"
);
}
#[tokio::test]
async fn newer_local_tag_deletion_survives_stale_inbound_tags() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "lww-tagging-deleted";
let object = "object";
let version_id = Uuid::new_v4().to_string();
make_bucket(&disk_stores, bucket).await;
// Local DeleteObjectTagging state: no tags, but a newer tagging timestamp.
let mut local = HashMap::new();
insert_str(&mut local, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
put_version(
&set_disks,
bucket,
object,
&version_id,
&inbound_tagging_opts(&version_id, "site=remote", T_OLD),
)
.await;
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert!(
info.user_tags.is_empty(),
"a newer local tag deletion must not be resurrected by older inbound tags"
);
assert_eq!(get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(), Some(T_LOCAL));
}
}
#[cfg(test)]
mod inline_put_commit_path_tests {
use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
crates/ecstore/src/storage_api_contracts/mod.rs
+6 -6
View File
@@ -27,12 +27,12 @@ pub(crate) mod internode {
NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY,
NS_SCANNER_LEADER_EPOCH_QUERY, NS_SCANNER_PROTOCOL_VERSION, NS_SCANNER_PROTOCOL_VERSION_QUERY,
NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY, NS_SCANNER_SESSION_ID_QUERY,
NS_SCANNER_SESSION_SEQUENCE_QUERY, NsScannerCapabilityResponse, PUT_FILE_AUTH_QUERY, PUT_FILE_AUTH_TRAILER_DIGEST_LEN,
PUT_FILE_AUTH_TRAILER_LEN, PUT_FILE_AUTH_TRAILER_MAC_LEN, PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_AUTH_V1,
PUT_FILE_CAPABILITY_CHALLENGE_QUERY, PUT_FILE_CAPABILITY_QUERY, PUT_FILE_CAPABILITY_VERSION, PUT_FILE_NONCE_QUERY,
PUT_FILE_SERVER_EPOCH_QUERY, PutFileCapabilityResponse, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION, WALK_DIR_BODY_SHA256_QUERY,
WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
NS_SCANNER_SESSION_SEQUENCE_QUERY, NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY, NsScannerCapabilityResponse,
PUT_FILE_AUTH_QUERY, PUT_FILE_AUTH_TRAILER_DIGEST_LEN, PUT_FILE_AUTH_TRAILER_LEN, PUT_FILE_AUTH_TRAILER_MAC_LEN,
PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_CHALLENGE_QUERY, PUT_FILE_CAPABILITY_QUERY,
PUT_FILE_CAPABILITY_VERSION, PUT_FILE_NONCE_QUERY, PUT_FILE_SERVER_EPOCH_QUERY, PutFileCapabilityResponse,
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
};
}
crates/ecstore/src/store/bucket.rs
+1 -1
View File
@@ -327,7 +327,7 @@ impl ECStore {
async fn cleanup_bucket_usage(&self, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) -> Result<()> {
run_bucket_usage_cleanup(guard, bucket, async {
crate::data_usage::prepare_bucket_usage_for_namespace_change(bucket, guard).await?;
crate::data_usage::remove_bucket_usage_from_backend_with_guard(self, bucket, guard).await
crate::data_usage::remove_bucket_usage_from_backend_with_guard_fenced(self, bucket, guard).await
})
.await
}
crates/ecstore/src/store/init.rs
+1795 -4
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/store/mod.rs
+149 -5
View File
@@ -44,7 +44,7 @@ use crate::error::{
use crate::runtime::global::DISK_RESERVE_FRACTION;
use crate::runtime::instance::InstanceContext;
use crate::runtime::sources as runtime_sources;
use crate::services::rebalance::RebalanceMeta;
use crate::services::rebalance::{RebalanceMeta, is_rebalance_conflicting_with_decommission};
use crate::storage_api_contracts::{
bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions},
@@ -349,16 +349,81 @@ impl ECStore {
/// remain suspended until an operator clears or retries them, so they are
/// a publication barrier even after the worker has stopped.
pub async fn scanner_data_usage_publication_blocked(&self) -> bool {
if self.scanner_data_movement_active().await {
let operation_gate = self.ctx.data_movement_operation_gate();
let _operation_guard = operation_gate.read_owned().await;
self.scanner_data_usage_publication_snapshot_blocked().await
}
async fn scanner_data_usage_publication_snapshot_blocked(&self) -> bool {
if self.ctx.data_movement_operation_epoch_exhausted() {
self.ctx.set_scanner_publication_state(true);
return true;
}
let decommission_cancelers = self.decommission_cancelers.read().await;
let decommission_active = decommission_cancelers
.iter()
.any(|canceler| canceler.as_ref().is_some_and(DecommissionCanceler::is_active));
let pool_meta = self.pool_meta.read().await;
pool_meta.pools.iter().any(|pool| {
let decommission_active = decommission_active
|| pool_meta.pools.iter().any(|pool| {
pool.decommission
.as_ref()
.is_some_and(|info| info.has_decommission_state() && !info.complete && !info.failed && !info.canceled)
});
let decommission_terminal = pool_meta.pools.iter().any(|pool| {
pool.decommission
.as_ref()
.is_some_and(|info| !info.queued && (info.failed || info.canceled))
})
});
drop(pool_meta);
let rebalance_active = self
.rebalance_meta
.read()
.await
.as_ref()
.is_some_and(is_rebalance_conflicting_with_decommission);
let blocked = decommission_active || decommission_terminal || rebalance_active;
self.ctx.set_scanner_publication_state(blocked);
blocked
}
/// Admit one short data-usage publication commit under the same
/// per-instance gate used by decommission side effects and transitions.
/// The epoch is sampled while the read guard is held, so a transition
/// cannot cross this admission without waiting for the commit to finish.
pub async fn scanner_data_usage_publication_read_guard(&self) -> (tokio::sync::OwnedRwLockReadGuard<()>, u64) {
let operation_gate = self.ctx.data_movement_operation_gate();
let operation_guard = operation_gate.read_owned().await;
let epoch = self.ctx.data_movement_operation_epoch();
(operation_guard, epoch)
}
/// Acquire the movement gate and inspect the movement owner once. The
/// state inspection is performed after acquiring the read guard so a
/// transition cannot update its durable state between the check and the
/// publication commit.
pub async fn scanner_data_usage_publication_admission_guard(&self) -> Option<(tokio::sync::OwnedRwLockReadGuard<()>, u64)> {
let operation_gate = self.ctx.data_movement_operation_gate();
let operation_guard = operation_gate.read_owned().await;
if self.ctx.data_movement_operation_epoch_exhausted() {
return None;
}
if self.scanner_data_usage_publication_snapshot_blocked().await {
return None;
}
Some((operation_guard, self.ctx.data_movement_operation_epoch()))
}
/// Capture the current publication epoch without holding the movement
/// gate across backend I/O. Callers must re-admit the same epoch before a
/// mutation commits.
pub(crate) async fn scanner_data_usage_publication_epoch(&self) -> Option<u64> {
let (operation_guard, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
drop(operation_guard);
Some(epoch)
}
}
@@ -995,6 +1060,85 @@ mod tests {
}
}
#[tokio::test]
async fn scanner_data_usage_publication_admission_is_fenced_and_epoch_monotonic() {
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
let operation_gate = store.ctx.data_movement_operation_gate();
let movement_guard = operation_gate.write().await;
let pending = {
let store = store.clone();
tokio::spawn(async move { store.scanner_data_usage_publication_admission_guard().await })
};
tokio::task::yield_now().await;
assert!(!pending.is_finished(), "publication admission must wait for a movement writer");
drop(movement_guard);
let (_, epoch) = pending
.await
.expect("publication admission task should not panic")
.expect("idle store should admit publication");
assert_eq!(epoch, 0);
assert_eq!(store.ctx.advance_data_movement_operation_epoch(), 1);
let (_, next_epoch) = store
.scanner_data_usage_publication_admission_guard()
.await
.expect("idle store should admit the next publication");
assert_eq!(next_epoch, 1);
}
#[tokio::test]
async fn scanner_data_usage_publication_epoch_releases_gate_before_backend_io() {
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
let epoch = store
.scanner_data_usage_publication_epoch()
.await
.expect("idle store should expose a publication epoch");
assert_eq!(epoch, 0);
let operation_gate = store.ctx.data_movement_operation_gate();
let _movement_guard = tokio::time::timeout(Duration::from_secs(1), operation_gate.write())
.await
.expect("epoch capture must not hold the movement gate across backend I/O");
}
#[tokio::test]
async fn scanner_data_usage_publication_admission_blocks_active_rebalance_snapshot() {
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
*store.rebalance_meta.write().await = Some(RebalanceMeta {
pool_stats: vec![crate::services::rebalance::RebalanceStats {
participating: true,
info: crate::services::rebalance::RebalanceInfo {
status: crate::services::rebalance::RebalStatus::Started,
..Default::default()
},
..Default::default()
}],
..Default::default()
});
assert!(
store.scanner_data_usage_publication_admission_guard().await.is_none(),
"active rebalance must fail closed at the storage-owned admission boundary"
);
}
#[tokio::test]
async fn scanner_publication_epoch_exhaustion_fails_closed_after_max() {
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
store.ctx.set_data_movement_operation_epoch_for_test(u64::MAX - 1);
assert_eq!(store.ctx.advance_data_movement_operation_epoch(), u64::MAX);
assert!(store.ctx.data_movement_operation_epoch_exhausted());
assert!(
store.scanner_data_usage_publication_admission_guard().await.is_none(),
"publication must fail closed at the reserved terminal epoch"
);
assert_eq!(store.ctx.advance_data_movement_operation_epoch(), u64::MAX);
assert!(store.ctx.data_movement_operation_epoch_exhausted());
assert!(store.scanner_data_usage_publication_blocked().await);
}
// The object graph is the isolation carrier: two ECStore instances holding
// distinct contexts report independent erasure state through their real
// `&self` accessors — no cross-contamination.
crates/ecstore/src/store/object.rs
+15 -7
View File
@@ -1203,7 +1203,8 @@ fn inject_batch_delete_pool_errors(
bucket: &str,
pool_idx: usize,
object_names: &[String],
result: &mut (Vec<DeletedObject>, Vec<Option<Error>>),
deleted: &[DeletedObject],
errors: &mut [Option<Error>],
) {
let state = BATCH_DELETE_POOL_ERROR_INJECTION
.get_or_init(|| std::sync::Mutex::new(None))
@@ -1220,8 +1221,8 @@ fn inject_batch_delete_pool_errors(
let Some(error) = state.errors.get(object_name) else {
continue;
};
if result.1[idx].is_none() && result.0[idx].found {
result.1[idx] = Some(error.clone());
if errors[idx].is_none() && deleted[idx].found {
errors[idx] = Some(error.clone());
state.observed.fetch_add(1, Ordering::AcqRel);
}
}
@@ -3194,7 +3195,7 @@ impl ECStore {
// Default return value
let mut del_objects = vec![DeletedObject::default(); objects.len()];
let accounting = vec![None; objects.len()];
let mut accounting = vec![None; objects.len()];
let mut del_errs = Vec::with_capacity(objects.len());
for _ in 0..objects.len() {
@@ -3333,11 +3334,12 @@ impl ECStore {
.iter()
.map(|object| object.object_name.clone())
.collect::<Vec<_>>();
let result = pool.delete_objects(bucket, pool_objects, pool_opts).await;
let result = pool.delete_objects_with_accounting(bucket, pool_objects, pool_opts).await;
#[cfg(test)]
let result = {
let mut result = result;
inject_batch_delete_pool_errors(bucket, pool.pool_idx, &pool_object_names, &mut result);
let (deleted, errors, _) = &mut result;
inject_batch_delete_pool_errors(bucket, pool.pool_idx, &pool_object_names, deleted, errors);
result
};
(object_indices, result)
@@ -3347,7 +3349,7 @@ impl ECStore {
let results = join_all(futures).await;
for idx in 0..del_objects.len() {
let pool_results = results.iter().filter_map(|(object_indices, (dels, errs))| {
let pool_results = results.iter().filter_map(|(object_indices, (dels, errs, _))| {
let pool_object_idx = object_indices.binary_search(&idx).ok()?;
Some((&dels[pool_object_idx], &errs[pool_object_idx]))
});
@@ -3367,6 +3369,12 @@ impl ECStore {
}
}
for (object_indices, (_, _, pool_accounting)) in &results {
for (pool_object_idx, object_idx) in object_indices.iter().enumerate() {
accounting[*object_idx] = pool_accounting.get(pool_object_idx).cloned().flatten();
}
}
#[cfg(test)]
for (idx, object) in objects.iter().enumerate() {
if del_errs[idx].is_none() && del_objects[idx].delete_marker {
crates/ecstore/src/store/rebalance.rs
+387 -10
View File
@@ -14,10 +14,15 @@
use super::*;
use crate::config::storageclass;
use crate::core::pools::merge_pool_status_refresh;
use crate::layout::pool_space::{ServerPoolsAvailableSpace, build_server_pools_available_space};
use crate::runtime::sources as runtime_sources;
use crate::storage_api_contracts::{admin::StorageAdminApi, namespace::NamespaceLocking as _, object::ObjectOperations as _};
pub(in crate::store) mod support;
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_POOLS: &str = "pools";
const EVENT_POOL_META_RELOAD: &str = "pool_meta_reload";
use support::{
LatestObjectInfoCandidate, PoolErr, PoolObjInfo, RebalanceDeletePoolResult, pool_lookup_not_found_error,
rebalance_disk_set_lookup_error, resolve_latest_object_info_candidates, resolve_rebalance_delete_from_all_pools_result,
@@ -684,17 +689,61 @@ impl ECStore {
)
}
pub async fn reload_pool_meta(&self) -> Result<()> {
let mut meta = PoolMeta::default();
/// Peer reload entry: refreshes in-memory pool metadata from the shared
/// persisted snapshot. Returns whether newer state was actually merged so
/// callers only trigger missing-worker recovery after a real state change;
/// delayed snapshots are merged monotonically and never blind-assigned.
pub async fn reload_pool_meta(&self) -> Result<bool> {
// Serialize the durable reload with local movement transitions. Loading
// before acquiring this gate would allow a stale disk snapshot to
// overwrite a newer local transition after the writer commits.
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
let mut reloaded = PoolMeta::default();
resolve_store_rebalance_pool_meta_reload_result(
meta.load(self.pools[0].clone(), self.pools.clone()).await,
reloaded.load(self.pools[0].clone(), self.pools.clone()).await,
"reload_pool_meta",
)?;
// Lock order: release the decommission_cancelers guard before taking
// the pool_meta write guard; neither is held without the movement gate.
let active_workers = {
let cancelers = self.decommission_cancelers.read().await;
cancelers
.iter()
.map(|canceler| canceler.as_ref().is_some_and(DecommissionCanceler::is_active))
.collect::<Vec<_>>()
};
let incoming_has_pools = !reloaded.pools.is_empty();
let mut pool_meta = self.pool_meta.write().await;
*pool_meta = meta;
// *self.pool_meta.write().expect("operation should succeed") = meta;
Ok(())
let movement_before = pool_meta.clone();
let merged_newer = merge_pool_status_refresh(&mut pool_meta, reloaded, &active_workers);
if crate::core::pools::pool_meta_movement_snapshot_changed(&movement_before, &pool_meta) {
self.ctx.advance_data_movement_operation_epoch();
}
if !merged_newer && !incoming_has_pools {
warn!(
event = EVENT_POOL_META_RELOAD,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
result = "ignored",
reason = "missing_metadata",
"Peer pool meta reload ignored because persisted metadata is missing"
);
} else if !merged_newer {
debug!(
event = EVENT_POOL_META_RELOAD,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
result = "ignored",
reason = "stale_snapshot",
"Peer pool meta reload ignored as a stale snapshot"
);
}
Ok(merged_newer)
}
/// Disk information deduplication function
@@ -861,6 +910,7 @@ mod tests {
use super::*;
use crate::bucket::replication::{ReplicationStatusType, VersionPurgeStatusType};
use crate::config::storageclass::{CLASS_RRS, CLASS_STANDARD, lookup_config_for_pools_without_env};
use crate::core::pools::{POOL_META_VERSION, PoolDecommissionInfo, PoolStatus};
use crate::disk::error::DiskError;
use crate::layout::endpoint::Endpoint;
use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
@@ -871,6 +921,7 @@ mod tests {
use rustfs_config::server_config::KVS;
use rustfs_filemeta::FileInfo;
use std::sync::Arc;
use time::{Duration as TimeDuration, OffsetDateTime};
use tokio_util::sync::CancellationToken;
async fn setup_multi_pool_test_store(
@@ -1587,10 +1638,6 @@ mod tests {
fn resolve_latest_object_info_candidates_rejects_equal_time_payload_identity_conflicts() {
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
let mut data_dir = base.clone();
data_dir.data_dir = Some(Uuid::from_u128(2));
assert_equal_time_identity_conflict(base.clone(), data_dir);
let mut size = base.clone();
size.size = 1;
assert_equal_time_identity_conflict(base.clone(), size);
@@ -1620,6 +1667,58 @@ mod tests {
);
}
#[test]
fn resolve_latest_object_info_candidates_accepts_data_movement_rewrites() {
let mut source = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
source.data_dir = Some(Uuid::from_u128(1));
let mut target = source.clone();
target.data_dir = Some(Uuid::from_u128(2));
rustfs_utils::http::insert_str(
Arc::make_mut(&mut target.user_defined),
rustfs_utils::http::SUFFIX_DATA_MOVED,
"true".to_string(),
);
rustfs_utils::http::insert_str(
Arc::make_mut(&mut target.user_defined),
rustfs_utils::http::SUFFIX_ACTUAL_SIZE,
"0".to_string(),
);
let (info, idx) = resolve_latest_object_info_candidates(
vec![
LatestObjectInfoCandidate {
info: Some(source),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: Some(target.clone()),
idx: 1,
err: None,
},
],
"bucket",
"object",
&ObjectOptions::default(),
)
.expect("a committed data-movement target must remain readable before source cleanup");
assert_eq!(idx, 1);
assert_eq!(info.data_dir, target.data_dir);
}
#[test]
fn resolve_latest_object_info_candidates_rejects_unmarked_data_dir_conflict() {
let mut left = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
left.data_dir = Some(Uuid::from_u128(1));
let mut right = left.clone();
right.data_dir = Some(Uuid::from_u128(2));
assert_equal_time_identity_conflict(left, right);
}
#[test]
fn resolve_latest_object_info_candidates_accepts_internal_metadata_aliases() {
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
@@ -2097,4 +2196,282 @@ mod tests {
.contains("failed to resolve rebalance disk set: pool index 2, set index 7, pool count 3")
);
}
fn reload_test_pool_status(decommission: Option<PoolDecommissionInfo>, last_update: time::OffsetDateTime) -> PoolStatus {
PoolStatus {
id: 0,
cmd_line: "pool-0".to_string(),
last_update,
decommission,
}
}
fn reload_test_pool_meta(pool: PoolStatus) -> PoolMeta {
PoolMeta {
version: POOL_META_VERSION,
pools: vec![pool],
dont_save: false,
}
}
async fn persist_reload_snapshot(store: &ECStore, snapshot: &PoolMeta) {
snapshot
.save(store.pools.clone())
.await
.expect("pool meta snapshot should persist to every pool");
}
#[tokio::test]
#[serial_test::serial]
async fn peer_pool_meta_reload_does_not_rollback_newer_local_states() {
let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-reload-stale", &[2]).await;
let stale_time = OffsetDateTime::now_utc();
let newer_time = stale_time + TimeDuration::seconds(30);
let progressed_states: [(&str, PoolDecommissionInfo); 4] = [
(
"queued",
PoolDecommissionInfo {
queued: true,
start_time: Some(stale_time),
..Default::default()
},
),
(
"canceled",
PoolDecommissionInfo {
canceled: true,
start_time: Some(stale_time),
..Default::default()
},
),
(
"failed",
PoolDecommissionInfo {
failed: true,
start_time: Some(stale_time),
..Default::default()
},
),
(
"complete",
PoolDecommissionInfo {
complete: true,
start_time: Some(stale_time),
..Default::default()
},
),
];
for (state_label, local_state) in progressed_states {
{
let mut pool_meta = store.pool_meta.write().await;
*pool_meta = reload_test_pool_meta(reload_test_pool_status(Some(local_state.clone()), newer_time));
}
// A delayed peer message carries a snapshot that predates the local progression.
let stale_snapshot = reload_test_pool_meta(reload_test_pool_status(
Some(PoolDecommissionInfo {
start_time: Some(stale_time),
..Default::default()
}),
stale_time,
));
persist_reload_snapshot(&store, &stale_snapshot).await;
let merged_newer = store.reload_pool_meta().await.expect("stale reload should succeed");
assert!(
!merged_newer,
"a delayed reload must not report merged newer state for the {state_label} progression"
);
let pool_meta = store.pool_meta.read().await;
let info = pool_meta.pools[0]
.decommission
.as_ref()
.expect("local decommission state should survive a stale reload");
assert_eq!(info.queued, local_state.queued, "{state_label} queued flag must not roll back");
assert_eq!(info.canceled, local_state.canceled, "{state_label} canceled flag must not roll back");
assert_eq!(info.failed, local_state.failed, "{state_label} failed flag must not roll back");
assert_eq!(info.complete, local_state.complete, "{state_label} complete flag must not roll back");
assert_eq!(
pool_meta.pools[0].last_update, newer_time,
"{state_label} progress timestamp must be kept"
);
}
shutdown.cancel();
}
#[tokio::test]
#[serial_test::serial]
async fn peer_pool_meta_reload_merges_newer_state_and_is_idempotent_on_duplicate_delivery() {
let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-reload-duplicate", &[2]).await;
let older_time = OffsetDateTime::now_utc();
let newer_time = older_time + TimeDuration::seconds(30);
{
let mut pool_meta = store.pool_meta.write().await;
*pool_meta = reload_test_pool_meta(reload_test_pool_status(
Some(PoolDecommissionInfo {
items_decommissioned: 1,
..Default::default()
}),
older_time,
));
}
let newer_snapshot = reload_test_pool_meta(reload_test_pool_status(
Some(PoolDecommissionInfo {
complete: true,
items_decommissioned: 10,
..Default::default()
}),
newer_time,
));
persist_reload_snapshot(&store, &newer_snapshot).await;
let merged_newer = store.reload_pool_meta().await.expect("first reload should succeed");
assert!(merged_newer, "a strictly newer persisted snapshot must merge");
{
let pool_meta = store.pool_meta.read().await;
let info = pool_meta.pools[0].decommission.as_ref().expect("merged decommission state");
assert!(info.complete);
assert_eq!(info.items_decommissioned, 10);
assert_eq!(pool_meta.pools[0].last_update, newer_time);
}
// Redelivering the same generation must be a no-op.
let duplicate_merged = store.reload_pool_meta().await.expect("duplicate reload should succeed");
assert!(!duplicate_merged, "a duplicate delivery must not re-apply merged state");
{
let pool_meta = store.pool_meta.read().await;
let info = pool_meta.pools[0].decommission.as_ref().expect("merged decommission state");
assert!(info.complete);
assert_eq!(info.items_decommissioned, 10);
assert_eq!(pool_meta.pools[0].last_update, newer_time);
}
shutdown.cancel();
}
#[tokio::test]
#[serial_test::serial]
async fn peer_pool_meta_reload_keeps_active_worker_progress_over_newer_snapshot() {
let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-reload-worker", &[2]).await;
*store.decommission_cancelers.write().await = vec![Some(crate::core::pools::DecommissionCanceler::new_for_test(
CancellationToken::new(),
))];
let worker_time = OffsetDateTime::now_utc();
let newer_time = worker_time + TimeDuration::seconds(30);
{
let mut pool_meta = store.pool_meta.write().await;
*pool_meta = reload_test_pool_meta(reload_test_pool_status(
Some(PoolDecommissionInfo {
start_time: Some(worker_time),
items_decommissioned: 10,
bytes_done: 1_024,
..Default::default()
}),
worker_time,
));
}
// Even a strictly newer terminal snapshot must not override a live worker.
let newer_terminal_snapshot = reload_test_pool_meta(reload_test_pool_status(
Some(PoolDecommissionInfo {
complete: true,
..Default::default()
}),
newer_time,
));
persist_reload_snapshot(&store, &newer_terminal_snapshot).await;
let merged_newer = store
.reload_pool_meta()
.await
.expect("reload under an active worker should succeed");
assert!(!merged_newer, "an active local worker must block snapshot replacement");
let pool_meta = store.pool_meta.read().await;
let info = pool_meta.pools[0]
.decommission
.as_ref()
.expect("worker progress should remain");
assert!(!info.complete);
assert_eq!(info.items_decommissioned, 10);
assert_eq!(info.bytes_done, 1_024);
assert_eq!(pool_meta.pools[0].last_update, worker_time);
shutdown.cancel();
}
#[tokio::test]
#[serial_test::serial]
async fn peer_pool_meta_reload_fails_closed_when_persisted_metadata_is_missing() {
let (temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-reload-missing", &[2]).await;
let kept_time = OffsetDateTime::now_utc();
{
let mut pool_meta = store.pool_meta.write().await;
*pool_meta = reload_test_pool_meta(reload_test_pool_status(
Some(PoolDecommissionInfo {
complete: true,
..Default::default()
}),
kept_time,
));
}
// Persist first so the test controls exactly what exists on disk.
persist_reload_snapshot(
&store,
&reload_test_pool_meta(reload_test_pool_status(
Some(PoolDecommissionInfo {
complete: true,
..Default::default()
}),
kept_time,
)),
)
.await;
let mut deleted_any = false;
for disk_index in 0..2 {
let pool_bin_dir = temp_dir
.path()
.join(format!("pool0-disk{disk_index}"))
.join(crate::disk::RUSTFS_META_BUCKET)
.join(crate::core::pools::POOL_META_NAME);
if pool_bin_dir.exists() {
tokio::fs::remove_dir_all(&pool_bin_dir)
.await
.expect("persisted pool metadata object dir should be removable");
deleted_any = true;
}
}
// The meta-bucket layout may nest objects per pool; fall back to removing
// every pool.bin object directory below the temp root.
if !deleted_any {
panic!("no pool.bin found under {:?}", temp_dir.path());
}
let merged_newer = store
.reload_pool_meta()
.await
.expect("reload with missing metadata should fail closed, not error");
assert!(!merged_newer, "missing persisted metadata must not count as merged state");
let pool_meta = store.pool_meta.read().await;
let info = pool_meta.pools[0]
.decommission
.as_ref()
.expect("missing persisted metadata must not default local state away");
assert!(info.complete);
assert_eq!(pool_meta.pools[0].last_update, kept_time);
shutdown.cancel();
}
}
crates/ecstore/src/store/rebalance/support.rs
+43 -23
View File
@@ -214,47 +214,67 @@ fn same_user_defined_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool {
}
}
/// Pool-specific erasure geometry is intentionally excluded: `get_object_info`
/// returns each pool's own `data_blocks`/`parity_blocks`, so those values can
/// differ for the same object version while the selected winner still carries
/// the chosen pool's layout. `put_object_reader` is also intentionally
/// excluded because it is a transient request handle that `ObjectInfo::clone`
/// drops. Every other ObjectInfo field is part of the production-visible
/// identity and must agree before the pool index can provide a deterministic
/// tie-break.
/// Pool-specific erasure geometry and data-movement rewrites are intentionally
/// excluded. The selected winner still carries the chosen pool's layout, while
/// the remaining read-visible fields must agree before the pool index can
/// provide a deterministic tie-break.
fn same_latest_object_info_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool {
left.bucket == right.bucket
let same_read_surface = left.bucket == right.bucket
&& left.name == right.name
&& left.storage_class == right.storage_class
&& left.is_dir == right.is_dir
&& left.restore_ongoing == right.restore_ongoing
&& left.restore_expires == right.restore_expires
&& left.is_latest == right.is_latest
&& left.content_type == right.content_type
&& left.content_encoding == right.content_encoding
&& left.num_versions == right.num_versions
&& left.successor_mod_time == right.successor_mod_time
&& left.inlined == right.inlined
&& left.metadata_only == right.metadata_only
&& left.version_only == right.version_only
&& left.replication_decision == right.replication_decision;
if !same_read_surface {
return false;
}
let exact_identity = left.storage_class == right.storage_class
&& left.mod_time == right.mod_time
&& left.size == right.size
&& left.actual_size == right.actual_size
&& left.is_dir == right.is_dir
&& same_user_defined_identity(left, right)
&& left.user_tags == right.user_tags
&& left.version_id == right.version_id
&& left.data_dir == right.data_dir
&& left.delete_marker == right.delete_marker
&& same_transition_identity(left, right)
&& left.restore_ongoing == right.restore_ongoing
&& left.restore_expires == right.restore_expires
&& left.parts == right.parts
&& left.is_latest == right.is_latest
&& left.content_type == right.content_type
&& left.content_encoding == right.content_encoding
&& left.expires == right.expires
&& left.num_versions == right.num_versions
&& left.successor_mod_time == right.successor_mod_time
&& left.etag == right.etag
&& left.inlined == right.inlined
&& left.metadata_only == right.metadata_only
&& left.version_only == right.version_only
&& left.replication_status_internal == right.replication_status_internal
&& left.replication_status == right.replication_status
&& left.version_purge_status_internal == right.version_purge_status_internal
&& left.version_purge_status == right.version_purge_status
&& left.replication_decision == right.replication_decision
&& left.checksum == right.checksum
&& left.checksum == right.checksum;
if exact_identity {
return true;
}
let left_moved =
rustfs_utils::http::get_consistent_str(&left.user_defined, rustfs_utils::http::SUFFIX_DATA_MOVED) == Some("true");
let right_moved =
rustfs_utils::http::get_consistent_str(&right.user_defined, rustfs_utils::http::SUFFIX_DATA_MOVED) == Some("true");
match (left_moved, right_moved) {
(false, false) => false,
(false, true) => crate::data_movement::is_equivalent_data_movement_object_identity(left, right, true, true),
(true, false) => crate::data_movement::is_equivalent_data_movement_object_identity(right, left, true, true),
(true, true) => {
crate::data_movement::is_equivalent_data_movement_object_identity(left, right, true, true)
|| crate::data_movement::is_equivalent_data_movement_object_identity(right, left, true, true)
}
}
}
pub(super) fn resolve_latest_object_info_candidates(
crates/filemeta/src/metacache.rs
+643 -1
View File
@@ -21,6 +21,7 @@ use arc_swap::ArcSwapOption;
use rmp::Marker;
use serde::{Deserialize, Serialize};
use std::cmp::Ordering;
use std::collections::{HashMap, HashSet};
use std::str::from_utf8;
use std::{
fmt::Debug,
@@ -37,8 +38,13 @@ use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
use tokio::spawn;
use tokio::sync::Mutex;
use tracing::{debug, warn};
use uuid::Uuid;
const SLASH_SEPARATOR: &str = "/";
pub const MAX_META_CACHE_HEAL_CANDIDATES: usize = 1024;
/// Keep truncation continuations bounded while still giving the scanner a
/// safe object-level retry for versions that did not fit in the candidate set.
pub const MAX_META_CACHE_HEAL_TRUNCATED_OBJECTS: usize = 64;
#[derive(Clone, Debug, Default)]
pub struct MetadataResolutionParams {
@@ -66,6 +72,50 @@ pub struct MetaCacheEntry {
pub reusable: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
pub enum MetaCacheHealCandidateKind {
Object,
DeleteMarker,
UnversionedObject,
}
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
pub struct MetaCacheHealCandidate {
pub object: String,
pub version_id: Option<Uuid>,
pub kind: MetaCacheHealCandidateKind,
/// Number of raw disk entries that carried this validated version.
pub replica_count: usize,
}
impl MetaCacheHealCandidate {
pub fn validated_version(&self) -> Option<Uuid> {
match self.kind {
MetaCacheHealCandidateKind::Object | MetaCacheHealCandidateKind::DeleteMarker => self.version_id,
MetaCacheHealCandidateKind::UnversionedObject => None,
}
}
pub fn is_unversioned(&self) -> bool {
self.kind == MetaCacheHealCandidateKind::UnversionedObject
}
}
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct MetaCacheHealDiscovery {
pub candidates: Vec<MetaCacheHealCandidate>,
pub unverified_count: usize,
pub truncated: bool,
/// Object names whose validated version set exceeded the candidate cap.
/// The scanner retries these names without a version and with destructive
/// healing disabled; this is an explicit bounded continuation, not a
/// version claim.
pub truncated_objects: Vec<String>,
/// Validated candidates beyond the main cap, retained with exact version
/// identities so callers never fall back to a latest-version request.
pub truncated_candidates: Vec<MetaCacheHealCandidate>,
}
impl MetaCacheEntry {
pub fn marshal_msg(&self) -> Result<Vec<u8>> {
let mut wr = Vec::new();
@@ -370,6 +420,185 @@ impl MetaCacheEntries {
})
}
/// Discover validated object/delete-marker versions and safe unversioned
/// inspection candidates in the raw entries without applying read quorum.
/// This is intentionally separate from [`Self::resolve`]: a sub-quorum
/// version is a valid heal target even though it must not participate in
/// normal reads or writes.
///
/// The validated list is bounded and deduplicated by object, version id,
/// and metadata kind; each candidate retains the number of raw disk
/// entries that carried it so callers can classify sub-quorum versions.
/// Entries whose xl.meta cannot be decoded are counted separately for
/// discovery accounting; they never become versionless destructive heal
/// requests and do not consume the validated quota. An
/// [`MetaCacheHealCandidateKind::UnversionedObject`] is always consumed by
/// a non-destructive scanner request.
pub fn discover_heal_candidates(&self, bucket: &str, max_candidates: usize) -> MetaCacheHealDiscovery {
let limit = max_candidates.min(MAX_META_CACHE_HEAL_CANDIDATES);
if limit == 0 || bucket.is_empty() {
return MetaCacheHealDiscovery::default();
}
let mut discovery = MetaCacheHealDiscovery {
candidates: Vec::<MetaCacheHealCandidate>::with_capacity(limit.min(self.0.len())),
unverified_count: 0,
truncated: false,
truncated_objects: Vec::with_capacity(MAX_META_CACHE_HEAL_TRUNCATED_OBJECTS.min(limit)),
truncated_candidates: Vec::new(),
};
let mut seen: HashMap<(String, Option<Uuid>, MetaCacheHealCandidateKind), usize> =
HashMap::with_capacity(limit.min(self.0.len()));
for entry in self.0.iter().flatten() {
if !valid_heal_candidate_name(bucket, entry) {
continue;
}
let meta = match FileMeta::load(&entry.metadata) {
Ok(meta) => meta,
Err(_) => {
discovery.unverified_count = discovery.unverified_count.saturating_add(1);
continue;
}
};
let mut entry_seen = HashSet::new();
for shallow in meta.versions {
let version = match shallow.parse_version_meta() {
Ok(version) if version.valid() => version,
Ok(_) | Err(_) => {
discovery.unverified_count = discovery.unverified_count.saturating_add(1);
continue;
}
};
if version.free_version() {
continue;
}
let payload_header = version.header();
if normalize_version_id(shallow.header.version_id) != normalize_version_id(payload_header.version_id)
|| shallow.header.version_type != payload_header.version_type
{
discovery.unverified_count = discovery.unverified_count.saturating_add(1);
continue;
}
let (kind, version_id) = match version.version_type {
VersionType::Object
if version.object.is_some() && version.delete_marker.is_none() && version.legacy_object.is_none() =>
{
match version.object.as_ref().and_then(|object| object.version_id) {
Some(id) if !id.is_nil() => (MetaCacheHealCandidateKind::Object, Some(id)),
Some(_) | None => (MetaCacheHealCandidateKind::UnversionedObject, None),
}
}
VersionType::Delete
if version.delete_marker.is_some() && version.object.is_none() && version.legacy_object.is_none() =>
{
let Some(id) = version.delete_marker.as_ref().and_then(|marker| marker.version_id) else {
discovery.unverified_count = discovery.unverified_count.saturating_add(1);
continue;
};
if id.is_nil() {
discovery.unverified_count = discovery.unverified_count.saturating_add(1);
continue;
}
(MetaCacheHealCandidateKind::DeleteMarker, Some(id))
}
VersionType::Legacy
if version.legacy_object.is_some() && version.object.is_none() && version.delete_marker.is_none() =>
{
let Some(legacy) = version.legacy_object.as_ref() else {
continue;
};
if legacy.version_id.is_empty() {
(MetaCacheHealCandidateKind::UnversionedObject, None)
} else {
let Ok(id) = Uuid::parse_str(&legacy.version_id) else {
discovery.unverified_count = discovery.unverified_count.saturating_add(1);
continue;
};
if id.is_nil() {
discovery.unverified_count = discovery.unverified_count.saturating_add(1);
continue;
}
(MetaCacheHealCandidateKind::Object, Some(id))
}
}
_ => {
discovery.unverified_count = discovery.unverified_count.saturating_add(1);
continue;
}
};
if normalize_version_id(payload_header.version_id) != version_id {
discovery.unverified_count = discovery.unverified_count.saturating_add(1);
continue;
}
// `all_parts=true` is the trust-boundary check for versioned
// candidates. A null/legacy object may still need the old
// non-destructive inspection fallback when its part arrays
// are parseable but incomplete; never use that fallback for
// a candidate carrying a real version id.
let file_info = match version.clone().into_fileinfo(bucket, &entry.name, true) {
Ok(file_info) => file_info,
Err(_) if version_id.is_none() && matches!(kind, MetaCacheHealCandidateKind::UnversionedObject) => {
discovery.unverified_count = discovery.unverified_count.saturating_add(1);
match version.into_fileinfo(bucket, &entry.name, false) {
Ok(file_info) => file_info,
Err(_) => continue,
}
}
Err(_) => {
discovery.unverified_count = discovery.unverified_count.saturating_add(1);
continue;
}
};
if file_info.volume != bucket || file_info.name != entry.name {
discovery.unverified_count = discovery.unverified_count.saturating_add(1);
continue;
}
let candidate = MetaCacheHealCandidate {
object: entry.name.clone(),
version_id,
kind,
replica_count: 1,
};
let key = (candidate.object.clone(), candidate.version_id, candidate.kind.clone());
if entry_seen.contains(&key) {
continue;
}
if let Some(index) = seen.get(&key).copied() {
entry_seen.insert(key);
discovery.candidates[index].replica_count = discovery.candidates[index].replica_count.saturating_add(1);
} else if discovery.candidates.len() >= limit {
// Keep the validated candidate list bounded, but retain a
// bounded object-level continuation so the scanner cannot
// silently lose every version of a busy object.
discovery.truncated = true;
if discovery.truncated_objects.len() < MAX_META_CACHE_HEAL_TRUNCATED_OBJECTS
&& !discovery.truncated_objects.iter().any(|object| object == &candidate.object)
{
discovery.truncated_objects.push(candidate.object.clone());
}
if discovery.truncated_objects.iter().any(|object| object == &candidate.object) {
discovery.truncated_candidates.push(candidate);
}
continue;
} else {
entry_seen.insert(key.clone());
seen.insert(key, discovery.candidates.len());
discovery.candidates.push(candidate);
}
}
}
discovery
}
fn resolve_inner(&self, mut params: MetadataResolutionParams, enforce_write_quorum: bool) -> Option<MetaCacheEntry> {
if self.0.is_empty() {
debug!(
@@ -546,6 +775,33 @@ impl MetaCacheEntries {
}
}
fn valid_heal_candidate_name(bucket: &str, entry: &MetaCacheEntry) -> bool {
if bucket.is_empty()
|| entry.name.is_empty()
|| entry.is_dir()
|| (cfg!(windows) && entry.name.contains('\\'))
|| entry.name.chars().any(char::is_control)
{
return false;
}
// Validate raw key components without normalizing them. The scanner maps
// accepted keys to filesystem paths later, so dot components and empty
// internal components must be rejected before that boundary. A final
// empty component is retained for valid keys ending in '/'.
let mut components = entry.name.split('/').peekable();
while let Some(component) = components.next() {
if component == "." || component == ".." || (component.is_empty() && components.peek().is_some()) {
return false;
}
}
true
}
fn normalize_version_id(version_id: Option<Uuid>) -> Option<Uuid> {
version_id.filter(|id| !id.is_nil())
}
#[derive(Debug, Default)]
pub struct MetaCacheEntriesSortedResult {
pub entries: Option<MetaCacheEntriesSorted>,
@@ -638,6 +894,7 @@ impl<W: AsyncWrite + Unpin> MetacacheWriter<W> {
}
pub async fn close(&mut self) -> Result<()> {
self.init().await?;
rmp::encode::write_bool(&mut self.buf, false).map_err(|e| Error::other(format!("{e:?}")))?;
self.flush().await?;
Ok(())
@@ -991,7 +1248,7 @@ impl<T: Clone + Debug + Send + Sync + 'static> Cache<T> {
mod tests {
use super::*;
use crate::test_data::create_real_xlmeta;
use crate::{FileMetaVersion, MetaDeleteMarker, TRANSITION_COMPLETE};
use crate::{FileMetaVersion, MetaDeleteMarker, MetaObjectV1, MetaObjectV1Erasure, MetaObjectV1Stat, TRANSITION_COMPLETE};
use std::collections::HashMap;
use std::io::Cursor;
use std::sync::{
@@ -1029,6 +1286,16 @@ mod tests {
assert_eq!(objs, nobjs);
}
#[tokio::test]
async fn empty_writer_emits_a_valid_stream() {
let mut output = Cursor::new(Vec::new());
let mut writer = MetacacheWriter::new(&mut output);
writer.close().await.expect("empty stream should close");
let mut reader = MetacacheReader::new(Cursor::new(output.into_inner()));
assert!(reader.read_all().await.expect("empty stream should decode").is_empty());
}
fn corrupt_stream_with_metadata_len(len_marker: u8, len: u32) -> Vec<u8> {
let mut data = Vec::new();
rmp::encode::write_u8(&mut data, METACACHE_STREAM_VERSION).unwrap();
@@ -1592,6 +1859,381 @@ mod tests {
);
}
#[test]
fn discover_heal_candidates_keeps_sub_quorum_versions_and_deduplicates() {
let now = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
let entries = MetaCacheEntries(vec![
Some(metacache_entry_single_version(1, now, "one")),
Some(metacache_entry_single_version(2, now, "two")),
Some(metacache_entry_single_version(2, now, "two")),
Some(metacache_entry_single_version(3, now, "three")),
]);
let discovery = entries.discover_heal_candidates("bucket", 16);
let ids: std::collections::HashSet<Uuid> = discovery
.candidates
.iter()
.filter_map(|candidate| candidate.version_id)
.collect();
assert_eq!(
ids,
[Uuid::from_u128(1), Uuid::from_u128(2), Uuid::from_u128(3)]
.into_iter()
.collect()
);
assert_eq!(discovery.candidates.len(), 3, "duplicate tied versions must be emitted once");
assert_eq!(
discovery
.candidates
.iter()
.find(|candidate| candidate.version_id == Some(Uuid::from_u128(2)))
.expect("duplicate version should be discovered")
.replica_count,
2
);
}
#[test]
fn discover_heal_candidates_does_not_count_duplicate_versions_within_one_entry() {
let now = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
let mut meta = FileMeta::load(&metacache_entry_single_version(1, now, "duplicate").metadata)
.expect("duplicate fixture should decode");
meta.versions.push(meta.versions[0].clone());
let entry = MetaCacheEntry {
name: "object".to_string(),
metadata: meta.marshal_msg().expect("duplicate metadata should marshal"),
cached: Some(meta),
reusable: false,
};
let discovery = MetaCacheEntries(vec![Some(entry)]).discover_heal_candidates("bucket", 16);
let candidate = discovery
.candidates
.iter()
.find(|candidate| candidate.version_id == Some(Uuid::from_u128(1)))
.expect("duplicate fixture should be discovered");
assert_eq!(candidate.replica_count, 1);
}
#[test]
fn discover_heal_candidates_covers_divergent_quorum_boundaries_n2_n4_n6() {
let now = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
for (disk_count, quorum) in [(2usize, 1usize), (4, 2), (6, 3)] {
let target_id = Uuid::from_u128(0x1000 + disk_count as u128);
for target_replicas in [quorum.saturating_sub(1), quorum, quorum + 1] {
let entries = (0..disk_count)
.map(|disk| {
let version_id = if disk < target_replicas {
target_id
} else {
Uuid::from_u128(0x2000 + disk as u128)
};
Some(metacache_entry_single_version(version_id.as_u128(), now, "divergent"))
})
.collect();
let discovery = MetaCacheEntries(entries).discover_heal_candidates("bucket", 32);
let target = discovery
.candidates
.iter()
.find(|candidate| candidate.version_id == Some(target_id));
assert_eq!(target.is_some(), target_replicas > 0, "N={disk_count}, replicas={target_replicas}");
if let Some(target) = target {
assert_eq!(target.replica_count, target_replicas);
}
}
}
}
#[test]
fn discover_heal_candidates_separates_delete_markers_and_preserves_unversioned_objects() {
let mut marker_meta = FileMeta::new();
marker_meta
.add_version(FileInfo {
volume: "bucket".to_string(),
name: "object".to_string(),
version_id: Some(Uuid::from_u128(99)),
deleted: true,
mod_time: Some(OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp")),
..Default::default()
})
.expect("delete marker should be added");
let marker = MetaCacheEntry {
name: "object".to_string(),
metadata: marker_meta.marshal_msg().expect("delete marker metadata should marshal"),
cached: Some(marker_meta),
reusable: false,
};
let unversioned_entry = metacache_entry_with_mod_time(
OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp"),
"unversioned",
);
let discovery = MetaCacheEntries(vec![Some(marker), Some(unversioned_entry)]).discover_heal_candidates("bucket", 16);
assert!(discovery.candidates.iter().any(|candidate| {
candidate.kind == MetaCacheHealCandidateKind::DeleteMarker && candidate.version_id == Some(Uuid::from_u128(99))
}));
assert!(discovery.candidates.iter().any(|candidate| {
candidate.kind == MetaCacheHealCandidateKind::UnversionedObject && candidate.version_id.is_none()
}));
}
#[test]
fn discover_heal_candidates_rejects_delete_markers_without_ids() {
let mut marker_meta = FileMeta::new();
marker_meta
.add_version(FileInfo {
volume: "bucket".to_string(),
name: "object".to_string(),
deleted: true,
mod_time: Some(OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp")),
..Default::default()
})
.expect("nil delete marker should be added");
let discovery = MetaCacheEntries(vec![Some(MetaCacheEntry {
name: "object".to_string(),
metadata: marker_meta.marshal_msg().expect("nil marker metadata should marshal"),
cached: Some(marker_meta),
reusable: false,
})])
.discover_heal_candidates("bucket", 16);
assert!(
!discovery
.candidates
.iter()
.any(|candidate| candidate.kind == MetaCacheHealCandidateKind::DeleteMarker)
);
assert!(discovery.unverified_count >= 1);
}
#[test]
fn discover_heal_candidates_skips_free_versions() {
let object_id = Uuid::from_u128(100);
let free_id = Uuid::from_u128(101);
let mut meta = FileMeta::new();
meta.add_version(FileInfo {
volume: "bucket".to_string(),
name: "object".to_string(),
version_id: Some(object_id),
transition_status: TRANSITION_COMPLETE.to_string(),
transitioned_objname: "remote/object".to_string(),
transition_version_id: Some(Uuid::from_u128(102)),
transition_tier: "WARM".to_string(),
mod_time: Some(OffsetDateTime::now_utc()),
..Default::default()
})
.expect("transitioned object should be added");
let mut delete = FileInfo {
volume: "bucket".to_string(),
name: "object".to_string(),
version_id: Some(object_id),
mod_time: Some(OffsetDateTime::now_utc()),
..Default::default()
};
delete.set_tier_free_version_id(&free_id.to_string());
meta.delete_version(&delete).expect("free version should be persisted");
let discovery = MetaCacheEntries(vec![Some(MetaCacheEntry {
name: "object".to_string(),
metadata: meta.marshal_msg().expect("free version metadata should marshal"),
cached: Some(meta),
reusable: false,
})])
.discover_heal_candidates("bucket", 16);
assert!(discovery.candidates.is_empty());
}
#[test]
fn discover_heal_candidates_preserves_unversioned_legacy_object() {
let legacy = MetaObjectV1 {
version: "1.0.1".to_string(),
format: "xl".to_string(),
stat: MetaObjectV1Stat {
size: 1,
mod_time: Some(OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp")),
name: "object".to_string(),
..Default::default()
},
erasure: MetaObjectV1Erasure {
data_blocks: 4,
parity_blocks: 2,
index: 1,
distribution: vec![1, 2, 3, 4, 5, 6],
..Default::default()
},
..Default::default()
};
let version = FileMetaVersion {
version_type: VersionType::Legacy,
legacy_object: Some(legacy),
..Default::default()
};
let mut meta = FileMeta::new();
meta.versions
.push(FileMetaShallowVersion::try_from(version).expect("legacy metadata should marshal"));
let discovery = MetaCacheEntries(vec![Some(MetaCacheEntry {
name: "object".to_string(),
metadata: meta.marshal_msg().expect("legacy metadata should marshal"),
cached: Some(meta),
reusable: false,
})])
.discover_heal_candidates("bucket", 16);
assert!(discovery.candidates.iter().any(|candidate| {
candidate.kind == MetaCacheHealCandidateKind::UnversionedObject && candidate.version_id.is_none()
}));
}
#[test]
fn discover_heal_candidates_rejects_nil_and_malformed_metadata_and_is_bounded() {
let now = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
let mut nil = metacache_entry_single_version(1, now, "nil");
let mut nil_meta = FileMeta::load(&nil.metadata).expect("nil fixture should decode");
let mut nil_version = nil_meta.versions[0]
.parse_version_meta()
.expect("nil fixture version should decode");
nil_version.object.as_mut().expect("object fixture").version_id = Some(Uuid::nil());
nil_meta.versions[0] = FileMetaShallowVersion::try_from(nil_version).expect("nil fixture should marshal");
nil.metadata = nil_meta.marshal_msg().expect("nil fixture metadata should marshal");
let mut mismatched = metacache_entry_single_version(2, now, "mismatched");
let mut mismatched_meta = FileMeta::load(&mismatched.metadata).expect("mismatched fixture should decode");
mismatched_meta.versions[0].header.version_id = Some(Uuid::from_u128(200));
mismatched.metadata = mismatched_meta.marshal_msg().expect("mismatched metadata should marshal");
let mut short_parts = metacache_entry_single_version(3, now, "short-parts");
let mut short_parts_meta = FileMeta::load(&short_parts.metadata).expect("short-parts fixture should decode");
let mut short_parts_version = short_parts_meta.versions[0]
.parse_version_meta()
.expect("short-parts fixture version should decode");
let object = short_parts_version.object.as_mut().expect("object fixture");
object.part_numbers = vec![1];
object.part_actual_sizes = vec![1];
object.part_sizes.clear();
short_parts_meta.versions[0] = FileMetaShallowVersion::try_from(short_parts_version).expect("short-parts should marshal");
short_parts.metadata = short_parts_meta.marshal_msg().expect("short-parts metadata should marshal");
let mut short_unversioned = metacache_entry_with_mod_time(now, "short-unversioned");
let mut short_unversioned_meta =
FileMeta::load(&short_unversioned.metadata).expect("short-unversioned fixture should decode");
let mut short_unversioned_version = short_unversioned_meta.versions[0]
.parse_version_meta()
.expect("short-unversioned version should decode");
let unversioned_object = short_unversioned_version.object.as_mut().expect("unversioned object fixture");
unversioned_object.part_numbers = vec![1];
unversioned_object.part_actual_sizes = vec![1];
unversioned_object.part_sizes.clear();
short_unversioned_meta.versions[0] =
FileMetaShallowVersion::try_from(short_unversioned_version).expect("short-unversioned should marshal");
short_unversioned.metadata = short_unversioned_meta
.marshal_msg()
.expect("short-unversioned metadata should marshal");
let mut malformed = nil.clone();
malformed.name = "malformed".to_string();
malformed.metadata = vec![1, 2, 3];
let entries = MetaCacheEntries(
std::iter::once(Some(nil))
.chain(std::iter::once(Some(mismatched)))
.chain(std::iter::once(Some(short_parts)))
.chain(std::iter::once(Some(short_unversioned)))
.chain(std::iter::once(Some(malformed)))
.chain((0..32).map(|id| Some(metacache_entry_single_version(id + 10, now, "bounded"))))
.collect(),
);
let discovery = entries.discover_heal_candidates("bucket", 5);
assert!(discovery.candidates.len() <= 5);
assert!(discovery.truncated, "bounded discovery must expose dropped candidates");
assert!(
discovery
.truncated_candidates
.iter()
.all(|candidate| candidate.version_id.is_some()),
"overflow candidates must retain exact version identities"
);
assert!(
discovery.truncated_objects.iter().any(|object| object == "object"),
"bounded discovery must expose an object-level safe continuation"
);
assert!(
!discovery
.candidates
.iter()
.any(|candidate| candidate.version_id == Some(Uuid::nil()))
);
assert!(
!discovery
.candidates
.iter()
.any(|candidate| candidate.version_id == Some(Uuid::from_u128(2)))
);
assert!(
!discovery
.candidates
.iter()
.any(|candidate| candidate.version_id == Some(Uuid::from_u128(3)))
);
assert!(discovery.candidates.iter().any(|candidate| {
candidate.kind == MetaCacheHealCandidateKind::UnversionedObject && candidate.version_id.is_none()
}));
assert!(
discovery.unverified_count >= 1,
"malformed and rejected metadata must remain observable during discovery"
);
for invalid_name in [
"../object",
"./object",
"object/../other",
"object//name",
"object\u{0001}name",
"object\0name",
] {
let mut entry = metacache_entry_single_version(400, now, invalid_name);
entry.name = invalid_name.to_string();
let discovery = MetaCacheEntries(vec![Some(entry)]).discover_heal_candidates("bucket", 5);
assert!(
discovery.candidates.is_empty(),
"invalid key should not become a heal candidate: {invalid_name:?}"
);
}
#[cfg(windows)]
{
let mut entry = metacache_entry_single_version(400, now, "object\\name");
entry.name = "object\\name".to_string();
assert!(
MetaCacheEntries(vec![Some(entry)])
.discover_heal_candidates("bucket", 5)
.candidates
.is_empty(),
"backslash is a path separator on Windows"
);
}
#[cfg(not(windows))]
{
let mut entry = metacache_entry_single_version(400, now, "object\\name");
entry.name = "object\\name".to_string();
assert_eq!(
MetaCacheEntries(vec![Some(entry)])
.discover_heal_candidates("bucket", 5)
.candidates
.len(),
1,
"backslash is object-key data on Unix"
);
}
for valid_name in ["trailing/", "prefix/object"] {
let mut entry = metacache_entry_single_version(401, now, valid_name);
entry.name = valid_name.to_string();
let discovery = MetaCacheEntries(vec![Some(entry)]).discover_heal_candidates("bucket", 5);
assert_eq!(discovery.candidates.len(), 1, "raw S3 key should remain opaque: {valid_name:?}");
}
}
#[test]
fn resolve_rejects_partial_latest_and_returns_committed_previous_metadata() {
let old_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
crates/heal/Cargo.toml
+1
View File
@@ -91,6 +91,7 @@ metrics = { workspace = true }
base64 = { workspace = true }
bytes = { workspace = true }
crc-fast = { workspace = true }
sha2 = { workspace = true }
[dev-dependencies]
serde_json = { workspace = true, features = ["raw_value"] }
crates/heal/src/heal/erasure_healer.rs
+519 -55
View File
@@ -13,10 +13,10 @@
// limitations under the License.
use crate::heal::{
progress::HealProgress,
progress::{HealProgress, add_bytes, increment_counter},
resume::{
CheckpointManager, ReplacementTargetIdentity, ResumeManager, ResumeUtils, compose_key,
replacement_target_identities_match,
CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, ReplacementTargetIdentity, ResumeManager,
ResumeUtils, compose_key, replacement_target_identities_match,
},
storage::{HealStorageAPI, next_heal_listing_token},
task::{demote_to_debug_when, is_missing_object_dir_heal_result, take_failure_log_sample},
@@ -373,6 +373,11 @@ impl ErasureSetHealer {
set_disk_id: &str,
buckets: &[String],
) -> Result<(ResumeManager, CheckpointManager)> {
if self.replacement_task_id.is_none() && CheckpointManager::is_blocked(&self.disk, task_id).await {
return Err(Error::TaskExecutionFailed {
message: format!("Resume task {task_id} has a blocked checkpoint"),
});
}
// check if resume state exists
let has_resume_state = if self.replacement_task_id.is_some() {
ResumeManager::has_replacement_intent(&self.disk, task_id).await
@@ -410,6 +415,9 @@ impl ErasureSetHealer {
&& state.successful_objects == 0
&& state.failed_objects == 0
&& state.skipped_objects == 0
&& state.skipped_new_versions == 0
&& state.skipped_ilm_expired == 0
&& state.processed_bytes == 0
{
// schedule_retry persists the authoritative resume reset before
// resetting the checkpoint. Reapply the checkpoint reset after
@@ -474,6 +482,23 @@ impl ErasureSetHealer {
// 2. initialize progress
self.initialize_progress(buckets, &state).await;
let (baseline_known, baseline_count, baseline_size, baseline_generation) = {
let baseline = self.progress.read().await;
(
baseline.baseline_known,
baseline.objects_total_count,
baseline.objects_total_size,
baseline.baseline_generation,
)
};
if baseline_known {
resume_manager
.set_progress_baseline(baseline_count, baseline_size, baseline_generation)
.await?;
checkpoint_manager
.set_progress_baseline(baseline_count, baseline_size, baseline_generation)
.await?;
}
// 3. continue from checkpoint
let current_bucket_index = checkpoint.current_bucket_index;
@@ -483,12 +508,66 @@ impl ErasureSetHealer {
let mut successful_objects = state.successful_objects;
let mut failed_objects = state.failed_objects;
let mut skipped_objects = state.skipped_objects;
let checkpoint_has_progress = checkpoint.baseline_known
|| checkpoint.successful_objects > 0
|| checkpoint.failed_object_count > 0
|| checkpoint.skipped_object_count > 0
|| checkpoint.skipped_new_versions > 0
|| checkpoint.skipped_ilm_expired > 0
|| checkpoint.processed_bytes > 0
|| checkpoint.total_objects > 0
|| checkpoint.total_bytes > 0
|| checkpoint.baseline_generation.is_some()
|| checkpoint.counter_unknown;
let checkpoint_generation_mismatch = checkpoint.baseline_known && checkpoint.baseline_generation != baseline_generation;
let mut restored_counter_unknown = state.counter_unknown || checkpoint.counter_unknown;
if checkpoint_has_progress {
successful_objects = checkpoint.successful_objects;
failed_objects = checkpoint.failed_object_count;
skipped_objects = checkpoint.skipped_object_count;
let restored_processed_objects = successful_objects
.checked_add(failed_objects)
.and_then(|value| value.checked_add(skipped_objects))
.and_then(|value| value.checked_add(checkpoint.skipped_new_versions))
.and_then(|value| value.checked_add(checkpoint.skipped_ilm_expired));
let checkpoint_counter_overflow = restored_processed_objects.is_none();
restored_counter_unknown |= checkpoint_counter_overflow;
processed_objects = restored_processed_objects.unwrap_or(u64::MAX);
let mut progress = self.progress.write().await;
progress.objects_scanned = processed_objects;
progress.objects_healed = successful_objects;
progress.objects_failed = failed_objects;
progress.skipped_objects = skipped_objects;
progress.skipped_new_versions = checkpoint.skipped_new_versions;
progress.skipped_ilm_expired = checkpoint.skipped_ilm_expired;
if checkpoint.baseline_known && !checkpoint_generation_mismatch {
progress.objects_total_count = checkpoint.total_objects;
progress.objects_total_size = checkpoint.total_bytes;
progress.baseline_generation = checkpoint.baseline_generation;
progress.baseline_known = true;
}
progress.bytes_processed = checkpoint.processed_bytes;
progress.counter_unknown = state.counter_unknown || checkpoint.counter_unknown;
progress.refresh_progress_percentage();
if checkpoint_generation_mismatch || checkpoint_counter_overflow || progress.counter_unknown {
progress.mark_unknown();
}
}
if checkpoint_generation_mismatch {
restored_counter_unknown = true;
}
if restored_counter_unknown {
checkpoint_manager.mark_counter_unknown().await?;
resume_manager.mark_counter_unknown().await?;
}
let mut failed_buckets = 0u64;
// 4. process remaining buckets
for (bucket_idx, bucket) in buckets.iter().enumerate().skip(current_bucket_index) {
// check if completed
if state.completed_buckets.contains(bucket) {
checkpoint_manager.complete_bucket(bucket_idx.saturating_add(1)).await?;
current_object_index = 0;
continue;
}
@@ -516,13 +595,42 @@ impl ErasureSetHealer {
return bucket_result;
}
// update checkpoint position
checkpoint_manager.update_position(bucket_idx, current_object_index).await?;
// update progress
resume_manager
.update_progress(processed_objects, successful_objects, failed_objects, skipped_objects)
let progress_snapshot = self.progress.read().await;
let bytes_processed = progress_snapshot.bytes_processed;
let skipped_new_versions = progress_snapshot.skipped_new_versions;
let skipped_ilm_expired = progress_snapshot.skipped_ilm_expired;
let counter_unknown = progress_snapshot.counter_unknown;
drop(progress_snapshot);
// The checkpoint is the recovery authority for object progress.
// Publish its counters and fence before the resume summary so a
// crash between the two stores cannot make recovery select newer
// summary bytes with an older checkpoint ledger.
if counter_unknown {
checkpoint_manager.mark_counter_unknown().await?;
}
checkpoint_manager
.update_progress(successful_objects, failed_objects, skipped_objects, bytes_processed)
.await?;
checkpoint_manager
.set_skipped_version_counts(skipped_new_versions, skipped_ilm_expired)
.await?;
checkpoint_manager.update_position(bucket_idx, current_object_index).await?;
resume_manager
.update_progress_with_bytes(
processed_objects,
successful_objects,
failed_objects,
skipped_objects,
bytes_processed,
)
.await?;
resume_manager
.set_skipped_version_counts(skipped_new_versions, skipped_ilm_expired)
.await?;
if counter_unknown {
resume_manager.mark_counter_unknown().await?;
}
// check cancel status
if self.cancel_token.is_cancelled() {
@@ -542,6 +650,7 @@ impl ErasureSetHealer {
match bucket_result {
Ok(_) => {
resume_manager.complete_bucket(bucket).await?;
checkpoint_manager.complete_bucket(bucket_idx.saturating_add(1)).await?;
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_BUCKET_STATE,
@@ -567,7 +676,9 @@ impl ErasureSetHealer {
error = %e,
"Erasure set bucket heal failed"
);
// continue to next bucket, do not interrupt the whole process
// A single durable cursor and ledger cannot safely preserve
// this bucket while processing a later one.
break;
}
}
@@ -775,20 +886,49 @@ impl ErasureSetHealer {
// Per-version dedup identity — the single canonical key.
let key = compose_key(&item.name, item.version_id.as_deref());
if checkpoint.processed_objects.contains(&key) || checkpoint.skipped_objects.contains(&key) {
if checkpoint.processed_objects.contains(&key)
|| checkpoint.failed_objects.contains(&key)
|| checkpoint.skipped_objects.contains(&key)
{
continue;
}
if should_skip_new_version(item.mod_time_unix_nanos, started_at_secs) {
checkpoint_manager.add_processed_object(key).await?;
*processed_objects = processed_objects.saturating_add(1);
let counter_ok = increment_counter(processed_objects);
completed_in_page = completed_in_page.saturating_add(1);
counter!("rustfs_heal_skipped_new_versions_total").increment(1);
{
let (outcome_record, counter_unknown) = {
let mut progress = self.progress.write().await;
progress.record_skipped_new_version();
progress.set_current_object(Some(format!("skipped_new: {bucket}/{}", item.name)));
progress.update_progress(*processed_objects, *successful_objects, *failed_objects, bytes_processed);
progress.update_object_progress(
*processed_objects,
*successful_objects,
*failed_objects,
*skipped_objects,
bytes_processed,
);
if !counter_ok {
progress.mark_unknown();
}
(
CheckpointObjectOutcomeRecord {
object: key,
outcome: CheckpointObjectOutcome::Processed,
successful: progress.objects_healed,
failed: progress.objects_failed,
skipped: progress.skipped_objects,
bytes: progress.bytes_processed,
skipped_new_versions: progress.skipped_new_versions,
skipped_ilm_expired: progress.skipped_ilm_expired,
counter_unknown: progress.counter_unknown,
},
progress.counter_unknown,
)
};
checkpoint_manager.record_object_outcome(outcome_record).await?;
if counter_unknown {
resume_manager.mark_counter_unknown().await?;
}
debug!(
target: "rustfs::heal::erasure_healer",
@@ -820,15 +960,41 @@ impl ErasureSetHealer {
)
.await?
{
checkpoint_manager.add_processed_object(key).await?;
*processed_objects = processed_objects.saturating_add(1);
let counter_ok = increment_counter(processed_objects);
completed_in_page = completed_in_page.saturating_add(1);
counter!("rustfs_heal_skipped_ilm_expired_total").increment(1);
{
let (outcome_record, counter_unknown) = {
let mut progress = self.progress.write().await;
progress.record_skipped_ilm_expired();
progress.set_current_object(Some(format!("skipped_ilm: {bucket}/{}", item.name)));
progress.update_progress(*processed_objects, *successful_objects, *failed_objects, bytes_processed);
progress.update_object_progress(
*processed_objects,
*successful_objects,
*failed_objects,
*skipped_objects,
bytes_processed,
);
if !counter_ok {
progress.mark_unknown();
}
(
CheckpointObjectOutcomeRecord {
object: key,
outcome: CheckpointObjectOutcome::Processed,
successful: progress.objects_healed,
failed: progress.objects_failed,
skipped: progress.skipped_objects,
bytes: progress.bytes_processed,
skipped_new_versions: progress.skipped_new_versions,
skipped_ilm_expired: progress.skipped_ilm_expired,
counter_unknown: progress.counter_unknown,
},
progress.counter_unknown,
)
};
checkpoint_manager.record_object_outcome(outcome_record).await?;
if counter_unknown {
resume_manager.mark_counter_unknown().await?;
}
debug!(
target: "rustfs::heal::erasure_healer",
@@ -954,11 +1120,11 @@ impl ErasureSetHealer {
while let Some((key, object, version_id, result)) = page_tasks.next().await {
let (object_size, result) = result;
match result {
let mut telemetry_unknown = false;
let checkpoint_outcome = match result {
Ok(true) => {
*successful_objects += 1;
bytes_processed = bytes_processed.saturating_add(object_size);
checkpoint_manager.add_processed_object(key).await?;
telemetry_unknown |= !increment_counter(successful_objects);
telemetry_unknown |= !add_bytes(&mut bytes_processed, object_size);
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
@@ -971,11 +1137,11 @@ impl ErasureSetHealer {
state = "healed",
"Erasure set object healed"
);
CheckpointObjectOutcome::Processed
}
Ok(false) => {
checkpoint_manager.add_processed_object(key).await?;
*successful_objects += 1;
bytes_processed = bytes_processed.saturating_add(object_size);
telemetry_unknown |= !increment_counter(successful_objects);
telemetry_unknown |= !add_bytes(&mut bytes_processed, object_size);
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
@@ -988,12 +1154,12 @@ impl ErasureSetHealer {
state = "missing_treated_as_ok",
"Erasure set missing object treated as ok"
);
CheckpointObjectOutcome::Processed
}
Err(err @ Error::TaskCancelled) | Err(err @ Error::TaskTimeout) => return Err(err),
Err(Error::TransientSkip { message }) => {
*skipped_objects += 1;
bytes_processed = bytes_processed.saturating_add(object_size);
checkpoint_manager.add_skipped_object(key).await?;
telemetry_unknown |= !increment_counter(skipped_objects);
telemetry_unknown |= !add_bytes(&mut bytes_processed, object_size);
demote_to_debug_when!(!take_failure_log_sample(&mut transient_skip_samples_logged), warn, target: "rustfs::heal::erasure_healer", {
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
component = LOG_COMPONENT_HEAL,
@@ -1006,11 +1172,11 @@ impl ErasureSetHealer {
error = %message,
"Erasure set object heal skipped due to transient error"
});
CheckpointObjectOutcome::Skipped
}
Err(err) => {
*failed_objects += 1;
bytes_processed = bytes_processed.saturating_add(object_size);
checkpoint_manager.add_failed_object(key).await?;
telemetry_unknown |= !increment_counter(failed_objects);
telemetry_unknown |= !add_bytes(&mut bytes_processed, object_size);
demote_to_debug_when!(!take_failure_log_sample(&mut failure_samples_logged), warn, target: "rustfs::heal::erasure_healer", {
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
component = LOG_COMPONENT_HEAL,
@@ -1023,15 +1189,43 @@ impl ErasureSetHealer {
error = %err,
"Erasure set object heal failed"
});
CheckpointObjectOutcome::Failed
}
}
};
*processed_objects += 1;
telemetry_unknown |= !increment_counter(processed_objects);
completed_in_page += 1;
{
let (outcome_record, counter_unknown) = {
let mut progress = self.progress.write().await;
progress.set_current_object(Some(format!("{bucket}/{object}")));
progress.update_progress(*processed_objects, *successful_objects, *failed_objects, bytes_processed);
progress.update_object_progress(
*processed_objects,
*successful_objects,
*failed_objects,
*skipped_objects,
bytes_processed,
);
if telemetry_unknown {
progress.mark_unknown();
}
(
CheckpointObjectOutcomeRecord {
object: key,
outcome: checkpoint_outcome,
successful: progress.objects_healed,
failed: progress.objects_failed,
skipped: progress.skipped_objects,
bytes: progress.bytes_processed,
skipped_new_versions: progress.skipped_new_versions,
skipped_ilm_expired: progress.skipped_ilm_expired,
counter_unknown: progress.counter_unknown,
},
progress.counter_unknown,
)
};
checkpoint_manager.record_object_outcome(outcome_record).await?;
if counter_unknown {
resume_manager.mark_counter_unknown().await?;
}
if completed_in_page.is_multiple_of(100) {
@@ -1041,16 +1235,22 @@ impl ErasureSetHealer {
*current_object_index = global_obj_idx;
// Persist the authoritative cursor FIRST (points at the next page
// boundary), then prune the per-version dedup sets. Both are
// idempotent under crash: heal_object re-heals safely.
let next_cursor = if is_truncated { next_token.clone() } else { None };
resume_manager.set_resume_cursor(next_cursor.clone()).await?;
checkpoint_manager.complete_page(bucket_index, *current_object_index).await?;
// Persist the checkpoint ledger and page position before exposing
// the next resume cursor. A crash before cursor publication keeps
// the page identities available for exact-once replay.
checkpoint_manager.advance_page(bucket_index, *current_object_index).await?;
// Check if there are more pages
if !is_truncated {
break;
}
continuation_token = next_heal_listing_token(bucket, "", next_token, is_truncated)?;
if continuation_token.is_none() {
// A truncated page without a continuation token is terminal.
// Retain its ledger until bucket completion is durable.
break;
}
resume_manager.set_resume_cursor(continuation_token.clone()).await?;
checkpoint_manager.prune_completed_page().await?;
// Anti-loop guard: an empty page reported as truncated cannot advance
// the cursor (there is no last identity to move past), so treat it as a
@@ -1069,12 +1269,6 @@ impl ErasureSetHealer {
)));
}
previous_page_last = page_last;
continuation_token = next_heal_listing_token(bucket, "", next_token, is_truncated)?;
if continuation_token.is_none() {
// Truncated but no continuation token: treat as end of listing.
break;
}
}
Ok(())
@@ -1083,10 +1277,66 @@ impl ErasureSetHealer {
/// initialize progress tracking
async fn initialize_progress(&self, _buckets: &[String], state: &crate::heal::resume::ResumeState) {
let mut progress = self.progress.write().await;
progress.objects_scanned = state.total_objects;
let existing_baseline = (
progress.objects_total_count,
progress.objects_total_size,
progress.baseline_generation,
progress.progress_state,
progress.baseline_known,
);
let baseline_generation_mismatch =
state.baseline_known && existing_baseline.4 && state.baseline_generation != existing_baseline.2;
let use_persisted_baseline = state.baseline_known && !baseline_generation_mismatch;
progress.objects_scanned = state.processed_objects;
progress.objects_healed = state.successful_objects;
progress.objects_failed = state.failed_objects;
progress.bytes_processed = 0; // Resume state tracks object counts, not byte counters.
progress.skipped_objects = state.skipped_objects;
progress.skipped_new_versions = state.skipped_new_versions;
progress.skipped_ilm_expired = state.skipped_ilm_expired;
progress.bytes_processed = state.processed_bytes;
progress.counter_unknown = state.counter_unknown;
if use_persisted_baseline
|| existing_baseline.0 > 0
|| existing_baseline.1 > 0
|| existing_baseline.2.is_some()
|| existing_baseline.4
{
progress.objects_total_count = if use_persisted_baseline {
state.total_objects
} else {
existing_baseline.0
};
progress.objects_total_size = if use_persisted_baseline {
state.total_bytes
} else {
existing_baseline.1
};
progress.baseline_generation = if use_persisted_baseline {
state.baseline_generation
} else {
existing_baseline.2
};
progress.baseline_known = use_persisted_baseline
|| existing_baseline.0 > 0
|| existing_baseline.1 > 0
|| existing_baseline.2.is_some()
|| existing_baseline.4;
}
progress.progress_state = if use_persisted_baseline
|| existing_baseline.0 > 0
|| existing_baseline.1 > 0
|| existing_baseline.2.is_some()
|| existing_baseline.4
{
crate::heal::progress::HealProgressState::Running
} else {
crate::heal::progress::HealProgressState::Indeterminate
};
if baseline_generation_mismatch || state.counter_unknown {
progress.mark_unknown();
}
progress.ledger_complete = false;
progress.refresh_progress_percentage();
progress.start_time = UNIX_EPOCH.checked_add(Duration::from_secs(state.start_time));
progress.last_update_time = UNIX_EPOCH.checked_add(Duration::from_secs(state.last_update));
progress.set_current_object(state.current_object.clone());
@@ -1264,8 +1514,8 @@ mod resume_loop_tests {
};
use crate::heal::progress::HealProgress;
use crate::heal::resume::{
CheckpointManager, RESUME_CHECKPOINT_FILE, ReplacementTargetIdentity, ResumeDeleteFailure, ResumeManager, ResumeUtils,
compose_key,
CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, RESUME_CHECKPOINT_FILE,
ReplacementTargetIdentity, ResumeDeleteFailure, ResumeManager, ResumeUtils, compose_key,
};
use crate::heal::storage::{HealLifecycleExpiryContext, HealListItem, HealObjectInfo, HealStorageAPI};
use crate::heal::storage_api::status::BucketInfo;
@@ -1405,6 +1655,7 @@ mod resume_loop_tests {
list_include_lifecycle_object_info: Mutex<Vec<bool>>,
replacement_target_identity_sequences: Mutex<VecDeque<Vec<ReplacementTargetIdentity>>>,
fail_listing: AtomicBool,
fail_listing_buckets: Mutex<HashSet<String>>,
}
impl FakeStorage {
@@ -1441,6 +1692,9 @@ mod resume_loop_tests {
fn fail_listing(&self) {
self.fail_listing.store(true, Ordering::SeqCst);
}
fn fail_bucket_listing(&self, bucket: &str) {
self.fail_listing_buckets.lock().unwrap().insert(bucket.to_string());
}
}
#[async_trait::async_trait]
@@ -1531,7 +1785,7 @@ mod resume_loop_tests {
}
async fn list_objects_for_heal_page(
&self,
_bucket: &str,
bucket: &str,
_prefix: &str,
continuation_token: Option<&str>,
include_lifecycle_object_info: bool,
@@ -1540,7 +1794,7 @@ mod resume_loop_tests {
.lock()
.unwrap()
.push(include_lifecycle_object_info);
if self.fail_listing.load(Ordering::SeqCst) {
if self.fail_listing.load(Ordering::SeqCst) || self.fail_listing_buckets.lock().unwrap().contains(bucket) {
return Err(Error::other("injected listing failure"));
}
let key = continuation_token.map(str::to_string);
@@ -1918,6 +2172,49 @@ mod resume_loop_tests {
assert!(state.completed_buckets.is_empty(), "the failed bucket must remain resumable");
}
#[tokio::test]
async fn bucket_failure_stops_before_a_later_bucket_checkpoint() {
let env = make_env().await;
let task_id = ResumeUtils::generate_task_id();
let buckets = vec!["a".to_string(), "b".to_string()];
let resume = ResumeManager::new(
env.healer.disk.clone(),
task_id.clone(),
"erasure_set".to_string(),
"pool_0_set_0".to_string(),
buckets.clone(),
)
.await
.unwrap();
let checkpoint = CheckpointManager::new(env.healer.disk.clone(), task_id.clone())
.await
.unwrap();
env.storage.fail_bucket_listing("a");
for _ in 0..3 {
assert!(resume.schedule_retry().await.unwrap());
}
env.healer
.execute_heal_with_resume(&buckets, "pool_0_set_0", &resume, &checkpoint)
.await
.expect_err("the first bucket failure must keep the pass incomplete");
let persisted = checkpoint.get_checkpoint().await;
assert_eq!(persisted.current_bucket_index, 0);
assert!(resume.get_state().await.completed_buckets.is_empty());
let resumed = ResumeManager::load_from_disk(env.healer.disk.clone(), &task_id)
.await
.unwrap();
let checkpoint = CheckpointManager::load_from_disk(env.healer.disk.clone(), &task_id)
.await
.unwrap();
env.healer
.execute_heal_with_resume(&buckets, "pool_0_set_0", &resumed, &checkpoint)
.await
.expect_err("recovery must retry the earlier failed bucket");
assert!(!resumed.get_state().await.completed);
}
#[tokio::test]
async fn completed_resume_state_is_not_selected_for_a_new_heal() {
let env = make_env().await;
@@ -2107,8 +2404,175 @@ mod resume_loop_tests {
let mut names: Vec<String> = env.storage.calls().into_iter().map(|(n, _)| n).collect();
names.sort();
assert_eq!(names, vec!["a", "b", "c", "d"], "every object exactly once, none dropped/doubled");
// Final page not truncated => cursor cleared.
assert_eq!(env.resume.resume_cursor().await, None);
// Keep the final page cursor until the outer loop durably completes the
// bucket, so a crash can replay only this page against its identities.
assert_eq!(env.resume.resume_cursor().await, Some("t1".to_string()));
}
#[tokio::test]
async fn persisted_failure_waits_for_the_bounded_retry_after_page_replay() {
let env = make_env().await;
env.storage.set_page(
None,
Page {
items: vec![item("object", Some("v1"), false)],
next: None,
truncated: false,
},
);
env.checkpoint
.record_object_outcome(CheckpointObjectOutcomeRecord {
object: compose_key("object", Some("v1")),
outcome: CheckpointObjectOutcome::Failed,
successful: 0,
failed: 1,
skipped: 0,
bytes: 0,
skipped_new_versions: 0,
skipped_ilm_expired: 0,
counter_unknown: false,
})
.await
.unwrap();
env.checkpoint.advance_page(0, 1).await.unwrap();
let resumed = ResumeManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
.await
.unwrap();
let checkpoint = CheckpointManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
.await
.unwrap();
env.healer
.execute_heal_with_resume(&["b".to_string()], "pool_0_set_0", &resumed, &checkpoint)
.await
.expect_err("the persisted failure must schedule a bounded retry");
assert!(
env.storage.calls().is_empty(),
"the failed identity must not be repeated in the same pass"
);
env.healer
.execute_heal_with_resume(&["b".to_string()], "pool_0_set_0", &resumed, &checkpoint)
.await
.expect("the bounded retry must heal the object");
assert_eq!(env.storage.calls(), vec![("object".to_string(), Some("v1".to_string()))]);
let state = resumed.get_state().await;
assert_eq!(state.successful_objects, 1);
assert_eq!(state.failed_objects, 0);
}
#[tokio::test]
async fn final_page_crash_replays_only_the_retained_page_identities() {
let env = make_env().await;
env.storage.set_page(
None,
Page {
items: vec![item("first", Some("v1"), false)],
next: Some("final-page".to_string()),
truncated: true,
},
);
env.storage.set_page(
Some("final-page"),
Page {
items: vec![item("last", Some("v1"), false)],
next: None,
truncated: false,
},
);
let (processed, successful, failed, skipped, result) = run(&env).await;
result.expect("the bucket pass must finish before the simulated crash");
assert_eq!((processed, successful, failed, skipped), (2, 2, 0, 0));
let resumed = ResumeManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
.await
.unwrap();
let checkpoint = CheckpointManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
.await
.unwrap();
env.healer
.execute_heal_with_resume(&["b".to_string()], "pool_0_set_0", &resumed, &checkpoint)
.await
.expect("the retained final-page ledger must make recovery exact");
assert_eq!(
env.storage.calls(),
vec![
("first".to_string(), Some("v1".to_string())),
("last".to_string(), Some("v1".to_string()))
]
);
let state = resumed.get_state().await;
assert_eq!(state.successful_objects, 2);
assert_eq!(state.processed_objects, 2);
}
#[tokio::test]
async fn truncated_page_without_token_retains_its_replay_ledger() {
let env = make_env().await;
env.storage.set_page(
None,
Page {
items: vec![item("object", Some("v1"), false)],
next: None,
truncated: true,
},
);
let (processed, successful, failed, skipped, result) = run(&env).await;
result.expect("the tokenless truncated page is a terminal page");
assert_eq!((processed, successful, failed, skipped), (1, 1, 0, 0));
let resumed = ResumeManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
.await
.unwrap();
let checkpoint = CheckpointManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
.await
.unwrap();
env.healer
.execute_heal_with_resume(&["b".to_string()], "pool_0_set_0", &resumed, &checkpoint)
.await
.expect("terminal-page recovery must not replay a durable identity");
assert_eq!(env.storage.calls(), vec![("object".to_string(), Some("v1".to_string()))]);
let state = resumed.get_state().await;
assert_eq!(state.successful_objects, 1);
assert_eq!(state.processed_objects, 1);
}
#[tokio::test]
async fn completed_bucket_reconciles_its_final_page_checkpoint_after_crash() {
let env = make_env().await;
env.storage.set_page(
None,
Page {
items: vec![item("object", Some("v1"), false)],
next: None,
truncated: false,
},
);
let (_, _, _, _, result) = run(&env).await;
result.expect("the bucket pass must finish before the simulated crash");
env.resume.complete_bucket("b").await.unwrap();
let resumed = ResumeManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
.await
.unwrap();
let checkpoint = CheckpointManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
.await
.unwrap();
env.healer
.execute_heal_with_resume(&["b".to_string()], "pool_0_set_0", &resumed, &checkpoint)
.await
.expect("recovery must finish the checkpoint transition without replaying the bucket");
assert_eq!(env.storage.calls(), vec![("object".to_string(), Some("v1".to_string()))]);
let checkpoint = checkpoint.get_checkpoint().await;
assert_eq!(checkpoint.current_bucket_index, 1);
assert!(checkpoint.processed_objects.is_empty());
}
#[tokio::test]
crates/heal/src/heal/manager.rs
+106 -30
View File
@@ -14,7 +14,7 @@
use crate::heal::{
progress::{HealProgress, HealStatistics},
resume::{ReplacementPhase, ResumeManager, ResumeState, ResumeUtils},
resume::{ReplacementPhase, ResumeGc, ResumeManager, ResumeState, ResumeUtils},
storage::HealStorageAPI,
task::{HealOptions, HealPriority, HealRequest, HealTask, HealTaskStatus, HealType, demote_to_debug_when},
};
@@ -53,9 +53,11 @@ const EVENT_HEAL_MAINLINE_THROTTLE: &str = "heal_mainline_throttle";
const EVENT_HEAL_SCHEDULER_STATE: &str = "heal_scheduler_state";
const EVENT_HEAL_QUEUE_STATE: &str = "heal_queue_state";
const EVENT_HEAL_UNCLEAN_SHUTDOWN: &str = "heal_unclean_shutdown";
const EVENT_HEAL_RESUME_GC: &str = "heal_resume_gc";
const LEGACY_ROOT_HEAL_PATH: &str = ".";
const MAX_RECOVERABLE_HEAL_RETRIES: u32 = 3;
const MAX_RECOVERABLE_HEAL_RETRY_DELAY: Duration = Duration::from_secs(30);
const RESUME_GC_INTERVAL: Duration = Duration::from_secs(60 * 60);
// Admission/scheduler outcomes for per-object requests (Object/Metadata/
// ECDecode) log via demote_to_debug_when! — MRF, autoheal, and scanner
@@ -119,6 +121,9 @@ struct MrfRepairNoticeTarget {
bucket: Arc<str>,
object: Arc<str>,
version_id: Option<[u8; 16]>,
kind: rustfs_common::mrf_channel::MrfKind,
scope: Option<rustfs_common::mrf_channel::MrfScope>,
lease: Option<rustfs_common::mrf_channel::MrfIngressLease>,
}
#[derive(Debug, Clone)]
@@ -889,7 +894,19 @@ impl HealManager {
}
fn remove_mrf_repair_notice_targets_for_task(&self, task_id: &str) {
lock_mrf_repair_notice_targets(&self.mrf_repair_notice_targets).remove(task_id);
let targets = lock_mrf_repair_notice_targets(&self.mrf_repair_notice_targets).remove(task_id);
if let Some(targets) = targets {
for target in targets {
rustfs_common::mrf_channel::release_mrf_identity(
target.kind,
&target.bucket,
&target.object,
target.version_id,
target.scope,
target.lease,
);
}
}
}
fn insert_mrf_repair_notice_target(
@@ -1150,6 +1167,49 @@ impl HealManager {
Ok(())
}
/// Start the bounded resume-state inspector. Destructive GC remains
/// disabled until the durable owner/CAS contract from backlog#1927 is
/// available; this task therefore cannot remove an active or stale file.
async fn start_resume_gc(&self) {
let cancel = self.cancel_token.clone();
tokio::spawn(async move {
let mut gc_by_disk = HashMap::<String, ResumeGc>::new();
let mut ticker = interval(RESUME_GC_INTERVAL);
loop {
tokio::select! {
_ = cancel.cancelled() => break,
_ = ticker.tick() => {
let disks = {
let local_disk_map = local_disk_map_read().await;
local_disk_map.values().flatten().cloned().collect::<Vec<_>>()
};
for disk in disks {
let disk_key = disk.endpoint().to_string();
let gc = gc_by_disk.entry(disk_key).or_default();
tokio::select! {
_ = cancel.cancelled() => return,
result = gc.inspect_disk(&disk) => {
if let Err(error) = result {
warn!(
target: "rustfs::heal::manager",
event = EVENT_HEAL_RESUME_GC,
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_MANAGER,
state = "inspect_failed",
endpoint = %disk.endpoint(),
error = %error,
"Heal resume GC inspection failed"
);
}
}
}
}
}
}
}
});
}
/// Create new HealManager
pub fn new(storage: Arc<dyn HealStorageAPI>, config: Option<HealConfig>) -> Self {
Self::new_with_workload_provider(storage, config, None)
@@ -1215,6 +1275,9 @@ impl HealManager {
// competing task for the same set.
self.process_unclean_shutdown().await;
// Inspect resume artifacts in a bounded, fail-closed background task.
self.start_resume_gc().await;
// start auto disk scanner to heal unformatted disks
if self.config.read().await.enable_auto_heal {
self.start_auto_disk_scanner().await?;
@@ -1279,7 +1342,20 @@ impl HealManager {
}
self.task_aliases.lock().await.clear();
self.retrying_heals.lock().await.clear();
lock_mrf_repair_notice_targets(&self.mrf_repair_notice_targets).clear();
let mrf_targets = {
let mut registry = lock_mrf_repair_notice_targets(&self.mrf_repair_notice_targets);
registry.drain().flat_map(|(_, targets)| targets).collect::<Vec<_>>()
};
for target in mrf_targets {
rustfs_common::mrf_channel::release_mrf_identity(
target.kind,
&target.bucket,
&target.object,
target.version_id,
target.scope,
target.lease,
);
}
crate::set_heal_queue_length(0);
// update state
@@ -1311,12 +1387,32 @@ impl HealManager {
.await
}
#[cfg(test)]
pub(crate) async fn submit_mrf_heal_request_with_receipt(
&self,
request: HealRequest,
bucket: Arc<str>,
object: Arc<str>,
version_id: Option<[u8; 16]>,
) -> Result<HealAdmissionReceipt> {
let kind = match &request.heal_type {
HealType::Metadata { .. } => rustfs_common::mrf_channel::MrfKind::MetadataCorruption,
HealType::ECDecode { .. } => rustfs_common::mrf_channel::MrfKind::DecodeFailure,
_ => rustfs_common::mrf_channel::MrfKind::PartialWrite,
};
self.submit_mrf_heal_request_with_receipt_and_identity(request, bucket, object, version_id, kind, None, None)
.await
}
pub(crate) async fn submit_mrf_heal_request_with_receipt_and_identity(
&self,
request: HealRequest,
bucket: Arc<str>,
object: Arc<str>,
version_id: Option<[u8; 16]>,
kind: rustfs_common::mrf_channel::MrfKind,
scope: Option<rustfs_common::mrf_channel::MrfScope>,
lease: Option<rustfs_common::mrf_channel::MrfIngressLease>,
) -> Result<HealAdmissionReceipt> {
self.submit_heal_request_with_receipt_alias_and_mrf_notice(
request,
@@ -1325,6 +1421,9 @@ impl HealManager {
bucket,
object,
version_id,
kind,
scope,
lease,
}),
)
.await
@@ -1539,7 +1638,7 @@ impl HealManager {
Self::insert_mrf_repair_notice_target(&mut targets, &task_id, target);
}
if let Some(displaced_task_id) = &displaced_task_id {
lock_mrf_repair_notice_targets(&self.mrf_repair_notice_targets).remove(displaced_task_id);
self.remove_mrf_repair_notice_targets_for_task(displaced_task_id);
}
drop(retrying_heals);
drop(queue);
@@ -2011,34 +2110,11 @@ impl HealManager {
return None;
}
let mut snapshot = HealProgress::default();
let mut progresses = Vec::with_capacity(active_tasks.len());
for task in active_tasks {
let progress = task.get_progress().await;
snapshot.objects_scanned = snapshot.objects_scanned.saturating_add(progress.objects_scanned);
snapshot.objects_healed = snapshot.objects_healed.saturating_add(progress.objects_healed);
snapshot.objects_failed = snapshot.objects_failed.saturating_add(progress.objects_failed);
snapshot.skipped_new_versions = snapshot.skipped_new_versions.saturating_add(progress.skipped_new_versions);
snapshot.skipped_ilm_expired = snapshot.skipped_ilm_expired.saturating_add(progress.skipped_ilm_expired);
snapshot.objects_total_count = snapshot.objects_total_count.saturating_add(progress.objects_total_count);
snapshot.objects_total_size = snapshot.objects_total_size.saturating_add(progress.objects_total_size);
snapshot.bytes_processed = snapshot.bytes_processed.saturating_add(progress.bytes_processed);
snapshot.start_time = match (snapshot.start_time, progress.start_time) {
(Some(current), Some(next)) => Some(current.min(next)),
(None, next) => next,
(current, None) => current,
};
snapshot.last_update_time = match (snapshot.last_update_time, progress.last_update_time) {
(Some(current), Some(next)) => Some(current.max(next)),
(None, next) => next,
(current, None) => current,
};
if progress.current_object.is_some() {
snapshot.current_object = progress.current_object;
}
progresses.push(task.get_progress().await);
}
snapshot.refresh_progress_percentage();
snapshot.refresh_estimated_completion_time();
Some(snapshot)
crate::heal::progress::aggregate_heal_progress(progresses)
}
}
crates/heal/src/heal/manager/auto_scan.rs
+12 -1
View File
@@ -506,7 +506,18 @@ impl HealManager {
&displaced_terminal,
)
.await;
lock_mrf_repair_notice_targets(&mrf_repair_notice_targets).remove(&displaced_task_id);
if let Some(targets) = lock_mrf_repair_notice_targets(&mrf_repair_notice_targets).remove(&displaced_task_id) {
for target in targets {
rustfs_common::mrf_channel::release_mrf_identity(
target.kind,
&target.bucket,
&target.object,
target.version_id,
target.scope,
target.lease,
);
}
}
}
if matches!(admission, HealAdmissionResult::Accepted) {
if should_notify {
crates/heal/src/heal/manager/queue.rs
+5 -1
View File
@@ -299,7 +299,11 @@ impl PriorityHealQueue {
/// Create a deduplication key from a heal request
pub(super) fn make_dedup_key(request: &HealRequest) -> String {
Self::make_dedup_key_for_type(&request.heal_type)
let base = Self::make_dedup_key_for_type(&request.heal_type);
match (&request.heal_type, request.options.set_key()) {
(HealType::Object { .. } | HealType::ECDecode { .. }, Some(scope)) => format!("{base}:scope:{scope}"),
_ => base,
}
}
pub(super) fn make_dedup_key_for_type(heal_type: &HealType) -> String {
crates/heal/src/heal/manager/scheduler.rs
+36 -1
View File
@@ -349,6 +349,8 @@ impl HealManager {
let notice_targets = take_mrf_repair_notice_targets(&mrf_repair_notice_targets_clone, &task_id);
if successful_completion {
emit_mrf_repaired_events(notice_targets);
} else {
release_mrf_repair_notice_targets(notice_targets);
}
task_aliases_clone
.lock()
@@ -638,7 +640,19 @@ pub(super) fn running_heal_set_counts(active_heals: &HashMap<String, Arc<HealTas
}
fn remove_mrf_repair_notice_targets(registry: &Arc<StdMutex<HashMap<String, Vec<MrfRepairNoticeTarget>>>>, task_id: &str) {
lock_mrf_repair_notice_targets(registry).remove(task_id);
let targets = lock_mrf_repair_notice_targets(registry).remove(task_id);
if let Some(targets) = targets {
for target in targets {
rustfs_common::mrf_channel::release_mrf_identity(
target.kind,
&target.bucket,
&target.object,
target.version_id,
target.scope,
target.lease,
);
}
}
}
fn take_mrf_repair_notice_targets(
@@ -671,6 +685,27 @@ fn move_mrf_repair_notice_targets(
fn emit_mrf_repaired_events(targets: Vec<MrfRepairNoticeTarget>) {
for target in targets {
rustfs_common::mrf_channel::note_mrf_repaired(&target.bucket, &target.object, target.version_id);
rustfs_common::mrf_channel::release_mrf_identity(
target.kind,
&target.bucket,
&target.object,
target.version_id,
target.scope,
target.lease,
);
}
}
fn release_mrf_repair_notice_targets(targets: Vec<MrfRepairNoticeTarget>) {
for target in targets {
rustfs_common::mrf_channel::release_mrf_identity(
target.kind,
&target.bucket,
&target.object,
target.version_id,
target.scope,
target.lease,
);
}
}
crates/heal/src/heal/mrf_queue.rs
+346 -63
View File
@@ -37,7 +37,7 @@ use crate::heal::manager::HealManager;
use metrics::{counter, gauge};
use rustfs_common::heal_channel::{HealAdmissionDropReason, HealAdmissionResult};
use rustfs_common::mrf_channel::{MRF_MAX_ATTEMPTS, MrfIntent};
use std::collections::VecDeque;
use std::collections::{HashSet, VecDeque};
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::mpsc;
@@ -48,15 +48,27 @@ use crate::heal::task::{HealOptions, HealPriority, HealRequest, HealType};
/// Journal location inside the metadata bucket, following the resume-state
/// layout.
pub(crate) const MRF_JOURNAL_PATH: &str = "buckets/.heal/mrf/journal.bin";
/// The scoped path is the authoritative snapshot for new readers and carries
/// both v1 and v2 records. The legacy path is only a v1 compatibility mirror;
/// older readers ignore the authoritative path, while new readers never merge
/// the two files. This prevents a partial two-file flush from fabricating a
/// mixed epoch.
pub(crate) const MRF_SCOPED_JOURNAL_PATH: &str = "buckets/.heal/mrf/journal-scoped.bin";
/// Record format tag.
const MRF_JOURNAL_FORMAT: u8 = 1;
/// Record layout version.
const MRF_JOURNAL_VERSION: u8 = 1;
const MRF_JOURNAL_VERSION_SCOPED: u8 = 2;
/// Fixed header size: format, version, kind, attempts, enqueued_at_ms,
/// has_version flag.
const MRF_RECORD_FIXED_HEAD: usize = 1 + 1 + 1 + 1 + 8 + 1;
const MRF_MAX_IDENTITY_COMPONENT: usize = 1024;
fn metric_f64(value: usize) -> f64 {
f64::from(u32::try_from(value).unwrap_or(u32::MAX))
}
#[derive(Debug, Clone)]
pub(crate) struct MrfConsumerConfig {
@@ -101,40 +113,90 @@ impl Default for MrfConsumerConfig {
/// incoming intent (never a resident one) and counts the loss.
pub(crate) struct MrfQueue {
pending: VecDeque<MrfIntent>,
pending_keys: HashSet<MrfQueueKey>,
bytes: usize,
capacity: usize,
byte_budget: usize,
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
struct MrfQueueKey {
kind: rustfs_common::mrf_channel::MrfKind,
bucket: Arc<str>,
object: Arc<str>,
version_id: Option<[u8; 16]>,
scope: Option<rustfs_common::mrf_channel::MrfScope>,
}
fn queue_key(intent: &MrfIntent) -> MrfQueueKey {
let version_id = intent.version_id.filter(|bytes| *bytes != [0; 16]);
let scope = (!matches!(intent.kind, rustfs_common::mrf_channel::MrfKind::MetadataCorruption))
.then_some(intent.scope)
.flatten();
MrfQueueKey {
kind: intent.kind,
bucket: intent.bucket.clone(),
object: intent.object.clone(),
version_id,
scope,
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum MrfQueuePushResult {
Enqueued,
Coalesced,
Rejected,
}
impl MrfQueue {
pub(crate) fn new(capacity: usize, byte_budget: usize) -> Self {
Self {
pending: VecDeque::new(),
pending_keys: HashSet::new(),
bytes: 0,
capacity,
byte_budget,
}
}
/// Returns `false` (after counting) when either ceiling would be crossed.
pub(crate) fn try_push(&mut self, intent: MrfIntent) -> bool {
pub(crate) fn try_push_typed(&mut self, intent: MrfIntent) -> MrfQueuePushResult {
if intent.bucket.len() > MRF_MAX_IDENTITY_COMPONENT || intent.object.len() > MRF_MAX_IDENTITY_COMPONENT {
counter!("rustfs_heal_mrf_dropped_total", "reason" => "identity_oversized").increment(1);
return MrfQueuePushResult::Rejected;
}
let key = queue_key(&intent);
if self.pending_keys.contains(&key) {
counter!("rustfs_heal_mrf_coalesced_total", "layer" => "queue").increment(1);
return MrfQueuePushResult::Coalesced;
}
let cost = intent.estimated_bytes();
if self.pending.len() >= self.capacity || self.bytes + cost > self.byte_budget {
counter!("rustfs_heal_mrf_dropped_total", "reason" => "queue_overflow").increment(1);
return false;
return MrfQueuePushResult::Rejected;
}
self.bytes += cost;
self.pending_keys.insert(key);
self.pending.push_back(intent);
true
MrfQueuePushResult::Enqueued
}
/// Bool compatibility adapter: only a newly executable queue item is
/// reported as accepted; a coalesced duplicate is not durable admission.
#[cfg(test)]
pub(crate) fn try_push(&mut self, intent: MrfIntent) -> bool {
matches!(self.try_push_typed(intent), MrfQueuePushResult::Enqueued)
}
pub(crate) fn pop_front(&mut self) -> Option<MrfIntent> {
let intent = self.pending.pop_front()?;
self.pending_keys.remove(&queue_key(&intent));
self.bytes = self.bytes.saturating_sub(intent.estimated_bytes());
Some(intent)
}
pub(crate) fn push_back(&mut self, intent: MrfIntent) {
self.pending_keys.insert(queue_key(&intent));
self.bytes += intent.estimated_bytes();
self.pending.push_back(intent);
}
@@ -157,10 +219,24 @@ impl MrfQueue {
// ---------------------------------------------------------------------------
/// Append one encoded record to `out`.
pub(crate) fn encode_intent(intent: &MrfIntent, out: &mut Vec<u8>) {
pub(crate) fn encode_intent(intent: &MrfIntent, out: &mut Vec<u8>) -> bool {
let Ok(bucket_len) = u32::try_from(intent.bucket.len()) else {
return false;
};
let Ok(object_len) = u32::try_from(intent.object.len()) else {
return false;
};
let scope = (!matches!(intent.kind, rustfs_common::mrf_channel::MrfKind::MetadataCorruption))
.then_some(intent.scope)
.flatten();
let version_id = intent.version_id.filter(|bytes| *bytes != [0; 16]);
let start = out.len();
out.push(MRF_JOURNAL_FORMAT);
out.push(MRF_JOURNAL_VERSION);
out.push(if scope.is_some() {
MRF_JOURNAL_VERSION_SCOPED
} else {
MRF_JOURNAL_VERSION
});
out.push(match intent.kind {
rustfs_common::mrf_channel::MrfKind::DecodeFailure => 1,
rustfs_common::mrf_channel::MrfKind::MetadataCorruption => 2,
@@ -168,27 +244,36 @@ pub(crate) fn encode_intent(intent: &MrfIntent, out: &mut Vec<u8>) {
});
out.push(intent.attempts);
out.extend_from_slice(&intent.enqueued_at_ms.to_le_bytes());
match intent.version_id {
match version_id {
Some(bytes) => {
out.push(1);
out.extend_from_slice(&bytes);
}
None => out.push(0),
}
out.extend_from_slice(&(intent.bucket.len() as u32).to_le_bytes());
out.extend_from_slice(&(intent.object.len() as u32).to_le_bytes());
if let Some(scope) = scope {
out.extend_from_slice(&scope.pool_index.to_le_bytes());
out.extend_from_slice(&scope.set_index.to_le_bytes());
}
out.extend_from_slice(&bucket_len.to_le_bytes());
out.extend_from_slice(&object_len.to_le_bytes());
out.extend_from_slice(intent.bucket.as_bytes());
out.extend_from_slice(intent.object.as_bytes());
let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc);
hasher.update(&out[start..]);
out.extend_from_slice(&(hasher.finalize() as u32).to_le_bytes());
let Ok(checksum) = u32::try_from(hasher.finalize()) else {
out.truncate(start);
return false;
};
out.extend_from_slice(&checksum.to_le_bytes());
true
}
fn decode_one(data: &[u8]) -> Option<(MrfIntent, usize)> {
if data.len() < MRF_RECORD_FIXED_HEAD + 8 {
return None;
}
if data[0] != MRF_JOURNAL_FORMAT || data[1] != MRF_JOURNAL_VERSION {
if data[0] != MRF_JOURNAL_FORMAT || !matches!(data[1], MRF_JOURNAL_VERSION | MRF_JOURNAL_VERSION_SCOPED) {
return None;
}
let kind = match data[2] {
@@ -198,24 +283,38 @@ fn decode_one(data: &[u8]) -> Option<(MrfIntent, usize)> {
_ => return None,
};
let attempts = data[3];
let enqueued_at_ms = u64::from_le_bytes(data[4..12].try_into().expect("slice length checked"));
let enqueued_at_ms = u64::from_le_bytes(data[4..12].try_into().ok()?);
let has_version = data[12] != 0;
let mut cursor = MRF_RECORD_FIXED_HEAD;
let version_id = if has_version {
if data.len() < cursor + 16 {
return None;
}
let bytes: [u8; 16] = data[cursor..cursor + 16].try_into().expect("slice length checked");
let bytes: [u8; 16] = data[cursor..cursor + 16].try_into().ok()?;
cursor += 16;
Some(bytes)
} else {
None
};
let scope = if data[1] == MRF_JOURNAL_VERSION_SCOPED {
if data.len() < cursor + 8 {
return None;
}
let pool_index = u32::from_le_bytes(data[cursor..cursor + 4].try_into().ok()?);
let set_index = u32::from_le_bytes(data[cursor + 4..cursor + 8].try_into().ok()?);
cursor += 8;
Some(rustfs_common::mrf_channel::MrfScope { pool_index, set_index })
} else {
None
};
if data.len() < cursor + 8 {
return None;
}
let bucket_len = u32::from_le_bytes(data[cursor..cursor + 4].try_into().expect("slice length checked")) as usize;
let object_len = u32::from_le_bytes(data[cursor + 4..cursor + 8].try_into().expect("slice length checked")) as usize;
let bucket_len = usize::try_from(u32::from_le_bytes(data[cursor..cursor + 4].try_into().ok()?)).ok()?;
let object_len = usize::try_from(u32::from_le_bytes(data[cursor + 4..cursor + 8].try_into().ok()?)).ok()?;
if bucket_len > MRF_MAX_IDENTITY_COMPONENT || object_len > MRF_MAX_IDENTITY_COMPONENT {
return None;
}
cursor += 8;
let body_end = cursor.checked_add(bucket_len)?.checked_add(object_len)?;
let record_end = body_end.checked_add(4)?;
@@ -224,7 +323,7 @@ fn decode_one(data: &[u8]) -> Option<(MrfIntent, usize)> {
}
let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc);
hasher.update(&data[..body_end]);
if (hasher.finalize() as u32) != u32::from_le_bytes(data[body_end..record_end].try_into().expect("slice length checked")) {
if u32::try_from(hasher.finalize()).ok()? != u32::from_le_bytes(data[body_end..record_end].try_into().ok()?) {
return None;
}
let bucket = std::sync::Arc::from(std::str::from_utf8(&data[cursor..cursor + bucket_len]).ok()?);
@@ -235,6 +334,12 @@ fn decode_one(data: &[u8]) -> Option<(MrfIntent, usize)> {
object,
version_id,
kind,
scope: if matches!(kind, rustfs_common::mrf_channel::MrfKind::MetadataCorruption) {
None
} else {
scope
},
lease: None,
enqueued_at_ms,
attempts,
},
@@ -269,9 +374,9 @@ async fn journal_disks() -> Vec<DiskStore> {
map.values().flatten().cloned().collect()
}
async fn read_journal() -> Option<Vec<u8>> {
async fn read_journal(path: &str) -> Option<Vec<u8>> {
for disk in journal_disks().await {
match disk.read_all(super::RUSTFS_META_BUCKET, MRF_JOURNAL_PATH).await {
match disk.read_all(super::RUSTFS_META_BUCKET, path).await {
Ok(bytes) => return Some(bytes.to_vec()),
Err(_) => continue,
}
@@ -282,35 +387,51 @@ async fn read_journal() -> Option<Vec<u8>> {
/// Write the snapshot to every local disk; returns true when at least one
/// disk accepted it, so a total write failure keeps the runtime dirty and
/// the next tick retries the persist.
async fn write_journal(data: &[u8]) -> bool {
async fn write_journal(path: &str, data: &[u8]) -> bool {
let payload = bytes::Bytes::copy_from_slice(data);
let mut any_persisted = false;
for disk in journal_disks().await {
match disk
.write_all(super::RUSTFS_META_BUCKET, MRF_JOURNAL_PATH, payload.clone())
.await
{
match disk.write_all(super::RUSTFS_META_BUCKET, path, payload.clone()).await {
Ok(()) => any_persisted = true,
Err(err) => warn_mrf_journal_write(&err),
}
}
if !data.is_empty() {
counter!("rustfs_heal_mrf_journal_fsync_total").increment(1);
}
gauge!("rustfs_heal_mrf_journal_bytes").set(data.len() as f64);
any_persisted
}
async fn delete_journal() {
for disk in journal_disks().await {
let _ = disk
async fn delete_journal(path: &str) -> bool {
let disks = journal_disks().await;
if disks.is_empty() {
counter!("rustfs_heal_mrf_journal_delete_failures_total").increment(1);
return false;
}
let mut all_deleted = true;
for disk in disks {
let result = disk
.delete(
super::RUSTFS_META_BUCKET,
MRF_JOURNAL_PATH,
path,
crate::heal::storage_api::owner::EcstoreDeleteOptions::default(),
)
.await;
if let Err(err) = result {
// Delete is idempotent: a compatibility mirror that was never
// written (or was already removed) is clean, not a retry state.
if !matches!(err, super::DiskError::FileNotFound | super::DiskError::VolumeNotFound) {
all_deleted = false;
}
}
}
if !all_deleted {
counter!("rustfs_heal_mrf_journal_delete_failures_total").increment(1);
}
all_deleted
}
async fn delete_journals() -> bool {
let authoritative_deleted = delete_journal(MRF_SCOPED_JOURNAL_PATH).await;
let legacy_deleted = delete_journal(MRF_JOURNAL_PATH).await;
authoritative_deleted && legacy_deleted
}
fn warn_mrf_journal_write(err: &super::DiskError) {
@@ -331,7 +452,10 @@ fn warn_mrf_journal_write(err: &super::DiskError) {
pub(crate) fn build_heal_request(intent: &MrfIntent) -> HealRequest {
let bucket = intent.bucket.to_string();
let object = intent.object.to_string();
let version_id = intent.version_id.map(|bytes| Uuid::from_bytes(bytes).to_string());
let version_id = intent
.version_id
.filter(|bytes| *bytes != [0; 16])
.map(|bytes| Uuid::from_bytes(bytes).to_string());
let (heal_type, priority) = match intent.kind {
rustfs_common::mrf_channel::MrfKind::DecodeFailure => (
HealType::ECDecode {
@@ -351,18 +475,28 @@ pub(crate) fn build_heal_request(intent: &MrfIntent) -> HealRequest {
HealPriority::Normal,
),
};
let mut request = HealRequest::new(heal_type, HealOptions::default(), priority);
let mut options = HealOptions::default();
if !matches!(intent.kind, rustfs_common::mrf_channel::MrfKind::MetadataCorruption)
&& let Some(scope) = intent.scope
{
options.pool_index = usize::try_from(scope.pool_index).ok();
options.set_index = usize::try_from(scope.set_index).ok();
}
let mut request = HealRequest::new(heal_type, options, priority);
request.source = rustfs_common::heal_channel::HealRequestSource::Mrf;
request
}
async fn submit_mrf_heal_request(manager: &HealManager, intent: &MrfIntent) -> crate::Result<HealAdmissionResult> {
let receipt = manager
.submit_mrf_heal_request_with_receipt(
.submit_mrf_heal_request_with_receipt_and_identity(
build_heal_request(intent),
intent.bucket.clone(),
intent.object.clone(),
intent.version_id,
intent.kind,
intent.scope,
intent.lease,
)
.await?;
Ok(receipt.result)
@@ -387,16 +521,38 @@ struct MrfRuntime {
}
impl MrfRuntime {
fn snapshot(&self) -> Vec<u8> {
let mut buf = Vec::new();
fn snapshot(&self) -> (Vec<u8>, Vec<u8>) {
let mut authoritative = Vec::new();
let mut legacy = Vec::new();
for intent in self.queue.intents() {
encode_intent(intent, &mut buf);
let scoped_identity =
!matches!(intent.kind, rustfs_common::mrf_channel::MrfKind::MetadataCorruption) && intent.scope.is_some();
if !encode_intent(intent, &mut authoritative) {
counter!("rustfs_heal_mrf_dropped_total", "reason" => "journal_identity_oversized").increment(1);
}
if !scoped_identity && !encode_intent(intent, &mut legacy) {
counter!("rustfs_heal_mrf_dropped_total", "reason" => "journal_identity_oversized").increment(1);
}
}
buf
(authoritative, legacy)
}
async fn flush(&mut self) {
let persisted = write_journal(&self.snapshot()).await;
let (authoritative, legacy) = self.snapshot();
let authoritative_persisted = write_journal(MRF_SCOPED_JOURNAL_PATH, &authoritative).await;
if !authoritative.is_empty() {
counter!("rustfs_heal_mrf_journal_fsync_total").increment(1);
}
gauge!("rustfs_heal_mrf_journal_bytes").set(metric_f64(authoritative.len()));
// Publish the compatibility mirror only after the authoritative
// snapshot has reached at least one disk. This ordering prevents an
// old reader from observing a newer epoch that a new reader cannot
// see when the canonical write is unavailable.
let legacy_persisted = authoritative_persisted && write_journal(MRF_JOURNAL_PATH, &legacy).await;
// Keep dirty until both the authoritative snapshot and its
// compatibility mirror have been accepted; otherwise a one-sided
// failure would never retry the missing file.
let persisted = authoritative_persisted && legacy_persisted;
self.new_since_flush = 0;
// Keep the dirty flag when every disk write failed: a clean backlog
// would otherwise never rewrite, losing the periodic persist retry a
@@ -404,7 +560,7 @@ impl MrfRuntime {
if persisted {
self.dirty = false;
}
self.journal_on_disk = true;
self.journal_on_disk |= authoritative_persisted || legacy_persisted;
}
/// Drain pending intents into the heal manager until it is full, the
@@ -430,6 +586,7 @@ impl MrfRuntime {
intent.attempts = intent.attempts.saturating_add(1);
if intent.attempts >= MRF_MAX_ATTEMPTS {
counter!("rustfs_heal_mrf_dropped_total", "reason" => "attempts_exhausted").increment(1);
rustfs_common::mrf_channel::release_mrf_intent(&intent);
continue;
}
self.queue.push_back(intent);
@@ -438,11 +595,13 @@ impl MrfRuntime {
}
Ok(HealAdmissionResult::Dropped(_)) => {
counter!("rustfs_heal_mrf_dropped_total", "reason" => "admission_policy").increment(1);
rustfs_common::mrf_channel::release_mrf_intent(&intent);
}
Err(_) => {
intent.attempts = intent.attempts.saturating_add(1);
if intent.attempts >= MRF_MAX_ATTEMPTS {
counter!("rustfs_heal_mrf_dropped_total", "reason" => "attempts_exhausted").increment(1);
rustfs_common::mrf_channel::release_mrf_intent(&intent);
continue;
}
self.queue.push_back(intent);
@@ -451,8 +610,8 @@ impl MrfRuntime {
}
}
}
gauge!("rustfs_heal_mrf_queue_depth").set(self.queue.depth() as f64);
gauge!("rustfs_heal_mrf_queue_bytes").set(self.queue.bytes() as f64);
gauge!("rustfs_heal_mrf_queue_depth").set(metric_f64(self.queue.depth()));
gauge!("rustfs_heal_mrf_queue_bytes").set(metric_f64(self.queue.bytes()));
}
}
@@ -496,7 +655,12 @@ pub async fn replay_journal_once(manager: &Arc<HealManager>) -> usize {
let config = MrfConsumerConfig::default();
let mut queue = MrfQueue::new(config.queue_capacity, config.journal_max_bytes);
let mut backoff_until: Option<tokio::time::Instant> = None;
replay_into(manager, &mut queue, &mut backoff_until).await
replay_into(manager, &mut queue, &mut backoff_until).await.replayed
}
struct ReplayOutcome {
replayed: usize,
journal_on_disk: bool,
}
/// Shared replay core: read + decode + re-arm + delete, then drain what fits.
@@ -504,11 +668,25 @@ async fn replay_into(
manager: &Arc<HealManager>,
queue: &mut MrfQueue,
backoff_until: &mut Option<tokio::time::Instant>,
) -> usize {
let Some(data) = read_journal().await else {
return 0;
) -> ReplayOutcome {
// The scoped file is a complete authoritative snapshot. Fall back to the
// legacy mirror only when the authoritative path is unavailable; merging
// both files could combine records from different flush epochs.
let data = match read_journal(MRF_SCOPED_JOURNAL_PATH).await {
Some(data) => data,
None => match read_journal(MRF_JOURNAL_PATH).await {
Some(data) => data,
None => {
return ReplayOutcome {
replayed: 0,
journal_on_disk: false,
};
}
},
};
let (intents, truncated) = decode_journal(&data);
let mut intents = Vec::new();
let (decoded, truncated) = decode_journal(&data);
intents.extend(decoded);
if truncated > 0 {
tracing::warn!(
target: "rustfs::heal::mrf",
@@ -516,12 +694,15 @@ async fn replay_into(
"MRF journal had a torn tail; truncated records were discarded"
);
}
counter!("rustfs_heal_mrf_replayed_total").increment(intents.len() as u64);
counter!("rustfs_heal_mrf_replayed_total").increment(u64::try_from(intents.len()).unwrap_or(u64::MAX));
let replayed = intents.len();
for intent in intents {
queue.try_push(intent);
let result = queue.try_push_typed(intent.clone());
if !matches!(result, MrfQueuePushResult::Enqueued) {
rustfs_common::mrf_channel::release_mrf_intent(&intent);
}
}
delete_journal().await;
let journal_on_disk = !delete_journals().await;
// Drain the replayed intents immediately; whatever the manager refuses
// stays armed in `queue` for the consumer's retry loop.
@@ -541,7 +722,10 @@ async fn replay_into(
}
}
}
replayed
ReplayOutcome {
replayed,
journal_on_disk,
}
}
/// Replay the journal, then keep draining the channel into the heal manager
@@ -559,7 +743,8 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
// Replay: read the journal, re-arm intents (duplicates are merged by the
// manager's dedup key), then drop the file so the next flush starts clean.
replay_into(&manager, &mut runtime.queue, &mut runtime.backoff_until).await;
let replay = replay_into(&manager, &mut runtime.queue, &mut runtime.backoff_until).await;
runtime.journal_on_disk = replay.journal_on_disk;
// The replay deleted the journal file; anything still pending (e.g. the
// manager was full and backoff armed) must be re-persisted by the next
// flush or a crash before it would lose those intents.
@@ -587,9 +772,14 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
return;
}
for intent in batch.drain(..) {
if runtime.queue.try_push(intent) {
runtime.new_since_flush += 1;
runtime.dirty = true;
match runtime.queue.try_push_typed(intent.clone()) {
MrfQueuePushResult::Enqueued => {
runtime.new_since_flush += 1;
runtime.dirty = true;
}
MrfQueuePushResult::Coalesced | MrfQueuePushResult::Rejected => {
rustfs_common::mrf_channel::release_mrf_intent(&intent);
}
}
}
runtime.dispatch(manager.as_ref()).await;
@@ -613,13 +803,14 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
TickAction::DeleteJournal => {
// All intents consumed: remove the journal so a restart
// replays nothing (mirrors MinIO's post-replay unlink).
delete_journal().await;
runtime.journal_on_disk = false;
gauge!("rustfs_heal_mrf_journal_bytes").set(0.0);
if delete_journals().await {
runtime.journal_on_disk = false;
gauge!("rustfs_heal_mrf_journal_bytes").set(0.0);
}
}
TickAction::Idle => {}
}
gauge!("rustfs_heal_mrf_queue_depth").set(runtime.queue.depth() as f64);
gauge!("rustfs_heal_mrf_queue_depth").set(metric_f64(runtime.queue.depth()));
}
}
}
@@ -664,6 +855,8 @@ mod tests {
object: StdArc::from(object),
version_id: Some([7u8; 16]),
kind: MrfKind::DecodeFailure,
scope: None,
lease: None,
enqueued_at_ms: 1_700_000_000_000,
attempts,
}
@@ -694,17 +887,101 @@ mod tests {
fn queue_enforces_count_and_byte_ceilings() {
let mut queue = MrfQueue::new(2, usize::MAX);
assert!(queue.try_push(intent("b", "o", 0)));
assert!(queue.try_push(intent("b", "o", 0)));
assert!(!queue.try_push(intent("b", "o", 0)), "count ceiling must drop");
assert!(queue.try_push(intent("b", "o2", 0)));
assert!(!queue.try_push(intent("b", "o3", 0)), "count ceiling must drop");
let mut tiny = MrfQueue::new(usize::MAX, intent("bucket", "object", 0).estimated_bytes());
assert!(tiny.try_push(intent("bucket", "object", 0)));
assert!(
!tiny.try_push(intent("bucket", "object", 0)),
!tiny.try_push(intent("bucket", "object2", 0)),
"byte budget must drop before the second intent fits"
);
}
#[test]
fn duplicate_mrf_intents_coalesce_to_one_execution() {
let mut queue = MrfQueue::new(1000, usize::MAX);
let mut enqueued = 0;
let mut coalesced = 0;
assert_eq!(queue.try_push_typed(intent("bucket", "object", 0)), MrfQueuePushResult::Enqueued);
enqueued += 1;
for _ in 0..999 {
match queue.try_push_typed(intent("bucket", "object", 0)) {
MrfQueuePushResult::Coalesced => coalesced += 1,
other => panic!("duplicate intent was not coalesced: {other:?}"),
}
}
assert_eq!(enqueued, 1);
assert_eq!(coalesced, 999);
assert_eq!(queue.depth(), 1);
}
#[test]
fn mrf_dedupe_does_not_merge_adjacent_version_pool_or_kind() {
let mut queue = MrfQueue::new(8, usize::MAX);
let mut first = intent("bucket", "object", 0);
first.kind = MrfKind::PartialWrite;
first.scope = Some(rustfs_common::mrf_channel::MrfScope {
pool_index: 1,
set_index: 1,
});
assert!(queue.try_push(first.clone()));
first.version_id = Some([8u8; 16]);
assert!(queue.try_push(first));
let mut other_scope = intent("bucket", "object", 0);
other_scope.kind = MrfKind::PartialWrite;
other_scope.scope = Some(rustfs_common::mrf_channel::MrfScope {
pool_index: 2,
set_index: 1,
});
assert!(queue.try_push(other_scope));
let mut other_kind = intent("bucket", "object", 0);
other_kind.kind = MrfKind::DecodeFailure;
other_kind.scope = None;
assert!(queue.try_push(other_kind));
assert_eq!(queue.depth(), 4);
}
#[test]
fn mrf_dedupe_full_returns_rejected_with_durable_pending() {
let mut queue = MrfQueue::new(1, usize::MAX);
assert_eq!(queue.try_push_typed(intent("bucket", "object", 0)), MrfQueuePushResult::Enqueued);
assert_eq!(queue.try_push_typed(intent("bucket", "other", 0)), MrfQueuePushResult::Rejected);
assert_eq!(queue.depth(), 1);
let mut snapshot = Vec::new();
assert!(encode_intent(queue.intents().next().expect("resident intent"), &mut snapshot));
assert!(!snapshot.is_empty(), "the resident intent remains journalable after rejection");
}
#[test]
fn mrf_dedupe_failure_releases_key_for_retry() {
let mut queue = MrfQueue::new(1, usize::MAX);
assert_eq!(queue.try_push_typed(intent("bucket", "object", 0)), MrfQueuePushResult::Enqueued);
let _failed = queue.pop_front().expect("queued intent");
assert_eq!(queue.try_push_typed(intent("bucket", "object", 1)), MrfQueuePushResult::Enqueued);
assert_eq!(queue.depth(), 1);
}
#[test]
fn mrf_dedupe_key_and_map_are_bounded() {
let mut queue = MrfQueue::new(2, usize::MAX);
assert_eq!(queue.try_push_typed(intent("bucket", "object", 0)), MrfQueuePushResult::Enqueued);
assert_eq!(queue.try_push_typed(intent("bucket", "other", 0)), MrfQueuePushResult::Enqueued);
assert_eq!(queue.pending_keys.len(), 2);
assert_eq!(queue.try_push_typed(intent("bucket", "third", 0)), MrfQueuePushResult::Rejected);
assert_eq!(queue.depth(), 2);
}
#[test]
fn cross_node_duplicate_execution_remains_idempotent() {
// Node-local ingress maps intentionally do not merge across nodes;
// the manager's existing identity key absorbs the duplicate later.
let mut node_a = MrfQueue::new(8, usize::MAX);
let mut node_b = MrfQueue::new(8, usize::MAX);
assert_eq!(node_a.try_push_typed(intent("bucket", "object", 0)), MrfQueuePushResult::Enqueued);
assert_eq!(node_b.try_push_typed(intent("bucket", "object", 0)), MrfQueuePushResult::Enqueued);
}
#[test]
fn journal_roundtrip_preserves_intents() {
let intents = vec![
@@ -715,6 +992,8 @@ mod tests {
object: StdArc::from("object/c"),
version_id: None,
kind: MrfKind::MetadataCorruption,
scope: None,
lease: None,
enqueued_at_ms: 5,
attempts: 1,
},
@@ -766,6 +1045,8 @@ mod tests {
object: StdArc::from("o"),
version_id: None,
kind: MrfKind::MetadataCorruption,
scope: None,
lease: None,
enqueued_at_ms: 0,
attempts: 0,
});
@@ -777,6 +1058,8 @@ mod tests {
object: StdArc::from("o"),
version_id: None,
kind: MrfKind::PartialWrite,
scope: None,
lease: None,
enqueued_at_ms: 0,
attempts: 0,
});
crates/heal/src/heal/progress.rs
+475 -34
View File
@@ -15,15 +15,91 @@
use serde::{Deserialize, Serialize};
use std::time::{Duration, SystemTime};
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
pub(crate) fn stable_generation(parts: &[&[u8]]) -> u64 {
let mut hash = 0xcbf29ce484222325u64;
for part in parts {
for byte in (part.len() as u64).to_be_bytes().into_iter().chain(part.iter().copied()) {
hash ^= u64::from(byte);
hash = hash.wrapping_mul(0x100000001b3);
}
}
hash
}
#[cfg(test)]
mod stable_generation_tests {
use super::stable_generation;
#[test]
fn stable_generation_has_a_fixed_vector() {
assert_eq!(stable_generation(&[b"rustfs", b"heal", b"42"]), 11_007_672_338_488_385_056);
}
}
pub(crate) fn increment_counter(counter: &mut u64) -> bool {
match counter.checked_add(1) {
Some(next) => {
*counter = next;
true
}
None => {
*counter = u64::MAX;
false
}
}
}
pub(crate) fn add_bytes(total: &mut u64, amount: u64) -> bool {
match total.checked_add(amount) {
Some(next) => {
*total = next;
true
}
None => {
*total = u64::MAX;
false
}
}
}
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum HealProgressKind {
#[default]
Unknown,
Stage,
ObjectSweep,
}
/// Whether the object ledger can produce a meaningful percentage.
///
/// A zero-valued baseline is not a completed scan: it means that no complete
/// usage snapshot was available. Keep this state explicit so callers do not
/// mistake the legacy `0.0` wire value for a measured zero-percent result.
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum HealProgressState {
#[default]
Unknown,
Indeterminate,
Running,
Completed,
}
#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)]
#[serde(default, rename_all = "camelCase")]
pub struct HealProgress {
#[serde(default)]
pub kind: HealProgressKind,
/// Objects scanned
pub objects_scanned: u64,
/// Objects healed
pub objects_healed: u64,
/// Objects failed
pub objects_failed: u64,
/// Versions deferred for a later retry pass.
#[serde(default)]
pub skipped_objects: u64,
/// Versions skipped because they were written after this heal started
pub skipped_new_versions: u64,
/// Versions skipped because lifecycle already selected them for expiry
@@ -44,11 +120,38 @@ pub struct HealProgress {
pub last_update_time: Option<SystemTime>,
/// Estimated completion time
pub estimated_completion_time: Option<SystemTime>,
/// Current stage number. Stage updates are intentionally independent from
/// the object ledger below.
#[serde(default)]
pub stage_current: u64,
/// Number of stages in the current task.
#[serde(default)]
pub stage_total: u64,
/// Explicitly distinguishes a missing usage baseline from measured 0%.
#[serde(default)]
pub progress_state: HealProgressState,
/// True only after the task's durable completion ledger was committed.
#[serde(default)]
pub ledger_complete: bool,
/// Generation of the usage snapshot used for the baseline, if available.
#[serde(default)]
pub baseline_generation: Option<u64>,
/// Whether the baseline was explicitly observed. This is separate from
/// the counters so a known empty scope (0 objects, 0 bytes) is not
/// confused with a legacy snapshot that omitted the baseline fields.
#[serde(default)]
pub baseline_known: bool,
/// Internal telemetry fence set when an aggregate counter overflows or
/// becomes inconsistent. It prevents a later refresh from fabricating a
/// percentage from the poisoned values.
#[serde(default)]
pub counter_unknown: bool,
}
impl HealProgress {
pub fn new() -> Self {
Self {
kind: HealProgressKind::Unknown,
start_time: Some(SystemTime::now()),
last_update_time: Some(SystemTime::now()),
..Default::default()
@@ -56,12 +159,87 @@ impl HealProgress {
}
pub fn update_progress(&mut self, scanned: u64, healed: u64, failed: u64, bytes: u64) {
self.update_object_sweep_progress(scanned, healed, failed, bytes);
}
pub fn update_object_sweep_progress(&mut self, scanned: u64, healed: u64, failed: u64, bytes: u64) {
self.kind = HealProgressKind::ObjectSweep;
self.objects_scanned = scanned;
self.objects_healed = healed;
self.objects_failed = failed;
self.bytes_processed = bytes;
self.last_update_time = Some(SystemTime::now());
let explicit_skipped = match self.skipped_new_versions.checked_add(self.skipped_ilm_expired) {
Some(value) => value,
None => {
self.mark_unknown();
0
}
};
let skipped = healed
.checked_add(failed)
.and_then(|value| value.checked_add(explicit_skipped))
.and_then(|value| scanned.checked_sub(value))
.unwrap_or(0);
self.update_object_progress(scanned, healed, failed, skipped, bytes);
}
/// Update task stage progress without modifying object counters.
pub fn update_stage(&mut self, current: u64, total: u64) {
let object_sweep_active = matches!(self.kind, HealProgressKind::ObjectSweep);
if !object_sweep_active {
self.kind = HealProgressKind::Stage;
}
self.ledger_complete = false;
self.stage_current = current.min(total);
self.stage_total = total;
if object_sweep_active {
self.last_update_time = Some(SystemTime::now());
self.refresh_progress_percentage();
return;
}
self.progress_state = if total == 0 {
HealProgressState::Indeterminate
} else {
HealProgressState::Running
};
self.progress_percentage = if total == 0 {
0.0
} else {
(current as f64 / total as f64 * 100.0).min(100.0)
};
self.last_update_time = Some(SystemTime::now());
}
/// Update the disjoint object ledger. `scanned` is the number of terminal
/// object outcomes and must equal healed + failed + deferred skipped plus
/// the two terminal skip classes. Overflow is a corrupt/unknown counter
/// state, not a reason to abort a completed heal.
pub fn update_object_progress(&mut self, scanned: u64, healed: u64, failed: u64, skipped: u64, bytes: u64) {
self.kind = HealProgressKind::ObjectSweep;
// `skipped` is the transient/deferred class. The two explicit skip
// counters are terminal classifications too, so include them in the
// same ledger without making callers maintain a second aggregate.
let outcomes = healed
.checked_add(failed)
.and_then(|value| value.checked_add(skipped))
.and_then(|value| value.checked_add(self.skipped_new_versions))
.and_then(|value| value.checked_add(self.skipped_ilm_expired));
self.objects_scanned = scanned;
self.objects_healed = healed;
self.objects_failed = failed;
self.skipped_objects = skipped;
self.bytes_processed = bytes;
self.last_update_time = Some(SystemTime::now());
self.ledger_complete = false;
if outcomes != Some(scanned) {
// Telemetry corruption must not abort a heal. Preserve the
// counters for diagnostics, but do not derive a percentage from a
// double-counted or overflowing ledger.
self.mark_unknown();
return;
}
self.refresh_progress_percentage();
self.refresh_estimated_completion_time();
}
@@ -69,50 +247,88 @@ impl HealProgress {
pub fn set_total_baseline(&mut self, objects_total_count: u64, objects_total_size: u64) {
self.objects_total_count = objects_total_count;
self.objects_total_size = objects_total_size;
self.baseline_known = true;
self.last_update_time = Some(SystemTime::now());
self.refresh_progress_percentage();
self.refresh_estimated_completion_time();
}
pub fn set_total_baseline_with_generation(&mut self, objects_total_count: u64, objects_total_size: u64, generation: u64) {
self.baseline_generation = Some(generation);
self.set_total_baseline(objects_total_count, objects_total_size);
}
pub fn record_skipped_new_version(&mut self) {
self.skipped_new_versions = self.skipped_new_versions.saturating_add(1);
let Some(next) = self.skipped_new_versions.checked_add(1) else {
self.mark_unknown();
return;
};
self.skipped_new_versions = next;
self.last_update_time = Some(SystemTime::now());
self.refresh_progress_percentage();
self.refresh_estimated_completion_time();
}
pub fn record_skipped_ilm_expired(&mut self) {
self.skipped_ilm_expired = self.skipped_ilm_expired.saturating_add(1);
let Some(next) = self.skipped_ilm_expired.checked_add(1) else {
self.mark_unknown();
return;
};
self.skipped_ilm_expired = next;
self.last_update_time = Some(SystemTime::now());
self.refresh_progress_percentage();
self.refresh_estimated_completion_time();
}
fn completed_for_baseline(&self) -> u64 {
fn completed_for_baseline(&self) -> Option<u64> {
self.objects_healed
.saturating_add(self.objects_failed)
.saturating_add(self.skipped_new_versions)
.saturating_add(self.skipped_ilm_expired)
.checked_add(self.objects_failed)?
.checked_add(self.skipped_objects)?
.checked_add(self.skipped_new_versions)?
.checked_add(self.skipped_ilm_expired)
}
pub(crate) fn refresh_progress_percentage(&mut self) {
if self.ledger_complete {
self.progress_state = HealProgressState::Completed;
self.progress_percentage = 100.0;
return;
}
if self.counter_unknown {
self.progress_state = HealProgressState::Unknown;
self.progress_percentage = 0.0;
return;
}
if !self.baseline_known {
self.progress_state = HealProgressState::Indeterminate;
self.progress_percentage = 0.0;
self.estimated_completion_time = None;
return;
}
if self.objects_total_size > 0 {
self.progress_percentage = ((self.bytes_processed as f64 / self.objects_total_size as f64) * 100.0).min(100.0);
self.progress_percentage = self.progress_percentage.min(99.999);
self.progress_state = HealProgressState::Running;
return;
}
if self.objects_total_count > 0 {
let completed = self.completed_for_baseline();
let Some(completed) = self.completed_for_baseline() else {
self.progress_state = HealProgressState::Unknown;
self.progress_percentage = 0.0;
return;
};
self.progress_percentage = ((completed as f64 / self.objects_total_count as f64) * 100.0).min(100.0);
self.progress_percentage = self.progress_percentage.min(99.999);
self.progress_state = HealProgressState::Running;
return;
}
let total = self
.objects_scanned
.saturating_add(self.objects_healed)
.saturating_add(self.objects_failed);
if total > 0 {
self.progress_percentage = (self.objects_healed as f64 / total as f64) * 100.0;
if self.baseline_known {
self.progress_state = HealProgressState::Running;
self.progress_percentage = 0.0;
return;
}
self.progress_state = HealProgressState::Indeterminate;
self.progress_percentage = 0.0;
}
pub fn set_current_object(&mut self, object: Option<String>) {
@@ -125,7 +341,11 @@ impl HealProgress {
self.estimated_completion_time = None;
return;
};
if self.is_completed() || !(0.0..100.0).contains(&self.progress_percentage) || self.bytes_processed == 0 {
if self.is_completed()
|| self.progress_percentage <= 0.0
|| self.progress_percentage >= 100.0
|| self.bytes_processed == 0
{
self.estimated_completion_time = None;
return;
}
@@ -142,18 +362,39 @@ impl HealProgress {
}
pub fn is_completed(&self) -> bool {
if self.progress_percentage >= 100.0 {
return true;
}
if self.objects_total_count > 0 || self.objects_total_size > 0 {
return false;
}
self.ledger_complete
}
self.objects_scanned > 0 && self.objects_healed.saturating_add(self.objects_failed) >= self.objects_scanned
/// Mark telemetry unknown while allowing the underlying heal operation to
/// continue. This is used for corrupt/overflowing counters at the
/// observability boundary; it must never turn a successful heal into an
/// execution error.
pub fn mark_unknown(&mut self) {
self.counter_unknown = true;
self.progress_state = HealProgressState::Unknown;
self.ledger_complete = false;
self.progress_percentage = 0.0;
self.estimated_completion_time = None;
self.last_update_time = Some(SystemTime::now());
}
/// Mark the object ledger terminal only after the enclosing task has
/// committed all durable resume state and cleanup fences.
pub fn mark_completed(&mut self) {
let telemetry_unknown = self.counter_unknown || self.progress_state == HealProgressState::Unknown;
self.ledger_complete = true;
if !telemetry_unknown {
self.progress_state = HealProgressState::Completed;
}
self.progress_percentage = 100.0;
self.last_update_time = Some(SystemTime::now());
self.estimated_completion_time = None;
}
pub fn get_success_rate(&self) -> f64 {
let total = self.objects_healed + self.objects_failed;
let Some(total) = self.objects_healed.checked_add(self.objects_failed) else {
return 0.0;
};
if total > 0 {
(self.objects_healed as f64 / total as f64) * 100.0
} else {
@@ -162,6 +403,101 @@ impl HealProgress {
}
}
pub fn aggregate_heal_progress(progresses: impl IntoIterator<Item = HealProgress>) -> Option<HealProgress> {
let mut snapshot = HealProgress::default();
let mut found = false;
let mut has_object_sweep = false;
let mut all_object_baselines_known = true;
let mut baseline_generation = None;
let mut baseline_generation_consistent = true;
let mut all_ledgers_complete = true;
let mut counter_overflow = false;
for progress in progresses {
found = true;
let object_sweep = matches!(progress.kind, HealProgressKind::ObjectSweep);
has_object_sweep |= object_sweep;
all_ledgers_complete &= progress.ledger_complete;
if object_sweep {
all_object_baselines_known &= progress.baseline_known;
match baseline_generation {
None => baseline_generation = Some(progress.baseline_generation),
Some(generation) => baseline_generation_consistent &= generation == progress.baseline_generation,
}
}
counter_overflow |= progress.counter_unknown || matches!(progress.progress_state, HealProgressState::Unknown);
for (target, value) in [
(&mut snapshot.objects_scanned, progress.objects_scanned),
(&mut snapshot.objects_healed, progress.objects_healed),
(&mut snapshot.objects_failed, progress.objects_failed),
(&mut snapshot.skipped_objects, progress.skipped_objects),
(&mut snapshot.skipped_new_versions, progress.skipped_new_versions),
(&mut snapshot.skipped_ilm_expired, progress.skipped_ilm_expired),
(&mut snapshot.objects_total_count, progress.objects_total_count),
(&mut snapshot.objects_total_size, progress.objects_total_size),
(&mut snapshot.bytes_processed, progress.bytes_processed),
(&mut snapshot.stage_current, progress.stage_current),
(&mut snapshot.stage_total, progress.stage_total),
] {
match target.checked_add(value) {
Some(sum) => *target = sum,
None => {
*target = u64::MAX;
counter_overflow = true;
}
}
}
snapshot.start_time = match (snapshot.start_time, progress.start_time) {
(Some(current), Some(next)) => Some(current.min(next)),
(None, next) => next,
(current, None) => current,
};
snapshot.last_update_time = match (snapshot.last_update_time, progress.last_update_time) {
(Some(current), Some(next)) => Some(current.max(next)),
(None, next) => next,
(current, None) => current,
};
if progress.current_object.is_some() {
snapshot.current_object = progress.current_object;
}
}
if !found {
return None;
}
snapshot.kind = if has_object_sweep {
HealProgressKind::ObjectSweep
} else {
HealProgressKind::Stage
};
snapshot.baseline_known = has_object_sweep && all_object_baselines_known && baseline_generation_consistent;
snapshot.baseline_generation = if snapshot.baseline_known && baseline_generation_consistent {
baseline_generation.flatten()
} else {
None
};
snapshot.ledger_complete = all_ledgers_complete;
snapshot.counter_unknown = counter_overflow;
if counter_overflow {
snapshot.progress_state = HealProgressState::Unknown;
snapshot.progress_percentage = if snapshot.ledger_complete { 100.0 } else { 0.0 };
} else if snapshot.ledger_complete {
snapshot.progress_state = HealProgressState::Completed;
snapshot.progress_percentage = 100.0;
} else if has_object_sweep {
snapshot.refresh_progress_percentage();
} else if snapshot.stage_total == 0 {
snapshot.progress_state = HealProgressState::Indeterminate;
snapshot.progress_percentage = 0.0;
} else {
snapshot.progress_state = HealProgressState::Running;
snapshot.progress_percentage = ((snapshot.stage_current as f64 / snapshot.stage_total as f64) * 100.0).min(99.999);
}
snapshot.refresh_estimated_completion_time();
Some(snapshot)
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct HealStatistics {
/// Total heal tasks
@@ -230,6 +566,7 @@ mod tests {
assert_eq!(progress.objects_scanned, 0);
assert_eq!(progress.objects_healed, 0);
assert_eq!(progress.objects_failed, 0);
assert_eq!(progress.skipped_objects, 0);
assert_eq!(progress.skipped_new_versions, 0);
assert_eq!(progress.skipped_ilm_expired, 0);
assert_eq!(progress.objects_total_count, 0);
@@ -250,10 +587,8 @@ mod tests {
assert_eq!(progress.objects_healed, 8);
assert_eq!(progress.objects_failed, 2);
assert_eq!(progress.bytes_processed, 1024);
// Progress percentage should be calculated based on healed/total
// total = scanned + healed + failed = 10 + 8 + 2 = 20
// healed/total = 8/20 = 0.4 = 40%
assert!((progress.progress_percentage - 40.0).abs() < 0.001);
assert_eq!(progress.progress_state, HealProgressState::Indeterminate);
assert_eq!(progress.progress_percentage, 0.0);
assert!(progress.last_update_time.is_some());
}
@@ -262,7 +597,8 @@ mod tests {
let mut progress = HealProgress::new();
progress.start_time = Some(SystemTime::now() - Duration::from_secs(10));
progress.update_progress(100, 25, 0, 4096);
progress.set_total_baseline(100, 16384);
progress.update_progress(25, 25, 0, 4096);
let eta = progress
.estimated_completion_time
@@ -275,7 +611,7 @@ mod tests {
let mut progress = HealProgress::new();
progress.set_total_baseline(10, 8192);
progress.update_progress(100, 25, 0, 4096);
progress.update_progress(25, 25, 0, 4096);
assert!((progress.progress_percentage - 50.0).abs() < 0.001);
}
@@ -285,7 +621,7 @@ mod tests {
let mut progress = HealProgress::new();
progress.set_total_baseline(10, 0);
progress.update_progress(100, 3, 2, 0);
progress.update_progress(5, 3, 2, 0);
assert!((progress.progress_percentage - 50.0).abs() < 0.001);
}
@@ -295,7 +631,7 @@ mod tests {
let mut progress = HealProgress::new();
progress.set_total_baseline(10, 0);
progress.update_progress(100, 3, 2, 0);
progress.update_progress(5, 3, 2, 0);
progress.record_skipped_new_version();
assert_eq!(progress.skipped_new_versions, 1);
@@ -336,7 +672,8 @@ mod tests {
fn test_heal_progress_update_progress_all_healed() {
let mut progress = HealProgress::new();
// When scanned=0, healed=10, failed=0: total=10, progress = 10/10 = 100%
progress.update_progress(0, 10, 0, 2048);
progress.update_progress(10, 10, 0, 2048);
progress.mark_completed();
// All healed, should be 100%
assert!((progress.progress_percentage - 100.0).abs() < 0.001);
@@ -394,6 +731,7 @@ mod tests {
assert_eq!(json["objectsScanned"], 10);
assert_eq!(json["objectsHealed"], 8);
assert_eq!(json["objectsFailed"], 2);
assert_eq!(json["skippedObjects"], 0);
assert_eq!(json["skippedNewVersions"], 0);
assert_eq!(json["skippedIlmExpired"], 0);
assert_eq!(json["bytesProcessed"], 1024);
@@ -405,6 +743,7 @@ mod tests {
fn test_heal_progress_is_completed_by_percentage() {
let mut progress = HealProgress::new();
progress.update_progress(10, 10, 0, 1024);
progress.mark_completed();
assert!(progress.is_completed());
}
@@ -415,7 +754,7 @@ mod tests {
progress.objects_scanned = 10;
progress.objects_healed = 8;
progress.objects_failed = 2;
// healed + failed = 8 + 2 = 10 >= scanned = 10
progress.mark_completed();
assert!(progress.is_completed());
}
@@ -455,6 +794,108 @@ mod tests {
assert!((progress.get_success_rate() - 100.0).abs() < 0.001);
}
#[test]
fn single_object_progress_reaches_terminal_100() {
let mut progress = HealProgress::new();
progress.update_object_progress(1, 1, 0, 0, 128);
assert!(!progress.is_completed());
progress.mark_completed();
assert!(progress.is_completed());
assert_eq!(progress.progress_percentage, 100.0);
}
#[test]
fn progress_without_baseline_is_indeterminate() {
let mut progress = HealProgress::new();
progress.update_object_progress(1, 1, 0, 0, 128);
assert_eq!(progress.progress_state, HealProgressState::Indeterminate);
assert_eq!(progress.progress_percentage, 0.0);
assert!(progress.estimated_completion_time.is_none());
}
#[test]
fn progress_retry_is_exactly_once() {
let mut progress = HealProgress::new();
progress.set_total_baseline(1, 128);
progress.update_object_progress(1, 1, 0, 0, 128);
progress.update_object_progress(1, 1, 0, 0, 128);
assert_eq!(progress.objects_scanned, 1);
assert_eq!(progress.objects_healed, 1);
assert_eq!(progress.bytes_processed, 128);
}
#[test]
fn progress_never_triggers_cleanup_before_terminal_ledger_empty() {
let mut progress = HealProgress::new();
progress.progress_percentage = 100.0;
assert!(!progress.is_completed());
progress.mark_completed();
assert!(progress.is_completed());
}
#[test]
fn progress_counter_overflow_is_marked_unknown_without_aborting_completed_heal() {
let mut progress = HealProgress::new();
progress.update_object_progress(u64::MAX, u64::MAX, 1, 0, 0);
assert_eq!(progress.progress_state, HealProgressState::Unknown);
progress.mark_completed();
assert!(progress.is_completed());
assert_eq!(progress.progress_state, HealProgressState::Unknown);
let aggregate = aggregate_heal_progress([progress]).expect("progress should aggregate");
assert!(aggregate.ledger_complete);
assert!(aggregate.counter_unknown);
assert_eq!(aggregate.progress_state, HealProgressState::Unknown);
assert_eq!(aggregate.progress_percentage, 100.0);
}
#[test]
fn aggregate_rejects_mixed_baseline_generations() {
let progress = |generation| HealProgress {
kind: HealProgressKind::ObjectSweep,
objects_scanned: 5,
objects_total_count: 10,
progress_state: HealProgressState::Running,
baseline_generation: Some(generation),
baseline_known: true,
..Default::default()
};
let aggregate = aggregate_heal_progress([progress(1), progress(2)]).expect("progress should aggregate");
assert!(!aggregate.baseline_known);
assert_eq!(aggregate.baseline_generation, None);
assert_eq!(aggregate.progress_state, HealProgressState::Indeterminate);
assert_eq!(aggregate.progress_percentage, 0.0);
}
#[test]
fn aggregate_accepts_multiple_sets_from_one_snapshot_generation() {
let progress = |objects_scanned| HealProgress {
kind: HealProgressKind::ObjectSweep,
objects_scanned,
objects_total_count: 10,
progress_state: HealProgressState::Running,
baseline_generation: Some(7),
baseline_known: true,
..Default::default()
};
let aggregate = aggregate_heal_progress([progress(5), progress(3)]).expect("progress should aggregate");
assert!(aggregate.baseline_known);
assert_eq!(aggregate.baseline_generation, Some(7));
}
#[test]
fn stage_updates_do_not_double_count_object_outcomes() {
let mut progress = HealProgress::new();
progress.update_object_progress(2, 1, 0, 1, 256);
progress.update_stage(3, 4);
assert_eq!(progress.kind, HealProgressKind::ObjectSweep);
assert_eq!(progress.objects_scanned, 2);
assert_eq!(progress.objects_healed, 1);
assert_eq!(progress.skipped_objects, 1);
}
#[test]
fn test_heal_statistics_new() {
let stats = HealStatistics::new();
crates/heal/src/heal/resume.rs
+133 -4
View File
@@ -28,10 +28,12 @@ use super::{
};
mod checkpoint;
mod gc;
mod replacement;
mod utils;
pub use checkpoint::{CheckpointManager, ResumeCheckpoint};
pub use checkpoint::{CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, ResumeCheckpoint};
pub(crate) use gc::ResumeGc;
pub(crate) use replacement::replacement_target_identities_match;
use replacement::replacement_targets_match_identities;
pub use replacement::{
@@ -51,6 +53,7 @@ const RESUME_STATE_FILE: &str = "ahm_resume_state.json";
const REPLACEMENT_INTENT_FILE: &str = "ahm_replacement_intent.json";
const RESUME_PROGRESS_FILE: &str = "ahm_progress.json";
pub(super) const RESUME_CHECKPOINT_FILE: &str = "ahm_checkpoint.json";
pub(super) const RESUME_CHECKPOINT_BLOCKED_FILE: &str = "ahm_checkpoint.blocked";
const REPLACEMENT_COMPLETION_PROOF_FILE: &str = "ahm_replacement_completion_proof.json";
const REPLACEMENT_RECOVERY_DIR: &str = "ahm-replacement";
const REPLACEMENT_INTENT_SEAL_FILE: &str = "ahm_replacement_intent_seal";
@@ -340,6 +343,12 @@ pub struct ResumeState {
pub failed_objects: u64,
/// skipped objects
pub skipped_objects: u64,
/// Terminal versions skipped because they were newer than the heal start.
#[serde(default)]
pub skipped_new_versions: u64,
/// Terminal versions handed to lifecycle expiry.
#[serde(default)]
pub skipped_ilm_expired: u64,
/// current bucket
pub current_bucket: Option<String>,
/// current object
@@ -354,6 +363,24 @@ pub struct ResumeState {
pub retry_count: u32,
/// max retries
pub max_retries: u32,
/// Bytes accounted by the object ledger; additive for old snapshots.
#[serde(default)]
pub processed_bytes: u64,
/// Total bytes from a complete usage snapshot, when available.
#[serde(default)]
pub total_bytes: u64,
/// Generation of the usage snapshot used for the baseline.
#[serde(default)]
pub baseline_generation: Option<u64>,
/// Whether the usage baseline is known. Missing in old snapshots means
/// indeterminate rather than a measured zero baseline.
#[serde(default)]
pub baseline_known: bool,
/// Persistent telemetry fence for counter/byte overflow or corruption.
/// It must survive a restart so a saturated snapshot is never presented as
/// a measured percentage on the next resume.
#[serde(default)]
pub counter_unknown: bool,
}
impl ResumeState {
@@ -377,6 +404,8 @@ impl ResumeState {
successful_objects: 0,
failed_objects: 0,
skipped_objects: 0,
skipped_new_versions: 0,
skipped_ilm_expired: 0,
current_bucket: None,
current_object: None,
completed_buckets: Vec::new(),
@@ -384,6 +413,11 @@ impl ResumeState {
error_message: None,
retry_count: 0,
max_retries: 3,
processed_bytes: 0,
total_bytes: 0,
baseline_generation: None,
baseline_known: false,
counter_unknown: false,
}
}
@@ -412,6 +446,39 @@ impl ResumeState {
self.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
}
pub fn update_progress_with_bytes(
&mut self,
processed: u64,
successful: u64,
failed: u64,
skipped: u64,
processed_bytes: u64,
) {
self.update_progress(processed, successful, failed, skipped);
self.processed_bytes = processed_bytes;
}
pub fn set_skipped_version_counts(&mut self, new_versions: u64, ilm_expired: u64) {
self.skipped_new_versions = new_versions;
self.skipped_ilm_expired = ilm_expired;
self.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
}
pub fn set_progress_baseline(&mut self, total_objects: u64, total_bytes: u64, generation: Option<u64>) {
self.total_objects = total_objects;
self.total_bytes = total_bytes;
self.baseline_generation = generation;
// This method is called only after a complete usage snapshot has been
// validated. A complete but empty snapshot is still a known baseline.
self.baseline_known = true;
self.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
}
pub fn mark_counter_unknown(&mut self) {
self.counter_unknown = true;
self.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
}
pub fn set_current_item(&mut self, bucket: Option<String>, object: Option<String>) {
self.current_bucket = bucket;
self.current_object = object;
@@ -437,6 +504,7 @@ impl ResumeState {
if let Some(pos) = self.pending_buckets.iter().position(|b| b == bucket) {
self.pending_buckets.remove(pos);
}
self.resume_cursor = None;
self.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
}
@@ -454,6 +522,10 @@ impl ResumeState {
self.successful_objects = 0;
self.failed_objects = 0;
self.skipped_objects = 0;
self.skipped_new_versions = 0;
self.skipped_ilm_expired = 0;
self.processed_bytes = 0;
self.counter_unknown = false;
self.completed = false;
// A retry re-scans every bucket from the beginning, so the version
// cursor must be cleared too — otherwise the retry would resume mid-scan.
@@ -476,14 +548,28 @@ impl ResumeState {
}
pub fn get_progress_percentage(&self) -> f64 {
if self.completed {
return 100.0;
}
if self.counter_unknown {
return 0.0;
}
if !self.baseline_known {
return 0.0;
}
if self.total_bytes > 0 {
return ((self.processed_bytes as f64 / self.total_bytes as f64) * 100.0).min(99.999);
}
if self.total_objects == 0 {
return 0.0;
}
(self.processed_objects as f64 / self.total_objects as f64) * 100.0
((self.processed_objects as f64 / self.total_objects as f64) * 100.0).min(99.999)
}
pub fn get_success_rate(&self) -> f64 {
let total = self.successful_objects + self.failed_objects;
let Some(total) = self.successful_objects.checked_add(self.failed_objects) else {
return 0.0;
};
if total == 0 {
return 0.0;
}
@@ -754,6 +840,14 @@ impl ResumeManager {
state.successful_objects = 0;
state.failed_objects = 0;
state.skipped_objects = 0;
state.skipped_new_versions = 0;
state.skipped_ilm_expired = 0;
state.processed_bytes = 0;
state.total_objects = 0;
state.total_bytes = 0;
state.baseline_generation = None;
state.baseline_known = false;
state.counter_unknown = false;
state.completed = false;
state.completed_buckets.clear();
state.schema_version = CURRENT_RESUME_SCHEMA;
@@ -838,6 +932,41 @@ impl ResumeManager {
self.save_state_throttled().await
}
pub async fn update_progress_with_bytes(
&self,
processed: u64,
successful: u64,
failed: u64,
skipped: u64,
processed_bytes: u64,
) -> Result<()> {
let mut state = self.state.write().await;
state.update_progress_with_bytes(processed, successful, failed, skipped, processed_bytes);
drop(state);
self.save_state_throttled().await
}
pub async fn set_progress_baseline(&self, total_objects: u64, total_bytes: u64, generation: Option<u64>) -> Result<()> {
let mut state = self.state.write().await;
state.set_progress_baseline(total_objects, total_bytes, generation);
drop(state);
self.save_state_throttled().await
}
pub async fn mark_counter_unknown(&self) -> Result<()> {
let mut state = self.state.write().await;
state.mark_counter_unknown();
drop(state);
self.save_state().await
}
pub async fn set_skipped_version_counts(&self, new_versions: u64, ilm_expired: u64) -> Result<()> {
let mut state = self.state.write().await;
state.set_skipped_version_counts(new_versions, ilm_expired);
drop(state);
self.save_state_throttled().await
}
/// Set current item. Called once per healed object, so persistence is
/// throttled: the in-memory state always updates, but the snapshot is only
/// written every `PERSIST_EVERY_MUTATIONS` calls or `PERSIST_INTERVAL`.
@@ -882,7 +1011,7 @@ impl ResumeManager {
let mut state = self.state.write().await;
state.complete_bucket(bucket);
drop(state);
self.save_state_throttled().await
self.save_state().await
}
/// mark task completed
crates/heal/src/heal/resume/checkpoint.rs
+514 -29
View File
@@ -13,26 +13,51 @@
// limitations under the License.
use crate::{Error, Result};
use base64::Engine as _;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::collections::HashSet;
use std::path::Path;
use std::sync::{Arc, Mutex};
use std::time::{SystemTime, UNIX_EPOCH};
use tokio::sync::RwLock;
use tokio::sync::{Mutex as AsyncMutex, RwLock};
use tracing::{debug, warn};
use super::super::{BUCKET_META_PREFIX, DiskStore, HealDiskExt as _, RUSTFS_META_BUCKET};
use super::super::storage_api::owner::{EcstoreConditionalFileUpdate, EcstoreDiskAPI, EcstoreDiskBytes};
use super::super::{BUCKET_META_PREFIX, DiskStore, HealDiskExt, RUSTFS_META_BUCKET};
use super::{
LOG_COMPONENT_HEAL, LOG_SUBSYSTEM_RESUME, PersistThrottle, RESUME_CHECKPOINT_FILE, delete_resume_file, path_to_str,
validate_resume_task_id,
LOG_COMPONENT_HEAL, LOG_SUBSYSTEM_RESUME, PersistThrottle, RESUME_CHECKPOINT_BLOCKED_FILE, RESUME_CHECKPOINT_FILE,
delete_resume_file, path_to_str, validate_resume_task_id,
};
const EVENT_HEAL_CHECKPOINT_STATE: &str = "heal_checkpoint_state";
const RESUME_CHECKPOINT_DIGEST_FILE: &str = "ahm_checkpoint.sha256";
const CHECKPOINT_PER_VERSION_SCHEMA: u32 = 5;
/// Current on-disk schema version for `ResumeCheckpoint`. Same rationale as
/// `CURRENT_RESUME_SCHEMA`: pre-per-version dedup identities are not comparable
/// to the new `compose_key` identities, so a stale checkpoint is discarded.
pub(super) const CURRENT_CHECKPOINT_SCHEMA: u32 = 5;
/// Current on-disk schema version for `ResumeCheckpoint`. Schema 5 could
/// persist dedup identities without the aggregate counters needed to restore
/// them safely, so stale checkpoints are discarded and replayed.
pub(super) const CURRENT_CHECKPOINT_SCHEMA: u32 = 6;
#[derive(Debug, Clone, Copy)]
pub enum CheckpointObjectOutcome {
Processed,
Failed,
Skipped,
}
#[derive(Debug)]
pub struct CheckpointObjectOutcomeRecord {
pub object: String,
pub outcome: CheckpointObjectOutcome,
pub successful: u64,
pub failed: u64,
pub skipped: u64,
pub bytes: u64,
pub skipped_new_versions: u64,
pub skipped_ilm_expired: u64,
pub counter_unknown: bool,
}
/// resume checkpoint
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -57,6 +82,35 @@ pub struct ResumeCheckpoint {
pub failed_objects: HashSet<String>,
/// skipped objects
pub skipped_objects: HashSet<String>,
/// Aggregate object ledger counters restored alongside the dedup sets.
#[serde(default)]
pub successful_objects: u64,
#[serde(default)]
pub failed_object_count: u64,
#[serde(default)]
pub skipped_object_count: u64,
#[serde(default)]
pub skipped_new_versions: u64,
#[serde(default)]
pub skipped_ilm_expired: u64,
#[serde(default)]
pub processed_bytes: u64,
#[serde(default)]
pub total_objects: u64,
#[serde(default)]
pub total_bytes: u64,
#[serde(default)]
pub baseline_generation: Option<u64>,
#[serde(default)]
pub baseline_known: bool,
/// Persistent telemetry fence for counter/byte overflow or corruption.
#[serde(default)]
pub counter_unknown: bool,
/// Integrity digest over the checkpoint with this field set to `None`.
/// Keeping it in the checkpoint makes the payload and its authentication
/// record one CAS generation instead of two independently-written files.
#[serde(default)]
pub integrity_digest: Option<String>,
}
impl ResumeCheckpoint {
@@ -70,6 +124,18 @@ impl ResumeCheckpoint {
processed_objects: HashSet::new(),
failed_objects: HashSet::new(),
skipped_objects: HashSet::new(),
successful_objects: 0,
failed_object_count: 0,
skipped_object_count: 0,
skipped_new_versions: 0,
skipped_ilm_expired: 0,
processed_bytes: 0,
total_objects: 0,
total_bytes: 0,
baseline_generation: None,
baseline_known: false,
counter_unknown: false,
integrity_digest: None,
}
}
@@ -91,6 +157,34 @@ impl ResumeCheckpoint {
self.skipped_objects.insert(object);
}
pub fn update_progress(&mut self, successful: u64, failed: u64, skipped: u64, bytes: u64) {
self.successful_objects = successful;
self.failed_object_count = failed;
self.skipped_object_count = skipped;
self.processed_bytes = bytes;
}
pub fn set_progress_baseline(&mut self, total_objects: u64, total_bytes: u64, generation: Option<u64>) {
self.total_objects = total_objects;
self.total_bytes = total_bytes;
self.baseline_generation = generation;
// The caller has already validated that this is a complete snapshot;
// preserve the distinction between a known empty scope and an old
// checkpoint that omitted all baseline fields.
self.baseline_known = true;
}
pub fn mark_counter_unknown(&mut self) {
self.counter_unknown = true;
self.checkpoint_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
}
pub fn set_skipped_version_counts(&mut self, new_versions: u64, ilm_expired: u64) {
self.skipped_new_versions = new_versions;
self.skipped_ilm_expired = ilm_expired;
self.checkpoint_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
}
/// Advance past a fully-processed page: objects below `object_index` are
/// skipped by position on resume, so the per-object sets no longer need
/// their entries and would otherwise grow with the whole bucket.
@@ -107,6 +201,17 @@ impl ResumeCheckpoint {
self.update_position(0, 0);
self.processed_objects.clear();
self.skipped_objects.clear();
self.successful_objects = 0;
self.failed_object_count = 0;
self.skipped_object_count = 0;
self.skipped_new_versions = 0;
self.skipped_ilm_expired = 0;
self.processed_bytes = 0;
self.total_objects = 0;
self.total_bytes = 0;
self.baseline_generation = None;
self.baseline_known = false;
self.counter_unknown = false;
self.failed_objects.clear();
}
}
@@ -116,17 +221,111 @@ pub struct CheckpointManager {
disk: DiskStore,
checkpoint: Arc<RwLock<ResumeCheckpoint>>,
throttle: Mutex<PersistThrottle>,
save_lock: AsyncMutex<()>,
last_saved: Mutex<Option<EcstoreDiskBytes>>,
}
impl CheckpointManager {
fn blocked_path(task_id: &str) -> std::path::PathBuf {
Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_BLOCKED_FILE}"))
}
/// Return whether a checkpoint was permanently isolated after a malformed
/// or unsupported snapshot was observed.
pub(crate) async fn is_blocked(disk: &DiskStore, task_id: &str) -> bool {
if validate_resume_task_id(task_id).is_err() {
return false;
}
let blocked_path = Self::blocked_path(task_id);
let Ok(path) = path_to_str(&blocked_path) else {
return false;
};
match HealDiskExt::read_all(disk.as_ref(), RUSTFS_META_BUCKET, path).await {
Ok(_) => true,
Err(crate::heal::DiskError::FileNotFound) => false,
Err(_) => true,
}
}
/// Validate the checkpoint while enumerating resumable state. This reads
/// the checkpoint once and also isolates malformed or unsupported data.
pub(crate) async fn is_resumable(disk: &DiskStore, task_id: &str) -> Result<bool> {
validate_resume_task_id(task_id)?;
if Self::is_blocked(disk, task_id).await {
return Err(Error::InvalidCheckpoint(format!("Resume task {task_id} has a blocked checkpoint")));
}
let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}"));
let Ok(path) = path_to_str(&file_path) else {
return Err(Error::InvalidCheckpoint("Resume checkpoint path is not valid UTF-8".to_string()));
};
match HealDiskExt::read_all(disk.as_ref(), RUSTFS_META_BUCKET, path).await {
Ok(bytes) if bytes.is_empty() => Ok(true),
Ok(bytes) => Self::load_from_data(disk.clone(), task_id, bytes.to_vec())
.await
.map(|_| true),
Err(crate::heal::DiskError::FileNotFound) => Ok(true),
Err(error) => Err(error.into()),
}
}
async fn block_invalid_snapshot(disk: &DiskStore, task_id: &str) {
// This marker is intentionally version-agnostic: an unsupported reader
// must stop selector retries until an operator cleans up the snapshot.
let blocked_path = Self::blocked_path(task_id);
let Ok(path) = path_to_str(&blocked_path) else {
return;
};
let result = EcstoreDiskAPI::compare_and_update_file(
disk.as_ref(),
RUSTFS_META_BUCKET,
path,
None,
Some(EcstoreDiskBytes::from_static(b"blocked")),
)
.await;
match result {
Ok(EcstoreConditionalFileUpdate::Updated | EcstoreConditionalFileUpdate::Mismatch) => {}
Ok(EcstoreConditionalFileUpdate::Missing) => warn!(
target: "rustfs::heal::resume",
event = EVENT_HEAL_CHECKPOINT_STATE,
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_RESUME,
task_id,
state = "blocked_marker_write_failed",
error = "marker target disappeared",
"Heal checkpoint could not persist its blocked marker"
),
Err(error) => warn!(
target: "rustfs::heal::resume",
event = EVENT_HEAL_CHECKPOINT_STATE,
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_RESUME,
task_id,
state = "blocked_marker_write_failed",
error = %error,
"Heal checkpoint could not persist its blocked marker"
),
}
}
/// create new checkpoint manager
pub async fn new(disk: DiskStore, task_id: String) -> Result<Self> {
validate_resume_task_id(&task_id)?;
let checkpoint_volume = format!("{RUSTFS_META_BUCKET}/{BUCKET_META_PREFIX}");
if let Err(error) = EcstoreDiskAPI::make_volume(disk.as_ref(), &checkpoint_volume).await
&& error != crate::heal::DiskError::VolumeExists
{
return Err(Error::TaskExecutionFailed {
message: format!("Failed to create checkpoint volume: {error}"),
});
}
let checkpoint = ResumeCheckpoint::new(task_id);
let manager = Self {
disk,
checkpoint: Arc::new(RwLock::new(checkpoint)),
throttle: Mutex::new(PersistThrottle::new()),
save_lock: AsyncMutex::new(()),
last_saved: Mutex::new(None),
};
// save initial checkpoint
@@ -140,6 +339,7 @@ impl CheckpointManager {
error = %e,
"Heal checkpoint persistence failed"
);
return Err(e);
}
Ok(manager)
}
@@ -148,21 +348,32 @@ impl CheckpointManager {
pub async fn load_from_disk(disk: DiskStore, task_id: &str) -> Result<Self> {
validate_resume_task_id(task_id)?;
let checkpoint_data = Self::read_checkpoint_file(&disk, task_id).await?;
let mut checkpoint: ResumeCheckpoint =
serde_json::from_slice(&checkpoint_data).map_err(|e| Error::TaskExecutionFailed {
message: format!("Failed to deserialize checkpoint: {e}"),
})?;
Self::load_from_data(disk, task_id, checkpoint_data).await
}
async fn load_from_data(disk: DiskStore, task_id: &str, checkpoint_data: Vec<u8>) -> Result<Self> {
validate_resume_task_id(task_id)?;
let mut checkpoint: ResumeCheckpoint = match serde_json::from_slice(&checkpoint_data) {
Ok(checkpoint) => checkpoint,
Err(error) => {
Self::block_invalid_snapshot(&disk, task_id).await;
return Err(Error::TaskExecutionFailed {
message: format!("Failed to deserialize checkpoint: {error}"),
});
}
};
if checkpoint.task_id != task_id {
Self::block_invalid_snapshot(&disk, task_id).await;
return Err(Error::TaskExecutionFailed {
message: "Resume checkpoint task id does not match filename".to_string(),
});
}
// A checkpoint from an older schema stored latest-only dedup identities
// that are not comparable to the new per-version `compose_key`
// identities. Discard the stale sets and position, then stamp the
// current schema so the scan restarts cleanly.
// Older checkpoints can contain identities that are not comparable to
// the current keys or lack their corresponding aggregate counters.
// Discard the stale sets and position so the scan restarts cleanly.
if checkpoint.schema_version > CURRENT_CHECKPOINT_SCHEMA {
Self::block_invalid_snapshot(&disk, task_id).await;
return Err(Error::TaskExecutionFailed {
message: format!(
"Checkpoint schema {} is newer than supported schema {CURRENT_CHECKPOINT_SCHEMA}",
@@ -170,7 +381,45 @@ impl CheckpointManager {
),
});
}
if checkpoint.schema_version < CURRENT_CHECKPOINT_SCHEMA {
let integrity_verified = if let Some(expected) = checkpoint.integrity_digest.as_deref() {
let actual = Self::checkpoint_digest(&Self::serialize_without_digest(&checkpoint)?);
if expected != actual {
Self::block_invalid_snapshot(&disk, task_id).await;
return Err(Error::InvalidCheckpoint(format!(
"Resume checkpoint digest does not match task {task_id}"
)));
}
true
} else if checkpoint.schema_version >= CURRENT_CHECKPOINT_SCHEMA {
Self::block_invalid_snapshot(&disk, task_id).await;
return Err(Error::InvalidCheckpoint(format!(
"Resume checkpoint digest is missing for task {task_id}"
)));
} else {
let digest_path = Self::digest_path(task_id);
let digest_path = path_to_str(&digest_path)?;
match HealDiskExt::read_all(disk.as_ref(), RUSTFS_META_BUCKET, digest_path).await {
Ok(expected) => {
let actual = Self::checkpoint_digest(&checkpoint_data);
if expected.as_ref() != actual.as_bytes() {
Self::block_invalid_snapshot(&disk, task_id).await;
return Err(Error::InvalidCheckpoint(format!(
"Resume checkpoint digest does not match task {task_id}"
)));
}
true
}
Err(crate::heal::DiskError::FileNotFound) => false,
Err(error) => {
return Err(Error::TaskExecutionFailed {
message: format!("Failed to read checkpoint digest: {error}"),
});
}
}
};
if checkpoint.schema_version < CHECKPOINT_PER_VERSION_SCHEMA || !integrity_verified {
warn!(
target: "rustfs::heal::resume",
event = EVENT_HEAL_CHECKPOINT_STATE,
@@ -185,15 +434,28 @@ impl CheckpointManager {
checkpoint.processed_objects.clear();
checkpoint.failed_objects.clear();
checkpoint.skipped_objects.clear();
checkpoint.successful_objects = 0;
checkpoint.failed_object_count = 0;
checkpoint.skipped_object_count = 0;
checkpoint.skipped_new_versions = 0;
checkpoint.skipped_ilm_expired = 0;
checkpoint.processed_bytes = 0;
checkpoint.total_objects = 0;
checkpoint.total_bytes = 0;
checkpoint.baseline_generation = None;
checkpoint.baseline_known = false;
checkpoint.counter_unknown = false;
checkpoint.current_bucket_index = 0;
checkpoint.current_object_index = 0;
checkpoint.schema_version = CURRENT_CHECKPOINT_SCHEMA;
}
checkpoint.schema_version = CURRENT_CHECKPOINT_SCHEMA;
Ok(Self {
disk,
checkpoint: Arc::new(RwLock::new(checkpoint)),
throttle: Mutex::new(PersistThrottle::new()),
save_lock: AsyncMutex::new(()),
last_saved: Mutex::new(Some(EcstoreDiskBytes::from(checkpoint_data))),
})
}
@@ -204,7 +466,7 @@ impl CheckpointManager {
}
let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}"));
match path_to_str(&file_path) {
Ok(path_str) => match disk.read_all(RUSTFS_META_BUCKET, path_str).await {
Ok(path_str) => match HealDiskExt::read_all(disk.as_ref(), RUSTFS_META_BUCKET, path_str).await {
Ok(data) => !data.is_empty(),
Err(_) => false,
},
@@ -225,7 +487,7 @@ impl CheckpointManager {
self.save_checkpoint_throttled().await
}
/// Advance past a completed page and prune the per-object sets, then persist.
/// Persist a completed page position while retaining its identities.
pub async fn complete_page(&self, bucket_index: usize, object_index: usize) -> Result<()> {
let mut checkpoint = self.checkpoint.write().await;
checkpoint.complete_page(bucket_index, object_index);
@@ -233,6 +495,35 @@ impl CheckpointManager {
self.save_checkpoint_throttled().await
}
/// Persist the page position while retaining identities until the resume
/// cursor is durable.
pub async fn advance_page(&self, bucket_index: usize, object_index: usize) -> Result<()> {
let mut checkpoint = self.checkpoint.write().await;
checkpoint.update_position(bucket_index, object_index);
drop(checkpoint);
self.save_checkpoint().await
}
/// Remove the previous page's dedup identities only after its resume cursor
/// has been durably exposed.
pub async fn prune_completed_page(&self) -> Result<()> {
let mut checkpoint = self.checkpoint.write().await;
checkpoint.processed_objects.clear();
checkpoint.skipped_objects.clear();
checkpoint.failed_objects.clear();
drop(checkpoint);
self.save_checkpoint().await
}
/// Advance to the next bucket and clear the final page identities after the
/// resume state has durably recorded the completed bucket.
pub async fn complete_bucket(&self, next_bucket_index: usize) -> Result<()> {
let mut checkpoint = self.checkpoint.write().await;
checkpoint.complete_page(next_bucket_index, 0);
drop(checkpoint);
self.save_checkpoint().await
}
/// Reset the checkpoint to the start of the scan for a retry, then persist.
pub async fn reset_for_retry(&self) -> Result<()> {
let mut checkpoint = self.checkpoint.write().await;
@@ -267,6 +558,62 @@ impl CheckpointManager {
self.save_checkpoint_if_due().await
}
/// Atomically persist an object's dedup identity with its aggregate result.
pub async fn record_object_outcome(&self, record: CheckpointObjectOutcomeRecord) -> Result<()> {
let CheckpointObjectOutcomeRecord {
object,
outcome,
successful,
failed,
skipped,
bytes,
skipped_new_versions,
skipped_ilm_expired,
counter_unknown,
} = record;
let mut checkpoint = self.checkpoint.write().await;
match outcome {
CheckpointObjectOutcome::Processed => checkpoint.add_processed_object(object),
CheckpointObjectOutcome::Failed => checkpoint.add_failed_object(object),
CheckpointObjectOutcome::Skipped => checkpoint.add_skipped_object(object),
}
checkpoint.update_progress(successful, failed, skipped, bytes);
checkpoint.set_skipped_version_counts(skipped_new_versions, skipped_ilm_expired);
if counter_unknown {
checkpoint.mark_counter_unknown();
}
drop(checkpoint);
self.save_checkpoint_if_due().await
}
pub async fn update_progress(&self, successful: u64, failed: u64, skipped: u64, bytes: u64) -> Result<()> {
let mut checkpoint = self.checkpoint.write().await;
checkpoint.update_progress(successful, failed, skipped, bytes);
drop(checkpoint);
self.save_checkpoint_if_due().await
}
pub async fn set_progress_baseline(&self, total_objects: u64, total_bytes: u64, generation: Option<u64>) -> Result<()> {
let mut checkpoint = self.checkpoint.write().await;
checkpoint.set_progress_baseline(total_objects, total_bytes, generation);
drop(checkpoint);
self.save_checkpoint_throttled().await
}
pub async fn mark_counter_unknown(&self) -> Result<()> {
let mut checkpoint = self.checkpoint.write().await;
checkpoint.mark_counter_unknown();
drop(checkpoint);
self.save_checkpoint().await
}
pub async fn set_skipped_version_counts(&self, new_versions: u64, ilm_expired: u64) -> Result<()> {
let mut checkpoint = self.checkpoint.write().await;
checkpoint.set_skipped_version_counts(new_versions, ilm_expired);
drop(checkpoint);
self.save_checkpoint_throttled().await
}
async fn save_checkpoint_if_due(&self) -> Result<()> {
let should_save = self.throttle.lock().map(|mut throttle| throttle.record()).unwrap_or(true);
if !should_save {
@@ -292,6 +639,8 @@ impl CheckpointManager {
let checkpoint_file = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}"));
delete_resume_file(&self.disk, &checkpoint_file).await?;
delete_resume_file(&self.disk, &Self::digest_path(&task_id)).await?;
delete_resume_file(&self.disk, &Self::blocked_path(&task_id)).await?;
debug!(
target: "rustfs::heal::resume",
@@ -307,21 +656,130 @@ impl CheckpointManager {
/// save checkpoint to disk
async fn save_checkpoint(&self) -> Result<()> {
let checkpoint = self.checkpoint.read().await;
// Serialize saves and take the snapshot only after acquiring the lock:
// a slower writer must not publish a snapshot taken before a newer one.
let _save_guard = self.save_lock.lock().await;
let checkpoint = self.checkpoint.read().await.clone();
validate_resume_task_id(&checkpoint.task_id)?;
let checkpoint_data = serde_json::to_vec(&*checkpoint).map_err(|e| Error::TaskExecutionFailed {
message: format!("Failed to serialize checkpoint: {e}"),
})?;
let unsigned_checkpoint_data = Self::serialize_without_digest(&checkpoint)?;
let digest = Self::checkpoint_digest(&unsigned_checkpoint_data);
let mut persisted_checkpoint = checkpoint.clone();
persisted_checkpoint.integrity_digest = Some(digest);
let checkpoint_data =
EcstoreDiskBytes::from(serde_json::to_vec(&persisted_checkpoint).map_err(|e| Error::TaskExecutionFailed {
message: format!("Failed to serialize checkpoint: {e}"),
})?);
let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{}_{}", checkpoint.task_id, RESUME_CHECKPOINT_FILE));
let path_str = path_to_str(&file_path)?;
self.disk
.write_all(RUSTFS_META_BUCKET, path_str, checkpoint_data.into())
let last_saved = self
.last_saved
.lock()
.map_err(|_| Error::TaskExecutionFailed {
message: "Checkpoint save state lock is poisoned; refusing to save".to_string(),
})?
.clone();
let update = EcstoreDiskAPI::compare_and_update_file(
self.disk.as_ref(),
RUSTFS_META_BUCKET,
path_str,
last_saved.clone(),
Some(checkpoint_data.clone()),
)
.await
.map_err(|e| Error::TaskExecutionFailed {
message: format!("Failed to save checkpoint: {e}"),
})?;
let expected = match update {
EcstoreConditionalFileUpdate::Updated => None,
EcstoreConditionalFileUpdate::Missing => {
return Err(Error::TaskExecutionFailed {
message: "Checkpoint was removed after this manager saved it; refusing to recreate it".to_string(),
});
}
EcstoreConditionalFileUpdate::Mismatch => {
// A healthy manager normally completes the CAS above without
// another read or JSON parse. Inspect only after a mismatch so
// corruption and future schemas cannot be overwritten blindly.
let existing = match HealDiskExt::read_all(self.disk.as_ref(), RUSTFS_META_BUCKET, path_str).await {
Ok(existing) => existing,
Err(crate::heal::DiskError::FileNotFound) => {
return Err(Error::TaskExecutionFailed {
message: "Checkpoint was removed after this manager saved it; refusing to recreate it".to_string(),
});
}
Err(error) => {
return Err(Error::TaskExecutionFailed {
message: format!("Failed to inspect checkpoint after CAS mismatch: {error}"),
});
}
};
if existing.is_empty() && last_saved.is_none() {
Some(existing)
} else {
let current: ResumeCheckpoint = match serde_json::from_slice(&existing) {
Ok(current) => current,
Err(error) => {
Self::block_invalid_snapshot(&self.disk, &checkpoint.task_id).await;
return Err(Error::TaskExecutionFailed {
message: format!("Existing checkpoint is corrupt: {error}"),
});
}
};
if current.task_id != checkpoint.task_id {
Self::block_invalid_snapshot(&self.disk, &checkpoint.task_id).await;
return Err(Error::TaskExecutionFailed {
message: "Existing checkpoint task id does not match filename".to_string(),
});
}
if current.schema_version > CURRENT_CHECKPOINT_SCHEMA {
Self::block_invalid_snapshot(&self.disk, &checkpoint.task_id).await;
return Err(Error::TaskExecutionFailed {
message: format!(
"Existing checkpoint schema {} is newer than supported schema {CURRENT_CHECKPOINT_SCHEMA}",
current.schema_version
),
});
}
if last_saved.as_ref().is_none_or(|saved| saved.as_ref() != existing.as_ref()) {
return Err(Error::TaskExecutionFailed {
message: "Checkpoint changed since this manager loaded it; refusing to overwrite newer progress"
.to_string(),
});
}
Some(existing)
}
}
};
if let Some(expected) = expected {
match EcstoreDiskAPI::compare_and_update_file(
self.disk.as_ref(),
RUSTFS_META_BUCKET,
path_str,
Some(expected),
Some(checkpoint_data.clone()),
)
.await
.map_err(|e| Error::TaskExecutionFailed {
message: format!("Failed to save checkpoint: {e}"),
})?;
message: format!("Failed to save checkpoint after CAS mismatch: {e}"),
})? {
EcstoreConditionalFileUpdate::Updated => {}
EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch => {
return Err(Error::TaskExecutionFailed {
message: "Checkpoint changed while saving; refusing to overwrite newer progress".to_string(),
});
}
}
}
let mut last_saved = self.last_saved.lock().map_err(|_| Error::TaskExecutionFailed {
message: "Checkpoint save state lock is poisoned after save".to_string(),
})?;
*last_saved = Some(checkpoint_data);
debug!(
target: "rustfs::heal::resume",
@@ -341,11 +799,38 @@ impl CheckpointManager {
let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}"));
let path_str = path_to_str(&file_path)?;
disk.read_all(RUSTFS_META_BUCKET, path_str)
HealDiskExt::read_all(disk.as_ref(), RUSTFS_META_BUCKET, path_str)
.await
.map(|bytes| bytes.to_vec())
.map_err(|e| Error::TaskExecutionFailed {
message: format!("Failed to read checkpoint file: {e}"),
})
}
fn serialize_without_digest(checkpoint: &ResumeCheckpoint) -> Result<Vec<u8>> {
let mut unsigned = checkpoint.clone();
unsigned.integrity_digest = None;
let mut value = serde_json::to_value(&unsigned).map_err(|e| Error::TaskExecutionFailed {
message: format!("Failed to serialize checkpoint: {e}"),
})?;
for field in ["processed_objects", "failed_objects", "skipped_objects"] {
let Some(values) = value.get_mut(field).and_then(serde_json::Value::as_array_mut) else {
return Err(Error::TaskExecutionFailed {
message: format!("Failed to canonicalize checkpoint field: {field}"),
});
};
values.sort_by(|left, right| left.as_str().cmp(&right.as_str()));
}
serde_json::to_vec(&value).map_err(|e| Error::TaskExecutionFailed {
message: format!("Failed to serialize checkpoint: {e}"),
})
}
fn checkpoint_digest(checkpoint_data: &[u8]) -> String {
base64::engine::general_purpose::STANDARD.encode(Sha256::digest(checkpoint_data))
}
fn digest_path(task_id: &str) -> std::path::PathBuf {
Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_DIGEST_FILE}"))
}
}
crates/heal/src/heal/resume/gc.rs
+666
View File
@@ -0,0 +1,666 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Bounded inspection of heal resume artifacts.
//!
//! The durable owner/CAS and quarantine primitives belong to backlog #1927 and
//! are not part of the current base revision. This module is therefore
//! deliberately inspect-only. In particular, it must never turn an age check
//! into a delete: ordinary heal writers still publish raw files on this base,
//! so a GC-side compare-and-delete would not fence a concurrent claim.
use metrics::counter;
use std::{
collections::BTreeMap,
path::{Component, Path},
time::{SystemTime, UNIX_EPOCH},
};
use tokio::io::AsyncReadExt;
use super::super::{BUCKET_META_PREFIX, DiskError, DiskStore, RUSTFS_META_BUCKET, storage_api::owner::EcstoreDiskAPI};
use super::{
LEGACY_REPLACEMENT_RECOVERY_MARKER_FILE, REPLACEMENT_COMPLETION_PROOF_FILE, REPLACEMENT_INTENT_FILE,
REPLACEMENT_INTENT_SEAL_FILE, RESUME_CHECKPOINT_FILE, RESUME_PROGRESS_FILE, RESUME_STATE_FILE, ResumeCheckpoint, ResumeState,
checkpoint::CURRENT_CHECKPOINT_SCHEMA,
};
use crate::{Error, Result};
const DEFAULT_ENTRY_BUDGET: usize = 256;
const DEFAULT_BYTE_BUDGET: usize = 4 * 1024 * 1024;
const GC_METRIC: &str = "rustfs_heal_resume_gc_inspected_total";
const GC_ERROR_METRIC: &str = "rustfs_heal_resume_gc_inspect_errors_total";
#[derive(Debug, Clone, Copy)]
pub(crate) struct ResumeGcConfig {
/// Maximum number of directory entries considered in one disk pass.
pub(crate) max_entries: usize,
/// Maximum number of bytes read in one disk pass.
pub(crate) max_bytes: usize,
}
impl Default for ResumeGcConfig {
fn default() -> Self {
Self {
max_entries: DEFAULT_ENTRY_BUDGET,
max_bytes: DEFAULT_BYTE_BUDGET,
}
}
}
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
pub(crate) struct ResumeGcReport {
/// Directory entries visited (including malformed entries).
pub(crate) inspected: usize,
pub(crate) active_skipped: usize,
pub(crate) orphaned: usize,
/// Records that must be handed to #1927's quarantine owner.
pub(crate) quarantine_required: usize,
pub(crate) generation_skipped: usize,
pub(crate) clock_skew: usize,
pub(crate) read_errors: usize,
pub(crate) retained: usize,
/// True while #1927's durable claim/quarantine capability is unavailable.
pub(crate) destructive_disabled: bool,
pub(crate) budget_exhausted: bool,
}
#[derive(Debug, Default)]
pub(crate) struct ResumeGc {
config: ResumeGcConfig,
/// Alternate the first namespace so a full ordinary page cannot starve
/// replacement recovery when the list API has no continuation token.
recovery_first: bool,
}
impl ResumeGc {
#[cfg(test)]
fn with_config(config: ResumeGcConfig) -> Self {
Self {
config,
recovery_first: false,
}
}
/// Inspect one bounded page from each resume namespace.
///
/// The caller owns scheduling and cancellation. A malformed or unreadable
/// artifact is reported and retained so a later pass can retry it; no
/// individual artifact error aborts the rest of the bounded page.
pub(crate) async fn inspect_disk(&mut self, disk: &DiskStore) -> Result<ResumeGcReport> {
let mut report = ResumeGcReport {
destructive_disabled: true,
..ResumeGcReport::default()
};
if self.config.max_entries == 0 || self.config.max_bytes == 0 {
report.budget_exhausted = true;
return Ok(report);
}
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
let mut bytes_read = 0usize;
let recovery_first = self.recovery_first;
self.recovery_first = !self.recovery_first;
if recovery_first {
inspect_namespace(self.config, disk, &replacement_prefix(), true, now, &mut bytes_read, &mut report).await?;
if !report.budget_exhausted {
inspect_namespace(self.config, disk, BUCKET_META_PREFIX, false, now, &mut bytes_read, &mut report).await?;
}
} else {
inspect_namespace(self.config, disk, BUCKET_META_PREFIX, false, now, &mut bytes_read, &mut report).await?;
if !report.budget_exhausted {
inspect_namespace(self.config, disk, &replacement_prefix(), true, now, &mut bytes_read, &mut report).await?;
}
}
counter!(GC_METRIC).increment(u64::try_from(report.inspected).unwrap_or(u64::MAX));
counter!(GC_ERROR_METRIC).increment(u64::try_from(report.read_errors).unwrap_or(u64::MAX));
Ok(report)
}
}
#[derive(Debug, Default, Clone, Copy)]
struct ArtifactSet {
state: bool,
checkpoint: bool,
progress: bool,
replacement_intent: bool,
proof: bool,
seal: bool,
legacy_marker: bool,
temporary: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ArtifactKind {
State,
Checkpoint,
Progress,
ReplacementIntent,
Proof,
Seal,
LegacyMarker,
}
impl ArtifactSet {
fn add(&mut self, kind: ArtifactKind, temporary: bool) {
self.temporary |= temporary;
match kind {
ArtifactKind::State => self.state = true,
ArtifactKind::Checkpoint => self.checkpoint = true,
ArtifactKind::Progress => self.progress = true,
ArtifactKind::ReplacementIntent => self.replacement_intent = true,
ArtifactKind::Proof => self.proof = true,
ArtifactKind::Seal => self.seal = true,
ArtifactKind::LegacyMarker => self.legacy_marker = true,
}
}
}
#[derive(Debug, Clone, Copy)]
struct InspectOptions {
max_bytes: usize,
now: u64,
}
struct InspectProgress<'a> {
bytes_read: &'a mut usize,
report: &'a mut ResumeGcReport,
}
fn replacement_prefix() -> String {
super::replacement_recovery_dir().to_string_lossy().into_owned()
}
async fn inspect_namespace(
config: ResumeGcConfig,
disk: &DiskStore,
prefix: &str,
replacement: bool,
now: u64,
bytes_read: &mut usize,
report: &mut ResumeGcReport,
) -> Result<()> {
let remaining = config.max_entries.saturating_sub(report.inspected);
if remaining == 0 {
report.budget_exhausted = true;
return Ok(());
}
let count = i32::try_from(remaining).unwrap_or(i32::MAX);
let mut entries = match EcstoreDiskAPI::list_dir(disk.as_ref(), "", RUSTFS_META_BUCKET, prefix, count).await {
Ok(entries) => entries,
Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => return Ok(()),
Err(error) => return Err(error.into()),
};
entries.sort_unstable();
let mut artifacts = BTreeMap::<String, ArtifactSet>::new();
for entry in entries {
if report.inspected >= config.max_entries {
report.budget_exhausted = true;
break;
}
report.inspected += 1;
let Some((task_id, kind, temporary)) = artifact_name(&entry, replacement) else {
report.quarantine_required += 1;
report.retained += 1;
continue;
};
artifacts.entry(task_id).or_default().add(kind, temporary);
}
if report.inspected >= config.max_entries {
report.budget_exhausted = true;
}
for (task_id, artifacts) in artifacts {
if *bytes_read >= config.max_bytes {
report.budget_exhausted = true;
break;
}
let options = InspectOptions {
max_bytes: config.max_bytes,
now,
};
let mut progress = InspectProgress { bytes_read, report };
inspect_task(options, disk, prefix, replacement, &task_id, artifacts, &mut progress).await?;
}
Ok(())
}
async fn inspect_task(
options: InspectOptions,
disk: &DiskStore,
prefix: &str,
replacement: bool,
task_id: &str,
artifacts: ArtifactSet,
progress: &mut InspectProgress<'_>,
) -> Result<()> {
let legacy_replacement = !replacement && !artifacts.state && artifacts.replacement_intent;
let state_suffix = if replacement || legacy_replacement {
REPLACEMENT_INTENT_FILE
} else {
RESUME_STATE_FILE
};
let state_path = artifact_path(prefix, task_id, state_suffix)?;
let state = match read_bounded(disk, &state_path, options.max_bytes, progress.bytes_read).await {
ReadOutcome::Missing => {
progress.report.orphaned += 1;
progress.report.retained += 1;
return Ok(());
}
ReadOutcome::TooLarge => {
progress.report.quarantine_required += 1;
progress.report.retained += 1;
progress.report.budget_exhausted = true;
return Ok(());
}
ReadOutcome::Error => {
progress.report.read_errors += 1;
progress.report.retained += 1;
return Ok(());
}
ReadOutcome::Bytes(bytes) => bytes,
};
let parsed: ResumeState = match serde_json::from_slice(&state) {
Ok(state) => state,
Err(_) => {
progress.report.quarantine_required += 1;
progress.report.retained += 1;
return Ok(());
}
};
if parsed.schema_version > super::CURRENT_RESUME_SCHEMA || parsed.task_id != task_id {
progress.report.quarantine_required += 1;
progress.report.retained += 1;
return Ok(());
}
if persistent_age_seconds(options.now, parsed.last_update).is_none() {
progress.report.clock_skew += 1;
progress.report.retained += 1;
return Ok(());
}
if let Some(generation) = parsed.replacement_generation.as_deref()
&& !claim_generation_matches(Some(generation), Some(task_id))
{
progress.report.generation_skipped += 1;
progress.report.retained += 1;
return Ok(());
}
if !replacement && artifacts.checkpoint {
let checkpoint_path = artifact_path(prefix, task_id, RESUME_CHECKPOINT_FILE)?;
match read_bounded(disk, &checkpoint_path, options.max_bytes, progress.bytes_read).await {
ReadOutcome::Bytes(bytes) => match serde_json::from_slice::<ResumeCheckpoint>(&bytes) {
Ok(checkpoint) if checkpoint.schema_version <= CURRENT_CHECKPOINT_SCHEMA && checkpoint.task_id == task_id => {}
_ => {
progress.report.quarantine_required += 1;
progress.report.retained += 1;
}
},
ReadOutcome::Missing => {
progress.report.orphaned += 1;
progress.report.retained += 1;
}
ReadOutcome::TooLarge => {
progress.report.quarantine_required += 1;
progress.report.retained += 1;
progress.report.budget_exhausted = true;
}
ReadOutcome::Error => {
progress.report.read_errors += 1;
progress.report.retained += 1;
}
}
}
if artifacts.state && artifacts.replacement_intent {
// A task cannot have two authoritative state records in one namespace;
// preserve both until the durable owner can resolve the generation.
progress.report.quarantine_required += 1;
}
if !parsed.completed {
progress.report.active_skipped += 1;
}
// The state and all associated evidence remain recoverable until #1927
// supplies a common generation/CAS transition and quarantine owner.
progress.report.retained += 1;
Ok(())
}
enum ReadOutcome {
Bytes(Vec<u8>),
Missing,
TooLarge,
Error,
}
async fn read_bounded(disk: &DiskStore, path: &str, max_bytes: usize, bytes_read: &mut usize) -> ReadOutcome {
let remaining = max_bytes.saturating_sub(*bytes_read);
if remaining == 0 {
return ReadOutcome::TooLarge;
}
let read_len = remaining.saturating_add(1);
let reader = match EcstoreDiskAPI::read_file(disk.as_ref(), RUSTFS_META_BUCKET, path).await {
Ok(reader) => reader,
Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => return ReadOutcome::Missing,
Err(_) => return ReadOutcome::Error,
};
let mut bytes = Vec::with_capacity(read_len.min(64 * 1024));
let Ok(read_len) = u64::try_from(read_len) else {
return ReadOutcome::TooLarge;
};
if reader.take(read_len).read_to_end(&mut bytes).await.is_err() {
return ReadOutcome::Error;
}
if bytes.len() > remaining {
*bytes_read = max_bytes;
return ReadOutcome::TooLarge;
}
*bytes_read = bytes_read.saturating_add(bytes.len());
ReadOutcome::Bytes(bytes)
}
fn artifact_path(prefix: &str, task_id: &str, suffix: &str) -> Result<String> {
if super::validate_resume_task_id(task_id).is_err() {
return Err(Error::other("invalid resume task id"));
}
Path::new(prefix)
.join(format!("{task_id}_{suffix}"))
.to_str()
.map(str::to_owned)
.ok_or_else(|| Error::other("invalid resume artifact path"))
}
/// Parse one directory entry without ever accepting a path component supplied
/// by a client. DiskAPI filters symlinks, but this check also protects remote
/// implementations and future mutating callers from traversal/reparse names.
fn artifact_name(entry: &str, replacement: bool) -> Option<(String, ArtifactKind, bool)> {
let path = Path::new(entry);
if entry.is_empty() || path.components().count() != 1 || !matches!(path.components().next(), Some(Component::Normal(_))) {
return None;
}
let (stem, temporary) = entry
.strip_suffix(".tmp")
.map(|stem| (stem, true))
.or_else(|| entry.strip_suffix(".bak").map(|stem| (stem, true)))
.unwrap_or((entry, false));
let suffixes: &[(&str, ArtifactKind)] = if replacement {
&[
(REPLACEMENT_INTENT_FILE, ArtifactKind::ReplacementIntent),
(REPLACEMENT_COMPLETION_PROOF_FILE, ArtifactKind::Proof),
(REPLACEMENT_INTENT_SEAL_FILE, ArtifactKind::Seal),
]
} else {
&[
(RESUME_STATE_FILE, ArtifactKind::State),
(RESUME_CHECKPOINT_FILE, ArtifactKind::Checkpoint),
(RESUME_PROGRESS_FILE, ArtifactKind::Progress),
(LEGACY_REPLACEMENT_RECOVERY_MARKER_FILE, ArtifactKind::LegacyMarker),
(REPLACEMENT_INTENT_FILE, ArtifactKind::ReplacementIntent),
(REPLACEMENT_COMPLETION_PROOF_FILE, ArtifactKind::Proof),
(REPLACEMENT_INTENT_SEAL_FILE, ArtifactKind::Seal),
]
};
suffixes.iter().find_map(|(suffix, kind)| {
stem.strip_suffix(&format!("_{suffix}"))
.filter(|task_id| super::validate_resume_task_id(task_id).is_ok())
.map(|task_id| (task_id.to_string(), *kind, temporary))
})
}
fn persistent_age_seconds(now: u64, updated: u64) -> Option<u64> {
now.checked_sub(updated)
}
fn claim_generation_matches(observed: Option<&str>, expected: Option<&str>) -> bool {
expected.is_none() || observed == expected
}
#[cfg(test)]
mod tests {
use super::*;
use crate::heal::{DiskOption, Endpoint, new_disk};
use tempfile::TempDir;
use uuid::Uuid;
async fn test_disk() -> (TempDir, DiskStore) {
let temp = TempDir::new().expect("test disk directory");
let endpoint = Endpoint::try_from(temp.path().to_string_lossy().as_ref()).expect("test endpoint");
let disk = new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("test disk");
match disk.make_volume(RUSTFS_META_BUCKET).await {
Ok(()) | Err(DiskError::VolumeExists) => {}
Err(error) => panic!("metadata volume: {error}"),
}
match disk.make_volume(&format!("{RUSTFS_META_BUCKET}/{BUCKET_META_PREFIX}")).await {
Ok(()) | Err(DiskError::VolumeExists) => {}
Err(error) => panic!("resume volume: {error}"),
}
(temp, disk)
}
async fn write_state(disk: &DiskStore, state: &ResumeState) {
let path = format!("{BUCKET_META_PREFIX}/{}_{}", state.task_id, RESUME_STATE_FILE);
disk.write_all(RUSTFS_META_BUCKET, &path, serde_json::to_vec(state).unwrap().into())
.await
.expect("resume state");
}
async fn write_replacement_state(disk: &DiskStore, state: &ResumeState) {
let volume = format!("{RUSTFS_META_BUCKET}/{BUCKET_META_PREFIX}/ahm-replacement");
match disk.make_volume(&volume).await {
Ok(()) | Err(DiskError::VolumeExists) => {}
Err(error) => panic!("replacement volume: {error}"),
}
let path = format!("{}/{}_{}", replacement_prefix(), state.task_id, REPLACEMENT_INTENT_FILE);
disk.write_all(RUSTFS_META_BUCKET, &path, serde_json::to_vec(state).unwrap().into())
.await
.expect("replacement state");
}
#[tokio::test]
async fn production_gc_does_not_delete_claimed_resume_state() {
let (_temp, disk) = test_disk().await;
let task_id = Uuid::new_v4().to_string();
write_state(&disk, &ResumeState::new(task_id.clone(), "set".into(), "disk".into(), vec![])).await;
let report = ResumeGc::default().inspect_disk(&disk).await.expect("inspect");
assert_eq!(report.active_skipped, 1);
assert!(
disk.read_all(RUSTFS_META_BUCKET, &format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_STATE_FILE}"))
.await
.is_ok()
);
}
#[tokio::test]
async fn production_gc_generation_mismatch_is_skip() {
let (_temp, disk) = test_disk().await;
let task_id = Uuid::new_v4().to_string();
let mut state = ResumeState::new(task_id.clone(), "set".into(), "disk".into(), vec![]);
state.replacement_generation = Some(Uuid::new_v4().to_string());
write_state(&disk, &state).await;
assert_eq!(ResumeGc::default().inspect_disk(&disk).await.unwrap().generation_skipped, 1);
}
#[cfg(unix)]
#[tokio::test]
async fn production_gc_rejects_symlink_or_outside_prefix() {
let (_temp, disk) = test_disk().await;
let id = Uuid::new_v4().to_string();
let root = EcstoreDiskAPI::path(disk.as_ref());
let outside = root.join("outside-resume-state");
std::fs::write(&outside, b"must remain").expect("outside fixture");
let symlink = root
.join(RUSTFS_META_BUCKET)
.join(BUCKET_META_PREFIX)
.join(format!("{id}_{RESUME_STATE_FILE}"));
std::os::unix::fs::symlink(&outside, &symlink).expect("symlink fixture");
let report = ResumeGc::default().inspect_disk(&disk).await.expect("inspect");
assert_eq!(report.inspected, 0, "symlinks are not eligible artifacts");
assert!(outside.exists());
assert!(artifact_name(&format!("{id}_{RESUME_STATE_FILE}"), false).is_some());
assert!(artifact_name(&format!("../{id}_{RESUME_STATE_FILE}"), false).is_none());
assert!(artifact_name(&format!("{id}/link_{RESUME_STATE_FILE}"), false).is_none());
}
#[cfg(not(unix))]
#[test]
fn production_gc_rejects_symlink_or_outside_prefix() {
let id = Uuid::new_v4().to_string();
assert!(artifact_name(&format!("{id}_{RESUME_STATE_FILE}"), false).is_some());
assert!(artifact_name(&format!("../{id}_{RESUME_STATE_FILE}"), false).is_none());
assert!(artifact_name(&format!("{id}/link_{RESUME_STATE_FILE}"), false).is_none());
}
#[tokio::test]
async fn production_gc_delete_failure_leaves_recoverable_state() {
let (_temp, disk) = test_disk().await;
let task_id = Uuid::new_v4().to_string();
let mut state = ResumeState::new(task_id.clone(), "set".into(), "disk".into(), vec![]);
state.mark_completed();
write_state(&disk, &state).await;
ResumeGc::default().inspect_disk(&disk).await.expect("inspect");
assert!(
disk.read_all(RUSTFS_META_BUCKET, &format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_STATE_FILE}"))
.await
.is_ok()
);
}
#[tokio::test]
async fn production_gc_handles_clock_skew_and_restart() {
let (_temp, disk) = test_disk().await;
let task_id = Uuid::new_v4().to_string();
let mut state = ResumeState::new(task_id, "set".into(), "disk".into(), vec![]);
state.last_update = u64::MAX;
write_state(&disk, &state).await;
assert_eq!(ResumeGc::default().inspect_disk(&disk).await.unwrap().clock_skew, 1);
assert!(persistent_age_seconds(1, 2).is_none());
}
#[tokio::test]
async fn production_gc_100k_states_respects_budget() {
let (_temp, disk) = test_disk().await;
for _ in 0..8 {
let state = ResumeState::new(Uuid::new_v4().to_string(), "set".into(), "disk".into(), vec![]);
write_state(&disk, &state).await;
}
let config = ResumeGcConfig {
max_entries: 2,
max_bytes: usize::MAX,
};
let report = ResumeGc::with_config(config).inspect_disk(&disk).await.unwrap();
assert!(report.inspected <= 2);
assert!(report.budget_exhausted);
}
#[tokio::test]
async fn production_gc_recovery_namespace_is_not_starved() {
let (_temp, disk) = test_disk().await;
let ordinary = ResumeState::new(Uuid::new_v4().to_string(), "set".into(), "disk".into(), vec![]);
write_state(&disk, &ordinary).await;
let replacement_id = Uuid::new_v4().to_string();
let mut replacement = ResumeState::new(replacement_id, "set".into(), "disk".into(), vec![]);
replacement.replacement_generation = Some(replacement.task_id.clone());
write_replacement_state(&disk, &replacement).await;
let mut gc = ResumeGc::with_config(ResumeGcConfig {
max_entries: 1,
max_bytes: usize::MAX,
});
assert_eq!(gc.inspect_disk(&disk).await.unwrap().inspected, 1);
let second = gc.inspect_disk(&disk).await.unwrap();
assert_eq!(second.inspected, 1, "the next bounded pass must start at recovery");
assert_eq!(second.active_skipped, 1);
}
#[tokio::test]
async fn production_gc_pairs_orphan_checkpoint_and_resume() {
let (_temp, disk) = test_disk().await;
let task_id = Uuid::new_v4().to_string();
let checkpoint = ResumeCheckpoint::new(task_id.clone());
let path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}");
disk.write_all(RUSTFS_META_BUCKET, &path, serde_json::to_vec(&checkpoint).unwrap().into())
.await
.expect("checkpoint");
assert_eq!(ResumeGc::default().inspect_disk(&disk).await.unwrap().orphaned, 1);
}
#[tokio::test]
async fn production_gc_does_not_delete_slow_active_task() {
let (_temp, disk) = test_disk().await;
let task_id = Uuid::new_v4().to_string();
let mut state = ResumeState::new(task_id, "set".into(), "disk".into(), vec![]);
state.last_update = 1;
write_state(&disk, &state).await;
assert_eq!(ResumeGc::default().inspect_disk(&disk).await.unwrap().active_skipped, 1);
}
#[tokio::test]
async fn production_gc_disables_on_mixed_version_capability() {
assert!(claim_generation_matches(None, None));
assert!(!claim_generation_matches(Some("new"), Some("old")));
// No #1927 capability means this implementation has no delete path.
assert!(ResumeGcConfig::default().max_entries > 0);
let (_temp, disk) = test_disk().await;
let report = ResumeGc::default().inspect_disk(&disk).await.expect("inspect");
assert!(report.destructive_disabled);
}
#[tokio::test]
async fn production_gc_future_schema_is_not_mtime_deleted() {
let (_temp, disk) = test_disk().await;
let task_id = Uuid::new_v4().to_string();
let mut state = ResumeState::new(task_id.clone(), "set".into(), "disk".into(), vec![]);
state.schema_version = super::super::CURRENT_RESUME_SCHEMA + 1;
write_state(&disk, &state).await;
let report = ResumeGc::default().inspect_disk(&disk).await.unwrap();
assert_eq!(report.quarantine_required, 1);
assert!(
disk.read_all(RUSTFS_META_BUCKET, &format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_STATE_FILE}"))
.await
.is_ok()
);
}
#[tokio::test]
async fn production_gc_quarantine_cleanup_is_bounded() {
let (_temp, disk) = test_disk().await;
for _ in 0..4 {
let task_id = Uuid::new_v4().to_string();
let path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_STATE_FILE}");
disk.write_all(RUSTFS_META_BUCKET, &path, b"corrupt".to_vec().into())
.await
.expect("corrupt state");
}
let report = ResumeGc::with_config(ResumeGcConfig {
max_entries: 2,
max_bytes: 1024,
})
.inspect_disk(&disk)
.await
.expect("inspect");
assert!(report.quarantine_required <= 2);
assert!(report.budget_exhausted);
}
}
crates/heal/src/heal/resume/tests.rs
+542 -4
View File
@@ -1296,6 +1296,7 @@ async fn test_resume_state_progress() {
assert_eq!(progress, 0.0); // total_objects is 0
state.total_objects = 100;
state.baseline_known = true;
let progress = state.get_progress_percentage();
assert_eq!(progress, 10.0);
}
@@ -1475,6 +1476,40 @@ fn test_checkpoint_object_sets_dedupe_and_prune() {
assert!(checkpoint.failed_objects.is_empty());
}
#[tokio::test]
async fn checkpoint_page_commit_keeps_ledger_until_cursor_is_durable() {
let (_temp_dir, disk) = schema_test_disk().await;
let task_id = ResumeUtils::generate_task_id();
let checkpoint = CheckpointManager::new(disk.clone(), task_id.clone()).await.unwrap();
checkpoint
.record_object_outcome(CheckpointObjectOutcomeRecord {
object: "bucket/object:v1".to_string(),
outcome: CheckpointObjectOutcome::Processed,
successful: 1,
failed: 0,
skipped: 0,
bytes: 128,
skipped_new_versions: 0,
skipped_ilm_expired: 0,
counter_unknown: false,
})
.await
.unwrap();
checkpoint.advance_page(0, 1).await.unwrap();
let reloaded = CheckpointManager::load_from_disk(disk.clone(), &task_id).await.unwrap();
let snapshot = reloaded.get_checkpoint().await;
assert_eq!(snapshot.current_object_index, 1);
assert_eq!(snapshot.successful_objects, 1);
assert_eq!(snapshot.processed_bytes, 128);
assert!(snapshot.processed_objects.contains("bucket/object:v1"));
checkpoint.prune_completed_page().await.unwrap();
let reloaded = CheckpointManager::load_from_disk(disk, &task_id).await.unwrap();
assert!(reloaded.get_checkpoint().await.processed_objects.is_empty());
}
#[test]
fn test_checkpoint_loads_legacy_vec_format() {
// Checkpoints written before the HashSet migration stored the object
@@ -1568,14 +1603,14 @@ async fn test_resumestate_schema_v0_discarded_on_load() {
}
#[tokio::test]
async fn test_checkpoint_schema_v4_discarded_on_load() {
async fn test_checkpoint_schema_v5_discarded_on_load() {
let (temp_dir, disk) = schema_test_disk().await;
// The previous checkpoint schema is unsafe once its paired resume
// state is discarded: retaining either position would skip work.
// Schema v5 can persist failed identities without the aggregate counters
// that make those identities safe to deduplicate after an upgrade.
let task_id = "00000000-0000-4000-8000-000000000002";
let legacy = r#"{
"schema_version": 4,
"schema_version": 5,
"task_id": "00000000-0000-4000-8000-000000000002",
"checkpoint_time": 1700000000,
"current_bucket_index": 2,
@@ -1600,6 +1635,32 @@ async fn test_checkpoint_schema_v4_discarded_on_load() {
temp_dir.close().expect("remove schema test directory");
}
#[tokio::test]
async fn downgraded_unsigned_checkpoint_resets_untrusted_progress() {
let (temp_dir, disk) = schema_test_disk().await;
let task_id = ResumeUtils::generate_task_id();
let manager = CheckpointManager::new(disk.clone(), task_id.clone()).await.unwrap();
manager.add_processed_object("victim-a".to_string()).await.unwrap();
manager.update_position(2, 500).await.unwrap();
let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}");
let bytes = disk.read_all(RUSTFS_META_BUCKET, &checkpoint_path).await.unwrap();
let mut downgraded: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
downgraded["schema_version"] = serde_json::json!(CURRENT_CHECKPOINT_SCHEMA - 1);
downgraded.as_object_mut().unwrap().remove("integrity_digest");
downgraded["processed_objects"] = serde_json::json!(["victim-b"]);
disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, serde_json::to_vec(&downgraded).unwrap().into())
.await
.expect("write downgraded checkpoint");
let manager = CheckpointManager::load_from_disk(disk, &task_id).await.unwrap();
let checkpoint = manager.get_checkpoint().await;
assert_eq!(checkpoint.schema_version, CURRENT_CHECKPOINT_SCHEMA);
assert_eq!(checkpoint.current_bucket_index, 0);
assert_eq!(checkpoint.current_object_index, 0);
assert!(checkpoint.processed_objects.is_empty());
temp_dir.close().unwrap();
}
#[tokio::test]
async fn current_normal_resume_schema_preserves_progress() {
let (temp_dir, disk) = schema_test_disk().await;
@@ -1639,6 +1700,120 @@ async fn current_normal_resume_schema_preserves_progress() {
temp_dir.close().expect("remove schema test directory");
}
#[test]
fn progress_checkpoint_restores_bytes_and_generation() {
let mut checkpoint = ResumeCheckpoint::new("progress-checkpoint".to_string());
checkpoint.set_progress_baseline(9, 4096, Some(77));
checkpoint.update_progress(4, 1, 2, 2048);
checkpoint.set_skipped_version_counts(3, 1);
checkpoint.mark_counter_unknown();
let restored: ResumeCheckpoint =
serde_json::from_slice(&serde_json::to_vec(&checkpoint).expect("serialize checkpoint")).expect("deserialize checkpoint");
assert_eq!(restored.processed_bytes, 2048);
assert_eq!(restored.total_objects, 9);
assert_eq!(restored.total_bytes, 4096);
assert_eq!(restored.baseline_generation, Some(77));
assert!(restored.baseline_known);
assert_eq!(restored.skipped_new_versions, 3);
assert_eq!(restored.skipped_ilm_expired, 1);
assert!(restored.counter_unknown);
}
#[test]
fn old_progress_schema_migrates_missing_fields_to_unknown() {
let state = ResumeState::new(
"legacy-progress".to_string(),
"erasure_set".to_string(),
"pool_0_set_0".to_string(),
Vec::new(),
);
let mut value = serde_json::to_value(state).expect("serialize legacy-compatible state");
let object = value.as_object_mut().expect("state must be an object");
for field in [
"processed_bytes",
"total_bytes",
"baseline_generation",
"baseline_known",
"skipped_new_versions",
"skipped_ilm_expired",
] {
object.remove(field);
}
object.insert("total_objects".to_string(), serde_json::json!(10));
object.insert("processed_objects".to_string(), serde_json::json!(5));
let restored: ResumeState = serde_json::from_value(value).expect("deserialize old progress state");
assert_eq!(restored.processed_bytes, 0);
assert_eq!(restored.total_bytes, 0);
assert_eq!(restored.baseline_generation, None);
assert!(!restored.baseline_known, "missing baseline must remain unknown");
assert_eq!(restored.get_progress_percentage(), 0.0);
assert_eq!(restored.skipped_new_versions, 0);
assert_eq!(restored.skipped_ilm_expired, 0);
}
#[test]
fn progress_counter_unknown_survives_resume_round_trip() {
let mut state = ResumeState::new(
"overflow-progress".to_string(),
"erasure_set".to_string(),
"pool_0_set_0".to_string(),
Vec::new(),
);
state.mark_counter_unknown();
let restored: ResumeState =
serde_json::from_slice(&serde_json::to_vec(&state).expect("serialize resume state")).expect("deserialize resume state");
assert!(restored.counter_unknown);
}
#[tokio::test]
async fn checkpoint_progress_survives_a_torn_resume_summary_write() {
let (_temp_dir, disk) = schema_test_disk().await;
let task_id = ResumeUtils::generate_task_id();
let _resume = ResumeManager::new(
disk.clone(),
task_id.clone(),
"erasure_set".to_string(),
"pool_0_set_0".to_string(),
vec!["bucket".to_string()],
)
.await
.expect("resume state should persist");
let checkpoint = CheckpointManager::new(disk.clone(), task_id.clone())
.await
.expect("checkpoint should persist");
// This is the ordering used by the erasure-set loop: the checkpoint is
// durable before the summary write. Stop here to model a crash in the
// inter-store window and verify that the recovery authority retains the
// telemetry fence and bytes.
checkpoint
.update_progress(3, 0, 0, 1024)
.await
.expect("checkpoint progress should persist");
checkpoint.mark_counter_unknown().await.expect("unknown fence should persist");
checkpoint
.update_position(0, 3)
.await
.expect("checkpoint position should persist");
let restored_checkpoint = CheckpointManager::load_from_disk(disk.clone(), &task_id)
.await
.expect("checkpoint should reload")
.get_checkpoint()
.await;
let restored_resume = ResumeManager::load_from_disk(disk, &task_id)
.await
.expect("resume summary should reload")
.get_state()
.await;
assert!(restored_checkpoint.counter_unknown);
assert_eq!(restored_checkpoint.processed_bytes, 1024);
assert_eq!(restored_checkpoint.current_object_index, 3);
assert!(!restored_resume.counter_unknown, "summary is intentionally the torn/older store");
}
#[tokio::test]
async fn future_resume_and_checkpoint_schemas_are_rejected() {
let (temp_dir, disk) = schema_test_disk().await;
@@ -1675,6 +1850,369 @@ async fn future_resume_and_checkpoint_schemas_are_rejected() {
temp_dir.close().expect("remove schema test directory");
}
#[tokio::test]
async fn checkpoint_save_does_not_replace_a_non_empty_truncated_snapshot() {
let (temp_dir, disk) = schema_test_disk().await;
let task_id = ResumeUtils::generate_task_id();
let manager = CheckpointManager::new(disk.clone(), task_id.clone())
.await
.expect("create checkpoint manager");
let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}");
let truncated = b"{\"schema_version\":5,\"task_id\":";
disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, truncated.as_slice().into())
.await
.expect("write truncated checkpoint fixture");
let error = manager
.update_position(2, 7)
.await
.expect_err("a truncated checkpoint must fail closed during save");
assert!(error.to_string().contains("Existing checkpoint is corrupt"));
assert_eq!(
disk.read_all(RUSTFS_META_BUCKET, &checkpoint_path)
.await
.expect("read truncated checkpoint fixture"),
truncated.as_slice()
);
assert!(CheckpointManager::is_blocked(&disk, &task_id).await);
temp_dir.close().expect("remove checkpoint save test directory");
}
#[tokio::test]
async fn checkpoint_save_does_not_replace_a_future_schema_snapshot() {
let (temp_dir, disk) = schema_test_disk().await;
let task_id = ResumeUtils::generate_task_id();
let manager = CheckpointManager::new(disk.clone(), task_id.clone())
.await
.expect("create checkpoint manager");
let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}");
let mut future = ResumeCheckpoint::new(task_id.clone());
future.schema_version = CURRENT_CHECKPOINT_SCHEMA + 1;
let future_bytes = serde_json::to_vec(&future).expect("serialize future checkpoint fixture");
disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, future_bytes.clone().into())
.await
.expect("write future checkpoint fixture");
let error = manager
.update_position(2, 7)
.await
.expect_err("a future schema must fail closed during save");
assert!(error.to_string().contains("Existing checkpoint schema"));
assert_eq!(
disk.read_all(RUSTFS_META_BUCKET, &checkpoint_path)
.await
.expect("read future checkpoint fixture"),
future_bytes
);
assert!(CheckpointManager::is_blocked(&disk, &task_id).await);
temp_dir.close().expect("remove future schema test directory");
}
#[tokio::test]
async fn checkpoint_digest_rejects_same_length_progress_tampering() {
let (temp_dir, disk) = schema_test_disk().await;
let task_id = ResumeUtils::generate_task_id();
let manager = CheckpointManager::new(disk.clone(), task_id.clone())
.await
.expect("create checkpoint manager");
manager
.add_processed_object("victim-a".to_string())
.await
.expect("persist checkpoint progress");
manager.update_position(1, 1).await.expect("flush checkpoint progress");
let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}");
let original = disk
.read_all(RUSTFS_META_BUCKET, &checkpoint_path)
.await
.expect("read checkpoint fixture");
let tampered = original
.windows(b"victim-a".len())
.position(|window| window == b"victim-a")
.map(|index| {
let mut bytes = original.to_vec();
bytes[index..index + b"victim-a".len()].copy_from_slice(b"victim-b");
bytes
})
.expect("checkpoint should contain the processed object");
disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, tampered.into())
.await
.expect("write tampered checkpoint fixture");
assert!(CheckpointManager::load_from_disk(disk.clone(), &task_id).await.is_err());
assert!(CheckpointManager::is_blocked(&disk, &task_id).await);
temp_dir.close().expect("remove digest test directory");
}
#[tokio::test]
async fn checkpoint_integrity_survives_missing_legacy_sidecar() {
let (temp_dir, disk) = schema_test_disk().await;
let task_id = ResumeUtils::generate_task_id();
let manager = CheckpointManager::new(disk.clone(), task_id.clone()).await.unwrap();
manager.update_position(2, 9).await.unwrap();
let digest_path = format!("{BUCKET_META_PREFIX}/{task_id}_ahm_checkpoint.sha256");
delete_resume_file(&disk, Path::new(&digest_path)).await.unwrap();
let restored = CheckpointManager::load_from_disk(disk, &task_id).await.unwrap();
let checkpoint = restored.get_checkpoint().await;
assert_eq!(checkpoint.current_bucket_index, 2);
assert_eq!(checkpoint.current_object_index, 9);
assert!(checkpoint.integrity_digest.is_some());
temp_dir.close().unwrap();
}
#[tokio::test]
async fn checkpoint_integrity_survives_multi_object_reload() {
let (temp_dir, disk) = schema_test_disk().await;
let task_id = ResumeUtils::generate_task_id();
let manager = CheckpointManager::new(disk.clone(), task_id.clone()).await.unwrap();
for index in 0..32 {
manager.add_processed_object(format!("processed-{index}")).await.unwrap();
manager.add_failed_object(format!("failed-{index}")).await.unwrap();
manager.add_skipped_object(format!("skipped-{index}")).await.unwrap();
}
manager.update_position(2, 9).await.unwrap();
CheckpointManager::load_from_disk(disk, &task_id)
.await
.expect("a healthy multi-object checkpoint must survive reload");
temp_dir.close().unwrap();
}
#[tokio::test]
async fn checkpoint_integrity_rejects_a_removed_embedded_digest() {
let (temp_dir, disk) = schema_test_disk().await;
let task_id = ResumeUtils::generate_task_id();
let manager = CheckpointManager::new(disk.clone(), task_id.clone()).await.unwrap();
manager.update_position(2, 9).await.unwrap();
let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}");
let bytes = disk
.read_all(RUSTFS_META_BUCKET, &checkpoint_path)
.await
.expect("read checkpoint fixture");
let mut value: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
value["current_object_index"] = serde_json::json!(10);
value.as_object_mut().unwrap().remove("integrity_digest");
disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, serde_json::to_vec(&value).unwrap().into())
.await
.expect("write tampered checkpoint fixture");
assert!(
CheckpointManager::load_from_disk(disk.clone(), &task_id).await.is_err(),
"a current checkpoint without its embedded digest must fail closed"
);
assert!(CheckpointManager::is_blocked(&disk, &task_id).await);
temp_dir.close().unwrap();
}
#[tokio::test]
async fn new_checkpoint_manager_rebuilds_an_empty_snapshot() {
let (temp_dir, disk) = schema_test_disk().await;
let task_id = ResumeUtils::generate_task_id();
let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}");
disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, EcstoreDiskBytes::new())
.await
.expect("write empty checkpoint fixture");
let manager = CheckpointManager::new(disk.clone(), task_id.clone())
.await
.expect("a new manager must rebuild an empty checkpoint");
manager
.update_position(3, 11)
.await
.expect("rebuilt checkpoint must remain writable");
assert!(CheckpointManager::has_checkpoint(&disk, &task_id).await);
temp_dir.close().expect("remove empty checkpoint test directory");
}
#[tokio::test]
async fn deleted_checkpoint_is_not_recreated_by_an_old_manager() {
let (temp_dir, disk) = schema_test_disk().await;
let task_id = ResumeUtils::generate_task_id();
let manager = CheckpointManager::new(disk.clone(), task_id.clone())
.await
.expect("create checkpoint manager");
manager.cleanup().await.expect("delete checkpoint fixture");
let error = manager
.update_position(1, 2)
.await
.expect_err("an old manager must not resurrect a deleted checkpoint");
assert!(error.to_string().contains("removed after this manager saved it"));
assert!(!CheckpointManager::has_checkpoint(&disk, &task_id).await);
temp_dir.close().expect("remove deleted checkpoint test directory");
}
#[cfg(unix)]
#[tokio::test]
async fn checkpoint_cleanup_leaves_no_task_specific_lock_artifact() {
let (temp_dir, disk) = schema_test_disk().await;
let task_id = ResumeUtils::generate_task_id();
let manager = CheckpointManager::new(disk.clone(), task_id.clone())
.await
.expect("create checkpoint manager");
let lock_path = Path::new(BUCKET_META_PREFIX)
.join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}"))
.with_extension("rustfs-cas.lock");
let lock_path = temp_dir.path().join(RUSTFS_META_BUCKET).join(lock_path);
manager.cleanup().await.expect("delete checkpoint fixture");
assert!(
!lock_path.exists(),
"successful checkpoint cleanup must not leave a task-specific lock artifact"
);
}
#[tokio::test]
async fn an_empty_blocked_marker_still_blocks_resume_selection() {
let (temp_dir, disk) = schema_test_disk().await;
let task_id = ResumeUtils::generate_task_id();
let manager = CheckpointManager::new(disk.clone(), task_id.clone())
.await
.expect("create checkpoint manager");
let blocked_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_BLOCKED_FILE}");
disk.write_all(RUSTFS_META_BUCKET, &blocked_path, EcstoreDiskBytes::new())
.await
.expect("write empty blocked marker fixture");
assert!(CheckpointManager::is_blocked(&disk, &task_id).await);
assert!(CheckpointManager::is_resumable(&disk, &task_id).await.is_err());
// Recovery requires replacing/cleaning the snapshot, then removing the
// marker; ordinary selector retries are intentionally not an unlock path.
manager.cleanup().await.expect("clean blocked checkpoint");
assert!(!CheckpointManager::is_blocked(&disk, &task_id).await);
temp_dir.close().expect("remove empty blocked marker test directory");
}
#[tokio::test]
async fn resumable_selector_skips_healthy_tasks_with_blocked_markers() {
let (temp_dir, disk) = schema_test_disk().await;
let tasks = [
(ResumeUtils::generate_task_id(), EcstoreDiskBytes::new()),
(ResumeUtils::generate_task_id(), EcstoreDiskBytes::from_static(b"blocked")),
];
for (task_id, marker) in &tasks {
ResumeManager::new(
disk.clone(),
task_id.clone(),
"erasure_set".to_string(),
"pool_0_set_0".to_string(),
vec!["bucket".to_string()],
)
.await
.expect("create healthy resume state");
CheckpointManager::new(disk.clone(), task_id.clone())
.await
.expect("create healthy checkpoint");
let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}");
let checkpoint_bytes = disk
.read_all(RUSTFS_META_BUCKET, &checkpoint_path)
.await
.expect("read healthy checkpoint before blocking");
let marker_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_BLOCKED_FILE}");
disk.write_all(RUSTFS_META_BUCKET, &marker_path, marker.clone())
.await
.expect("write blocked marker");
assert!(ResumeUtils::get_resumable_tasks(&disk).await.is_err());
assert_eq!(
disk.read_all(RUSTFS_META_BUCKET, &checkpoint_path)
.await
.expect("read healthy checkpoint after blocking"),
checkpoint_bytes
);
}
temp_dir.close().expect("remove blocked selector test directory");
}
#[tokio::test]
async fn stale_checkpoint_manager_cannot_overwrite_newer_progress() {
let (temp_dir, disk) = schema_test_disk().await;
let task_id = ResumeUtils::generate_task_id();
let first = CheckpointManager::new(disk.clone(), task_id.clone())
.await
.expect("create first checkpoint manager");
let second = CheckpointManager::load_from_disk(disk.clone(), &task_id)
.await
.expect("load second checkpoint manager");
second
.update_position(4, 20)
.await
.expect("persist newer checkpoint progress");
let error = first
.update_position(1, 3)
.await
.expect_err("stale checkpoint manager must not overwrite newer progress");
assert!(error.to_string().contains("newer progress"));
let persisted = CheckpointManager::load_from_disk(disk.clone(), &task_id)
.await
.expect("load newer checkpoint progress")
.get_checkpoint()
.await;
assert_eq!(persisted.current_bucket_index, 4);
assert_eq!(persisted.current_object_index, 20);
temp_dir.close().expect("remove stale manager test directory");
}
#[tokio::test]
async fn resumable_selector_isolates_future_and_corrupt_checkpoints() {
let (temp_dir, disk) = schema_test_disk().await;
let future_task = ResumeUtils::generate_task_id();
let corrupt_task = ResumeUtils::generate_task_id();
for task_id in [&future_task, &corrupt_task] {
ResumeManager::new(
disk.clone(),
task_id.to_string(),
"erasure_set".to_string(),
"pool_0_set_0".to_string(),
vec!["bucket".to_string()],
)
.await
.expect("create resumable state fixture");
}
let future_path = format!("{BUCKET_META_PREFIX}/{future_task}_{RESUME_CHECKPOINT_FILE}");
let mut future = ResumeCheckpoint::new(future_task.clone());
future.schema_version = CURRENT_CHECKPOINT_SCHEMA + 1;
let future_bytes = serde_json::to_vec(&future).expect("serialize future checkpoint fixture");
disk.write_all(RUSTFS_META_BUCKET, &future_path, future_bytes.clone().into())
.await
.expect("write future checkpoint fixture");
let corrupt_path = format!("{BUCKET_META_PREFIX}/{corrupt_task}_{RESUME_CHECKPOINT_FILE}");
let corrupt_bytes = b"{truncated";
disk.write_all(RUSTFS_META_BUCKET, &corrupt_path, corrupt_bytes.as_slice().into())
.await
.expect("write corrupt checkpoint fixture");
assert!(CheckpointManager::is_resumable(&disk, &future_task).await.is_err());
assert!(CheckpointManager::is_resumable(&disk, &corrupt_task).await.is_err());
assert!(ResumeUtils::get_resumable_tasks(&disk).await.is_err());
for (task_id, path, bytes) in [
(&future_task, future_path, future_bytes),
(&corrupt_task, corrupt_path, corrupt_bytes.to_vec()),
] {
assert_eq!(
disk.read_all(RUSTFS_META_BUCKET, &path)
.await
.expect("read isolated checkpoint bytes"),
bytes
);
let blocked_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_BLOCKED_FILE}");
assert!(
!disk
.read_all(RUSTFS_META_BUCKET, &blocked_path)
.await
.expect("read checkpoint blocked marker")
.is_empty()
);
}
temp_dir.close().expect("remove selector isolation test directory");
}
#[test]
fn test_persist_throttle_batches_until_threshold() {
let mut throttle = PersistThrottle::new();
crates/heal/src/heal/resume/utils.rs
+2 -1
View File
@@ -21,7 +21,7 @@ use uuid::Uuid;
use super::super::{BUCKET_META_PREFIX, DiskError, DiskStore, HealDiskExt as _, RUSTFS_META_BUCKET};
use super::replacement::{ReplacementPhase, ReplacementRecoveryRecord};
use super::{
EVENT_HEAL_RESUME_STATE, LOG_COMPONENT_HEAL, LOG_SUBSYSTEM_RESUME, REPLACEMENT_COMPLETION_PROOF_FILE,
CheckpointManager, EVENT_HEAL_RESUME_STATE, LOG_COMPONENT_HEAL, LOG_SUBSYSTEM_RESUME, REPLACEMENT_COMPLETION_PROOF_FILE,
REPLACEMENT_INTENT_FILE, RESUME_STATE_FILE, ResumeManager, ResumeStateFile, is_replacement_intent, path_to_str,
replacement_recovery_corruption_for_state_load, replacement_recovery_dir, validate_resume_task_id,
};
@@ -67,6 +67,7 @@ impl ResumeUtils {
// Extract task ID from filename: {task_id}_ahm_resume_state.json
if let Some(task_id) = entry.strip_suffix(&format!("_{RESUME_STATE_FILE}"))
&& validate_resume_task_id(task_id).is_ok()
&& CheckpointManager::is_resumable(disk, task_id).await?
{
task_ids.push(task_id.to_string());
}
crates/heal/src/heal/storage.rs
+47 -2
View File
@@ -22,6 +22,7 @@ use serde::{Deserialize, Serialize};
use std::sync::Arc;
use tracing::{debug, error, warn};
use super::progress::stable_generation;
use super::storage_api::owner::{EcstoreHealLifecycleExpiryContext, ecstore_load_admin_data_usage_from_backend_cached};
use super::storage_api::storage::{
BucketInfo, BucketOperations, DiskSetSelector, HealOperations as _, ListOperations as _, ObjectIO as _,
@@ -34,6 +35,9 @@ pub use super::{HealObjectInfo, HealObjectOptions, HealPutObjReader};
pub struct HealBucketUsageBaseline {
pub objects_count: u64,
pub bytes: u64,
/// Stable identity of the validated usage snapshot and selected scope.
/// `None` is retained for test/legacy providers that cannot expose one.
pub generation: Option<u64>,
}
pub struct HealLifecycleExpiryContext {
@@ -785,11 +789,52 @@ impl HealStorageAPI for ECStoreHealStorage {
let mut baseline = HealBucketUsageBaseline::default();
for bucket in buckets {
if let Some(usage) = info.buckets_usage.get(bucket) {
baseline.objects_count = baseline.objects_count.saturating_add(usage.objects_count);
baseline.bytes = baseline.bytes.saturating_add(usage.size);
baseline.objects_count = match baseline.objects_count.checked_add(usage.objects_count) {
Some(total) => total,
// A corrupt/overflowing usage snapshot is not a usable
// denominator. Leave progress indeterminate instead of
// turning saturation into a plausible percentage.
None => return Ok(None),
};
baseline.bytes = match baseline.bytes.checked_add(usage.size) {
Some(total) => total,
None => return Ok(None),
};
}
}
let identity = info.snapshot_identity();
let mut canonical = Vec::new();
match identity.last_update {
Some(last_update) => {
canonical.push(1);
canonical.extend_from_slice(
&last_update
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_nanos()
.to_be_bytes(),
);
}
None => canonical.push(0),
}
for value in [identity.scanner_cycle, identity.scanner_epoch] {
match value {
Some(value) => {
canonical.push(1);
canonical.extend_from_slice(&value.to_be_bytes());
}
None => canonical.push(0),
}
}
let mut scope = buckets.to_vec();
scope.sort_unstable();
for bucket in scope {
canonical.extend_from_slice(&(bucket.len() as u64).to_be_bytes());
canonical.extend_from_slice(bucket.as_bytes());
}
baseline.generation = Some(stable_generation(&[&canonical]));
Ok(Some(baseline))
}
crates/heal/src/heal/task.rs
+7 -3
View File
@@ -649,7 +649,7 @@ impl HealTask {
let mut progress = self.progress.write().await;
progress.set_current_object(Some(format!("skipped: {bucket}/{object}")));
progress.update_progress(0, 1, 0, 0);
progress.update_stage(1, 1);
Ok(())
}
@@ -733,7 +733,7 @@ impl HealTask {
"Heal object skipped for data usage cache after transient error"
);
let mut progress = self.progress.write().await;
progress.update_progress(3, 3, 0, 0);
progress.update_stage(3, 3);
true
}
@@ -757,7 +757,7 @@ impl HealTask {
);
let mut progress = self.progress.write().await;
progress.set_current_object(Some(format!("skipped: {bucket}/{object}")));
progress.update_progress(4, 4, 0, 0);
progress.update_stage(4, 4);
true
}
@@ -831,6 +831,10 @@ impl HealTask {
match &result {
Ok(_) => {
// A stage can reach its final step before the durable resume
// ledger and cleanup fences commit. Publish terminal 100 only
// after the enclosing operation has returned success.
self.progress.write().await.mark_completed();
let mut status = self.status.write().await;
*status = HealTaskStatus::Completed;
demote_to_debug_when!(self.heal_type.is_per_object(), info, target: "rustfs::heal::task", {
crates/heal/src/heal/task/heal_bucket.rs
+40 -15
View File
@@ -13,6 +13,7 @@
// limitations under the License.
/// bucket/cluster/prefix heal: the recursive bucket-objects sweep and the erasure-set usage baseline
use super::*;
use crate::heal::progress::{add_bytes, increment_counter, stable_generation};
impl HealTask {
pub(super) async fn heal_bucket(&self, bucket: &str) -> Result<()> {
@@ -32,7 +33,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.set_current_object(Some(format!("bucket: {bucket}")));
progress.update_progress(0, 3, 0, 0);
progress.update_stage(0, 3);
}
// Step 1: Check if bucket exists
@@ -66,7 +67,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.update_progress(1, 3, 0, 0);
progress.update_stage(1, 3);
}
// Step 2: Perform bucket heal using ecstore
@@ -122,7 +123,7 @@ impl HealTask {
if !self.options.recursive {
let mut progress = self.progress.write().await;
progress.update_progress(3, 3, 0, 0);
progress.update_stage(3, 3);
}
Ok(())
}
@@ -142,7 +143,7 @@ impl HealTask {
);
{
let mut progress = self.progress.write().await;
progress.update_progress(3, 3, 0, 0);
progress.update_stage(3, 3);
}
Err(Error::TaskExecutionFailed {
message: format!("Failed to heal bucket {bucket}: {e}"),
@@ -245,6 +246,7 @@ impl HealTask {
let mut scanned = 0u64;
let mut healed = 0u64;
let mut failed = 0u64;
let mut skipped = 0u64;
let mut retryable_failed = 0u64;
let mut permanent_failed = 0u64;
let mut bytes = 0u64;
@@ -286,16 +288,14 @@ impl HealTask {
let mut retry = Vec::with_capacity(pending.len());
for item in pending {
self.check_control_flags().await?;
let mut telemetry_unknown = false;
let object = item.name.as_str();
if retry_attempt == 0 {
scanned = scanned.saturating_add(1);
}
{
let mut progress = self.progress.write().await;
progress.set_current_object(Some(format!("{bucket}/{object}")));
progress.update_progress(scanned, healed, failed, bytes);
}
let mut terminal_outcome = true;
let error = match self
.await_with_control(
self.storage
@@ -304,13 +304,13 @@ impl HealTask {
.await
{
Ok((result, None)) => {
healed = healed.saturating_add(1);
bytes = bytes.saturating_add(u64::try_from(result.object_size).unwrap_or_default());
telemetry_unknown |= !increment_counter(&mut healed);
telemetry_unknown |= !add_bytes(&mut bytes, u64::try_from(result.object_size).unwrap_or(u64::MAX));
self.record_result_item(result).await;
None
}
Ok((_, Some(err))) if is_missing_object_dir_heal_result(object, &err) => {
healed = healed.saturating_add(1);
telemetry_unknown |= !increment_counter(&mut healed);
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_BUCKET_RESULT,
@@ -329,6 +329,7 @@ impl HealTask {
if let Some(err) = error {
if Self::should_skip_data_usage_cache_heal_error(bucket, object, &err) {
telemetry_unknown |= !increment_counter(&mut skipped);
warn!(
target: "rustfs::heal::task",
event = EVENT_HEAL_BUCKET_RESULT,
@@ -342,6 +343,7 @@ impl HealTask {
"Heal bucket object repair skipped due to transient metadata error"
);
} else if err.is_recoverable_heal() && retry_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES {
terminal_outcome = false;
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_BUCKET_RESULT,
@@ -357,7 +359,7 @@ impl HealTask {
);
retry.push(item);
} else {
failed = failed.saturating_add(1);
telemetry_unknown |= !increment_counter(&mut failed);
if err.is_recoverable_heal() {
retryable_failed = retryable_failed.saturating_add(1);
} else {
@@ -383,8 +385,19 @@ impl HealTask {
}
}
if terminal_outcome {
telemetry_unknown |= !increment_counter(&mut scanned);
}
if !terminal_outcome {
continue;
}
let mut progress = self.progress.write().await;
progress.update_progress(scanned, healed, failed, bytes);
progress.update_object_progress(scanned, healed, failed, skipped, bytes);
if telemetry_unknown {
progress.mark_unknown();
}
}
pending = retry;
retry_attempt = retry_attempt.saturating_add(1);
@@ -432,6 +445,9 @@ impl HealTask {
}
pub(super) async fn apply_erasure_set_usage_baseline(&self, buckets: &[String]) -> Result<()> {
if matches!(self.options.scan_mode, HealScanMode::Deep) || matches!(self.source, HealRequestSource::AutoHeal) {
return Ok(());
}
let baseline = match self
.await_with_control(self.storage.erasure_set_usage_baseline(buckets))
.await
@@ -442,9 +458,18 @@ impl HealTask {
Err(_) => return Ok(()),
};
let HealBucketUsageBaseline { objects_count, bytes } = baseline;
let HealBucketUsageBaseline {
objects_count,
bytes,
generation,
} = baseline;
let generation = generation.map(|snapshot_generation| stable_generation(&[&snapshot_generation.to_be_bytes()]));
let mut progress = self.progress.write().await;
progress.set_total_baseline(objects_count, bytes);
if let Some(generation) = generation {
progress.set_total_baseline_with_generation(objects_count, bytes, generation);
} else {
progress.set_total_baseline(objects_count, bytes);
}
Ok(())
}
}
crates/heal/src/heal/task/heal_erasure_set.rs
+7 -9
View File
@@ -32,7 +32,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.set_current_object(Some(format!("erasure_set: {} ({} buckets)", set_disk_id, buckets.len())));
progress.update_progress(0, 4, 0, 0);
progress.update_stage(0, 4);
}
let is_auto_replacement = matches!(self.source, HealRequestSource::AutoHeal) && !self.heal_endpoints.is_empty();
@@ -248,7 +248,7 @@ impl HealTask {
);
{
let mut progress = self.progress.write().await;
progress.update_progress(4, 4, 0, 0);
progress.update_stage(4, 4);
}
return Err(Error::TaskExecutionFailed {
message: format!("Failed to heal disk format for {set_disk_id}: {error}"),
@@ -304,7 +304,7 @@ impl HealTask {
);
{
let mut progress = self.progress.write().await;
progress.update_progress(4, 4, 0, 0);
progress.update_stage(4, 4);
}
return Err(Error::TaskExecutionFailed {
message: format!("Failed to heal disk format for {set_disk_id}: {e}"),
@@ -314,7 +314,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.update_progress(1, 4, 0, 0);
progress.update_stage(1, 4);
}
// The rebuilt disks are formatted now: mark them as healing so
@@ -343,7 +343,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.update_progress(2, 4, 0, 0);
progress.update_stage(2, 4);
}
// Step 3: Heal bucket structure
@@ -427,7 +427,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.update_progress(3, 4, 0, 0);
progress.update_stage(3, 4);
}
// Step 4: Execute erasure set heal with resume
@@ -470,9 +470,7 @@ impl HealTask {
};
{
let mut progress = self.progress.write().await;
let bytes_processed = progress.bytes_processed;
progress.update_progress(4, 4, 0, bytes_processed);
self.progress.write().await.update_stage(4, 4);
}
match result {
crates/heal/src/heal/task/heal_metadata.rs
+10 -10
View File
@@ -32,7 +32,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.set_current_object(Some(format!("metadata: {bucket}/{object}")));
progress.update_progress(0, 3, 0, 0);
progress.update_stage(0, 3);
}
// Step 1: Check if object exists
@@ -74,7 +74,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.update_progress(1, 3, 0, 0);
progress.update_stage(1, 3);
}
// Step 2: Perform metadata heal using ecstore
@@ -122,7 +122,7 @@ impl HealTask {
);
{
let mut progress = self.progress.write().await;
progress.update_progress(3, 3, 0, 0);
progress.update_stage(3, 3);
}
return Err(Error::TaskExecutionFailed {
message: format!("Failed to heal metadata {bucket}/{object}: {e}"),
@@ -145,7 +145,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.update_progress(3, 3, 0, 0);
progress.update_stage(3, 3);
}
self.record_result_item(result).await;
Ok(())
@@ -167,7 +167,7 @@ impl HealTask {
);
{
let mut progress = self.progress.write().await;
progress.update_progress(3, 3, 0, 0);
progress.update_stage(3, 3);
}
Err(Error::TaskExecutionFailed {
message: format!("Failed to heal metadata {bucket}/{object}: {e}"),
@@ -194,7 +194,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.set_current_object(Some(format!("ec_decode: {bucket}/{object}")));
progress.update_progress(0, 3, 0, 0);
progress.update_stage(0, 3);
}
// Step 1: Check if object exists
@@ -236,7 +236,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.update_progress(1, 3, 0, 0);
progress.update_stage(1, 3);
}
// Step 2: Perform EC decode heal using ecstore
@@ -284,7 +284,7 @@ impl HealTask {
);
{
let mut progress = self.progress.write().await;
progress.update_progress(3, 3, 0, 0);
progress.update_stage(3, 3);
}
return Err(Error::TaskExecutionFailed {
message: format!("Failed to heal EC decode {bucket}/{object}: {e}"),
@@ -309,7 +309,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.update_progress(3, 3, 0, object_size);
progress.update_object_progress(1, 1, 0, 0, object_size);
}
self.record_result_item(result).await;
Ok(())
@@ -331,7 +331,7 @@ impl HealTask {
);
{
let mut progress = self.progress.write().await;
progress.update_progress(3, 3, 0, 0);
progress.update_stage(3, 3);
}
Err(Error::TaskExecutionFailed {
message: format!("Failed to heal EC decode {bucket}/{object}: {e}"),
crates/heal/src/heal/task/heal_object.rs
+8 -8
View File
@@ -36,7 +36,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.set_current_object(Some(format!("{bucket}/{object}")));
progress.update_progress(0, 4, 0, 0);
progress.update_stage(0, 4);
}
// Step 1: Check if object exists and get metadata
@@ -132,7 +132,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.update_progress(1, 3, 0, 0);
progress.update_stage(1, 3);
}
// Step 2: directly call ecstore to perform heal
@@ -187,7 +187,7 @@ impl HealTask {
);
{
let mut progress = self.progress.write().await;
progress.update_progress(3, 3, 0, 0);
progress.update_stage(3, 3);
}
return Ok(());
}
@@ -207,7 +207,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.update_progress(3, 3, 0, 0);
progress.update_stage(3, 3);
}
if Self::should_return_typed_heal_error(&e) {
@@ -249,7 +249,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.update_progress(3, 3, 0, object_size);
progress.update_object_progress(1, 1, 0, 0, object_size);
}
self.record_result_item(result).await;
Ok(())
@@ -275,7 +275,7 @@ impl HealTask {
);
{
let mut progress = self.progress.write().await;
progress.update_progress(3, 3, 0, 0);
progress.update_stage(3, 3);
}
return Ok(());
}
@@ -295,7 +295,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.update_progress(3, 3, 0, 0);
progress.update_stage(3, 3);
}
if Self::should_return_typed_heal_error(&e) {
@@ -414,7 +414,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.update_progress(4, 4, 0, object_size);
progress.update_object_progress(1, 1, 0, 0, object_size);
}
self.record_result_item(result).await;
Ok(())
crates/heal/src/heal/task/tests.rs
+96
View File
@@ -22,6 +22,7 @@ use std::sync::Mutex;
use tempfile::TempDir;
use super::super::storage_api::status::BucketInfo;
use crate::heal::progress::{HealProgressState, aggregate_heal_progress};
#[tokio::test]
async fn retry_request_carries_remaining_timeout_budget() {
@@ -2124,6 +2125,7 @@ async fn erasure_set_heal_applies_usage_baseline_to_progress() {
usage_baseline: Mutex::new(Some(HealBucketUsageBaseline {
objects_count: 10,
bytes: 8,
generation: Some(1),
})),
..Default::default()
});
@@ -2147,10 +2149,104 @@ async fn erasure_set_heal_applies_usage_baseline_to_progress() {
let progress = task.get_progress().await;
assert_eq!(progress.objects_total_count, 10);
assert_eq!(progress.objects_total_size, 8);
assert!(progress.baseline_generation.is_some());
assert!(progress.baseline_known);
assert_eq!(progress.bytes_processed, 2);
assert!((progress.progress_percentage - 25.0).abs() < 0.001);
}
#[tokio::test]
async fn erasure_sets_from_one_usage_snapshot_share_baseline_generation() {
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage {
usage_baseline: Mutex::new(Some(HealBucketUsageBaseline {
objects_count: 10,
bytes: 8,
generation: Some(7),
})),
..Default::default()
});
let buckets = vec!["bucket-a".to_string()];
let task_for_set = |set_disk_id: &str| {
HealTask::from_request(
HealRequest::new(
HealType::ErasureSet {
buckets: buckets.clone(),
set_disk_id: set_disk_id.to_string(),
},
HealOptions::default(),
HealPriority::Normal,
),
storage.clone(),
)
};
let first = task_for_set("pool_0_set_0");
let second = task_for_set("pool_0_set_1");
first
.apply_erasure_set_usage_baseline(&buckets)
.await
.expect("first baseline");
second
.apply_erasure_set_usage_baseline(&buckets)
.await
.expect("second baseline");
first.progress.write().await.update_object_progress(0, 0, 0, 0, 0);
second.progress.write().await.update_object_progress(0, 0, 0, 0, 0);
let first = first.get_progress().await;
let second = second.get_progress().await;
let expected_generation = first.baseline_generation;
assert!(expected_generation.is_some());
assert_eq!(second.baseline_generation, expected_generation);
let aggregate = aggregate_heal_progress([first, second]).expect("aggregate progress");
assert!(aggregate.baseline_known);
assert_eq!(aggregate.baseline_generation, expected_generation);
assert_eq!(aggregate.progress_state, HealProgressState::Running);
}
#[tokio::test]
async fn erasure_set_disk_walk_keeps_cluster_usage_baseline_indeterminate() {
for (scan_mode, source) in [
(HealScanMode::Deep, HealRequestSource::Admin),
(HealScanMode::Normal, HealRequestSource::AutoHeal),
] {
let temp = TempDir::new().expect("temporary directory should be created");
let disk = make_resume_disk(&temp).await;
let storage = Arc::new(MockStorage {
resume_disk: Mutex::new(Some(disk)),
usage_baseline: Mutex::new(Some(HealBucketUsageBaseline {
objects_count: 10,
bytes: 8,
generation: Some(1),
})),
..Default::default()
});
let mut request = HealRequest::new(
HealType::ErasureSet {
buckets: vec!["bucket-a".to_string()],
set_disk_id: "pool_0_set_0".to_string(),
},
HealOptions {
scan_mode,
timeout: None,
..Default::default()
},
HealPriority::Normal,
);
request.source = source;
let task = HealTask::from_request(request, storage);
task.heal_erasure_set(vec!["bucket-a".to_string()], "pool_0_set_0".to_string())
.await
.expect("erasure set heal should complete");
let progress = task.get_progress().await;
assert!(!progress.baseline_known);
assert_eq!(progress.baseline_generation, None);
assert_eq!(progress.progress_state, HealProgressState::Indeterminate);
}
}
#[tokio::test]
async fn erasure_set_heal_ignores_usage_baseline_errors() {
let temp = TempDir::new().expect("temporary directory should be created");
crates/heal/src/lib.rs
+1 -1
View File
@@ -19,7 +19,7 @@ pub use error::{Error, Result};
pub use heal::{
HealManager, HealOperationsSnapshot, HealOptions, HealPriority, HealPriorityCounts, HealRequest, HealSourceCounts, HealType,
channel::HealChannelProcessor,
progress::HealProgress,
progress::{HealProgress, aggregate_heal_progress},
resume::{ReplacementRecoveryRecord, ReplacementRecoveryState, ResumeUtils},
};
use rustfs_concurrency::WorkloadAdmissionSnapshotProvider;
crates/heal/tests/mrf_pipeline_test.rs
+72 -2
View File
@@ -35,6 +35,7 @@ use storage_api::endpoint_index::{Endpoint, EndpointServerPools, Endpoints, Pool
const META_BUCKET: &str = ".rustfs.sys";
const JOURNAL_REL: &str = "buckets/.heal/mrf/journal.bin";
const SCOPED_JOURNAL_REL: &str = "buckets/.heal/mrf/journal-scoped.bin";
async fn heal_env() -> (Vec<std::path::PathBuf>, Arc<dyn HealStorageAPI>) {
let env = rustfs_test_utils::TestECStoreEnv::builder()
@@ -79,14 +80,18 @@ fn journal_record(kind: u8, bucket: &str, object: &str, version: Option<[u8; 16]
body
}
fn write_journal_to_disks(disk_paths: &[std::path::PathBuf], data: &[u8]) {
fn write_journal_path_to_disks(disk_paths: &[std::path::PathBuf], relative_path: &str, data: &[u8]) {
for path in disk_paths {
let journal = path.join(META_BUCKET).join(JOURNAL_REL);
let journal = path.join(META_BUCKET).join(relative_path);
std::fs::create_dir_all(journal.parent().expect("journal parent")).expect("create journal dir");
std::fs::write(&journal, data).expect("write journal fixture");
}
}
fn write_journal_to_disks(disk_paths: &[std::path::PathBuf], data: &[u8]) {
write_journal_path_to_disks(disk_paths, JOURNAL_REL, data);
}
async fn wait_until<F, Fut>(deadline: Duration, mut probe: F) -> bool
where
F: FnMut() -> Fut,
@@ -187,8 +192,73 @@ async fn journal_replay_arms_intents_and_deletes_the_file() {
.all(|path| !Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()),
"the journal file must be removed after a successful replay"
);
assert!(
disk_paths
.iter()
.all(|path| !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()),
"the authoritative journal file must also be removed after replay"
);
let snapshot = manager.operations_snapshot().await;
assert_eq!(snapshot.queued_by_priority.urgent, 1, "the decode-failure record must replay as Urgent");
assert!(snapshot.queued_by_priority.normal >= 1, "the partial-write record must replay as Normal");
}
/// A canonical snapshot and its compatibility mirror may differ after a
/// partial flush. Replay must choose the complete canonical epoch instead of
/// combining records that never coexisted in memory.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn authoritative_journal_is_not_merged_with_legacy_mirror() {
let (disk_paths, storage) = heal_env().await;
let mut endpoints: Vec<Endpoint> = disk_paths
.iter()
.map(|p| Endpoint::try_from(p.to_string_lossy().as_ref()).expect("endpoint from disk path"))
.collect();
for (i, endpoint) in endpoints.iter_mut().enumerate() {
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(i);
}
let pool = PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: endpoints.len(),
endpoints: Endpoints::from(endpoints),
cmd_line: "mrf-authoritative-test".to_string(),
platform: String::new(),
};
init_local_disks(EndpointServerPools::from(vec![pool]))
.await
.expect("local disks should register");
let authoritative = journal_record(1, "authoritative-bucket", "authoritative-object", None, 0);
let legacy = journal_record(1, "legacy-bucket", "legacy-object", None, 0);
write_journal_path_to_disks(&disk_paths, SCOPED_JOURNAL_REL, &authoritative);
write_journal_path_to_disks(&disk_paths, JOURNAL_REL, &legacy);
let manager = make_manager(storage);
let replayed = mrf_queue::replay_journal_once(&manager).await;
assert_eq!(replayed, 1, "only the authoritative snapshot epoch may replay");
let snapshot = manager.operations_snapshot().await;
assert_eq!(snapshot.queued_by_source.mrf, 1);
assert!(
disk_paths.iter().all(|path| {
!Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
&& !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
}),
"replay cleanup must remove both journal paths"
);
// A scoped-only snapshot is valid during a rollout where no legacy
// compatibility mirror was written. Missing legacy files must not leave
// the runtime in a permanent cleanup-retry state.
let scoped_only = journal_record(1, "scoped-only-bucket", "scoped-only-object", None, 0);
write_journal_path_to_disks(&disk_paths, SCOPED_JOURNAL_REL, &scoped_only);
assert_eq!(mrf_queue::replay_journal_once(&manager).await, 1);
assert!(disk_paths.iter().all(|path| {
!Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
&& !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
}));
}
crates/io-metrics/src/internode_metrics.rs
+4
View File
@@ -61,6 +61,8 @@ pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_JSON_ENCODE: &str = "batch
pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MSGPACK_ENCODE: &str = "batch_read_version_response_msgpack_encode";
pub const INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP: &str = "batch_read_version_rpc_roundtrip";
pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE: &str = "batch_read_version_response_decode";
pub const INTERNODE_STAGE_BATCH_READ_VERSION_COALESCER_WAIT: &str = "batch_read_version_coalescer_wait";
pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MAP: &str = "batch_read_version_response_map";
const OPERATION_LABEL: &str = "operation";
const BACKEND_LABEL: &str = "backend";
@@ -1431,6 +1433,8 @@ mod tests {
);
assert_eq!(INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP, "read_version_rpc_roundtrip");
assert_eq!(INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE, "read_version_response_decode");
assert_eq!(INTERNODE_STAGE_BATCH_READ_VERSION_COALESCER_WAIT, "batch_read_version_coalescer_wait");
assert_eq!(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MAP, "batch_read_version_response_map");
assert_eq!(
INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL,
"rustfs_system_network_internode_signature_v1_fallback_total"
crates/lifecycle/src/core.rs
+33 -4
View File
@@ -54,6 +54,7 @@ const ERR_LIFECYCLE_INVALID_EXPIRED_OBJECT_ALL_VERSIONS: &str =
"Days must be a positive integer and Date must not be specified inside Expiration with ExpiredObjectAllVersions";
const ERR_LIFECYCLE_INVALID_DEL_MARKER_EXPIRATION_DAYS: &str = "Days must be a positive integer with DelMarkerExpiration";
const ERR_LIFECYCLE_INVALID_RULE_ID_TOO_LONG: &str = "Rule ID must be at most 255 characters";
const ERR_LIFECYCLE_INVALID_RULE_ID_EMPTY: &str = "Rule ID must not be empty";
const ERR_LIFECYCLE_INVALID_RULE_STATUS: &str = "Rule status must be either Enabled or Disabled";
const ERR_LIFECYCLE_DEL_MARKER_WITH_TAGS: &str = "Rule with DelMarkerExpiration cannot have tags based filtering";
const ERR_LIFECYCLE_EXPIRED_OBJECT_DELETE_MARKER_WITH_TAGS: &str =
@@ -402,10 +403,13 @@ impl Lifecycle for BucketLifecycleConfiguration {
NoncurrentVersionTransitionOps::validate(transition)?;
}
}
if let Some(id) = &r.id
&& id.len() > 255
{
return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_RULE_ID_TOO_LONG));
if let Some(id) = &r.id {
if id.is_empty() {
return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_RULE_ID_EMPTY));
}
if id.len() > 255 {
return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_RULE_ID_TOO_LONG));
}
}
r.validate()?;
if let Some(object_lock_enabled) = lr.object_lock_enabled.as_ref()
@@ -3730,6 +3734,31 @@ mod tests {
.expect("empty prefix with filter should be valid");
}
#[tokio::test]
async fn validate_rejects_empty_rule_id() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
days: Some(30),
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some(String::new()),
noncurrent_version_expiration: None,
noncurrent_version_transitions: None,
prefix: None,
transitions: None,
}],
};
let error = lc.validate(&ObjectLockConfiguration::default()).await.unwrap_err();
assert_eq!(error.to_string(), ERR_LIFECYCLE_INVALID_RULE_ID_EMPTY);
}
// --- TASK-004 tests: ExpiredObjectAllVersions ---
#[tokio::test]
crates/madmin/src/metrics.rs
+75
View File
@@ -302,10 +302,28 @@ pub struct ScannerUsageFreshnessSnapshot {
pub last_usage_save_result: String,
#[serde(rename = "last_usage_save_result_code", default)]
pub last_usage_save_result_code: u64,
#[serde(rename = "last_durable_success_unix_secs", default)]
pub last_durable_success_unix_secs: u64,
#[serde(rename = "last_publication_unix_secs", default)]
pub last_publication_unix_secs: u64,
#[serde(rename = "last_publication_state", default)]
pub last_publication_state: String,
#[serde(rename = "last_publication_reason", default)]
pub last_publication_reason: String,
#[serde(rename = "deferred_pending", default)]
pub deferred_pending: bool,
#[serde(rename = "deferred_total", default)]
pub deferred_total: u64,
#[serde(rename = "last_deferred_unix_secs", default)]
pub last_deferred_unix_secs: u64,
#[serde(rename = "last_deferred_reason", default)]
pub last_deferred_reason: String,
}
impl ScannerUsageFreshnessSnapshot {
fn merge(&mut self, other: &Self) {
let self_deferred_state_at = self.last_deferred_unix_secs.max(self.last_durable_success_unix_secs);
let other_deferred_state_at = other.last_deferred_unix_secs.max(other.last_durable_success_unix_secs);
self.dirty_pending_buckets = self.dirty_pending_buckets.saturating_add(other.dirty_pending_buckets);
self.last_dirty_mark_unix_secs = self.last_dirty_mark_unix_secs.max(other.last_dirty_mark_unix_secs);
self.last_dirty_clear_unix_secs = self.last_dirty_clear_unix_secs.max(other.last_dirty_clear_unix_secs);
@@ -318,6 +336,22 @@ impl ScannerUsageFreshnessSnapshot {
self.last_usage_save_result = other.last_usage_save_result.clone();
self.last_usage_save_result_code = other.last_usage_save_result_code;
}
self.last_durable_success_unix_secs = self.last_durable_success_unix_secs.max(other.last_durable_success_unix_secs);
if other.last_publication_unix_secs > self.last_publication_unix_secs {
self.last_publication_unix_secs = other.last_publication_unix_secs;
self.last_publication_state = other.last_publication_state.clone();
self.last_publication_reason = other.last_publication_reason.clone();
}
self.deferred_total = self.deferred_total.saturating_add(other.deferred_total);
if other_deferred_state_at > self_deferred_state_at {
self.deferred_pending = other.deferred_pending;
} else if other_deferred_state_at == self_deferred_state_at {
self.deferred_pending |= other.deferred_pending;
}
if other.last_deferred_unix_secs > self.last_deferred_unix_secs {
self.last_deferred_unix_secs = other.last_deferred_unix_secs;
self.last_deferred_reason = other.last_deferred_reason.clone();
}
}
}
@@ -1566,6 +1600,47 @@ mod tests {
assert_eq!(explicit_false.current_cycle_active, Some(false));
}
#[test]
fn usage_freshness_deferred_fields_are_backward_compatible_and_merge() {
let legacy: ScannerUsageFreshnessSnapshot = serde_json::from_value(serde_json::json!({
"dirty_pending_buckets": 2,
"last_usage_save_result": "success"
}))
.expect("legacy usage freshness should decode");
assert!(!legacy.deferred_pending);
assert_eq!(legacy.deferred_total, 0);
let mut merged = ScannerUsageFreshnessSnapshot::default();
merged.merge(&ScannerUsageFreshnessSnapshot {
deferred_pending: true,
deferred_total: 2,
last_deferred_unix_secs: 20,
last_deferred_reason: "data_movement".to_string(),
..Default::default()
});
merged.merge(&ScannerUsageFreshnessSnapshot {
deferred_total: 1,
last_deferred_unix_secs: 10,
last_deferred_reason: "older".to_string(),
..Default::default()
});
assert!(merged.deferred_pending);
assert_eq!(merged.deferred_total, 3);
assert_eq!(merged.last_deferred_unix_secs, 20);
assert_eq!(merged.last_deferred_reason, "data_movement");
merged.merge(&ScannerUsageFreshnessSnapshot {
deferred_pending: false,
last_durable_success_unix_secs: 30,
last_publication_unix_secs: 30,
last_publication_state: "success".to_string(),
..Default::default()
});
assert!(!merged.deferred_pending);
assert_eq!(merged.last_durable_success_unix_secs, 30);
assert_eq!(merged.last_publication_state, "success");
}
#[test]
fn scanner_metrics_merge_prefers_an_active_first_cycle() {
let collected_at = Timestamp::now();
crates/madmin/src/site_replication.rs
+6
View File
@@ -431,6 +431,12 @@ pub struct SRBucketMeta {
pub cors: Option<String>,
#[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")]
pub api_version: Option<String>,
/// Set by a sender that merges replication configs under the derived
/// site-rule contract (operator rule priorities verbatim, `site-repl-*`
/// ids classified by id/ARN). A receiver merges a payload without it the
/// pre-contract way; a pre-contract receiver ignores the field.
#[serde(rename = "derivedRuleContract", default, skip_serializing_if = "std::ops::Not::not")]
pub derived_rule_contract: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]

Some files were not shown because too many files have changed in this diff Show More