Merge branch 'main' into test/distributed-e2e-hardening

This commit is contained in:
Zhengchao An
2026-09-05 22:21:31 +08:00
committed by GitHub
190 changed files with 23423 additions and 3611 deletions
+1 -1
View File
@@ -1,2 +1,2 @@
sha256-darwin=a881fd7d3f5cb94654221ca85b8b30cce1b95e608824a55a15339cbc294e6d34 sha256-darwin=a881fd7d3f5cb94654221ca85b8b30cce1b95e608824a55a15339cbc294e6d34
sha256-linux=e9a8d64e73f627c4d26c236dbbba690c9ee03a9e26d42a4244515b4439365535 sha256-linux=a2933d83dfe74ffa03410a0959333a1c48288b8469ca9f17273d449d7510c24b
+72
View File
@@ -0,0 +1,72 @@
{
"lane": "ci/test-and-lint",
"tests": [
{
"invariant": "write-quorum",
"suite": "rustfs-ecstore",
"name": "set_disk::ops::object::inline_put_commit_path_tests::inline_put_direct_commit_accepts_exact_quorum_and_rejects_quorum_minus_one"
},
{
"invariant": "metadata-rollback",
"suite": "rustfs-ecstore",
"name": "set_disk::core::io_primitives::tests::write_unique_file_info_reverts_metadata_when_write_quorum_fails"
},
{
"invariant": "stale-writer",
"suite": "rustfs-ecstore",
"name": "set_disk::ops::object::put_object_tmp_cleanup_tests::put_object_no_lock_aborts_after_outer_namespace_lock_loss"
},
{
"invariant": "range-body",
"suite": "rustfs-ecstore",
"name": "set_disk::ops::object::transition_upload_integrity_tests::transitioned_compressed_object_range_get_returns_plaintext_slice"
},
{
"invariant": "multipart-cancellation",
"suite": "rustfs-ecstore",
"name": "set_disk::ops::multipart::tests::cancelled_complete_keeps_upload_lock_through_tail_cleanup"
},
{
"invariant": "list-uncommitted-version",
"suite": "rustfs-filemeta",
"name": "metacache::tests::resolve_with_write_quorum_slack_keeps_partial_latest_hidden_during_merge"
},
{
"invariant": "minio-object-fixture",
"suite": "rustfs-filemeta",
"name": "filemeta::test::parses_real_minio_object_xlmeta"
},
{
"invariant": "corrupt-part-arrays",
"suite": "rustfs-filemeta",
"name": "filemeta::test::crc_valid_but_part_arrays_corrupt_into_fileinfo_errors_not_panics"
}
],
"fixtures": [
{
"path": "crates/filemeta/tests/fixtures/minio/object_large_bin.xlmeta.hex",
"sha256": "e8093767806d701e639b48d023190e858fbc4cde69bcfd83c22af8cba8452ce5",
"source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
},
{
"path": "crates/filemeta/tests/fixtures/minio/object_small_txt.xlmeta.hex",
"sha256": "2a415ad3a3be5a9440035d4026ff880e0e8c1ec1701be9f4e077734e8dce03da",
"source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
},
{
"path": "crates/filemeta/tests/fixtures/minio/object_versioned_txt.xlmeta.hex",
"sha256": "7f21f50c326dd8b0228deb6dbdb7052b3d0a3f8ee6c85d43486f0e6bb7a97261",
"source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
},
{
"path": "crates/ecstore/tests/fixtures/minio/bucket_metadata.blob.hex",
"sha256": "f2b6e260aff106adf6039feb1c645686e84e75404ff725491fb18668be5db203",
"source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
},
{
"path": "crates/ecstore/tests/fixtures/minio/bucket_metadata_full.xlmeta.hex",
"sha256": "3b6de589519c08a1614c8bd409bb8199c17d42043861b07bce513075e6fbfc12",
"source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
}
]
}
+3 -2
View File
@@ -3,9 +3,10 @@
.NOTPARALLEL: pre-commit pre-pr dev-check .NOTPARALLEL: pre-commit pre-pr dev-check
.PHONY: setup-hooks .PHONY: setup-hooks
setup-hooks: ## Set up git hooks setup-hooks: ## Install the configured pre-commit hooks
@echo "🔧 Setting up git hooks..." @echo "🔧 Setting up git hooks..."
chmod +x .git/hooks/pre-commit pre-commit validate-config
pre-commit install
@echo "✅ Git hooks setup complete!" @echo "✅ Git hooks setup complete!"
.PHONY: doc-paths-check .PHONY: doc-paths-check
+1
View File
@@ -40,6 +40,7 @@ script-tests: ## Run shell script tests
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test $(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test $(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test $(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/test_security_workflow.py
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py $(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
bash -n ./scripts/validate_object_data_cache_cold_stampede.sh bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test $(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test
+115
View File
@@ -0,0 +1,115 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
name: Quick Checks
description: Run the shared compile-free RustFS quality checks.
runs:
using: composite
steps:
- name: Install quality tools
uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
with:
tool: |
ripgrep@15.2.0
shellcheck@0.11.0
- name: Install actionlint
shell: bash
run: |
actionlint_dir="$(mktemp -d "${RUNNER_TEMP}/actionlint.XXXXXX")"
curl --fail --location --silent --show-error \
--output "$actionlint_dir/actionlint.tar.gz" \
https://github.com/rhysd/actionlint/releases/download/v1.7.12/actionlint_1.7.12_linux_amd64.tar.gz
echo "8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8 $actionlint_dir/actionlint.tar.gz" | sha256sum --check --status
tar -xzf "$actionlint_dir/actionlint.tar.gz" -C "$actionlint_dir" actionlint
rm "$actionlint_dir/actionlint.tar.gz"
echo "$actionlint_dir" >> "$GITHUB_PATH"
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
with:
components: rustfmt
- name: Check workflow syntax and shell scripts
shell: bash
run: shellcheck --version && actionlint
- name: Check code formatting
shell: bash
run: cargo fmt --all --check
- name: Check unsafe code allowances
shell: bash
run: ./scripts/check_unsafe_code_allowances.sh
- name: Check layered dependencies
shell: bash
run: ./scripts/check_layer_dependencies.sh
- name: Check architecture migration rules
shell: bash
run: ./scripts/check_architecture_migration_rules.sh
- name: Check logging guardrails
shell: bash
run: ./scripts/check_logging_guardrails.sh
- name: Check error other(format!) ratchet
shell: bash
run: ./scripts/check_error_other_format_ratchet.sh
- name: Check tokio io-uring feature guard
shell: bash
run: ./scripts/check_no_tokio_io_uring.sh
- name: Check extension schema boundaries
shell: bash
run: ./scripts/check_extension_schema_boundaries.sh
- name: Check body-cache whitelist guard
shell: bash
run: ./scripts/check_body_cache_whitelist.sh
- name: Check s3s footprint ratchet
shell: bash
run: ./scripts/check_s3s_footprint.sh
- name: Check cryptographic capability wording
shell: bash
run: ./scripts/check_fips_wording.sh
- name: Check no embedded secret material
shell: bash
run: ./scripts/check_embedded_secrets.sh
- name: Check test wiring
shell: bash
run: |
python3 ./scripts/check_test_wiring.py --self-test
python3 ./scripts/check_scheduled_validation_freshness.py --self-test
python3 ./scripts/test_security_workflow.py
python3 ./scripts/check_test_wiring.py
- name: Check no planning docs committed
shell: bash
run: ./scripts/check_no_planning_docs.sh
- name: Check CI paths stay in sync
shell: bash
run: ./scripts/check_ci_paths_sync.sh
- name: Check io_uring lane --lib precondition
shell: bash
run: ./scripts/check_uring_lane_lib_only.sh
+5 -5
View File
@@ -10,16 +10,16 @@ Use N/A when there is no related issue.
## Summary of Changes ## Summary of Changes
<!-- <!--
Briefly explain what changed and why reviewers should accept it. Describe the concrete problem and resulting behavior. For a behavior change, name the input or state that triggers it and the expected outcome. Explain any new dependency or abstraction that the change needs.
Focus on behavior, compatibility, and review-relevant context.
--> -->
## Verification ## Verification
<!-- <!--
List the commands or checks you ran, for example: Give 13 concrete pieces of evidence for the changed behavior: the test or command, its observed result, and the regression it catches. For a bug fix, record a failing-before/passing-after check or explain why it was unavailable.
- `make pre-commit`
Use N/A only when verification is not applicable. Identify the tested commit and any local changes. When testing a prebuilt binary or external service, include its source/version and artifact identity; a successful run against a different build is not evidence for this change.
List relevant checks not run and the remaining risk. Use the validation tier in AGENTS.md; do not run broader checks solely to fill this section. For documentation-only changes, list the applicable documentation checks. Use N/A only when verification is not applicable.
--> -->
## Impact ## Impact
+6 -88
View File
@@ -12,24 +12,10 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# Companion to ci.yml for required status checks. # Reports the existing required checks for paths excluded by ci.yml.
# # Mixed PRs can trigger both workflows; their Quick Checks jobs use one shared
# ci.yml skips docs-only pull requests via paths-ignore, but the branch ruleset # action to keep validation coverage aligned. Keep this paths list in sync with
# requires a check named "Test and Lint" — without this workflow a docs-only PR # ci.yml's pull_request.paths-ignore via scripts/check_ci_paths_sync.sh.
# would wait on it forever. This workflow triggers on exactly the paths ci.yml
# ignores and reports success under the same job name. Mixed PRs trigger both
# workflows and the real check still gates: a required check with any failing
# run blocks the merge.
# https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/defining-the-mergeability-of-pull-requests/troubleshooting-required-status-checks#handling-skipped-but-required-checks
#
# "Quick Checks" is mirrored here ahead of the ruleset change that will make it
# required too (rustfs/backlog#1599). Until that change lands this job is
# inert; mirroring it first is what lets the ruleset change happen without
# stranding docs-only PRs on a check nobody reports.
#
# Keep the paths list below in sync with the pull_request paths-ignore list
# in ci.yml, and keep the quick-checks steps below byte-identical to the
# quick-checks job in ci.yml.
name: Continuous Integration (docs only) name: Continuous Integration (docs only)
@@ -59,19 +45,6 @@ permissions:
contents: read contents: read
jobs: jobs:
# Deliberately NOT a bare `echo`. Once "Quick Checks" becomes a required
# check, ci.yml gates every expensive job behind it, so a mixed PR reports
# two check runs with this name: the real one (45-51s) and this companion.
# GitHub has no written contract for how it picks between same-named
# required check runs ("latest wins" vs "any failure blocks"), so instead of
# relying on ordering we make both runs execute the same commands against
# the same merge ref — their conclusions are then necessarily identical and
# the choice does not matter. Keep these steps byte-identical to the
# quick-checks job in ci.yml (a guard script that asserts this, and the paths
# sync below, is tracked in rustfs/backlog#1603).
#
# For a genuinely docs-only PR this adds no strictness (no code changed, so
# fmt and the guards always pass) and costs ~50s of ubuntu-latest.
quick-checks: quick-checks:
name: Quick Checks name: Quick Checks
runs-on: ubuntu-latest runs-on: ubuntu-latest
@@ -82,63 +55,8 @@ jobs:
with: with:
persist-credentials: false persist-credentials: false
- name: Install ripgrep - name: Run shared quick checks
uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2 uses: ./.github/actions/quick-checks
with:
tool: ripgrep@15.2.0
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
with:
components: rustfmt
- name: Check code formatting
run: cargo fmt --all --check
- name: Check unsafe code allowances
run: ./scripts/check_unsafe_code_allowances.sh
- name: Check layered dependencies
run: ./scripts/check_layer_dependencies.sh
- name: Check architecture migration rules
run: ./scripts/check_architecture_migration_rules.sh
- name: Check logging guardrails
run: ./scripts/check_logging_guardrails.sh
- name: Check tokio io-uring feature guard
run: ./scripts/check_no_tokio_io_uring.sh
- name: Check extension schema boundaries
run: ./scripts/check_extension_schema_boundaries.sh
- name: Check body-cache whitelist guard
run: ./scripts/check_body_cache_whitelist.sh
- name: Check s3s footprint ratchet
run: ./scripts/check_s3s_footprint.sh
- name: Check cryptographic capability wording
run: ./scripts/check_fips_wording.sh
- name: Check no embedded secret material
run: ./scripts/check_embedded_secrets.sh
- name: Check test wiring
run: |
python3 ./scripts/check_test_wiring.py --self-test
python3 ./scripts/check_scheduled_validation_freshness.py --self-test
python3 ./scripts/check_test_wiring.py
- name: Check no planning docs committed
run: ./scripts/check_no_planning_docs.sh
- name: Check CI paths stay in sync
run: ./scripts/check_ci_paths_sync.sh
- name: Check io_uring lane --lib precondition
run: ./scripts/check_uring_lane_lib_only.sh
test-and-lint: test-and-lint:
name: Test and Lint name: Test and Lint
+10 -66
View File
@@ -100,12 +100,7 @@ jobs:
- name: Typos check with custom config file - name: Typos check with custom config file
uses: crate-ci/typos@37bb98842b0d8c4ffebdb75301a13db0267cef89 # master uses: crate-ci/typos@37bb98842b0d8c4ffebdb75301a13db0267cef89 # master
# Fast, compile-free checks that fail early so contributors get feedback in # Fail early with compile-free checks shared with docs-only CI.
# ~1 minute instead of waiting for the full test job.
#
# These steps are mirrored byte-for-byte in ci-docs-only.yml so that a mixed
# PR, which reports two check runs named "Quick Checks", cannot get one red
# and one green. Edit both jobs together.
quick-checks: quick-checks:
name: Quick Checks name: Quick Checks
if: github.event_name != 'pull_request' || github.event.action != 'closed' if: github.event_name != 'pull_request' || github.event.action != 'closed'
@@ -117,66 +112,8 @@ jobs:
with: with:
persist-credentials: false persist-credentials: false
- name: Install ripgrep - name: Run shared quick checks
uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2 uses: ./.github/actions/quick-checks
with:
tool: ripgrep@15.2.0
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
with:
components: rustfmt
- name: Check code formatting
run: cargo fmt --all --check
- name: Check unsafe code allowances
run: ./scripts/check_unsafe_code_allowances.sh
- name: Check layered dependencies
run: ./scripts/check_layer_dependencies.sh
- name: Check architecture migration rules
run: ./scripts/check_architecture_migration_rules.sh
- name: Check logging guardrails
run: ./scripts/check_logging_guardrails.sh
- name: Check error other(format!) ratchet
run: ./scripts/check_error_other_format_ratchet.sh
- name: Check tokio io-uring feature guard
run: ./scripts/check_no_tokio_io_uring.sh
- name: Check extension schema boundaries
run: ./scripts/check_extension_schema_boundaries.sh
- name: Check body-cache whitelist guard
run: ./scripts/check_body_cache_whitelist.sh
- name: Check s3s footprint ratchet
run: ./scripts/check_s3s_footprint.sh
- name: Check cryptographic capability wording
run: ./scripts/check_fips_wording.sh
- name: Check no embedded secret material
run: ./scripts/check_embedded_secrets.sh
- name: Check test wiring
run: |
python3 ./scripts/check_test_wiring.py --self-test
python3 ./scripts/check_scheduled_validation_freshness.py --self-test
python3 ./scripts/check_test_wiring.py
- name: Check no planning docs committed
run: ./scripts/check_no_planning_docs.sh
- name: Check CI paths stay in sync
run: ./scripts/check_ci_paths_sync.sh
- name: Check io_uring lane --lib precondition
run: ./scripts/check_uring_lane_lib_only.sh
test-and-lint: test-and-lint:
name: Test and Lint name: Test and Lint
@@ -269,6 +206,7 @@ jobs:
CARGO_BUILD_JOBS: ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && '3' || '2' }} CARGO_BUILD_JOBS: ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && '3' || '2' }}
run: | run: |
mkdir -p artifacts/test-and-lint mkdir -p artifacts/test-and-lint
rm -f target/nextest/ci/junit.xml
./scripts/ci/resource_sampler.sh start nextest ./scripts/ci/resource_sampler.sh start nextest
trap './scripts/ci/resource_sampler.sh stop' EXIT trap './scripts/ci/resource_sampler.sh stop' EXIT
set +e set +e
@@ -277,6 +215,12 @@ jobs:
--status-level all --final-status-level all \ --status-level all --final-status-level all \
2>&1 | tee artifacts/test-and-lint/nextest.log 2>&1 | tee artifacts/test-and-lint/nextest.log
status=${PIPESTATUS[0]} status=${PIPESTATUS[0]}
if [[ "${status}" -eq 0 ]]; then
cargo nextest list --profile ci --all --exclude e2e_test --message-format json \
> artifacts/test-and-lint/core-test-listing.json \
&& python3 scripts/check_test_wiring.py --check-core artifacts/test-and-lint/core-test-listing.json \
&& test -s target/nextest/ci/junit.xml || status=$?
fi
{ {
echo "command=cargo nextest run --profile ci --all --exclude e2e_test" echo "command=cargo nextest run --profile ci --all --exclude e2e_test"
echo "exit_status=${status}" echo "exit_status=${status}"
-3
View File
@@ -54,9 +54,6 @@ env:
jobs: jobs:
heal-test: heal-test:
runs-on: smoke-testing runs-on: smoke-testing
# Requirement: a failing suite must not fail the workflow; failures
# are filed to rustfs/backlog and the chain continues.
continue-on-error: true
timeout-minutes: 480 timeout-minutes: 480
# Standalone manual run, or one link of the nightly functional chain # Standalone manual run, or one link of the nightly functional chain
# (storage -> heal -> pool). Pool expansion no longer re-runs heal. # (storage -> heal -> pool). Pool expansion no longer re-runs heal.
-2
View File
@@ -49,7 +49,6 @@ env:
jobs: jobs:
kms-test: kms-test:
runs-on: smoke-testing runs-on: smoke-testing
continue-on-error: true
timeout-minutes: 420 timeout-minutes: 420
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps: steps:
@@ -109,7 +108,6 @@ jobs:
- name: Run KMS suite - name: Run KMS suite
id: test id: test
continue-on-error: true
env: env:
LOG_FILE: /tmp/rustfs-kms.log LOG_FILE: /tmp/rustfs-kms.log
run: | run: |
@@ -84,9 +84,6 @@ env:
jobs: jobs:
performance-test: performance-test:
runs-on: pf-testing runs-on: pf-testing
# Requirement: a failing benchmark must not fail the workflow;
# failures are filed to rustfs/backlog.
continue-on-error: true
timeout-minutes: 900 timeout-minutes: 900
# Run on manual dispatch, or when the nightly build completed successfully. # Run on manual dispatch, or when the nightly build completed successfully.
# Skipped when nightly failed. # Skipped when nightly failed.
@@ -76,9 +76,6 @@ jobs:
pool-expansion-test: pool-expansion-test:
name: Pool expansion / decommission test name: Pool expansion / decommission test
runs-on: smoke-testing runs-on: smoke-testing
# Requirement: a failing suite must not fail the workflow; failures
# are filed to rustfs/backlog and the chain continues.
continue-on-error: true
timeout-minutes: 360 timeout-minutes: 360
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
env: env:
@@ -62,9 +62,6 @@ env:
jobs: jobs:
replication-test: replication-test:
runs-on: smoke-testing runs-on: smoke-testing
# A failed replication run must not break the chain or the workflow: the
# failure is reported to rustfs/backlog instead (see the issue step).
continue-on-error: true
timeout-minutes: 360 timeout-minutes: 360
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps: steps:
@@ -116,7 +113,6 @@ jobs:
- name: Run replication suite - name: Run replication suite
id: test id: test
continue-on-error: true
env: env:
LOG_FILE: /tmp/rustfs-replication.log LOG_FILE: /tmp/rustfs-replication.log
run: | run: |
@@ -37,7 +37,6 @@ env:
jobs: jobs:
s3-compat-test: s3-compat-test:
runs-on: smoke-testing runs-on: smoke-testing
continue-on-error: true
timeout-minutes: 360 timeout-minutes: 360
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps: steps:
@@ -88,7 +87,6 @@ jobs:
- name: Run S3 compatibility suite - name: Run S3 compatibility suite
id: test id: test
continue-on-error: true
env: env:
LOG_FILE: /tmp/rustfs-s3-compat.log LOG_FILE: /tmp/rustfs-s3-compat.log
run: | run: |
+55 -28
View File
@@ -74,10 +74,23 @@ env:
jobs: jobs:
security-test: security-test:
runs-on: smoke-testing runs-on: smoke-testing
continue-on-error: true
timeout-minutes: 360 timeout-minutes: 360
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps: steps:
- name: Checkout repository (for the OIDC live gate script)
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Initialize security evidence
id: evidence
run: |
set -euo pipefail
umask 077
SECURITY_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-security-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
mkdir -- "${SECURITY_ARTIFACTS_DIR}"
printf 'SECURITY_ARTIFACTS_DIR=%s\n' "${SECURITY_ARTIFACTS_DIR}" >> "${GITHUB_ENV}"
# auto-testing is private: clone it with the dedicated PF token (not # auto-testing is private: clone it with the dedicated PF token (not
# GITHUB_TOKEN) and retry transient GitHub/network failures. # GITHUB_TOKEN) and retry transient GitHub/network failures.
- name: Checkout auto-testing scripts (with retry) - name: Checkout auto-testing scripts (with retry)
@@ -98,11 +111,6 @@ jobs:
echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2 echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
exit 1 exit 1
- name: Checkout repository (for the OIDC live gate script)
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Show environment - name: Show environment
run: | run: |
uname -a uname -a
@@ -135,7 +143,8 @@ jobs:
id: test id: test
continue-on-error: true continue-on-error: true
env: env:
REPORT_FILE: /tmp/rustfs-security-report.md REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md
TMPDIR: ${{ env.SECURITY_ARTIFACTS_DIR }}
RUSTFS_SECURITY_OIDC_LIVE_SCRIPT: ${{ github.workspace }}/scripts/test/oidc_keycloak_live.sh RUSTFS_SECURITY_OIDC_LIVE_SCRIPT: ${{ github.workspace }}/scripts/test/oidc_keycloak_live.sh
run: | run: |
set -euo pipefail set -euo pipefail
@@ -159,29 +168,48 @@ jobs:
else else
ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}") ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
fi fi
./auto-testing/rustfs-security-test.sh "${ARGS[@]}" GITHUB_STEP_SUMMARY=/dev/null ./auto-testing/rustfs-security-test.sh "${ARGS[@]}"
- name: Generate report - name: Generate report
if: always() id: report
if: ${{ always() && steps.evidence.outcome == 'success' }}
env:
TEST_OUTCOME: ${{ steps.test.outcome }}
run: | run: |
set -euo pipefail set -euo pipefail
if [ ! -f /tmp/rustfs-security-report.md ]; then RESULT=failure
{ if [ "${TEST_OUTCOME}" = "success" ] && [ -s "${SECURITY_ARTIFACTS_DIR}/suite-report.md" ]; then
echo "# RustFS security test report" RESULT=success
echo ""
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
echo "- Trigger: ${{ github.event_name }}"
echo "- Test Step Outcome: failure (suite did not produce a report)"
} > /tmp/rustfs-security-report.md
fi fi
cat /tmp/rustfs-security-report.md >> "${GITHUB_STEP_SUMMARY}" {
echo "# RustFS security test report"
echo ""
echo "- Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
echo "- Workflow Commit: ${GITHUB_SHA}"
echo "- Trigger: ${GITHUB_EVENT_NAME}"
echo "- Test Step Outcome: ${RESULT}"
echo "- Suite Step Outcome: ${TEST_OUTCOME}"
echo ""
# The dashboard prioritizes case rows over the step outcome.
# Keep partial case results in the artifact when the suite fails.
if [ "${RESULT}" = "success" ]; then
cat "${SECURITY_ARTIFACTS_DIR}/suite-report.md"
elif [ -s "${SECURITY_ARTIFACTS_DIR}/suite-report.md" ]; then
echo "The suite did not complete successfully. See suite-report.md in this run's artifact for diagnostics."
else
echo "The suite did not produce a non-empty report."
fi
} > "${SECURITY_ARTIFACTS_DIR}/report.md"
cat "${SECURITY_ARTIFACTS_DIR}/report.md" >> "${GITHUB_STEP_SUMMARY}"
[ "${RESULT}" = "success" ]
- name: Upload functional report to dashboard - name: Upload functional report to dashboard
if: always() if: ${{ always() && steps.evidence.outcome == 'success' }}
continue-on-error: true continue-on-error: true
env: env:
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }} GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
REPORT_FILE: /tmp/rustfs-security-report.md REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/report.md
SUITE: security SUITE: security
run: | run: |
set -euo pipefail set -euo pipefail
@@ -210,8 +238,9 @@ jobs:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
SUITE: 'security' SUITE: 'security'
SUITE_LABEL: 'Security' SUITE_LABEL: 'Security'
EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
REPORT_FILE: '/tmp/rustfs-security-report.md' REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/report.md
LOG_FILE: '' LOG_FILE: ''
run: | run: |
set -euo pipefail set -euo pipefail
@@ -245,7 +274,7 @@ jobs:
echo "" echo ""
echo "## Report (errors and symptoms)" echo "## Report (errors and symptoms)"
echo "" echo ""
if [ -s "${REPORT_FILE}" ]; then if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
redact < "${REPORT_FILE}" redact < "${REPORT_FILE}"
elif [ -s "${LOG_FILE:-}" ]; then elif [ -s "${LOG_FILE:-}" ]; then
echo "(report file missing; log tail below)" echo "(report file missing; log tail below)"
@@ -263,14 +292,12 @@ jobs:
echo "filed backlog issue for suite ${SUITE}" echo "filed backlog issue for suite ${SUITE}"
- name: Upload report and logs - name: Upload report and logs
if: always() if: ${{ always() && steps.evidence.outcome == 'success' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with: with:
name: rustfs-security-test-${{ github.run_id }} name: rustfs-security-test-${{ github.run_id }}-${{ github.run_attempt }}
path: | path: ${{ env.SECURITY_ARTIFACTS_DIR }}/
/tmp/rustfs-security-report.md if-no-files-found: error
/tmp/rustfs-security.*/*
if-no-files-found: ignore
retention-days: 3 retention-days: 3
- name: Cleanup environment (after) - name: Cleanup environment (after)
@@ -46,7 +46,6 @@ env:
jobs: jobs:
storage-test: storage-test:
runs-on: smoke-testing runs-on: smoke-testing
continue-on-error: true
timeout-minutes: 360 timeout-minutes: 360
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps: steps:
@@ -97,7 +96,6 @@ jobs:
- name: Run storage engine suite - name: Run storage engine suite
id: test id: test
continue-on-error: true
env: env:
LOG_FILE: /tmp/rustfs-storage.log LOG_FILE: /tmp/rustfs-storage.log
run: | run: |
-3
View File
@@ -61,9 +61,6 @@ env:
jobs: jobs:
tier-test: tier-test:
runs-on: smoke-testing runs-on: smoke-testing
# Requirement: a failing suite must not fail the workflow; failures
# are filed to rustfs/backlog and the chain continues.
continue-on-error: true
timeout-minutes: 420 timeout-minutes: 420
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps: steps:
+26 -4
View File
@@ -18,7 +18,7 @@ on:
workflow_dispatch: workflow_dispatch:
inputs: inputs:
from_version: from_version:
description: 'OLD RustFS release tag (must ship a .deb asset, e.g. 1.0.0-rc.3)' description: 'OLD RustFS release tag, e.g. 1.0.0-rc.3 (its release must ship a .deb asset). Leave empty for the default.'
required: false required: false
default: '1.0.0-rc.3' default: '1.0.0-rc.3'
from_url: from_url:
@@ -26,7 +26,7 @@ on:
required: false required: false
type: string type: string
to_version: to_version:
description: 'NEW RustFS release tag (leave empty for latest nightly)' description: 'NEW RustFS release tag, e.g. 1.0.0-rc.5 (any version with a .deb asset). Leave empty for latest nightly.'
required: false required: false
to_url: to_url:
description: 'NEW .deb URL. Overrides to_version / nightly default.' description: 'NEW .deb URL. Overrides to_version / nightly default.'
@@ -79,7 +79,6 @@ env:
jobs: jobs:
upgrade-test: upgrade-test:
runs-on: smoke-testing runs-on: smoke-testing
continue-on-error: true
timeout-minutes: 420 timeout-minutes: 420
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps: steps:
@@ -142,9 +141,9 @@ jobs:
- name: Run upgrade compatibility suite - name: Run upgrade compatibility suite
id: test id: test
continue-on-error: true
env: env:
LOG_FILE: /tmp/rustfs-upgrade.log LOG_FILE: /tmp/rustfs-upgrade.log
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
run: | run: |
set -euo pipefail set -euo pipefail
chmod +x auto-testing/rustfs-upgrade-test.sh chmod +x auto-testing/rustfs-upgrade-test.sh
@@ -175,6 +174,29 @@ jobs:
else else
ARGS+=(--to-url "${RUSTFS_NIGHTLY_PACKAGE_URL}") ARGS+=(--to-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
fi fi
# Fail fast with a clear message when a requested release tag has
# no .deb asset (e.g. 1.0.0-rc.4 ships only zips), instead of
# letting the suite die mid-run on a 404.
check_release_asset() {
local version="$1" tag asset url
[ -n "${version}" ] && [ "${version}" != "null" ] || return 0
tag="${version#v}"
asset="rustfs_${tag//-/.}_amd64.deb"
url="https://github.com/rustfs/rustfs/releases/download/${tag}/${asset}"
if ! gh api "repos/rustfs/rustfs/releases/tags/${tag}" --jq '.assets[].name' 2>/dev/null | grep -qxF "${asset}"; then
echo "ERROR: release ${tag} has no downloadable asset ${asset}:" >&2
echo " ${url}" >&2
echo "Pick a tag whose release ships a .deb (check its release assets)." >&2
exit 1
fi
echo "resolved ${tag} -> ${url}"
}
if [ -z "${FROM_URL}" ]; then
check_release_asset "${FROM_VERSION}"
fi
if [ -z "${TO_URL}" ]; then
check_release_asset "${TO_VERSION}"
fi
./auto-testing/rustfs-upgrade-test.sh "${ARGS[@]}" ./auto-testing/rustfs-upgrade-test.sh "${ARGS[@]}"
- name: Generate report - name: Generate report
@@ -42,6 +42,7 @@ jobs:
- name: Check latest scheduled runs - name: Check latest scheduled runs
env: env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
RUSTFS_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
run: | run: |
set +e set +e
python3 scripts/check_scheduled_validation_freshness.py \ python3 scripts/check_scheduled_validation_freshness.py \
+1
View File
@@ -33,6 +33,7 @@ profile.json
*.zst *.zst
.secrets .secrets
*.go *.go
!crates/zip/tests/fixtures/snowball/**/generate/*.go
*.pb *.pb
*.svg *.svg
deploy/logs/*.log.* deploy/logs/*.log.*
+3 -3
View File
@@ -3,9 +3,9 @@
repos: repos:
- repo: local - repo: local
hooks: hooks:
- id: rustfs-dev-check - id: rustfs-fmt-check
name: rustfs dev-check name: Rust formatting
entry: make dev-check entry: cargo fmt --all --check
language: system language: system
types: [rust] types: [rust]
pass_filenames: false pass_filenames: false
+11 -37
View File
@@ -109,24 +109,17 @@ affected boundaries and risks. CI still runs its configured repository gates.
### 🔒 Git Pre-commit Hooks (optional) ### 🔒 Git Pre-commit Hooks (optional)
Git hooks are **not** versioned in this repository, so a fresh clone has no The optional hook uses the checked-in `.pre-commit-config.yaml`. Install [pre-commit](https://pre-commit.com/#installation), then run this from the checkout or a linked worktree:
active pre-commit hook. If you add your own `.git/hooks/pre-commit` (a good
choice is a one-liner that runs `make pre-commit`), you can mark it executable
with:
```bash ```bash
make setup-hooks make setup-hooks
``` ```
Or manually: The hook runs `cargo fmt --all --check` when staged files include Rust source. It does not compile the workspace or run tests. Fix formatting with `cargo fmt --all`, inspect and stage the result, then commit again.
```bash `pre-commit install` resolves Git's hook directory for linked worktrees and preserves an existing hook in migration mode. If you use `core.hooksPath`, keep that hook manager and integrate `pre-commit run` there; the installer refuses to silently replace that configuration.
chmod +x .git/hooks/pre-commit
```
With or without a hook, follow the verification tiers in `AGENTS.md`. Run the A local hook provides early formatting feedback. With or without it, follow the verification tiers in `AGENTS.md`, run relevant behavioral tests, and satisfy the CI merge gates. `make pre-commit` and `make dev-check` remain explicit broader commands.
applicable scoped checks, and reserve `make pre-pr` for broad cross-module
changes whose impact cannot be bounded by those checks.
### 📝 Formatting Configuration ### 📝 Formatting Configuration
@@ -138,31 +131,11 @@ fn_call_width = 90
single_line_let_else_max_width = 100 single_line_let_else_max_width = 100
``` ```
### 🚫 Commit Prevention
If you set up a pre-commit hook and your code doesn't meet the formatting requirements, the hook will:
1. **Block the commit** and show clear error messages
2. **Provide exact commands** to fix the issues
3. **Guide you through** the resolution process
Example output when formatting fails:
```
❌ Code formatting check failed!
💡 Please run 'cargo fmt --all' to format your code before committing.
🔧 Quick fix:
cargo fmt --all
git add .
git commit
```
### 🔄 Development Workflow ### 🔄 Development Workflow
1. **Make your changes** 1. **Make your changes**
2. **Format your code**: `make fmt` or `cargo fmt --all` 2. **Format your code**: `make fmt` or `cargo fmt --all`
3. **Run the fast gate**: `make pre-commit` (no clippy, no tests) 3. **Select relevant checks** using the validation tier in `AGENTS.md`; use `make pre-commit` when its broader fast gate adds useful coverage
4. **Commit your changes**: `git commit -m "your message"` 4. **Commit your changes**: `git commit -m "your message"`
5. **Complete the applicable multi-role adversarial review** for non-exempt changes (see `AGENTS.md`) 5. **Complete the applicable multi-role adversarial review** for non-exempt changes (see `AGENTS.md`)
6. **Run applicable scoped checks before opening/updating a PR**; consider 6. **Run applicable scoped checks before opening/updating a PR**; consider
@@ -206,11 +179,12 @@ Configure your IDE to:
#### Pre-commit hook not running? #### Pre-commit hook not running?
```bash ```bash
# Check if hook is executable pre-commit validate-config
ls -la .git/hooks/pre-commit pre-commit run --all-files
# Inspect any configured hook manager; do not overwrite it.
# Make it executable if needed git config --get core.hooksPath
chmod +x .git/hooks/pre-commit # Install if no separate hook manager is configured.
make setup-hooks
``` ```
#### Formatting issues? #### Formatting issues?
Generated
+356 -125
View File
File diff suppressed because it is too large Load Diff
+15 -11
View File
@@ -168,7 +168,7 @@ reqwest = "0.13.4"
rustfs-kafka-async = { version = "1.3.1" } rustfs-kafka-async = { version = "1.3.1" }
socket2 = { version = "0.6.5" } socket2 = { version = "0.6.5" }
tokio = { version = "1.53.1" } tokio = { version = "1.53.1" }
tokio-rustls = { default-features = false, version = "0.26.4" } tokio-rustls = { default-features = false, version = "0.26.5" }
tokio-stream = { version = "0.1.19" } tokio-stream = { version = "0.1.19" }
tokio-test = "0.4.5" tokio-test = "0.4.5"
tokio-util = { version = "0.7.19" } tokio-util = { version = "0.7.19" }
@@ -199,10 +199,10 @@ serde_urlencoded = "0.7.1"
# matching stable releases are not available yet, while previous stable lines # matching stable releases are not available yet, while previous stable lines
# have incompatible APIs. Keep them exact-pinned and monitor upstream for stable # have incompatible APIs. Keep them exact-pinned and monitor upstream for stable
# releases. # releases.
aes-gcm = { version = "=0.11.1" } aes-gcm = { version = "0.11.1" }
argon2 = { version = "=0.6.0" } argon2 = { version = "0.6.0" }
blake2 = "=0.11.0" blake2 = "0.11.0"
chacha20poly1305 = { version = "=0.11.0" } chacha20poly1305 = { version = "0.11.0" }
crc-fast = "1.10.0" crc-fast = "1.10.0"
hmac = { version = "0.13.0" } hmac = { version = "0.13.0" }
jsonwebtoken = { version = "11.0.0" } jsonwebtoken = { version = "11.0.0" }
@@ -234,15 +234,19 @@ tokio-postgres-rustls = "0.14.0"
# Utilities and Tools # Utilities and Tools
anyhow = "1.0.104" anyhow = "1.0.104"
arc-swap = "1.9.2" arc-swap = "1.9.2"
# RUSTFS_COMPAT_TODO(tokio-tar-extension-limits): keep the fork pin until every parser hardening used by Snowball is released upstream. Remove after astral-sh/tokio-tar#118 is merged and a published release includes extension, physical-entry, and sparse limits, cancellation-safe sparse parsing, and error-fused entry streams. # RUSTFS_COMPAT_TODO(tokio-tar-extension-limits): keep the fork pin while Snowball and Swift still depend on it. Remove after Snowball uses a released tar-codec/tar-framing API that exposes precedence-resolved MinIO vendor records, RustFS preserves cancellation-safe ownership of large streamed members, footerless minio-go input is accepted only at an authenticated complete request boundary, the existing resource-limit, cancellation, and error-fuse regressions pass, and Swift no longer needs this fork.
astral-tokio-tar = { git = "https://github.com/cxymds/tokio-tar.git", rev = "603756478b7668436e464519c77ccac22a99ba96" } astral-tokio-tar = { git = "https://github.com/cxymds/tokio-tar.git", rev = "603756478b7668436e464519c77ccac22a99ba96" }
# Candidate Snowball parser versions exercised by rustfs-zip compatibility fixtures.
tar-codec = "0.0.14"
tar-framing = "0.0.14"
atoi = "3.1.0" atoi = "3.1.0"
atomic_enum = "0.3.0" atomic_enum = "0.3.0"
aws-config = { version = "1.11.0" } aws-config = { version = "1.12.0" }
aws-credential-types = { version = "1.3.0" } aws-credential-types = { version = "1.3.0" }
aws-sdk-kms = { default-features = false, version = "1.117.0" } aws-sdk-kms = { default-features = false, version = "1.118.0" }
aws-sdk-s3 = { default-features = false, version = "1.144.0" } aws-sdk-s3 = { default-features = false, version = "1.145.0" }
aws-sdk-sts = { default-features = false, version = "1.113.0" } aws-sdk-sts = { default-features = false, version = "1.114.0" }
aws-smithy-async = { version = "1.3.0" }
aws-smithy-http-client = { default-features = false, version = "1.4.0" } aws-smithy-http-client = { default-features = false, version = "1.4.0" }
aws-smithy-runtime-api = { version = "1.16.0" } aws-smithy-runtime-api = { version = "1.16.0" }
aws-smithy-types = { version = "1.6.3" } aws-smithy-types = { version = "1.6.3" }
@@ -339,7 +343,7 @@ windows = { version = "0.62.2" }
windows-sys = "0.61.2" windows-sys = "0.61.2"
xxhash-rust = { version = "0.8.18" } xxhash-rust = { version = "0.8.18" }
zip = "8.6.0" zip = "8.6.0"
zstd = "0.13.3" zstd = "0.14.0"
# Observability and Metrics # Observability and Metrics
metrics = "0.24.6" metrics = "0.24.6"
+15
View File
@@ -130,6 +130,21 @@ Scanner cycle budget controls:
- timeout returns S3 `SlowDown`, so clients should use normal SDK retry handling. - timeout returns S3 `SlowDown`, so clients should use normal SDK retry handling.
- this is not a fdatasync or group-commit switch. Track fdatasync batching separately with `rustfs_s3_put_object_rename_fdatasync_batch_files`. - this is not a fdatasync or group-commit switch. Track fdatasync batching separately with `rustfs_s3_put_object_rename_fdatasync_batch_files`.
## Remote tier timeout environment variables
- `RUSTFS_TIER_REMOTE_CONNECT_TIMEOUT_SECS`
- remote tier TCP connect timeout.
- default is `10`.
- must be positive; zero fails tier client initialization, while an invalid integer is logged and falls back to the default.
- `RUSTFS_TIER_REMOTE_REQUEST_TIMEOUT_SECS`
- remote tier request timeout through response headers.
- default is `86400` so large transition uploads keep a production-safe budget.
- must be positive; zero fails tier client initialization, while an invalid integer is logged and falls back to the default. Very large values are accepted and act as a correspondingly long budget.
- `RUSTFS_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS`
- maximum idle time between remote tier response-body chunks.
- default is `60`; the timer resets only when non-empty body data keeps progressing.
- must be positive; zero fails tier client initialization, while an invalid integer is logged and falls back to the default.
## Drive timeout environment variables ## Drive timeout environment variables
- `RUSTFS_DRIVE_METADATA_TIMEOUT_SECS` - `RUSTFS_DRIVE_METADATA_TIMEOUT_SECS`
+32
View File
@@ -137,6 +137,28 @@ pub const DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED: bool = false;
const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE); const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE);
const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED); const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED);
/// Environment variable for remote tier TCP connect timeout in seconds.
pub const ENV_TIER_REMOTE_CONNECT_TIMEOUT_SECS: &str = "RUSTFS_TIER_REMOTE_CONNECT_TIMEOUT_SECS";
/// Default remote tier TCP connect timeout in seconds.
pub const DEFAULT_TIER_REMOTE_CONNECT_TIMEOUT_SECS: u64 = 10;
/// Environment variable for the remote tier request timeout in seconds.
///
/// This bounds upload/download request progress through response headers. The
/// default is intentionally large so multi-TiB transition uploads keep their
/// previous production budget while black-hole remotes no longer wait forever.
pub const ENV_TIER_REMOTE_REQUEST_TIMEOUT_SECS: &str = "RUSTFS_TIER_REMOTE_REQUEST_TIMEOUT_SECS";
/// Default remote tier request timeout in seconds.
pub const DEFAULT_TIER_REMOTE_REQUEST_TIMEOUT_SECS: u64 = 24 * 60 * 60;
/// Environment variable for remote tier response-body idle timeout in seconds.
///
/// The timer is re-armed on every non-empty response-body chunk, so slow but
/// progressing remotes can continue while silent response bodies are cancelled.
pub const ENV_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS: &str = "RUSTFS_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS";
/// Default remote tier response-body idle timeout in seconds.
pub const DEFAULT_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS: u64 = 60;
/// Request the object-transaction fencing contract used by storage-owned /// Request the object-transaction fencing contract used by storage-owned
/// cleanup receipts and lock-window optimizations. /// cleanup receipts and lock-window optimizations.
/// ///
@@ -812,6 +834,16 @@ mod remote_version_state_tests {
); );
} }
#[test]
fn remote_tier_timeout_env_names_are_stable() {
assert_eq!(super::ENV_TIER_REMOTE_CONNECT_TIMEOUT_SECS, "RUSTFS_TIER_REMOTE_CONNECT_TIMEOUT_SECS");
assert_eq!(super::ENV_TIER_REMOTE_REQUEST_TIMEOUT_SECS, "RUSTFS_TIER_REMOTE_REQUEST_TIMEOUT_SECS");
assert_eq!(
super::ENV_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS,
"RUSTFS_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS"
);
}
#[test] #[test]
fn data_movement_part_checksum_gate_uses_stable_environment_names() { fn data_movement_part_checksum_gate_uses_stable_environment_names() {
assert_eq!(super::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, "RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_WRITE"); assert_eq!(super::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, "RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_WRITE");
@@ -6743,6 +6743,99 @@ async fn test_site_replication_replicates_object_with_bucket_versioning_real_dua
Ok(()) Ok(())
} }
#[tokio::test]
async fn test_site_replication_replays_bucket_created_during_peer_outage_real_dual_node() -> TestResult {
init_logging();
// Keep compilation outside the scenario timeout. Recovery itself waits
// for the production 30-second lightweight retry tick.
let _rustfs_binary = rustfs_binary_path();
match timeout(Duration::from_secs(150), async {
let mut site_env = replication_fast_env();
site_env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
let mut site_a_env = RustFSTestEnvironment::new().await?;
site_a_env.start_rustfs_server_with_env(vec![], &site_env).await?;
let mut site_b_env = RustFSTestEnvironment::new().await?;
site_b_env.start_rustfs_server_without_cleanup_with_env(&site_env).await?;
let site_a_client = site_a_env.create_s3_client();
let site_b_client = site_b_env.create_s3_client();
let bucket = "site-repl-peer-outage";
let key = "after-recovery.txt";
let payload = b"site replication recovered the missed bucket".to_vec();
let add_status = site_replication_add(
&site_a_env,
&[
PeerSite {
name: "outage-site-a".to_string(),
endpoint: site_a_env.url.clone(),
access_key: site_a_env.access_key.clone(),
secret_key: site_a_env.secret_key.clone(),
..Default::default()
},
PeerSite {
name: "outage-site-b".to_string(),
endpoint: site_b_env.url.clone(),
access_key: site_b_env.access_key.clone(),
secret_key: site_b_env.secret_key.clone(),
..Default::default()
},
],
)
.await?;
assert!(add_status.success, "unexpected site add result: {add_status:?}");
wait_for_site_replication_enabled(&site_a_env, 2).await?;
wait_for_site_replication_enabled(&site_b_env, 2).await?;
site_b_env.stop_server();
site_a_client.create_bucket().bucket(bucket).send().await?;
site_a_client.head_bucket().bucket(bucket).send().await?;
let queued = site_replication_info(&site_a_env)
.await?
.retry_stats
.ok_or("peer outage did not persist a site replication retry event")?;
assert!(queued.pending + queued.failed > 0, "peer outage retry queue was unexpectedly empty");
site_b_env.restart_server_preserving_data(vec![], &site_env).await?;
let recovery_deadline = tokio::time::Instant::now() + Duration::from_secs(75);
loop {
let bucket_recovered = site_b_client.head_bucket().bucket(bucket).send().await.is_ok();
let queue_empty = site_replication_info(&site_a_env).await?.retry_stats.is_none();
if bucket_recovered && queue_empty {
break;
}
if tokio::time::Instant::now() >= recovery_deadline {
return Err(format!(
"site replication retry did not settle after peer recovery; bucket_recovered={bucket_recovered}, queue_empty={queue_empty}"
)
.into());
}
sleep(Duration::from_millis(250)).await;
}
site_a_client
.put_object()
.bucket(bucket)
.key(key)
.body(ByteStream::from(payload.clone()))
.send()
.await?;
assert_eq!(wait_for_object_on_target(&site_b_client, bucket, key).await?, payload);
Ok(())
})
.await
{
Ok(result) => result,
Err(_) => Err("site replication peer-outage recovery timed out after 150 seconds".into()),
}
}
/// Re-applying a site's own replication config must not disable the peer's reverse direction. /// Re-applying a site's own replication config must not disable the peer's reverse direction.
/// ///
/// `PutBucketReplication` broadcasts the config to every peer — the console's replication /// `PutBucketReplication` broadcasts the config to every peer — the console's replication
+2
View File
@@ -215,6 +215,7 @@ serde_urlencoded.workspace = true
google-cloud-storage = { workspace = true } google-cloud-storage = { workspace = true }
google-cloud-auth = { workspace = true } google-cloud-auth = { workspace = true }
faster-hex = { workspace = true } faster-hex = { workspace = true }
quick-xml = { workspace = true }
ratelimit = { workspace = true } ratelimit = { workspace = true }
aws-smithy-http-client = { workspace = true, default-features = false, features = ["rustls-aws-lc"] } aws-smithy-http-client = { workspace = true, default-features = false, features = ["rustls-aws-lc"] }
@@ -244,6 +245,7 @@ windows-sys = { workspace = true, features = [
windows-sys = { workspace = true, features = ["Win32_System_Ioctl"] } windows-sys = { workspace = true, features = ["Win32_System_Ioctl"] }
[dev-dependencies] [dev-dependencies]
aws-smithy-async.workspace = true
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util", "fs"] } tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util", "fs"] }
criterion = { workspace = true, features = ["html_reports"] } criterion = { workspace = true, features = ["html_reports"] }
temp-env = { workspace = true, features = ["async_closure"] } temp-env = { workspace = true, features = ["async_closure"] }
+28 -19
View File
@@ -153,12 +153,13 @@ pub mod bucket {
LastSourceError, LatencyBucketSnapshot, NEGATIVE_CACHE_MAX_ENTRIES, NegativeCache, OdmBucketSnapshot, OdmLookup, LastSourceError, LatencyBucketSnapshot, NEGATIVE_CACHE_MAX_ENTRIES, NegativeCache, OdmBucketSnapshot, OdmLookup,
OdmOp, OdmOutcome, OdmStateError, OdmStats, OdmStatsSnapshot, OnDemandMigrationSys, PullError, PullFailureReason, OdmOp, OdmOutcome, OdmStateError, OdmStats, OdmStatsSnapshot, OnDemandMigrationSys, PullError, PullFailureReason,
PullFollower, PullLeader, PullOutcome, PullPath, PullResult, PullSlot, SOURCE_LATENCY_BUCKET_BOUNDS_MS, PullFollower, PullLeader, PullOutcome, PullPath, PullResult, PullSlot, SOURCE_LATENCY_BUCKET_BOUNDS_MS,
SourceLatencySnapshot, source_client_spec, SourceLatencySnapshot, source_backend_spec, source_client_spec,
}; };
pub use crate::bucket::on_demand_migration::{ pub use crate::bucket::on_demand_migration::{
ConfigPublishHook, FilterConfig, HeadPolicy, ON_DEMAND_MIGRATION_CONFIG_HOOK, ON_DEMAND_MIGRATION_CONFIG_VERSION, AzureSourceConfig, ConfigPublishHook, FilterConfig, GcsSourceConfig, HeadPolicy, ON_DEMAND_MIGRATION_CONFIG_HOOK,
OnDemandMigrationConfig, OnDemandMigrationConfigError, PathStyle, PolicyConfig, Provider, RangeGetPolicy, ON_DEMAND_MIGRATION_CONFIG_VERSION, OnDemandMigrationConfig, OnDemandMigrationConfigError, PathStyle, PolicyConfig,
SourceConfig, SourceCredentials, SourceErrorPolicy, SourceTimeout, TlsConfig, ValidationContext, Provider, RangeGetPolicy, SourceConfig, SourceCredentials, SourceErrorPolicy, SourceTimeout, TlsConfig,
ValidationContext,
}; };
pub use crate::bucket::on_demand_migration::{ pub use crate::bucket::on_demand_migration::{
EnqueueOutcome, LocalObject, MAX_MULTIPART_PARTS, OdmWriteBack, PULL_MAX_RETRIES, PULL_RETRY_BASE_DELAYS, EnqueueOutcome, LocalObject, MAX_MULTIPART_PARTS, OdmWriteBack, PULL_MAX_RETRIES, PULL_RETRY_BASE_DELAYS,
@@ -167,9 +168,10 @@ pub mod bucket {
idle_guarded_body, idle_guarded_body,
}; };
pub use crate::bucket::on_demand_migration::{ pub use crate::bucket::on_demand_migration::{
FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListThroughCursor, ListThroughMerger, ListThroughToken, FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger,
ListThroughTokenError, MAX_LIST_FETCHES_PER_SIDE, MergeOutcome, MergePick, MergeSide, SOURCE_LIST_MAX_RATE_WAIT, ListThroughToken, ListThroughTokenError, MAX_LIST_FETCHES_PER_SIDE, MAX_LIST_NO_PROGRESS_PAGES, MergeOutcome,
SOURCE_LIST_RATE_PER_SEC, SourceListPlan, SourceListRateLimiter, decode_continuation_token, source_list_plan, MergePick, MergeSide, SOURCE_LIST_MAX_RATE_WAIT, SOURCE_LIST_RATE_PER_SEC, SourceListPlan, SourceListRateLimiter,
decode_continuation_token, source_list_plan,
}; };
pub mod backfill { pub mod backfill {
pub use crate::bucket::on_demand_migration::backfill::{ pub use crate::bucket::on_demand_migration::backfill::{
@@ -184,9 +186,9 @@ pub mod bucket {
} }
pub mod source_client { pub mod source_client {
pub use crate::bucket::on_demand_migration::source_client::{ pub use crate::bucket::on_demand_migration::source_client::{
SourceClient, SourceClientSpec, SourceError, SourceGet, SourceHead, SourceListRequest, SourceObject, SourcePage, AzureAuth, AzureSourceSpec, GcsSourceSpec, SourceBackendSpec, SourceClient, SourceClientSpec, SourceError,
SourceProbe, SourceProvider, SourceSse, SourceTimeouts, USER_AGENT_SUFFIX, is_multipart_etag, range_header_value, SourceGet, SourceHead, SourceListRequest, SourceObject, SourcePage, SourceProbe, SourceProvider, SourceSse,
resolve_path_style, SourceTimeouts, USER_AGENT_SUFFIX, is_multipart_etag, range_header_value, resolve_path_style,
}; };
} }
} }
@@ -196,15 +198,16 @@ pub mod bucket {
pub use crate::bucket::metadata_sys::ConfigWriteLockProbe; pub use crate::bucket::metadata_sys::ConfigWriteLockProbe;
pub use crate::bucket::metadata_sys::{ pub use crate::bucket::metadata_sys::{
BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock, BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
acquire_bucket_metadata_transaction_lock_for_incarnation, capture_bucket_metadata_incarnation, delete, acquire_bucket_metadata_transaction_lock_for_incarnation, acquire_scanner_bucket_incarnation_fence,
delete_if_incarnation, delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy, capture_bucket_metadata_incarnation, delete, delete_if_incarnation, delete_under_transaction_lock, get,
get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config, get_accelerate_config, get_bucket_policy, get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk,
get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config, get_cors_config, get_durability_config, get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config,
get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config, get_public_access_block_config, get_notification_config, get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config,
get_quota_config, get_replication_config, get_request_payment_config, get_sse_config, get_tagging_config, get_public_access_block_config, get_quota_config, get_replication_config, get_request_payment_config, get_sse_config,
get_versioning_config, get_website_config, init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, get_tagging_config, get_versioning_config, get_website_config, init_bucket_metadata_sys, list_bucket_targets,
remove_bucket_metadata, set_bucket_metadata, update, update_bucket_targets_under_transaction_lock, reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata, update,
update_config_with, update_if_incarnation, update_quota_if_incarnation, update_under_transaction_lock, update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation, update_quota_if_incarnation,
update_under_transaction_lock,
}; };
} }
@@ -560,6 +563,12 @@ pub mod set_disk {
pub mod test_util { pub mod test_util {
pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test; pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test;
pub use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause}; pub use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause};
/// Keep a namespace commit pending until the returned owner is dropped.
#[must_use]
pub fn hold_namespace_commit(store: &crate::store::ECStore) -> impl Send + Sync {
store.ctx.begin_namespace_commit()
}
} }
} }
+84 -25
View File
@@ -59,7 +59,7 @@ use rustfs_utils::http::{
insert_header, insert_header,
}; };
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::collections::HashMap; use std::collections::{HashMap, HashSet};
use std::error::Error; use std::error::Error;
use std::fmt; use std::fmt;
use std::str::FromStr as _; use std::str::FromStr as _;
@@ -376,6 +376,11 @@ pub struct BucketTargetSys {
/// [`SsecPassthroughCapability`]; reset alongside `arn_remotes_map`. /// [`SsecPassthroughCapability`]; reset alongside `arn_remotes_map`.
ssec_passthrough_map: Arc<RwLock<HashMap<String, SsecPassthroughRecord>>>, ssec_passthrough_map: Arc<RwLock<HashMap<String, SsecPassthroughRecord>>>,
pub targets_map: Arc<RwLock<HashMap<String, Vec<BucketTarget>>>>, pub targets_map: Arc<RwLock<HashMap<String, Vec<BucketTarget>>>>,
/// Buckets whose persisted `bucket-targets.json` exists but cannot be
/// decoded (rustfs/backlog#2282). Written under the bucket's update mutex
/// alongside `targets_map`, and read before it so an unreadable
/// configuration surfaces as a typed error instead of an empty target set.
unreadable_targets: Arc<RwLock<HashSet<String>>>,
pub h_mutex: Arc<RwLock<HashMap<String, EpHealth>>>, pub h_mutex: Arc<RwLock<HashMap<String, EpHealth>>>,
target_h_mutex: Arc<RwLock<HashMap<String, EpHealth>>>, target_h_mutex: Arc<RwLock<HashMap<String, EpHealth>>>,
pub hc_client: Arc<HttpClient>, pub hc_client: Arc<HttpClient>,
@@ -419,6 +424,7 @@ impl BucketTargetSys {
arn_remotes_map: Arc::new(RwLock::new(HashMap::new())), arn_remotes_map: Arc::new(RwLock::new(HashMap::new())),
ssec_passthrough_map: Arc::new(RwLock::new(HashMap::new())), ssec_passthrough_map: Arc::new(RwLock::new(HashMap::new())),
targets_map: Arc::new(RwLock::new(HashMap::new())), targets_map: Arc::new(RwLock::new(HashMap::new())),
unreadable_targets: Arc::new(RwLock::new(HashSet::new())),
h_mutex: Arc::new(RwLock::new(HashMap::new())), h_mutex: Arc::new(RwLock::new(HashMap::new())),
target_h_mutex: Arc::new(RwLock::new(HashMap::new())), target_h_mutex: Arc::new(RwLock::new(HashMap::new())),
hc_client: Arc::new(build_health_check_client()), hc_client: Arc::new(build_health_check_client()),
@@ -628,30 +634,40 @@ impl BucketTargetSys {
health_map.clone() health_map.clone()
} }
pub async fn list_targets(&self, bucket: &str, arn_type: &str) -> Vec<BucketTarget> { /// Targets of one bucket, or of every bucket when `bucket` is empty.
///
/// A bucket that simply has no targets yields an empty list; a bucket
/// whose persisted configuration cannot be decoded is an error, so an
/// admin listing reports the fault instead of an empty list that reads as
/// "replication is not configured" (rustfs/backlog#2282).
pub async fn list_targets(&self, bucket: &str, arn_type: &str) -> Result<Vec<BucketTarget>, BucketTargetError> {
let health_stats = self.target_health_stats().await; let health_stats = self.target_health_stats().await;
let mut targets = Vec::new(); let mut targets = Vec::new();
if !bucket.is_empty() { if !bucket.is_empty() {
if let Ok(bucket_targets) = self.list_bucket_targets(bucket).await { match self.list_bucket_targets(bucket).await {
for mut target in bucket_targets.targets { Ok(bucket_targets) => {
if arn_type.is_empty() || target.target_type.to_string() == arn_type { for mut target in bucket_targets.targets {
if let Some(health) = health_stats.get(&target.arn) { if arn_type.is_empty() || target.target_type.to_string() == arn_type {
target.total_downtime = health.offline_duration; if let Some(health) = health_stats.get(&target.arn) {
target.online = health.online; target.total_downtime = health.offline_duration;
target.last_online = health.last_online; target.online = health.online;
target.latency = target::LatencyStat { target.last_online = health.last_online;
curr: health.latency.curr, target.latency = target::LatencyStat {
avg: health.latency.avg, curr: health.latency.curr,
max: health.latency.peak, avg: health.latency.avg,
}; max: health.latency.peak,
target.offline_count = health.offline_count; };
target.offline_count = health.offline_count;
}
targets.push(target);
} }
targets.push(target);
} }
} }
Err(BucketTargetError::BucketRemoteTargetNotFound { .. }) => {}
Err(err) => return Err(err),
} }
return targets; return Ok(targets);
} }
let targets_map = self.targets_map.read().await; let targets_map = self.targets_map.read().await;
@@ -674,10 +690,16 @@ impl BucketTargetSys {
} }
} }
targets Ok(targets)
} }
pub async fn list_bucket_targets(&self, bucket: &str) -> Result<BucketTargets, BucketTargetError> { pub async fn list_bucket_targets(&self, bucket: &str) -> Result<BucketTargets, BucketTargetError> {
if self.unreadable_targets.read().await.contains(bucket) {
return Err(BucketTargetError::BucketRemoteTargetsUnreadable {
bucket: bucket.to_string(),
});
}
let targets_map = self.targets_map.read().await; let targets_map = self.targets_map.read().await;
if let Some(targets) = targets_map.get(bucket) { if let Some(targets) = targets_map.get(bucket) {
Ok(BucketTargets { Ok(BucketTargets {
@@ -690,13 +712,30 @@ impl BucketTargetSys {
} }
} }
/// Record that this bucket's persisted targets configuration exists but
/// cannot be decoded (rustfs/backlog#2282).
///
/// Any snapshot published from an earlier readable load is deliberately
/// left in place: withdrawing it would produce exactly the silent "no
/// targets configured" state this marker exists to prevent. The marker is
/// cleared by the next successful publish, which is what makes a repaired
/// configuration take effect without a restart.
pub async fn mark_targets_unreadable(&self, bucket: &str) {
let update_mutex = self.target_update_mutex(bucket).await;
let _update_guard = update_mutex.lock().await;
self.unreadable_targets.write().await.insert(bucket.to_string());
}
pub async fn delete(&self, bucket: &str) { pub async fn delete(&self, bucket: &str) {
let update_mutex = self.target_update_mutex(bucket).await; let update_mutex = self.target_update_mutex(bucket).await;
let _update_guard = update_mutex.lock().await; let _update_guard = update_mutex.lock().await;
// Lock order: targets_map, then arn_remotes_map, then target_h_mutex, // Lock order: unreadable_targets, then targets_map, then
// then ssec_passthrough_map (always last; also taken standalone by the // arn_remotes_map, then target_h_mutex, then ssec_passthrough_map
// capability accessors). // (always last; also taken standalone by the capability accessors).
self.unreadable_targets.write().await.remove(bucket);
let mut targets_map = self.targets_map.write().await; let mut targets_map = self.targets_map.write().await;
let mut arn_remotes_map = self.arn_remotes_map.write().await; let mut arn_remotes_map = self.arn_remotes_map.write().await;
let mut health_map = self.target_h_mutex.write().await; let mut health_map = self.target_h_mutex.write().await;
@@ -1093,6 +1132,11 @@ impl BucketTargetSys {
/// Keeping persisted-config reads under the same mutex prevents a stale /// Keeping persisted-config reads under the same mutex prevents a stale
/// reload from overwriting a concurrent credential rotation. /// reload from overwriting a concurrent credential rotation.
async fn update_all_targets_locked(&self, bucket: &str, targets: Option<&BucketTargets>) { async fn update_all_targets_locked(&self, bucket: &str, targets: Option<&BucketTargets>) {
// Reaching here means the persisted configuration decoded, so the
// unreadable marker (if any) is stale. Cleared before the maps below
// so `unreadable_targets` stays the outermost of this module's locks.
self.unreadable_targets.write().await.remove(bucket);
let mut clients = Vec::new(); let mut clients = Vec::new();
if let Some(new_targets) = targets { if let Some(new_targets) = targets {
for target in &new_targets.targets { for target in &new_targets.targets {
@@ -1100,9 +1144,9 @@ impl BucketTargetSys {
} }
} }
// Lock order: targets_map, then arn_remotes_map, then target_h_mutex, // Lock order: unreadable_targets (above), then targets_map, then
// then ssec_passthrough_map (always last; also taken standalone by the // arn_remotes_map, then target_h_mutex, then ssec_passthrough_map
// capability accessors). // (always last; also taken standalone by the capability accessors).
let mut targets_map = self.targets_map.write().await; let mut targets_map = self.targets_map.write().await;
let mut arn_remotes_map = self.arn_remotes_map.write().await; let mut arn_remotes_map = self.arn_remotes_map.write().await;
let mut health_map = self.target_h_mutex.write().await; let mut health_map = self.target_h_mutex.write().await;
@@ -1161,6 +1205,11 @@ impl BucketTargetSys {
} }
pub async fn set(&self, bucket: &str, meta: &BucketMetadata) { pub async fn set(&self, bucket: &str, meta: &BucketMetadata) {
if meta.bucket_targets_unreadable() {
self.mark_targets_unreadable(bucket).await;
return;
}
let Some(config) = &meta.bucket_target_config else { let Some(config) = &meta.bucket_target_config else {
return; return;
}; };
@@ -2276,6 +2325,13 @@ pub enum BucketTargetError {
BucketRemoteTargetNotFound { BucketRemoteTargetNotFound {
bucket: String, bucket: String,
}, },
/// The bucket's persisted targets configuration exists but cannot be
/// decoded. Distinct from `BucketRemoteTargetNotFound`, which means the
/// bucket genuinely has no targets: callers must not degrade this one to
/// an empty target set (rustfs/backlog#2282).
BucketRemoteTargetsUnreadable {
bucket: String,
},
BucketRemoteArnTypeInvalid { BucketRemoteArnTypeInvalid {
bucket: String, bucket: String,
}, },
@@ -2309,6 +2365,9 @@ impl fmt::Display for BucketTargetError {
BucketTargetError::BucketRemoteTargetNotFound { bucket } => { BucketTargetError::BucketRemoteTargetNotFound { bucket } => {
write!(f, "Remote target not found for bucket: {bucket}") write!(f, "Remote target not found for bucket: {bucket}")
} }
BucketTargetError::BucketRemoteTargetsUnreadable { bucket } => {
write!(f, "Persisted replication target configuration is unreadable for bucket: {bucket}")
}
BucketTargetError::BucketRemoteArnTypeInvalid { bucket } => { BucketTargetError::BucketRemoteArnTypeInvalid { bucket } => {
write!(f, "Invalid ARN type for bucket: {bucket}") write!(f, "Invalid ARN type for bucket: {bucket}")
} }
@@ -3256,7 +3315,7 @@ mod tests {
}], }],
); );
let targets = sys.list_targets("", "").await; let targets = sys.list_targets("", "").await.expect("listing every bucket's targets");
assert_eq!(targets.len(), 1); assert_eq!(targets.len(), 1);
assert!(!targets[0].online); assert!(!targets[0].online);
@@ -584,33 +584,173 @@ impl ExpiryOp for FreeVersionTask {
} }
} }
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum TransitionDeleteVersionPlan {
Direct { version_id_exact: bool },
ProbeLegacyUnknown,
}
fn legacy_transition_version_state_missing(oi: &ObjectInfo) -> Result<bool, std::io::Error> {
use rustfs_utils::http::metadata_compat::{
SUFFIX_TRANSITIONED_VERSION_ID, SUFFIX_TRANSITIONED_VERSION_STATE, contains_key_str, get_consistent_str,
};
if !contains_key_str(&oi.user_defined, SUFFIX_TRANSITIONED_VERSION_STATE) {
let version_key_present = contains_key_str(&oi.user_defined, SUFFIX_TRANSITIONED_VERSION_ID);
if version_key_present {
if oi.transitioned_object.version_id.is_empty() {
let has_non_empty_version = oi.user_defined.iter().any(|(key, value)| {
rustfs_utils::http::metadata_compat::strip_internal_prefix_preserving_case(key)
.is_some_and(|suffix| suffix.eq_ignore_ascii_case(SUFFIX_TRANSITIONED_VERSION_ID))
&& !value.is_empty()
});
if !has_non_empty_version {
// MinIO writes the transitioned-versionID key with an empty value
// for unversioned tier objects. The backend probe remains the proof.
return Ok(true);
}
} else if get_consistent_str(&oi.user_defined, SUFFIX_TRANSITIONED_VERSION_ID)
== Some(oi.transitioned_object.version_id.as_str())
{
return Ok(true);
}
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"legacy remote tier version metadata is conflicting or malformed",
));
}
if !oi.transitioned_object.version_id.is_empty() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"legacy remote tier version metadata is missing or inconsistent",
));
}
return Ok(true);
}
let persisted = get_consistent_str(&oi.user_defined, SUFFIX_TRANSITIONED_VERSION_STATE).ok_or_else(|| {
std::io::Error::new(
std::io::ErrorKind::InvalidData,
"remote tier object has conflicting transition version state metadata",
)
})?;
if persisted != oi.transition_version_state.as_str() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"remote tier object transition version state metadata changed during decoding",
));
}
Ok(false)
}
fn transition_remote_version_delete_plan(oi: &ObjectInfo) -> Result<TransitionDeleteVersionPlan, std::io::Error> {
match oi.transition_version_state {
rustfs_filemeta::TransitionVersionState::Unknown => {
if legacy_transition_version_state_missing(oi)? {
Ok(TransitionDeleteVersionPlan::ProbeLegacyUnknown)
} else {
validate_transition_remote_version(oi)
.map(|version_id_exact| TransitionDeleteVersionPlan::Direct { version_id_exact })
}
}
_ => validate_transition_remote_version(oi)
.map(|version_id_exact| TransitionDeleteVersionPlan::Direct { version_id_exact }),
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct ResolvedTransitionDeleteVersion {
version_id_exact: bool,
remote_already_missing: bool,
}
async fn acquire_free_version_tier_lease( async fn acquire_free_version_tier_lease(
oi: &ObjectInfo, oi: &ObjectInfo,
tier_config_mgr: &Arc<RwLock<TierConfigMgr>>, tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
) -> Result<(TierOperationLease, bool), std::io::Error> { ) -> Result<(TierOperationLease, TransitionDeleteVersionPlan), std::io::Error> {
let version_id_exact = validate_transition_remote_version(oi)?; let delete_plan = transition_remote_version_delete_plan(oi)?;
let identity = tier_destination_id_from_metadata(&oi.user_defined)? let identity = tier_destination_id_from_metadata(&oi.user_defined)?
.ok_or_else(|| std::io::Error::other("tier free-version has no durable backend identity"))?; .ok_or_else(|| std::io::Error::other("tier free-version has no durable backend identity"))?;
let lease = let lease =
TierConfigMgr::acquire_operation_lease_for_backend_identity(tier_config_mgr, &oi.transitioned_object.tier, identity) TierConfigMgr::acquire_operation_lease_for_backend_identity(tier_config_mgr, &oi.transitioned_object.tier, identity)
.await .await
.map_err(std::io::Error::other)?; .map_err(std::io::Error::other)?;
Ok((lease, version_id_exact)) Ok((lease, delete_plan))
}
async fn resolve_transition_delete_version_plan(
oi: &ObjectInfo,
lease: &TierOperationLease,
delete_plan: TransitionDeleteVersionPlan,
) -> Result<ResolvedTransitionDeleteVersion, std::io::Error> {
match delete_plan {
TransitionDeleteVersionPlan::Direct { version_id_exact } => Ok(ResolvedTransitionDeleteVersion {
version_id_exact,
remote_already_missing: false,
}),
TransitionDeleteVersionPlan::ProbeLegacyUnknown => {
let expected_version = oi.transitioned_object.version_id.as_str();
if expected_version.is_empty() {
return Err(std::io::Error::new(
std::io::ErrorKind::WouldBlock,
"remote tier cannot safely delete a legacy object without an exact version ID",
));
}
let probe = lease
.probe_transition_version(&oi.transitioned_object.name, expected_version)
.await?;
match (expected_version, probe) {
(expected, crate::services::tier::warm_backend::TransitionCandidateProbe::VersionedPresent(actual))
if expected == actual =>
{
lease.validate_remote_version_id(expected)?;
Ok(ResolvedTransitionDeleteVersion {
version_id_exact: true,
remote_already_missing: false,
})
}
(_, crate::services::tier::warm_backend::TransitionCandidateProbe::Missing) => {
Ok(ResolvedTransitionDeleteVersion {
version_id_exact: false,
remote_already_missing: true,
})
}
(_, crate::services::tier::warm_backend::TransitionCandidateProbe::Unsupported) => Err(std::io::Error::new(
std::io::ErrorKind::Unsupported,
"remote tier cannot prove legacy transition delete state",
)),
_ => Err(std::io::Error::new(
std::io::ErrorKind::WouldBlock,
"remote tier object version state is unknown",
)),
}
}
}
}
async fn execute_resolved_transition_delete(
oi: &ObjectInfo,
lease: &TierOperationLease,
resolved: ResolvedTransitionDeleteVersion,
) -> Result<(), std::io::Error> {
if !resolved.remote_already_missing {
delete_object_from_remote_tier_with_lease_idempotent(
&oi.transitioned_object.name,
&oi.transitioned_object.version_id,
lease,
resolved.version_id_exact,
)
.await?;
}
Ok(())
} }
async fn delete_free_version_remote_object_with_lease( async fn delete_free_version_remote_object_with_lease(
oi: &ObjectInfo, oi: &ObjectInfo,
lease: &TierOperationLease, lease: &TierOperationLease,
version_id_exact: bool, delete_plan: TransitionDeleteVersionPlan,
) -> Result<(), std::io::Error> { ) -> Result<(), std::io::Error> {
delete_object_from_remote_tier_with_lease_idempotent( let resolved = resolve_transition_delete_version_plan(oi, lease, delete_plan).await?;
&oi.transitioned_object.name, execute_resolved_transition_delete(oi, lease, resolved).await
&oi.transitioned_object.version_id,
lease,
version_id_exact,
)
.await?;
Ok(())
} }
fn free_version_physical_topology_generation(api: &ECStore) -> String { fn free_version_physical_topology_generation(api: &ECStore) -> String {
@@ -641,6 +781,16 @@ fn free_version_remote_tuple_matches(candidate: &ObjectInfo, expected: &ObjectIn
if candidate.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown if candidate.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
|| expected.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown || expected.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
{ {
let candidate_legacy_missing = legacy_transition_version_state_missing(candidate)?;
let expected_legacy_missing = legacy_transition_version_state_missing(expected)?;
if candidate.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
&& expected.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
&& candidate_legacy_missing
&& expected_legacy_missing
&& candidate.transitioned_object.version_id == expected.transitioned_object.version_id
{
return Ok(true);
}
return Err(std::io::Error::new( return Err(std::io::Error::new(
std::io::ErrorKind::WouldBlock, std::io::ErrorKind::WouldBlock,
"tier free-version remote version state is unknown", "tier free-version remote version state is unknown",
@@ -716,7 +866,7 @@ async fn cleanup_free_version_exact(api: Arc<ECStore>, oi: &ObjectInfo, cancel:
.acquire_bucket_lifecycle_read_lock(&oi.bucket) .acquire_bucket_lifecycle_read_lock(&oi.bucket)
.await .await
.map_err(std::io::Error::other)?; .map_err(std::io::Error::other)?;
let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, &api.tier_config_mgr()).await?; let (lease, delete_plan) = acquire_free_version_tier_lease(oi, &api.tier_config_mgr()).await?;
let local_object = encode_dir_object(&oi.name); let local_object = encode_dir_object(&oi.name);
let object_guards = api let object_guards = api
.acquire_all_physical_object_write_locks("tier_free_version_cleanup", &oi.bucket, &local_object) .acquire_all_physical_object_write_locks("tier_free_version_cleanup", &oi.bucket, &local_object)
@@ -734,16 +884,30 @@ async fn cleanup_free_version_exact(api: Arc<ECStore>, oi: &ObjectInfo, cancel:
"tier free-version cleanup fence is invalid before remote delete", "tier free-version cleanup fence is invalid before remote delete",
)); ));
} }
let resolved = tokio::select! {
_ = cancel.cancelled() => {
return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "tier free-version cleanup was cancelled"));
}
result = tokio::time::timeout_at(deadline, resolve_transition_delete_version_plan(oi, &lease, delete_plan)) => {
result.map_err(|_| {
std::io::Error::new(std::io::ErrorKind::TimedOut, "tier free-version remote probe timed out")
})??
}
};
if !free_version_cleanup_fences_current(&topology_generation, &api, &bucket_guard, &object_guards, &lease, cancel, deadline) {
return Err(std::io::Error::new(
std::io::ErrorKind::WouldBlock,
"tier free-version cleanup fence changed after remote probe",
));
}
tokio::select! { tokio::select! {
_ = cancel.cancelled() => { _ = cancel.cancelled() => {
return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "tier free-version cleanup was cancelled")); return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "tier free-version cleanup was cancelled"));
} }
result = tokio::time::timeout_at( result = tokio::time::timeout_at(deadline, execute_resolved_transition_delete(oi, &lease, resolved)) => {
deadline, result.map_err(|_| {
delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact), std::io::Error::new(std::io::ErrorKind::TimedOut, "tier free-version remote delete timed out")
) => { })??;
result
.map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "tier free-version remote delete timed out"))??;
} }
} }
if !free_version_cleanup_fences_current(&topology_generation, &api, &bucket_guard, &object_guards, &lease, cancel, deadline) { if !free_version_cleanup_fences_current(&topology_generation, &api, &bucket_guard, &object_guards, &lease, cancel, deadline) {
@@ -791,8 +955,8 @@ async fn delete_free_version_remote_object(
oi: &ObjectInfo, oi: &ObjectInfo,
tier_config_mgr: &Arc<RwLock<TierConfigMgr>>, tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
) -> Result<(), std::io::Error> { ) -> Result<(), std::io::Error> {
let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?; let (lease, delete_plan) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?;
delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact).await delete_free_version_remote_object_with_lease(oi, &lease, delete_plan).await
} }
#[allow( #[allow(
@@ -808,8 +972,8 @@ where
F: FnOnce() -> Fut, F: FnOnce() -> Fut,
Fut: std::future::Future<Output = T>, Fut: std::future::Future<Output = T>,
{ {
let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?; let (lease, delete_plan) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?;
delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact).await?; delete_free_version_remote_object_with_lease(oi, &lease, delete_plan).await?;
let result = delete_local().await; let result = delete_local().await;
drop(lease); drop(lease);
Ok(result) Ok(result)
@@ -4688,6 +4852,39 @@ fn validate_transition_remote_version(oi: &ObjectInfo) -> Result<bool, std::io::
} }
} }
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum TransitionReadVersionPlan {
Direct,
ProbeLegacyUnversioned,
}
const LEGACY_TRANSITION_READ_PROBE_TIMEOUT: StdDuration = StdDuration::from_secs(30);
fn transition_remote_version_read_plan(oi: &ObjectInfo) -> Result<TransitionReadVersionPlan, std::io::Error> {
let version = oi.transitioned_object.version_id.as_str();
match oi.transition_version_state {
rustfs_filemeta::TransitionVersionState::Unknown => {
if !legacy_transition_version_state_missing(oi)? {
return validate_transition_remote_version(oi).map(|_| TransitionReadVersionPlan::Direct);
}
if version.is_empty() {
Ok(TransitionReadVersionPlan::ProbeLegacyUnversioned)
} else {
Ok(TransitionReadVersionPlan::Direct)
}
}
rustfs_filemeta::TransitionVersionState::KnownDisabled if version.is_empty() => Ok(TransitionReadVersionPlan::Direct),
rustfs_filemeta::TransitionVersionState::SuspendedNull if version == "null" => Ok(TransitionReadVersionPlan::Direct),
rustfs_filemeta::TransitionVersionState::Exact if !version.is_empty() && version != "null" => {
Ok(TransitionReadVersionPlan::Direct)
}
_ => Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"remote tier object version state conflicts with its version ID",
)),
}
}
// The resolver joins the tier manager as the second injected port this read // The resolver joins the tier manager as the second injected port this read
// needs; grouping the request half into a struct would churn every call site of // needs; grouping the request half into a struct would churn every call site of
// a bug fix. // a bug fix.
@@ -4702,7 +4899,12 @@ pub(crate) async fn get_transitioned_object_reader_with_tier_manager(
tier_config_mgr: &Arc<RwLock<TierConfigMgr>>, tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
resolver: Option<&dyn ObjectEncryptionResolver>, resolver: Option<&dyn ObjectEncryptionResolver>,
) -> Result<GetObjectReader, std::io::Error> { ) -> Result<GetObjectReader, std::io::Error> {
validate_transition_remote_version(oi)?; let read_plan = transition_remote_version_read_plan(oi)?;
// Reject invalid ranges and encryption requests before a compatibility
// probe can amplify them into remote listing work.
let plan = ReadPlan::build_for_request(rs.clone(), oi, opts, h, resolver)
.await
.map_err(|err| std::io::Error::other(format!("building the read plan for {bucket}/{object} failed: {err}")))?;
let expected_identity = tier_destination_id_from_metadata(&oi.user_defined)?; let expected_identity = tier_destination_id_from_metadata(&oi.user_defined)?;
let lease = match expected_identity { let lease = match expected_identity {
Some(identity) => { Some(identity) => {
@@ -4716,7 +4918,36 @@ pub(crate) async fn get_transitioned_object_reader_with_tier_manager(
Err(err) => return Err(std::io::Error::other(err)), Err(err) => return Err(std::io::Error::other(err)),
}; };
tgt_client.validate_remote_version_id(&oi.transitioned_object.version_id)?; match read_plan {
TransitionReadVersionPlan::Direct => {
tgt_client.validate_remote_version_id(&oi.transitioned_object.version_id)?;
}
TransitionReadVersionPlan::ProbeLegacyUnversioned => {
// RUSTFS_COMPAT_TODO(backlog#2203): remove operation-time probing
// after an admin reconcile can persist every proven legacy state.
let probe = tokio::time::timeout(
LEGACY_TRANSITION_READ_PROBE_TIMEOUT,
tgt_client.probe_transition_candidate(&oi.transitioned_object.name),
)
.await
.map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "legacy remote tier version probe timed out"))??;
match probe {
crate::services::tier::warm_backend::TransitionCandidateProbe::UnversionedPresent => {}
crate::services::tier::warm_backend::TransitionCandidateProbe::Unsupported => {
return Err(std::io::Error::new(
std::io::ErrorKind::Unsupported,
"remote tier cannot prove legacy unversioned transition state",
));
}
_ => {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"remote tier object version state is unknown",
));
}
}
}
}
// The same read plan the local path uses, so the tier fetch is positioned in // The same read plan the local path uses, so the tier fetch is positioned in
// the object's *stored* coordinate system and the stream is handed the same // the object's *stored* coordinate system and the stream is handed the same
@@ -4724,9 +4955,6 @@ pub(crate) async fn get_transitioned_object_reader_with_tier_manager(
// through a plaintext-coordinate range and skipping the transform is how a // through a plaintext-coordinate range and skipping the transform is how a
// transitioned SSE object used to come back as silently corrupt bytes of the // transitioned SSE object used to come back as silently corrupt bytes of the
// right length (rustfs/rustfs#6025). // right length (rustfs/rustfs#6025).
let plan = ReadPlan::build_for_request(rs.clone(), oi, opts, h, resolver)
.await
.map_err(|err| std::io::Error::other(format!("building the read plan for {bucket}/{object} failed: {err}")))?;
let (off, length) = (plan.storage_offset() as i64, plan.storage_length()); let (off, length) = (plan.storage_offset() as i64, plan.storage_length());
let mut gopts = WarmBackendGetOpts::default(); let mut gopts = WarmBackendGetOpts::default();
@@ -5599,11 +5827,13 @@ mod tests {
use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}; use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
use crate::object_api::{ObjectInfo, ObjectOptions, PutObjReader}; use crate::object_api::{ObjectInfo, ObjectOptions, PutObjReader};
#[cfg(feature = "test-util")] #[cfg(feature = "test-util")]
use crate::services::tier::test_util::MockWarmOp;
#[cfg(feature = "test-util")]
use crate::services::tier::test_util::register_mock_tier; use crate::services::tier::test_util::register_mock_tier;
#[cfg(feature = "test-util")] #[cfg(feature = "test-util")]
use crate::services::tier::tier::TierConfigMgr; use crate::services::tier::tier::TierConfigMgr;
#[cfg(feature = "test-util")] #[cfg(feature = "test-util")]
use crate::services::tier::warm_backend::WarmBackend as _; use crate::services::tier::warm_backend::{TransitionCandidateProbe, WarmBackend as _};
use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause}; use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause};
use crate::set_disk::{RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY}; use crate::set_disk::{RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY};
use crate::storage_api_contracts::namespace::NamespaceLocking as _; use crate::storage_api_contracts::namespace::NamespaceLocking as _;
@@ -6299,7 +6529,75 @@ mod tests {
#[cfg(feature = "test-util")] #[cfg(feature = "test-util")]
#[tokio::test] #[tokio::test]
async fn transitioned_get_rejects_unknown_version_state_before_backend_io() { async fn transitioned_get_allows_legacy_unknown_exact_version_for_non_destructive_read() {
let manager = TierConfigMgr::new();
let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
let backend = register_mock_tier(&manager, &tier).await;
let remote_object = format!("remote/{}", Uuid::new_v4());
let body = Bytes::from_static(b"legacy transitioned object body");
let remote_version = backend
.put(
&remote_object,
ReaderImpl::Body(body.clone()),
i64::try_from(body.len()).expect("body length should fit"),
)
.await
.expect("mock remote object should be stored");
let mut user_defined = HashMap::new();
insert_legacy_transition_version_id(&mut user_defined, &remote_version);
let object_info = ObjectInfo {
bucket: "bucket".to_string(),
name: "object".to_string(),
size: i64::try_from(body.len()).expect("body length should fit"),
transitioned_object: TransitionedObject {
name: remote_object,
version_id: remote_version,
status: crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string(),
tier: tier.clone(),
..Default::default()
},
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
user_defined: user_defined.into(),
..Default::default()
};
let range = Some(crate::storage_api_contracts::range::HTTPRangeSpec {
is_suffix_length: false,
start: 7,
end: 18,
});
let mut reader = get_transitioned_object_reader_with_tier_manager(
&object_info.bucket,
&object_info.name,
&range,
&HeaderMap::new(),
&object_info,
&ObjectOptions::default(),
&manager,
None,
)
.await
.expect("legacy unknown state should still allow a non-destructive read");
let mut got = Vec::new();
reader
.stream
.read_to_end(&mut got)
.await
.expect("transitioned reader should drain");
assert_eq!(got, &body.as_ref()[7..=18]);
assert_eq!(backend.get_count().await, 1);
assert_eq!(backend.remove_count().await, 0);
assert_eq!(
TierConfigMgr::active_operation_lease_count(&manager, &tier).await,
0,
"tier generation lease should release after EOF"
);
}
#[cfg(feature = "test-util")]
#[tokio::test]
async fn transitioned_get_rejects_explicit_unknown_version_state_before_backend_io() {
let manager = TierConfigMgr::new(); let manager = TierConfigMgr::new();
let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
let backend = register_mock_tier(&manager, &tier).await; let backend = register_mock_tier(&manager, &tier).await;
@@ -6315,6 +6613,181 @@ mod tests {
..Default::default() ..Default::default()
}, },
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown, transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
user_defined: user_defined_with_transition_version_state(rustfs_filemeta::TransitionVersionState::Unknown).into(),
..Default::default()
};
let err = match get_transitioned_object_reader_with_tier_manager(
&object_info.bucket,
&object_info.name,
&None,
&HeaderMap::new(),
&object_info,
&ObjectOptions::default(),
&manager,
None,
)
.await
{
Ok(_) => panic!("explicit unknown remote version state must fail before backend IO"),
Err(err) => err,
};
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
assert_eq!(backend.op_log().await, Vec::<MockWarmOp>::new());
assert_eq!(backend.get_count().await, 0);
}
#[cfg(feature = "test-util")]
#[tokio::test]
async fn transitioned_get_rejects_present_but_invalid_legacy_version_metadata() {
let manager = TierConfigMgr::new();
let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
let backend = register_mock_tier(&manager, &tier).await;
for persisted_version in [
Uuid::nil().to_string(),
"\u{fffd}".to_string(),
"bad\u{0001}version".to_string(),
] {
let mut user_defined = HashMap::new();
insert_legacy_transition_version_id(&mut user_defined, &persisted_version);
let object_info = ObjectInfo {
bucket: "bucket".to_string(),
name: "object".to_string(),
size: 1,
transitioned_object: TransitionedObject {
name: "remote/object".to_string(),
version_id: String::new(),
status: crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string(),
tier: tier.clone(),
..Default::default()
},
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
user_defined: user_defined.into(),
..Default::default()
};
let err = match get_transitioned_object_reader_with_tier_manager(
&object_info.bucket,
&object_info.name,
&None,
&HeaderMap::new(),
&object_info,
&ObjectOptions::default(),
&manager,
None,
)
.await
{
Ok(_) => panic!("present but invalid legacy version metadata must fail before backend IO"),
Err(err) => err,
};
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
}
assert_eq!(backend.op_log().await, Vec::<MockWarmOp>::new());
}
#[cfg(feature = "test-util")]
#[tokio::test]
async fn transitioned_get_probes_legacy_empty_unknown_state_before_unversioned_read() {
let manager = TierConfigMgr::new();
let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
let backend = register_mock_tier(&manager, &tier).await;
backend.set_put_remote_version(Some(String::new())).await;
let remote_object = format!("remote/{}", Uuid::new_v4());
let body = Bytes::from_static(b"legacy unversioned transitioned object body");
let remote_version = backend
.put(
&remote_object,
ReaderImpl::Body(body.clone()),
i64::try_from(body.len()).expect("body length should fit"),
)
.await
.expect("mock remote object should be stored");
assert!(remote_version.is_empty());
let object_info = ObjectInfo {
bucket: "bucket".to_string(),
name: "object".to_string(),
size: i64::try_from(body.len()).expect("body length should fit"),
transitioned_object: TransitionedObject {
name: remote_object.clone(),
version_id: String::new(),
status: crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string(),
tier: tier.clone(),
..Default::default()
},
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
user_defined: HashMap::from([("x-minio-internal-transitioned-versionID".to_string(), String::new())]).into(),
..Default::default()
};
let mut reader = get_transitioned_object_reader_with_tier_manager(
&object_info.bucket,
&object_info.name,
&None,
&HeaderMap::new(),
&object_info,
&ObjectOptions::default(),
&manager,
None,
)
.await
.expect("probe-proven legacy unversioned state should allow a non-destructive read");
let mut got = Vec::new();
reader
.stream
.read_to_end(&mut got)
.await
.expect("transitioned reader should drain");
assert_eq!(got, body.as_ref());
assert_eq!(backend.remove_count().await, 0);
assert_eq!(
backend.op_log().await,
vec![
MockWarmOp::Put {
object: remote_object.clone()
},
MockWarmOp::Probe {
object: remote_object.clone()
},
MockWarmOp::Get { object: remote_object },
]
);
assert_eq!(
TierConfigMgr::active_operation_lease_count(&manager, &tier).await,
0,
"tier generation lease should release after EOF"
);
}
#[cfg(feature = "test-util")]
#[tokio::test]
async fn transitioned_get_rejects_ambiguous_empty_unknown_state_without_backend_get() {
let manager = TierConfigMgr::new();
let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
let backend = register_mock_tier(&manager, &tier).await;
let remote_object = format!("remote/{}", Uuid::new_v4());
backend
.set_transition_candidate_probe_override(Some(TransitionCandidateProbe::VersionedPresent(
"versioned-candidate".to_string(),
)))
.await;
let object_info = ObjectInfo {
bucket: "bucket".to_string(),
name: "object".to_string(),
size: 1,
transitioned_object: TransitionedObject {
name: remote_object.clone(),
version_id: String::new(),
status: crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string(),
tier,
..Default::default()
},
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
..Default::default() ..Default::default()
}; };
@@ -6330,19 +6803,28 @@ mod tests {
) )
.await .await
{ {
Ok(_) => panic!("unknown remote version state must fail before backend IO"), Ok(_) => panic!("versioned legacy unknown state without stored version must fail before backend GET"),
Err(err) => err, Err(err) => err,
}; };
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
assert_eq!(backend.op_log().await, vec![MockWarmOp::Probe { object: remote_object }]);
assert_eq!(backend.get_count().await, 0); assert_eq!(backend.get_count().await, 0);
assert_eq!(backend.remove_count().await, 0);
} }
#[cfg(feature = "test-util")] #[cfg(feature = "test-util")]
#[tokio::test] #[tokio::test]
async fn free_version_delete_rejects_unknown_version_state_before_backend_io() { async fn free_version_delete_rejects_explicit_unknown_before_backend_io() {
let manager = TierConfigMgr::new(); let manager = TierConfigMgr::new();
let backend = register_mock_tier(&manager, "WARM").await; let backend = register_mock_tier(&manager, "WARM").await;
let identity = test_tier_destination_identity(&manager, "WARM").await;
let mut user_defined = user_defined_with_tier_destination_identity(identity);
rustfs_utils::http::metadata_compat::insert_str(
&mut user_defined,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
rustfs_filemeta::TransitionVersionState::Unknown.as_str().to_string(),
);
let object_info = ObjectInfo { let object_info = ObjectInfo {
transitioned_object: TransitionedObject { transitioned_object: TransitionedObject {
name: "remote/object".to_string(), name: "remote/object".to_string(),
@@ -6351,17 +6833,251 @@ mod tests {
..Default::default() ..Default::default()
}, },
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown, transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
user_defined: user_defined.into(),
..Default::default() ..Default::default()
}; };
let err = super::delete_free_version_remote_object(&object_info, &manager) let err = super::delete_free_version_remote_object(&object_info, &manager)
.await .await
.expect_err("unknown remote version state must fail before backend IO"); .expect_err("explicit unknown cleanup must fail before backend IO");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
assert!(err.to_string().contains("version state is unknown"));
assert_eq!(backend.op_log().await, Vec::<MockWarmOp>::new());
assert_eq!(backend.remove_count().await, 0); assert_eq!(backend.remove_count().await, 0);
} }
#[cfg(feature = "test-util")]
async fn test_tier_destination_identity(
manager: &Arc<tokio::sync::RwLock<TierConfigMgr>>,
tier: &str,
) -> crate::services::tier::tier::TierDestinationId {
TierConfigMgr::acquire_operation_lease(manager, tier)
.await
.expect("test tier lease should be available")
.backend_identity()
}
#[cfg(feature = "test-util")]
fn user_defined_with_tier_destination_identity(
identity: crate::services::tier::tier::TierDestinationId,
) -> HashMap<String, String> {
let mut user_defined = HashMap::new();
rustfs_utils::http::metadata_compat::insert_str(
&mut user_defined,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
rustfs_utils::crypto::hex(identity),
);
user_defined
}
#[cfg(feature = "test-util")]
fn user_defined_with_transition_version_state(state: rustfs_filemeta::TransitionVersionState) -> HashMap<String, String> {
let mut user_defined = HashMap::new();
rustfs_utils::http::metadata_compat::insert_str(
&mut user_defined,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
state.as_str().to_string(),
);
user_defined
}
#[cfg(feature = "test-util")]
fn insert_legacy_transition_version_id(user_defined: &mut HashMap<String, String>, version_id: &str) {
rustfs_utils::http::metadata_compat::insert_str(
user_defined,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_ID,
version_id.to_string(),
);
}
#[cfg(feature = "test-util")]
#[tokio::test]
async fn free_version_tuple_rejects_mixed_legacy_missing_and_explicit_unknown() {
let manager = TierConfigMgr::new();
register_mock_tier(&manager, "WARM").await;
let identity = test_tier_destination_identity(&manager, "WARM").await;
let mut legacy_metadata = user_defined_with_tier_destination_identity(identity);
insert_legacy_transition_version_id(&mut legacy_metadata, "legacy-version");
let mut explicit_metadata = legacy_metadata.clone();
rustfs_utils::http::metadata_compat::insert_str(
&mut explicit_metadata,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
rustfs_filemeta::TransitionVersionState::Unknown.as_str().to_string(),
);
let make_info = |user_defined: HashMap<String, String>| ObjectInfo {
transitioned_object: TransitionedObject {
name: "remote/object".to_string(),
version_id: "legacy-version".to_string(),
tier: "WARM".to_string(),
..Default::default()
},
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
user_defined: user_defined.into(),
..Default::default()
};
let err = super::free_version_remote_tuple_matches(&make_info(legacy_metadata), &make_info(explicit_metadata))
.expect_err("mixed legacy-missing and explicit unknown provenance must fail closed");
assert_eq!(err.kind(), std::io::ErrorKind::WouldBlock);
}
#[cfg(feature = "test-util")]
#[tokio::test]
async fn free_version_delete_probes_exact_version_hidden_by_current_delete_marker() {
let manager = TierConfigMgr::new();
let tier = "WARM";
let backend = register_mock_tier(&manager, tier).await;
let identity = test_tier_destination_identity(&manager, tier).await;
let remote_object = format!("remote/{}", Uuid::new_v4());
let body = Bytes::from_static(b"legacy exact cleanup body");
let remote_version = backend
.put(
&remote_object,
ReaderImpl::Body(body),
i64::try_from(b"legacy exact cleanup body".len()).expect("body length should fit"),
)
.await
.expect("mock remote object should be stored");
let mut user_defined = user_defined_with_tier_destination_identity(identity);
insert_legacy_transition_version_id(&mut user_defined, &remote_version);
backend
.set_transition_candidate_probe_override(Some(TransitionCandidateProbe::Missing))
.await;
assert_eq!(
backend
.probe_transition_candidate_state(&remote_object)
.await
.expect("current remote view should be readable"),
TransitionCandidateProbe::Missing,
"a current delete marker must hide the historical data version from an unversioned probe"
);
backend.clear_op_log().await;
let object_info = ObjectInfo {
transitioned_object: TransitionedObject {
name: remote_object.clone(),
version_id: remote_version,
tier: tier.to_string(),
..Default::default()
},
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
user_defined: user_defined.into(),
..Default::default()
};
super::delete_free_version_remote_object(&object_info, &manager)
.await
.expect("probe-proven legacy exact cleanup should delete the remote version");
super::delete_free_version_remote_object(&object_info, &manager)
.await
.expect("a retry after the exact remote version is already missing should be idempotent");
assert_eq!(
backend.op_log().await,
vec![
MockWarmOp::Get {
object: remote_object.clone()
},
MockWarmOp::Remove {
object: remote_object.clone()
},
MockWarmOp::Get {
object: remote_object.clone()
},
]
);
assert_eq!(
backend.remove_versions().await,
vec![(remote_object, object_info.transitioned_object.version_id)]
);
}
#[cfg(feature = "test-util")]
#[tokio::test]
async fn free_version_delete_retains_legacy_unknown_unversioned_object() {
let manager = TierConfigMgr::new();
let tier = "WARM";
let backend = register_mock_tier(&manager, tier).await;
backend.set_put_remote_version(Some(String::new())).await;
let identity = test_tier_destination_identity(&manager, tier).await;
let remote_object = format!("remote/{}", Uuid::new_v4());
let body = Bytes::from_static(b"legacy unversioned cleanup body");
let remote_version = backend
.put(
&remote_object,
ReaderImpl::Body(body),
i64::try_from(b"legacy unversioned cleanup body".len()).expect("body length should fit"),
)
.await
.expect("mock remote object should be stored");
assert!(remote_version.is_empty());
backend.clear_op_log().await;
let mut user_defined = user_defined_with_tier_destination_identity(identity);
user_defined.insert("x-minio-internal-transitioned-versionID".to_string(), String::new());
let object_info = ObjectInfo {
transitioned_object: TransitionedObject {
name: remote_object.clone(),
version_id: String::new(),
tier: tier.to_string(),
..Default::default()
},
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
user_defined: user_defined.into(),
..Default::default()
};
let err = super::delete_free_version_remote_object(&object_info, &manager)
.await
.expect_err("legacy unversioned cleanup cannot exclude a versioning-state race");
assert_eq!(err.kind(), std::io::ErrorKind::WouldBlock);
assert!(backend.op_log().await.is_empty());
assert_eq!(backend.remove_count().await, 0);
assert!(backend.remove_versions().await.is_empty());
}
#[cfg(feature = "test-util")]
#[tokio::test]
async fn free_version_delete_does_not_remove_a_different_remote_version() {
let manager = TierConfigMgr::new();
let tier = "WARM";
let backend = register_mock_tier(&manager, tier).await;
let identity = test_tier_destination_identity(&manager, tier).await;
let remote_object = format!("remote/{}", Uuid::new_v4());
backend.set_put_remote_version(Some("different-version".to_string())).await;
backend
.put(
&remote_object,
ReaderImpl::Body(Bytes::from_static(b"different remote version")),
i64::try_from(b"different remote version".len()).expect("body length should fit"),
)
.await
.expect("different remote version should be stored");
backend.clear_op_log().await;
let mut user_defined = user_defined_with_tier_destination_identity(identity);
insert_legacy_transition_version_id(&mut user_defined, "legacy-version");
let object_info = ObjectInfo {
transitioned_object: TransitionedObject {
name: remote_object.clone(),
version_id: "legacy-version".to_string(),
tier: tier.to_string(),
..Default::default()
},
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
user_defined: user_defined.into(),
..Default::default()
};
super::delete_free_version_remote_object(&object_info, &manager)
.await
.expect("a missing exact legacy version should be an idempotent cleanup success");
assert_eq!(backend.op_log().await, vec![MockWarmOp::Get { object: remote_object }]);
assert_eq!(backend.remove_count().await, 0);
assert!(backend.remove_versions().await.is_empty());
}
#[cfg(feature = "test-util")] #[cfg(feature = "test-util")]
#[tokio::test] #[tokio::test]
async fn free_version_remote_delete_requires_persisted_destination_identity() { async fn free_version_remote_delete_requires_persisted_destination_identity() {
@@ -1170,6 +1170,7 @@ pub async fn save_manual_transition_job_record_if_current(
data.clone(), data.clone(),
&ObjectOptions { &ObjectOptions {
max_parity: true, max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions { http_preconditions: Some(HTTPPreconditions {
if_match: Some(current_etag.to_string()), if_match: Some(current_etag.to_string()),
..Default::default() ..Default::default()
@@ -1242,6 +1243,7 @@ pub(crate) async fn save_manual_transition_worker_result_if_absent(
data, data,
&ObjectOptions { &ObjectOptions {
max_parity: true, max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions { http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("*".to_string()), if_none_match: Some("*".to_string()),
..Default::default() ..Default::default()
@@ -1270,6 +1272,7 @@ pub(crate) async fn save_manual_transition_task_if_absent(
data, data,
&ObjectOptions { &ObjectOptions {
max_parity: true, max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions { http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("*".to_string()), if_none_match: Some("*".to_string()),
..Default::default() ..Default::default()
@@ -1621,6 +1624,7 @@ pub async fn save_manual_transition_scope_admission_if_absent(
data.clone(), data.clone(),
&ObjectOptions { &ObjectOptions {
max_parity: true, max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions { http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("*".to_string()), if_none_match: Some("*".to_string()),
..Default::default() ..Default::default()
@@ -1672,6 +1676,7 @@ pub async fn save_manual_transition_scope_admission_if_current(
data.clone(), data.clone(),
&ObjectOptions { &ObjectOptions {
max_parity: true, max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions { http_preconditions: Some(HTTPPreconditions {
if_match: Some(current_etag.to_string()), if_match: Some(current_etag.to_string()),
..Default::default() ..Default::default()
@@ -1733,6 +1733,7 @@ async fn save_config_if_none_fenced(
data.clone(), data.clone(),
&ObjectOptions { &ObjectOptions {
max_parity: true, max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions { http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("*".to_string()), if_none_match: Some("*".to_string()),
..Default::default() ..Default::default()
@@ -1832,6 +1833,7 @@ async fn save_decommission_manifest_checkpoint_if_match(
let mut opts = ObjectOptions { let mut opts = ObjectOptions {
max_parity: true, max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
no_lock: true, no_lock: true,
http_preconditions: Some(HTTPPreconditions { http_preconditions: Some(HTTPPreconditions {
if_match: Some(observed_etag), if_match: Some(observed_etag),
@@ -1960,6 +1962,7 @@ async fn save_config_if_match_fenced(
data.clone(), data.clone(),
&ObjectOptions { &ObjectOptions {
max_parity: true, max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions { http_preconditions: Some(HTTPPreconditions {
if_match: Some(etag.to_string()), if_match: Some(etag.to_string()),
..Default::default() ..Default::default()
@@ -3780,6 +3783,7 @@ where
data.clone(), data.clone(),
&ObjectOptions { &ObjectOptions {
max_parity: true, max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions { http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("*".to_string()), if_none_match: Some("*".to_string()),
..Default::default() ..Default::default()
@@ -3869,6 +3873,7 @@ where
data.clone(), data.clone(),
&ObjectOptions { &ObjectOptions {
max_parity: true, max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions { http_preconditions: Some(HTTPPreconditions {
if_match: Some(etag), if_match: Some(etag),
..Default::default() ..Default::default()
@@ -3893,6 +3898,7 @@ where
data.clone(), data.clone(),
&ObjectOptions { &ObjectOptions {
max_parity: true, max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions { http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("*".to_string()), if_none_match: Some("*".to_string()),
..Default::default() ..Default::default()
@@ -15,8 +15,6 @@
#![allow(unused_variables)] #![allow(unused_variables)]
#![allow(unused_mut)] #![allow(unused_mut)]
#![allow(unused_assignments)] #![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use super::runtime_boundary as runtime_sources; use super::runtime_boundary as runtime_sources;
use crate::bucket::lifecycle::bucket_lifecycle_ops::ExpiryOp; use crate::bucket::lifecycle::bucket_lifecycle_ops::ExpiryOp;
@@ -72,9 +70,11 @@ static REMOTE_DELETE_BREAKER: LazyLock<Mutex<RemoteDeleteBreaker>> = LazyLock::n
}); });
#[cfg(test)] #[cfg(test)]
static REMOTE_TIER_DELETE_TEST_HOOK: std::sync::LazyLock< type RemoteTierDeleteTestHook = Box<dyn Fn(&str, &str, &str) -> std::io::Result<()> + Send + Sync>;
std::sync::Mutex<Option<Box<dyn Fn(&str, &str, &str) -> std::io::Result<()> + Send + Sync>>>,
> = std::sync::LazyLock::new(|| std::sync::Mutex::new(None)); #[cfg(test)]
static REMOTE_TIER_DELETE_TEST_HOOK: std::sync::LazyLock<std::sync::Mutex<Option<RemoteTierDeleteTestHook>>> =
std::sync::LazyLock::new(|| std::sync::Mutex::new(None));
#[derive(Debug)] #[derive(Debug)]
struct RemoteDeleteBreaker { struct RemoteDeleteBreaker {
@@ -107,7 +107,7 @@ impl RemoteDeleteBreaker {
fn prune(&mut self, now: Instant) { fn prune(&mut self, now: Instant) {
while let Some(ts) = self.failures.front().copied() { while let Some(ts) = self.failures.front().copied() {
if now.duration_since(ts) > self.window { if now.duration_since(ts) > self.window {
self.failures.pop_front(); let _ = self.failures.pop_front();
} else { } else {
break; break;
} }
@@ -137,10 +137,10 @@ fn is_signer_header_error(err: &std::io::Error) -> bool {
return false; return false;
} }
if let Some(source) = err.get_ref() { if let Some(source) = err.get_ref()
if error_chain_contains_signer_header_marker(source) { && error_chain_contains_signer_header_marker(source)
return true; {
} return true;
} }
let message = err.to_string().to_ascii_lowercase(); let message = err.to_string().to_ascii_lowercase();
@@ -205,7 +205,7 @@ impl ObjSweeper {
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
pub fn with_version(&mut self, vid: Option<Uuid>) -> &Self { pub fn with_version(&mut self, vid: Option<Uuid>) -> &Self {
self.version_id = vid.clone(); self.version_id = vid;
self self
} }
@@ -219,7 +219,7 @@ impl ObjSweeper {
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
pub fn get_opts(&self) -> lifecycle::ObjectOpts { pub fn get_opts(&self) -> lifecycle::ObjectOpts {
let mut opts = ObjectOpts { let mut opts = ObjectOpts {
version_id: self.version_id.clone(), version_id: self.version_id,
versioned: self.versioned, versioned: self.versioned,
version_suspended: self.suspended, version_suspended: self.suspended,
..Default::default() ..Default::default()
@@ -388,8 +388,8 @@ impl Jentry {
impl ExpiryOp for Jentry { impl ExpiryOp for Jentry {
fn op_hash(&self) -> u64 { fn op_hash(&self) -> u64 {
let mut hasher = Sha256::new(); let mut hasher = Sha256::new();
hasher.update(format!("{}", self.tier_name).as_bytes()); hasher.update(self.tier_name.as_bytes());
hasher.update(format!("{}", self.obj_name).as_bytes()); hasher.update(self.obj_name.as_bytes());
xxh64::xxh64(hasher.finalize().as_slice(), XXHASH_SEED) xxh64::xxh64(hasher.finalize().as_slice(), XXHASH_SEED)
} }
@@ -436,7 +436,7 @@ async fn delete_object_from_remote_tier_raw_with_manager(
tier_name: &str, tier_name: &str,
tier_config_mgr: &Arc<tokio::sync::RwLock<TierConfigMgr>>, tier_config_mgr: &Arc<tokio::sync::RwLock<TierConfigMgr>>,
) -> Result<(), std::io::Error> { ) -> Result<(), std::io::Error> {
let lease = TierConfigMgr::acquire_operation_lease(&tier_config_mgr, tier_name) let lease = TierConfigMgr::acquire_operation_lease(tier_config_mgr, tier_name)
.await .await
.map_err(std::io::Error::other)?; .map_err(std::io::Error::other)?;
delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, &lease, false, true).await delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, &lease, false, true).await
@@ -612,6 +612,7 @@ pub(crate) async fn save_transition_transaction_record(
data.clone(), data.clone(),
&ObjectOptions { &ObjectOptions {
max_parity: true, max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions { http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("*".to_string()), if_none_match: Some("*".to_string()),
..Default::default() ..Default::default()
@@ -658,6 +659,7 @@ pub(crate) async fn save_transition_transaction_record_if_current(
data.clone(), data.clone(),
&ObjectOptions { &ObjectOptions {
max_parity: true, max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions { http_preconditions: Some(HTTPPreconditions {
if_match: Some(etag), if_match: Some(etag),
..Default::default() ..Default::default()
+162 -8
View File
@@ -477,6 +477,18 @@ impl BucketMetadata {
!self.table_bucket_config_json.is_empty() !self.table_bucket_config_json.is_empty()
} }
/// `bucket-targets.json` is stored for this bucket but this build cannot
/// decode it.
///
/// Keeps "no replication targets configured" and "the target
/// configuration cannot be read" apart, the same distinction the
/// `fabricated` marker draws for the bucket metadata as a whole. Only
/// meaningful after [`Self::parse_all_configs`] has run; readers must fail
/// closed on `true` instead of serving an empty target set.
pub fn bucket_targets_unreadable(&self) -> bool {
!self.bucket_targets_config_json.is_empty() && self.bucket_target_config.is_none()
}
/// Parsed per-bucket durability override, if a valid one is stored. /// Parsed per-bucket durability override, if a valid one is stored.
/// ///
/// Absent/empty/unparsable payloads all mean "no override" (the bucket /// Absent/empty/unparsable payloads all mean "no override" (the bucket
@@ -964,7 +976,32 @@ impl BucketMetadata {
Ok(()) Ok(())
} }
fn parse_all_configs(&mut self) -> Result<()> { /// Decode every stored sub-configuration into its typed field.
///
/// A decode failure never fails the whole load: this runs on every bucket
/// metadata read, including startup and peer reload, so one bucket's
/// corrupt sub-configuration must not make the bucket — or the node —
/// unloadable. Instead the failure is *retained*: the raw bytes stay
/// untouched and the typed field stays `None`, so `!raw.is_empty() &&
/// typed.is_none()` is the durable "exists but cannot be read" signal that
/// each accessor keys off. Which accessors must fail closed on it:
///
/// | Config | Verdict |
/// |---|---|
/// | policy | Fails closed: `get_bucket_policy` re-parses the raw JSON and propagates the error; `get_bucket_policy_raw` returns the stored bytes. |
/// | object lock | Fails closed in `object_lock_config_state_from_authoritative_metadata`; a retention decision may never be taken on a guess. |
/// | versioning | Fails closed in `get_versioning_config`; guessing Unversioned would make delete markers and version ids diverge from what is on disk. |
/// | replication | Fails closed in `get_replication_config`. |
/// | bucket targets | Fails closed in `get_bucket_targets_config`, and `sync_bucket_target_sys` marks the bucket unreadable in `BucketTargetSys` instead of publishing an empty target set (rustfs/backlog#2282). |
/// | encryption | Fails closed in `get_sse_config`: degrading to "no default encryption" stores plaintext objects the operator required to be encrypted. |
/// | public access block | Fails closed in `get_public_access_block_config`: degrading grants the anonymous access the operator asked to block. |
/// | quota | Fails closed in `get_quota_config`; the enforcement path in `quota::checker` already re-parses the raw JSON and refuses on error. |
/// | lifecycle | Safe to degrade: no rules means no expiration and no transition, so nothing is deleted or moved on the strength of an unreadable rule set. The bucket keeps serving reads and writes. |
/// | notification | Safe to degrade: events are an outbound side channel; no consumer draws a durability or authorization conclusion from their absence. |
/// | tagging | Safe to degrade: bucket tags are cost-allocation labels here; object-level tag conditions come from object metadata, not this blob. |
/// | CORS | Safe to degrade: an absent CORS configuration rejects cross-origin browser requests, which is already the restrictive direction. |
/// | logging, website, accelerate, request payment, bucket ACL | Safe to degrade: each only shapes an optional response or an optional side channel, and none of them authorizes an action or decides whether data is retained. |
pub(super) fn parse_all_configs(&mut self) -> Result<()> {
if let Err(e) = self.parse_policy_config() { if let Err(e) = self.parse_policy_config() {
tracing::warn!( tracing::warn!(
event = "bucket_metadata_parse_failed", event = "bucket_metadata_parse_failed",
@@ -1088,20 +1125,26 @@ impl BucketMetadata {
"Failed to parse bucket metadata config" "Failed to parse bucket metadata config"
); );
} }
// A stored targets blob that cannot be decoded must not collapse into
// the empty target set: that is indistinguishable from "no replication
// configured", so replication stops and no caller ever sees an error
// (rustfs/backlog#2282). Leaving the typed field `None` while the raw
// bytes stay non-empty is the retained parse failure every targets
// reader keys off; the bytes are preserved so the configuration is
// still recoverable.
self.bucket_target_config = None;
if !self.bucket_targets_config_json.is_empty() { if !self.bucket_targets_config_json.is_empty() {
if let Err(e) = serde_json::from_slice::<BucketTargets>(&self.bucket_targets_config_json) match serde_json::from_slice::<BucketTargets>(&self.bucket_targets_config_json) {
.map(|t| self.bucket_target_config = Some(t)) Ok(targets) => self.bucket_target_config = Some(targets),
{ Err(e) => tracing::error!(
tracing::warn!(
event = "bucket_metadata_parse_failed", event = "bucket_metadata_parse_failed",
component = "ecstore", component = "ecstore",
subsystem = "bucket_metadata", subsystem = "bucket_metadata",
bucket = %self.name, bucket = %self.name,
config = "bucket_targets", config = "bucket_targets",
error = %e, error = %e,
"Failed to parse bucket metadata config" "Bucket replication targets are unreadable; replication for this bucket fails closed"
); ),
self.bucket_target_config = Some(BucketTargets::default());
} }
} else { } else {
self.bucket_target_config = Some(BucketTargets::default()); self.bucket_target_config = Some(BucketTargets::default());
@@ -1535,6 +1578,117 @@ mod test {
assert_eq!(bucket_targets.targets[0].target_bucket, "target-bucket"); assert_eq!(bucket_targets.targets[0].target_bucket, "target-bucket");
} }
/// rustfs/backlog#2282: a stored targets blob this build cannot decode
/// must not become the empty target set, and must stay distinguishable
/// from a bucket that never configured a target.
#[test]
fn unreadable_bucket_targets_never_degrade_to_an_empty_target_set() {
let truncated = br#"{"targets":[{"endpoint":"s3.example.com","#.to_vec();
let mut corrupt = BucketMetadata::new("corrupt-targets");
corrupt.bucket_targets_config_json = truncated.clone();
corrupt
.parse_all_configs()
.expect("one unreadable sub-config must not fail the whole metadata load");
assert!(
corrupt.bucket_target_config.is_none(),
"an undecodable targets blob must not produce a target set at all"
);
assert!(corrupt.bucket_targets_unreadable());
assert_eq!(
corrupt.bucket_targets_config_json, truncated,
"the raw bytes must survive so the configuration stays recoverable"
);
// The genuinely-absent case is unchanged, and the two now diverge.
let mut absent = BucketMetadata::new("no-targets");
absent.parse_all_configs().expect("absent targets parse");
assert!(
absent.bucket_target_config.as_ref().is_some_and(BucketTargets::is_empty),
"a bucket that configured no target still reads as an empty target set"
);
assert!(!absent.bucket_targets_unreadable());
}
/// `Credentials` carries no struct-level `serde(default)`, so one target
/// missing `secretKey` is a hard parse error for the whole document. That
/// must surface as "unreadable", never as "no targets configured".
#[test]
fn bucket_targets_missing_secret_key_are_unreadable_not_empty() {
let mut bm = BucketMetadata::new("missing-secret-key");
bm.bucket_targets_config_json = br#"{"targets":[{"endpoint":"s3.example.com","targetbucket":"remote","arn":"arn:rustfs:replication:us-east-1:src:1","credentials":{"accessKey":"AKIAEXAMPLE"}}]}"#.to_vec();
bm.parse_all_configs()
.expect("a rejected targets document must not fail the whole metadata load");
assert!(
bm.bucket_targets_unreadable(),
"a targets document rejected for a missing secretKey is unreadable, not empty"
);
assert!(bm.bucket_target_config.is_none());
}
/// The invariant every branch of `parse_all_configs` shares: a stored but
/// undecodable payload keeps its raw bytes and leaves the typed field
/// `None`, so no branch fabricates a value. What a reader may then do with
/// that state is decided per config; see the table on `parse_all_configs`.
#[test]
fn every_config_branch_retains_its_parse_failure_instead_of_defaulting() {
let malformed_xml = b"<not-a-valid-document".to_vec();
let malformed_json = b"{not-json".to_vec();
let mut bm = BucketMetadata::new("all-configs-malformed");
bm.policy_config_json = malformed_json.clone();
bm.quota_config_json = malformed_json.clone();
bm.bucket_targets_config_json = malformed_json.clone();
bm.notification_config_xml = malformed_xml.clone();
bm.lifecycle_config_xml = malformed_xml.clone();
bm.object_lock_config_xml = malformed_xml.clone();
bm.versioning_config_xml = malformed_xml.clone();
bm.encryption_config_xml = malformed_xml.clone();
bm.tagging_config_xml = malformed_xml.clone();
bm.replication_config_xml = malformed_xml.clone();
bm.cors_config_xml = malformed_xml.clone();
bm.logging_config_xml = malformed_xml.clone();
bm.website_config_xml = malformed_xml.clone();
bm.accelerate_config_xml = malformed_xml.clone();
bm.request_payment_config_xml = malformed_xml.clone();
bm.public_access_block_config_xml = malformed_xml.clone();
// `bucket_acl_config_json` is only checked for UTF-8, so only invalid
// UTF-8 exercises its failure branch.
bm.bucket_acl_config_json = vec![0xff, 0xfe];
bm.parse_all_configs()
.expect("a bucket whose every config is corrupt must still load its metadata");
let cleared: [(&str, bool); 17] = [
("policy", bm.policy_config.is_none()),
("quota", bm.quota_config.is_none()),
("bucket_targets", bm.bucket_target_config.is_none()),
("notification", bm.notification_config.is_none()),
("lifecycle", bm.lifecycle_config.is_none()),
("object_lock", bm.object_lock_config.is_none()),
("versioning", bm.versioning_config.is_none()),
("encryption", bm.sse_config.is_none()),
("tagging", bm.tagging_config.is_none()),
("replication", bm.replication_config.is_none()),
("cors", bm.cors_config.is_none()),
("logging", bm.logging_config.is_none()),
("website", bm.website_config.is_none()),
("accelerate", bm.accelerate_config.is_none()),
("request_payment", bm.request_payment_config.is_none()),
("public_access_block", bm.public_access_block_config.is_none()),
("bucket_acl", bm.bucket_acl_config.is_none()),
];
for (config, is_cleared) in cleared {
assert!(is_cleared, "{config}: a corrupt payload must not be replaced by a default");
}
assert_eq!(bm.bucket_targets_config_json, malformed_json, "raw bytes are retained");
assert_eq!(bm.lifecycle_config_xml, malformed_xml, "raw bytes are retained");
}
#[test] #[test]
fn lifecycle_update_config_clears_parsed_config_on_delete() { fn lifecycle_update_config_clears_parsed_config_on_delete() {
let mut bm = BucketMetadata::new("test-bucket"); let mut bm = BucketMetadata::new("test-bucket");
+275 -11
View File
@@ -360,6 +360,16 @@ async fn refresh_buckets_metadata_once(sys: Arc<RwLock<BucketMetadataSys>>) {
} }
async fn sync_bucket_target_sys(bucket: &str, bm: &BucketMetadata) { async fn sync_bucket_target_sys(bucket: &str, bm: &BucketMetadata) {
if bm.bucket_targets_unreadable() {
// "The configuration cannot be read" is not "no targets configured".
// Publishing an empty snapshot here is what silently stopped
// replication (rustfs/backlog#2282): mark the bucket instead, so every
// targets reader gets a typed error, and leave any snapshot from an
// earlier readable load in place rather than withdrawing it.
BucketTargetSys::get().mark_targets_unreadable(bucket).await;
return;
}
BucketTargetSys::get() BucketTargetSys::get()
.update_all_targets(bucket, bm.bucket_target_config.as_ref()) .update_all_targets(bucket, bm.bucket_target_config.as_ref())
.await; .await;
@@ -645,6 +655,12 @@ pub struct BucketMetadataMutationGuard {
} }
impl BucketMetadataMutationGuard { impl BucketMetadataMutationGuard {
/// Returns the storage-verified identity while both incarnation fences remain valid.
pub fn checked_bucket_incarnation(&self) -> Result<(&str, Uuid)> {
self.ensure_valid(&self.bucket)?;
Ok((&self.bucket, self.incarnation_id))
}
fn ensure_valid(&self, bucket: &str) -> Result<()> { fn ensure_valid(&self, bucket: &str) -> Result<()> {
if self.bucket != bucket { if self.bucket != bucket {
return Err(Error::other("bucket metadata mutation guard does not match bucket")); return Err(Error::other("bucket metadata mutation guard does not match bucket"));
@@ -664,6 +680,29 @@ async fn acquire_config_write_guard_for_incarnation(
sys: Arc<RwLock<BucketMetadataSys>>, sys: Arc<RwLock<BucketMetadataSys>>,
bucket: &str, bucket: &str,
expected_incarnation_id: Option<Uuid>, expected_incarnation_id: Option<Uuid>,
) -> Result<BucketMetadataMutationGuard> {
acquire_config_write_guard_with_migration(sys, bucket, expected_incarnation_id, true).await
}
/// Scanner probes must not create an incarnation to make a capability available.
pub async fn acquire_scanner_bucket_incarnation_fence(
bucket: &str,
expected_incarnation_id: Uuid,
expected_owner_id: Uuid,
) -> Result<BucketMetadataMutationGuard> {
super::utils::check_valid_bucket_name(bucket)?;
let sys = get_bucket_metadata_sys()?;
if expected_owner_id.is_nil() || sys.read().await.api.id != expected_owner_id || expected_incarnation_id.is_nil() {
return Err(Error::other("scanner bucket incarnation owner does not match"));
}
acquire_config_write_guard_with_migration(sys, bucket, Some(expected_incarnation_id), false).await
}
async fn acquire_config_write_guard_with_migration(
sys: Arc<RwLock<BucketMetadataSys>>,
bucket: &str,
expected_incarnation_id: Option<Uuid>,
migrate: bool,
) -> Result<BucketMetadataMutationGuard> { ) -> Result<BucketMetadataMutationGuard> {
let metadata_sys = sys.read().await.clone(); let metadata_sys = sys.read().await.clone();
let lifecycle_guard = metadata_sys.api.acquire_bucket_lifecycle_read_lock(bucket).await?; let lifecycle_guard = metadata_sys.api.acquire_bucket_lifecycle_read_lock(bucket).await?;
@@ -671,13 +710,15 @@ async fn acquire_config_write_guard_for_incarnation(
// Legacy buckets are migrated while the lifecycle fence prevents a // Legacy buckets are migrated while the lifecycle fence prevents a
// same-name replacement. The second read under the write transaction is // same-name replacement. The second read under the write transaction is
// the CAS source of truth for the actual rewrite. // the CAS source of truth for the actual rewrite.
await_bucket_namespace_operation( if migrate {
Some(&lifecycle_guard), await_bucket_namespace_operation(
bucket, Some(&lifecycle_guard),
"bucket config incarnation migration", bucket,
metadata_sys.get_bucket_incarnation_id(bucket), "bucket config incarnation migration",
) metadata_sys.get_bucket_incarnation_id(bucket),
.await?; )
.await?;
}
let transaction_guard = await_bucket_namespace_operation( let transaction_guard = await_bucket_namespace_operation(
Some(&lifecycle_guard), Some(&lifecycle_guard),
bucket, bucket,
@@ -2118,7 +2159,9 @@ impl BucketMetadataSys {
pub async fn get_public_access_block_config(&self, bucket: &str) -> Result<(PublicAccessBlockConfiguration, OffsetDateTime)> { pub async fn get_public_access_block_config(&self, bucket: &str) -> Result<(PublicAccessBlockConfiguration, OffsetDateTime)> {
let (bm, _) = self.get_config(bucket).await?; let (bm, _) = self.get_config(bucket).await?;
if let Some(config) = &bm.public_access_block_config { if !bm.public_access_block_config_xml.is_empty() && bm.public_access_block_config.is_none() {
Err(Error::other("persisted bucket public access block configuration is invalid"))
} else if let Some(config) = &bm.public_access_block_config {
Ok((config.clone(), bm.public_access_block_config_updated_at)) Ok((config.clone(), bm.public_access_block_config_updated_at))
} else { } else {
Err(Error::ConfigNotFound) Err(Error::ConfigNotFound)
@@ -2429,7 +2472,9 @@ impl BucketMetadataSys {
pub async fn get_sse_config(&self, bucket: &str) -> Result<(ServerSideEncryptionConfiguration, OffsetDateTime)> { pub async fn get_sse_config(&self, bucket: &str) -> Result<(ServerSideEncryptionConfiguration, OffsetDateTime)> {
let (bm, _) = self.get_config(bucket).await?; let (bm, _) = self.get_config(bucket).await?;
if let Some(config) = &bm.sse_config { if !bm.encryption_config_xml.is_empty() && bm.sse_config.is_none() {
Err(Error::other("persisted bucket encryption configuration is invalid"))
} else if let Some(config) = &bm.sse_config {
Ok((config.clone(), bm.encryption_config_updated_at)) Ok((config.clone(), bm.encryption_config_updated_at))
} else { } else {
Err(Error::ConfigNotFound) Err(Error::ConfigNotFound)
@@ -2500,7 +2545,9 @@ impl BucketMetadataSys {
pub async fn get_quota_config(&self, bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> { pub async fn get_quota_config(&self, bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> {
let (bm, _) = self.get_config(bucket).await?; let (bm, _) = self.get_config(bucket).await?;
if let Some(config) = &bm.quota_config { if !bm.quota_config_json.is_empty() && bm.quota_config.is_none() {
Err(Error::other("persisted bucket quota configuration is invalid"))
} else if let Some(config) = &bm.quota_config {
Ok((config.clone(), bm.quota_config_updated_at)) Ok((config.clone(), bm.quota_config_updated_at))
} else { } else {
Err(Error::ConfigNotFound) Err(Error::ConfigNotFound)
@@ -2522,7 +2569,9 @@ impl BucketMetadataSys {
pub async fn get_bucket_targets_config(&self, bucket: &str) -> Result<BucketTargets> { pub async fn get_bucket_targets_config(&self, bucket: &str) -> Result<BucketTargets> {
let (bm, _) = self.get_config(bucket).await?; let (bm, _) = self.get_config(bucket).await?;
if let Some(config) = &bm.bucket_target_config { if bm.bucket_targets_unreadable() {
Err(Error::other("persisted bucket replication target configuration is invalid"))
} else if let Some(config) = &bm.bucket_target_config {
Ok(config.clone()) Ok(config.clone())
} else { } else {
Err(Error::ConfigNotFound) Err(Error::ConfigNotFound)
@@ -2593,6 +2642,7 @@ pub(crate) mod test_support {
mod tests { mod tests {
use super::test_support::isolated_store_over_temp_disks; use super::test_support::isolated_store_over_temp_disks;
use super::*; use super::*;
use crate::bucket::bucket_target_sys::BucketTargetError;
use crate::bucket::metadata::{ use crate::bucket::metadata::{
BUCKET_ACCELERATE_CONFIG, BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_LOGGING_CONFIG, BUCKET_NOTIFICATION_CONFIG, BUCKET_ACCELERATE_CONFIG, BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_LOGGING_CONFIG, BUCKET_NOTIFICATION_CONFIG,
BUCKET_POLICY_CONFIG, BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, BUCKET_REPLICATION_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, BUCKET_REPLICATION_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG,
@@ -2788,6 +2838,36 @@ mod tests {
); );
} }
/// The `parse_all_configs` audit (rustfs/backlog#2282): every accessor
/// whose configuration grants something — plaintext storage, anonymous
/// access, capacity, replication targets — reports a corrupt payload as
/// invalid rather than as absent, because "absent" is what grants it.
#[tokio::test]
async fn malformed_permissive_configs_are_not_reported_as_absent() {
let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
let sys = BucketMetadataSys::new(ecstore);
let bucket = "malformed-permissive-config";
let mut metadata = BucketMetadata::new(bucket);
metadata.encryption_config_xml = b"<ServerSideEncryptionConfiguration".to_vec();
metadata.public_access_block_config_xml = b"<PublicAccessBlockConfiguration".to_vec();
metadata.quota_config_json = b"{not-json".to_vec();
metadata.bucket_targets_config_json = b"{not-json".to_vec();
metadata
.parse_all_configs()
.expect("a corrupt sub-config must not fail the load");
sys.set(bucket.to_string(), Arc::new(metadata)).await;
for (config, result) in [
("encryption", sys.get_sse_config(bucket).await.err()),
("public access block", sys.get_public_access_block_config(bucket).await.err()),
("quota", sys.get_quota_config(bucket).await.err()),
("bucket targets", sys.get_bucket_targets_config(bucket).await.err()),
] {
let err = result.unwrap_or_else(|| panic!("malformed {config} metadata must not read as a value"));
assert_ne!(err, Error::ConfigNotFound, "malformed {config} metadata must not be reported as absent");
}
}
#[tokio::test] #[tokio::test]
async fn config_states_distinguish_authoritative_absence_from_fabricated_metadata() { async fn config_states_distinguish_authoritative_absence_from_fabricated_metadata() {
use std::sync::atomic::Ordering; use std::sync::atomic::Ordering;
@@ -3127,6 +3207,82 @@ mod tests {
); );
} }
#[tokio::test]
async fn scoped_dirty_usage_incarnation_probe_does_not_migrate_legacy_metadata() {
let (dirs, store) = isolated_store_over_temp_disks().await;
let sys = Arc::new(RwLock::new(BucketMetadataSys::new(store.clone())));
let bucket = "scoped-ack-legacy";
for dir in &dirs {
std::fs::create_dir_all(dir.path().join(bucket)).expect("create legacy bucket");
}
let mut metadata = BucketMetadata::new(bucket);
metadata.bucket_incarnation_id = Uuid::nil();
sys.read()
.await
.persist_and_set(metadata)
.await
.expect("persist legacy metadata");
assert!(
acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(Uuid::new_v4()), false)
.await
.is_err()
);
assert!(load_bucket_incarnation(store, bucket).await.expect("read sidecar").is_none());
assert!(
sys.read()
.await
.get_config_from_disk(bucket)
.await
.expect("read metadata")
.bucket_incarnation_id
.is_nil()
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn scoped_dirty_usage_incarnation_rejects_deleted_and_recreated_bucket() {
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let sys = bucket_metadata_sys_of(&store.ctx).expect("metadata owner");
let bucket = "scoped-ack-recreated";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
let old = store.bucket_incarnation_id_from_disk(bucket).await.expect("old incarnation");
let guard = acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(old), false)
.await
.expect("trusted incarnation fence");
assert_eq!(guard.checked_bucket_incarnation().expect("valid fences"), (bucket, old));
drop(guard);
store
.delete_bucket(bucket, &DeleteBucketOptions::default())
.await
.expect("delete bucket");
assert!(
acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(old), false)
.await
.is_err()
);
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("recreate bucket");
let new = store.bucket_incarnation_id_from_disk(bucket).await.expect("new incarnation");
assert_ne!(old, new);
assert!(
acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(old), false)
.await
.is_err()
);
assert!(
acquire_config_write_guard_with_migration(sys, bucket, Some(new), false)
.await
.is_ok()
);
}
#[tokio::test] #[tokio::test]
async fn old_node_metadata_rewrite_cannot_replace_bucket_incarnation_sidecar() { async fn old_node_metadata_rewrite_cannot_replace_bucket_incarnation_sidecar() {
let (dirs, ecstore) = isolated_store_over_temp_disks().await; let (dirs, ecstore) = isolated_store_over_temp_disks().await;
@@ -4066,6 +4222,114 @@ mod tests {
target_sys.delete(bucket).await; target_sys.delete(bucket).await;
} }
/// rustfs/backlog#2282: an unreadable `bucket-targets.json` reaches every
/// targets reader as a typed error; it neither withdraws a snapshot a
/// previous readable load published, nor collapses into the "no targets
/// configured" state that a bucket with an absent configuration reports.
#[tokio::test]
#[serial]
async fn unreadable_bucket_targets_fail_closed_and_stay_distinct_from_absent() {
let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
let sys = BucketMetadataSys::new(ecstore);
let target_sys = BucketTargetSys::get();
let unreadable = "targets-unreadable";
let absent = "targets-absent";
target_sys.delete(unreadable).await;
target_sys.delete(absent).await;
// A readable load publishes this bucket's targets.
let mut readable = BucketMetadata::new(unreadable);
readable.bucket_target_config = Some(BucketTargets {
targets: vec![target(unreadable, "live")],
});
sync_bucket_target_sys(unreadable, &readable).await;
assert_eq!(
target_sys
.list_bucket_targets(unreadable)
.await
.expect("readable targets publish")
.targets
.len(),
1
);
// The same bucket reloaded with a blob that cannot be decoded.
let mut corrupt = BucketMetadata::new(unreadable);
corrupt.bucket_targets_config_json = br#"{"targets":[{"endpoint":"#.to_vec();
corrupt
.parse_all_configs()
.expect("an unreadable targets blob must not fail the metadata load");
sys.set(unreadable.to_string(), Arc::new(corrupt)).await;
assert!(
matches!(
target_sys.list_bucket_targets(unreadable).await,
Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. })
),
"an unreadable configuration must not read as an empty or a missing target set"
);
assert!(
target_sys.list_targets(unreadable, "").await.is_err(),
"the admin listing must surface the fault instead of an empty list"
);
let err = sys
.get_bucket_targets_config(unreadable)
.await
.expect_err("an unreadable targets configuration must not read as a value");
assert_ne!(err, Error::ConfigNotFound, "unreadable must not be reported as absent");
// A bucket that never configured a target keeps its previous behavior.
let mut no_targets = BucketMetadata::new(absent);
no_targets.parse_all_configs().expect("absent targets parse");
sys.set(absent.to_string(), Arc::new(no_targets)).await;
assert!(
matches!(
target_sys.list_bucket_targets(absent).await,
Err(BucketTargetError::BucketRemoteTargetNotFound { .. })
),
"an absent configuration must still report as a missing target set"
);
assert!(
target_sys
.list_targets(absent, "")
.await
.expect("an absent configuration lists no targets")
.is_empty()
);
assert!(
sys.get_bucket_targets_config(absent)
.await
.expect("an absent targets configuration still reads as an empty set")
.is_empty(),
"the absent path must keep returning an empty target set, exactly as before"
);
// One bucket's unreadable configuration does not reach another bucket.
assert!(!matches!(
target_sys.list_bucket_targets(absent).await,
Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. })
));
// A repaired configuration takes effect on the next load, no restart.
let mut repaired = BucketMetadata::new(unreadable);
repaired.bucket_target_config = Some(BucketTargets {
targets: vec![target(unreadable, "repaired")],
});
sync_bucket_target_sys(unreadable, &repaired).await;
assert_eq!(
target_sys
.list_bucket_targets(unreadable)
.await
.expect("a repaired configuration clears the unreadable marker")
.targets
.len(),
1
);
target_sys.delete(unreadable).await;
target_sys.delete(absent).await;
}
#[tokio::test] #[tokio::test]
#[serial] #[serial]
async fn metadata_reload_clears_stale_bucket_targets_when_config_is_removed() { async fn metadata_reload_clears_stale_bucket_targets_when_config_is_removed() {
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,172 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! One contract every [`SourceBackend`] implementation must satisfy.
//!
//! The migration pipeline talks to a source only through the trait, so a new
//! provider is correct exactly when it answers the same questions the same way:
//! the same head fields, the same range semantics, the same page shape, the
//! same error classes. Each backend supplies a fixture that answers this fixed
//! corpus in its own dialect and then runs [`assert_backend_contract`], so a
//! provider-specific mapping bug shows up as a contract failure rather than as
//! a surprise in the pull pipeline.
//!
//! Backends differ in two documented ways, declared through
//! [`BackendCapabilities`]: whether the provider's ETag is a content digest,
//! and whether the provider can resume a listing from a key.
use super::source_client::{SourceBackend, SourceError, SourceListRequest};
use crate::storage_api_contracts::range::HTTPRangeSpec;
use std::collections::HashMap;
/// The single object every fixture serves.
pub(super) const OBJECT_KEY: &str = "dir/a.txt";
pub(super) const OBJECT_BODY: &[u8] = b"hello";
/// MD5 of [`OBJECT_BODY`]; the ETag of the object on a digest provider.
pub(super) const OBJECT_MD5: &str = "5d41402abc4b2a76b9719d911017c592";
/// The second key the fixture's listing returns, on its second page.
pub(super) const SECOND_KEY: &str = "dir/b.txt";
pub(super) const COMMON_PREFIX: &str = "dir/sub/";
pub(super) const LIST_CURSOR: &str = "cursor-1";
/// A key the fixture answers with the provider's "no such object".
pub(super) const MISSING_KEY: &str = "missing";
/// A key the fixture answers with the provider's "not authorized".
pub(super) const FORBIDDEN_KEY: &str = "secret";
/// Where backends are allowed to differ.
#[derive(Clone, Copy, Debug)]
pub(super) struct BackendCapabilities {
/// The provider's ETag is an opaque token, not a digest of the bytes.
pub(super) etag_is_opaque: bool,
/// The provider can resume a listing from a key rather than only from an
/// opaque cursor.
pub(super) supports_start_after: bool,
/// The provider has an object-tagging concept at all. GCS does not, and
/// answers with an empty map instead of failing a pull.
pub(super) supports_tagging: bool,
}
/// Drives `backend` through the shared corpus. Fixtures are scripted in
/// request order, so the call order here is part of the contract.
pub(super) async fn assert_backend_contract(backend: &dyn SourceBackend, caps: BackendCapabilities) {
// 1. HEAD maps the object's shared fields.
let head = backend.head(OBJECT_KEY).await.expect("HEAD of the fixture object");
assert_eq!(head.size, OBJECT_BODY.len() as u64, "HEAD reports the object size");
assert_eq!(head.content_type.as_deref(), Some("text/plain"));
assert_eq!(
head.user_metadata,
HashMap::from([("owner".to_string(), "alice".to_string())]),
"user metadata is keyed without the provider prefix"
);
assert!(head.storage_class.is_some(), "the provider's tier is recorded");
assert!(head.last_modified.is_some(), "the provider's timestamp is parsed");
assert!(head.sse.is_none(), "the fixture object is not server-side encrypted");
assert!(!head.is_multipart_etag);
assert_eq!(head.etag_is_opaque, caps.etag_is_opaque);
match caps.etag_is_opaque {
false => assert_eq!(head.etag.as_deref(), Some(OBJECT_MD5), "a digest ETag is mapped verbatim"),
true => assert!(head.etag.is_some(), "an opaque ETag is still recorded"),
}
// 2. An unranged GET streams the whole object and reports no range.
let got = backend.get(OBJECT_KEY, None).await.expect("unranged GET");
assert_eq!(got.head.size, OBJECT_BODY.len() as u64);
assert!(got.content_range.is_none(), "an unranged GET has no content-range");
assert_eq!(got.head.etag_is_opaque, caps.etag_is_opaque, "GET and HEAD agree about the ETag");
let body = got.body.collect().await.expect("body streams").into_bytes();
assert_eq!(body.as_ref(), OBJECT_BODY);
// 3. A ranged GET returns exactly the requested interval, and `size` is
// the length of the returned bytes rather than of the object.
let range = HTTPRangeSpec {
is_suffix_length: false,
start: 1,
end: 3,
};
let got = backend.get(OBJECT_KEY, Some(&range)).await.expect("ranged GET");
assert_eq!(got.head.size, 3, "a ranged GET reports the range length");
assert_eq!(got.content_range.as_deref(), Some("bytes 1-3/5"));
let body = got.body.collect().await.expect("body streams").into_bytes();
assert_eq!(body.as_ref(), &OBJECT_BODY[1..=3]);
// 4. A delimiter listing rolls prefixes up and hands back a cursor.
let page = backend
.list(&SourceListRequest {
prefix: Some("dir/"),
delimiter: Some("/"),
max_keys: 2,
..Default::default()
})
.await
.expect("first listing page");
assert_eq!(page.objects.len(), 1, "the first page holds one object");
assert_eq!(page.objects[0].key, OBJECT_KEY, "listing keys are in the source namespace");
assert_eq!(page.objects[0].size, OBJECT_BODY.len() as u64);
assert!(page.objects[0].last_modified.is_some());
assert_eq!(page.common_prefixes, vec![COMMON_PREFIX.to_string()]);
assert!(page.is_truncated);
assert_eq!(page.next_continuation_token.as_deref(), Some(LIST_CURSOR));
// 5. The cursor is passed back verbatim and the last page ends the walk.
let page = backend
.list(&SourceListRequest {
prefix: Some("dir/"),
delimiter: Some("/"),
continuation_token: Some(LIST_CURSOR),
max_keys: 2,
..Default::default()
})
.await
.expect("second listing page");
assert_eq!(page.objects.len(), 1);
assert_eq!(page.objects[0].key, SECOND_KEY);
assert!(!page.is_truncated);
assert!(page.next_continuation_token.is_none(), "a complete listing carries no cursor");
// 6. Tags come back as a flat map, empty on a provider without tags.
let tags = backend.tagging(OBJECT_KEY).await.expect("object tags");
match caps.supports_tagging {
true => assert_eq!(tags, HashMap::from([("env".to_string(), "prod".to_string())])),
false => assert!(tags.is_empty(), "a provider without tags reports none: {tags:?}"),
}
// 7. The probe confirms the bucket or container answers.
backend.probe().await.expect("probe of the fixture bucket");
// 8. A missing object is `NotFound`, and never retried.
let err = backend.head(MISSING_KEY).await.expect_err("a missing object must fail");
assert!(matches!(err, SourceError::NotFound), "{err:?}");
assert_eq!(err.class_label(), "not_found");
assert!(!err.is_retryable());
// 9. A denied object is `AccessDenied`, and never retried.
let err = backend.head(FORBIDDEN_KEY).await.expect_err("a denied object must fail");
assert!(matches!(err, SourceError::AccessDenied), "{err:?}");
assert_eq!(err.class_label(), "access_denied");
assert!(!err.is_retryable());
// 10. A provider without a key cursor must refuse one instead of listing
// from the wrong position. This issues no request either way.
if !caps.supports_start_after {
let err = backend
.list(&SourceListRequest {
start_after: Some(OBJECT_KEY),
max_keys: 1,
..Default::default()
})
.await
.expect_err("a backend without a key cursor must refuse start_after");
assert!(matches!(err, SourceError::Unsupported(_)), "{err:?}");
}
}
@@ -25,8 +25,8 @@
//! [`BACKFILL_SAVE_INTERVAL`], and at every page end, with an `If-Match` //! [`BACKFILL_SAVE_INTERVAL`], and at every page end, with an `If-Match`
//! compare-and-set so a concurrent cancel or takeover is never overwritten. //! compare-and-set so a concurrent cancel or takeover is never overwritten.
//! - The `continuation_token` only advances once every pull queued from the //! - The `continuation_token` only advances once every pull queued from the
//! page before it has reported back, so a crash re-lists at most one page //! page before it has succeeded. After a failure it stays at that page,
//! (already-present keys are then skipped, never re-pulled). //! so crash recovery cannot skip failed pulls (existing keys are skipped).
//! - The owner holds a lease of [`BACKFILL_LEASE`] renewed by every save. The //! - The owner holds a lease of [`BACKFILL_LEASE`] renewed by every save. The
//! recovery loop ([`run_backfill_recovery_loop`]) scans the buckets this //! recovery loop ([`run_backfill_recovery_loop`]) scans the buckets this
//! node has an ODM state for every [`BACKFILL_RECOVERY_INTERVAL`] and takes //! node has an ODM state for every [`BACKFILL_RECOVERY_INTERVAL`] and takes
@@ -367,9 +367,8 @@ pub struct LocalBackfillObject {
pub source_etag: Option<String>, pub source_etag: Option<String>,
} }
/// Receiver of one queued pull's report; `None` when the pull was coalesced /// Shared report of a new or coalesced pull; absent only when not admitted.
/// into one already running. pub type PullReport = Option<super::pull::QueuedPullReport>;
pub type PullReport = Option<oneshot::Receiver<QueuedPullOutcome>>;
/// Everything the job needs from its bucket, so the loop can run against a /// Everything the job needs from its bucket, so the loop can run against a
/// mock in unit tests. Production: [`BucketBackfillContext`]. /// mock in unit tests. Production: [`BucketBackfillContext`].
@@ -684,6 +683,7 @@ async fn write_checkpoint(
}; };
let opts = ObjectOptions { let opts = ObjectOptions {
max_parity: true, max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(preconditions), http_preconditions: Some(preconditions),
..Default::default() ..Default::default()
}; };
@@ -1190,9 +1190,11 @@ impl Job {
} }
async fn main_loop(&mut self) -> Result<(), Stop> { async fn main_loop(&mut self) -> Result<(), Stop> {
let mut cursor = self.checkpoint.continuation_token.clone();
let failed_at_resume = self.checkpoint.failed;
loop { loop {
self.check_cancel()?; self.check_cancel()?;
let page = self.list_page().await?; let page = self.list_page(cursor.as_deref()).await?;
for object in &page.objects { for object in &page.objects {
self.check_cancel()?; self.check_cancel()?;
self.checkpoint.listed += 1; self.checkpoint.listed += 1;
@@ -1204,10 +1206,13 @@ impl Job {
self.drain_ready(); self.drain_ready();
self.tick(false).await?; self.tick(false).await?;
} }
// Only advance the cursor once every pull of this page reported // A persisted cursor certifies successful work, not just listing
// back, so a takeover re-lists at most this page. // progress. Keep it at the first failed page for crash recovery.
self.drain_all().await?; self.drain_all().await?;
self.checkpoint.continuation_token = page.next_continuation_token.clone(); cursor = page.next_continuation_token;
if self.checkpoint.failed == failed_at_resume {
self.checkpoint.continuation_token = cursor.clone();
}
self.tick(true).await?; self.tick(true).await?;
if !page.is_truncated { if !page.is_truncated {
return Ok(()); return Ok(());
@@ -1222,7 +1227,7 @@ impl Job {
} }
} }
async fn list_page(&mut self) -> Result<SourcePage, Stop> { async fn list_page(&mut self, cursor: Option<&str>) -> Result<SourcePage, Stop> {
let mut attempt = 0; let mut attempt = 0;
loop { loop {
while !self.context.source_available() { while !self.context.source_available() {
@@ -1230,7 +1235,7 @@ impl Job {
self.tick(false).await?; self.tick(false).await?;
} }
let prefix = self.checkpoint.prefix.clone(); let prefix = self.checkpoint.prefix.clone();
let token = self.checkpoint.continuation_token.clone(); let token = cursor.map(str::to_string);
match self match self
.context .context
.list_page(prefix.as_deref(), token.as_deref(), BACKFILL_LIST_PAGE_SIZE) .list_page(prefix.as_deref(), token.as_deref(), BACKFILL_LIST_PAGE_SIZE)
@@ -1304,9 +1309,10 @@ impl Job {
} }
loop { loop {
match self.context.enqueue(key) { match self.context.enqueue(key) {
(EnqueueOutcome::Enqueued, report) => { (EnqueueOutcome::Enqueued | EnqueueOutcome::Coalesced, report) => {
self.checkpoint.enqueued += 1; self.checkpoint.enqueued += 1;
if let Some(rx) = report { let rx = report.ok_or(Stop::Unavailable)?;
{
let key = key.to_string(); let key = key.to_string();
self.outstanding.push(Box::pin(async move { (key, rx.await) })); self.outstanding.push(Box::pin(async move { (key, rx.await) }));
} }
@@ -1321,11 +1327,6 @@ impl Job {
); );
return Ok(()); return Ok(());
} }
(EnqueueOutcome::Coalesced, _) => {
// Someone else pulls it; its result is not ours to count.
self.checkpoint.enqueued += 1;
return Ok(());
}
(EnqueueOutcome::QueueFull, _) => { (EnqueueOutcome::QueueFull, _) => {
// Wait, never drop: one completion frees a slot. // Wait, never drop: one completion frees a slot.
if self.outstanding.is_empty() { if self.outstanding.is_empty() {
@@ -1639,6 +1640,7 @@ mod tests {
queue_capacity: usize, queue_capacity: usize,
pending: Mutex<Vec<(String, oneshot::Sender<QueuedPullOutcome>)>>, pending: Mutex<Vec<(String, oneshot::Sender<QueuedPullOutcome>)>>,
fail_keys: HashSet<String>, fail_keys: HashSet<String>,
coalesced: bool,
auto_complete: AtomicBool, auto_complete: AtomicBool,
cancel: CancellationToken, cancel: CancellationToken,
config_updated_at: Mutex<Option<OffsetDateTime>>, config_updated_at: Mutex<Option<OffsetDateTime>>,
@@ -1666,6 +1668,7 @@ mod tests {
queue_capacity: usize::MAX, queue_capacity: usize::MAX,
pending: Mutex::new(Vec::new()), pending: Mutex::new(Vec::new()),
fail_keys: HashSet::new(), fail_keys: HashSet::new(),
coalesced: false,
auto_complete: AtomicBool::new(true), auto_complete: AtomicBool::new(true),
cancel: CancellationToken::new(), cancel: CancellationToken::new(),
config_updated_at: Mutex::new(Some(ts(1_700_000_000))), config_updated_at: Mutex::new(Some(ts(1_700_000_000))),
@@ -1745,7 +1748,12 @@ mod tests {
} else { } else {
self.pending.lock().push((key.to_string(), tx)); self.pending.lock().push((key.to_string(), tx));
} }
(EnqueueOutcome::Enqueued, Some(rx)) let outcome = if self.coalesced {
EnqueueOutcome::Coalesced
} else {
EnqueueOutcome::Enqueued
};
(outcome, Some(futures::FutureExt::shared(rx)))
} }
fn cancel_token(&self) -> CancellationToken { fn cancel_token(&self) -> CancellationToken {
@@ -1911,7 +1919,7 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn failed_pulls_are_counted_hashed_and_finish_with_failures() { async fn failed_pulls_are_counted_hashed_and_finish_with_failures() {
let bucket = "backfill-failed"; let bucket = "backfill-failed";
let mut context = MockContext::new(5, 1000); let mut context = MockContext::new(5, 2);
Arc::get_mut(&mut context) Arc::get_mut(&mut context)
.expect("unshared") .expect("unshared")
.fail_keys .fail_keys
@@ -1926,12 +1934,52 @@ mod tests {
.checkpoint; .checkpoint;
assert_eq!(cp.state, BackfillState::CompletedWithFailures); assert_eq!(cp.state, BackfillState::CompletedWithFailures);
assert_eq!((cp.pulled, cp.failed), (4, 1)); assert_eq!((cp.pulled, cp.failed), (4, 1));
assert_eq!(cp.continuation_token.as_deref(), Some("2"), "retain the first failed page for recovery");
assert_eq!(cp.failed_keys, vec![key_hash("k/00002")]); assert_eq!(cp.failed_keys, vec![key_hash("k/00002")]);
let last = cp.last_error.expect("last error"); let last = cp.last_error.expect("last error");
assert_eq!(last.class, "local_write"); assert_eq!(last.class, "local_write");
assert_eq!(last.key_hash.as_deref(), Some(key_hash("k/00002").as_str())); assert_eq!(last.key_hash.as_deref(), Some(key_hash("k/00002").as_str()));
} }
#[tokio::test]
async fn coalesced_pulls_block_the_checkpoint_and_report_failures() {
let bucket = "backfill-coalesced";
let mut context = MockContext::new(1, 1);
{
let ctx = Arc::get_mut(&mut context).expect("unshared");
ctx.coalesced = true;
ctx.auto_complete = AtomicBool::new(false);
ctx.fail_keys.insert("k/00000".to_string());
}
let (_dirs, store, runner) = runner_with("node-a", bucket, Arc::clone(&context)).await;
runner.start(bucket, BackfillRequest::default()).await.expect("start");
tokio::time::timeout(Duration::from_secs(10), async {
while context.pending.lock().is_empty() {
tokio::task::yield_now().await;
}
})
.await
.expect("job enqueued");
assert!(runner.is_running_locally(bucket), "coalescing is not completion");
let cp = read_checkpoint(&store, bucket)
.await
.expect("read")
.expect("checkpoint")
.checkpoint;
assert!(cp.state.is_active());
assert!(cp.continuation_token.is_none());
context.complete_pending();
runner.wait_until_idle(bucket).await;
let cp = read_checkpoint(&store, bucket)
.await
.expect("read")
.expect("checkpoint")
.checkpoint;
assert_eq!(cp.state, BackfillState::CompletedWithFailures);
assert_eq!((cp.enqueued, cp.pulled, cp.failed), (1, 0, 1));
assert_eq!(cp.failed_keys, vec![key_hash("k/00000")]);
}
#[tokio::test] #[tokio::test]
async fn listing_failure_marks_the_job_failed_with_the_error_class() { async fn listing_failure_marks_the_job_failed_with_the_error_class() {
let bucket = "backfill-list-error"; let bucket = "backfill-list-error";
@@ -2144,6 +2192,68 @@ mod tests {
assert_eq!(runner.recover_once().await.taken_over, 0, "a finished job is not recovered"); assert_eq!(runner.recover_once().await.taken_over, 0, "a finished job is not recovered");
} }
#[tokio::test]
async fn recovery_advances_past_historical_failures_but_pins_new_failures() {
let bucket = "backfill-takeover-failed";
let mut context = MockContext::new(8, 2);
{
let ctx = Arc::get_mut(&mut context).expect("unshared");
ctx.auto_complete = AtomicBool::new(false);
ctx.fail_keys.insert("k/00004".to_string());
}
let (_dirs, store, runner) = runner_with("node-b", bucket, Arc::clone(&context)).await;
let crashed_at = OffsetDateTime::now_utc() - Duration::from_secs(300);
let mut crashed = BackfillCheckpoint::new(&BackfillRequest::default(), ts(1_700_000_000), "node-a", crashed_at);
crashed.continuation_token = Some("2".to_string());
crashed.failed = 1;
crashed.record_failure("local_write", Some("k/00002"), crashed_at);
write_checkpoint(&store, bucket, &crashed, None)
.await
.expect("seed failed page with an expired lease");
assert_eq!(runner.recover_once().await.taken_over, 1);
for (page_start, durable_token, failures) in [(2, "2", 1), (4, "4", 1), (6, "4", 2)] {
tokio::time::timeout(Duration::from_secs(10), async {
loop {
if context.pending.lock().len() == 2 {
break;
}
tokio::task::yield_now().await;
}
})
.await
.expect("resumed page enqueued before its reports complete");
assert_eq!(
context.pending.lock().iter().map(|(key, _)| key.clone()).collect::<Vec<_>>(),
vec![format!("k/{page_start:05}"), format!("k/{:05}", page_start + 1)]
);
let cp = read_checkpoint(&store, bucket)
.await
.expect("read persisted page boundary")
.expect("checkpoint")
.checkpoint;
assert_eq!(cp.job_id, crashed.job_id);
assert_eq!(cp.owner.as_ref().map(|owner| owner.node.as_str()), Some("node-b"));
assert_eq!(cp.continuation_token.as_deref(), Some(durable_token));
assert_eq!(cp.failed, failures);
context.complete_pending();
}
runner.wait_until_idle(bucket).await;
let cp = read_checkpoint(&store, bucket)
.await
.expect("read completed checkpoint")
.expect("checkpoint")
.checkpoint;
assert_eq!(cp.state, BackfillState::CompletedWithFailures);
assert_eq!((cp.pulled, cp.failed), (5, 2));
assert_eq!(cp.continuation_token.as_deref(), Some("4"));
assert_eq!(cp.failed_keys, vec![key_hash("k/00002"), key_hash("k/00004")]);
assert_eq!(
context.list_requests.lock().as_slice(),
&[Some("2".to_string()), Some("4".to_string()), Some("6".to_string())]
);
}
#[tokio::test] #[tokio::test]
async fn recovery_cancels_a_job_whose_config_changed_and_reclaims_own_node_jobs() { async fn recovery_cancels_a_job_whose_config_changed_and_reclaims_own_node_jobs() {
let bucket = "backfill-recovery-config"; let bucket = "backfill-recovery-config";
@@ -86,7 +86,12 @@ impl BreakerVerdict {
Some(SourceError::Throttled | SourceError::Timeout | SourceError::Connect(_) | SourceError::ServerError(_)) => { Some(SourceError::Throttled | SourceError::Timeout | SourceError::Connect(_) | SourceError::ServerError(_)) => {
BreakerVerdict::Failure BreakerVerdict::Failure
} }
Some(SourceError::AccessDenied | SourceError::Unsupported(_) | SourceError::Other(_)) => BreakerVerdict::Neutral, Some(
SourceError::AccessDenied
| SourceError::Unsupported(_)
| SourceError::InvalidPagination(_)
| SourceError::Other(_),
) => BreakerVerdict::Neutral,
} }
} }
} }
@@ -30,6 +30,10 @@ pub const ON_DEMAND_MIGRATION_CONFIG_VERSION: u32 = 1;
const REDACTED: &str = "REDACTED"; const REDACTED: &str = "REDACTED";
const AUTO_REGION: &str = "auto"; const AUTO_REGION: &str = "auto";
const AUTO_REGION_FALLBACK: &str = "us-east-1"; const AUTO_REGION_FALLBACK: &str = "us-east-1";
/// Public Azure Blob host suffix; the account name is the first label.
pub const AZURE_BLOB_SUFFIX: &str = "blob.core.windows.net";
/// Public Google Cloud Storage endpoint for the native provider.
pub const GCS_DEFAULT_ENDPOINT: &str = "https://storage.googleapis.com";
const KIB: u64 = 1024; const KIB: u64 = 1024;
const MIB: u64 = 1024 * KIB; const MIB: u64 = 1024 * KIB;
@@ -75,14 +79,25 @@ pub struct SourceConfig {
pub bucket: String, pub bucket: String,
#[serde(default)] #[serde(default)]
pub path_style: PathStyle, pub path_style: PathStyle,
/// `None` means anonymous access to a public source bucket. /// `None` means anonymous access to a public source bucket. Only the
/// SigV4 providers read it; `azure` and `gcs_native` carry their own
/// credentials in `azure` / `gcs`.
#[serde(default)] #[serde(default)]
pub credentials: Option<SourceCredentials>, pub credentials: Option<SourceCredentials>,
#[serde(default)] #[serde(default)]
pub tls: TlsConfig, pub tls: TlsConfig,
/// Required for [`Provider::Azure`] and rejected for every other
/// provider.
#[serde(default)]
pub azure: Option<AzureSourceConfig>,
/// Required for [`Provider::GcsNative`] and rejected for every other
/// provider. [`Provider::Gcs`] keeps using `credentials` because it
/// speaks the S3 interoperability API.
#[serde(default)]
pub gcs: Option<GcsSourceConfig>,
} }
/// Source vendor family. `azure` is deliberately absent from this version. /// Source vendor family.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")] #[serde(rename_all = "lowercase")]
pub enum Provider { pub enum Provider {
@@ -94,6 +109,12 @@ pub enum Provider {
R2, R2,
/// GCS XML interoperability API with HMAC keys. /// GCS XML interoperability API with HMAC keys.
Gcs, Gcs,
/// Native Azure Blob service; parameters in `source.azure`.
Azure,
/// Native GCS JSON API with a service-account key; parameters in
/// `source.gcs`.
#[serde(rename = "gcs_native")]
GcsNative,
} }
impl Provider { impl Provider {
@@ -105,13 +126,22 @@ impl Provider {
Provider::Rustfs => "rustfs", Provider::Rustfs => "rustfs",
Provider::R2 => "r2", Provider::R2 => "r2",
Provider::Gcs => "gcs", Provider::Gcs => "gcs",
Provider::Azure => "azure",
Provider::GcsNative => "gcs_native",
} }
} }
/// Providers that do not speak S3 and therefore ignore `region`,
/// `path_style` and `credentials`.
pub fn is_native(&self) -> bool {
matches!(self, Provider::Azure | Provider::GcsNative)
}
/// Providers whose SDKs accept `region = "auto"`; RustFS maps it to /// Providers whose SDKs accept `region = "auto"`; RustFS maps it to
/// `us-east-1` for signing. /// `us-east-1` for signing. The native providers never sign with a
/// region, so they accept it as well.
fn accepts_auto_region(&self) -> bool { fn accepts_auto_region(&self) -> bool {
matches!(self, Provider::R2 | Provider::Minio | Provider::Rustfs) matches!(self, Provider::R2 | Provider::Minio | Provider::Rustfs) || self.is_native()
} }
} }
@@ -164,6 +194,73 @@ impl fmt::Debug for SourceCredentials {
} }
} }
/// Native Azure Blob source parameters. The container is `source.bucket`,
/// so a config never carries two names for the same container. Exactly one
/// of `account_key` and `sas_token` must be set: the account key signs with
/// Shared Key, the SAS token is appended to every request URL.
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct AzureSourceConfig {
/// Storage account name; also derives the default `blob.core.windows.net`
/// endpoint when `source.endpoint` is absent.
pub account: String,
/// Base64 shared key of the storage account.
#[serde(default)]
pub account_key: Option<String>,
/// SAS query string without the leading `?`.
#[serde(default)]
pub sas_token: Option<String>,
}
impl AzureSourceConfig {
/// A copy safe to return to admin clients or log: both secrets are
/// replaced by `REDACTED`, and whether each is set stays visible.
pub fn redacted(&self) -> Self {
Self {
account: self.account.clone(),
account_key: self.account_key.as_ref().map(|_| REDACTED.to_string()),
sas_token: self.sas_token.as_ref().map(|_| REDACTED.to_string()),
}
}
}
impl fmt::Debug for AzureSourceConfig {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("AzureSourceConfig")
.field("account", &self.account)
.field("account_key", &self.account_key.as_ref().map(|_| REDACTED))
.field("sas_token", &self.sas_token.as_ref().map(|_| REDACTED))
.finish()
}
}
/// Native Google Cloud Storage source parameters. The bucket is
/// `source.bucket`; only the service-account key lives here.
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct GcsSourceConfig {
/// Service-account key JSON, verbatim as downloaded from Google Cloud.
pub service_account_json: String,
}
impl GcsSourceConfig {
/// A copy safe to return to admin clients or log: the whole key JSON is
/// a secret (it embeds the private key), so it is replaced wholesale.
pub fn redacted(&self) -> Self {
Self {
service_account_json: REDACTED.to_string(),
}
}
}
impl fmt::Debug for GcsSourceConfig {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("GcsSourceConfig")
.field("service_account_json", &REDACTED)
.finish()
}
}
#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] #[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
#[serde(deny_unknown_fields)] #[serde(deny_unknown_fields)]
pub struct TlsConfig { pub struct TlsConfig {
@@ -354,6 +451,14 @@ pub enum OnDemandMigrationConfigError {
InvalidBucket(&'static str), InvalidBucket(&'static str),
#[error("source credentials field {0} must not be empty")] #[error("source credentials field {0} must not be empty")]
EmptyCredential(&'static str), EmptyCredential(&'static str),
#[error("source.{0} is required for provider {1}")]
MissingProviderBlock(&'static str, Provider),
#[error("source.{0} is not valid for provider {1}")]
UnexpectedProviderBlock(&'static str, Provider),
/// Carries only the reason: the block holds account keys, SAS tokens and
/// service-account JSON, so no value of it is ever echoed.
#[error("source.{0} is invalid: {1}")]
InvalidProviderBlock(&'static str, &'static str),
#[error("source tls.ca_cert_pem is not a PEM certificate")] #[error("source tls.ca_cert_pem is not a PEM certificate")]
InvalidCaCert, InvalidCaCert,
#[error("filter.{0} must be null or a non-empty string")] #[error("filter.{0} must be null or a non-empty string")]
@@ -388,6 +493,8 @@ impl OnDemandMigrationConfig {
pub fn redacted(&self) -> Self { pub fn redacted(&self) -> Self {
let mut copy = self.clone(); let mut copy = self.clone();
copy.source.credentials = self.source.credentials.as_ref().map(SourceCredentials::redacted); copy.source.credentials = self.source.credentials.as_ref().map(SourceCredentials::redacted);
copy.source.azure = self.source.azure.as_ref().map(AzureSourceConfig::redacted);
copy.source.gcs = self.source.gcs.as_ref().map(GcsSourceConfig::redacted);
copy copy
} }
@@ -433,6 +540,12 @@ impl SourceConfig {
match (&self.endpoint, self.provider) { match (&self.endpoint, self.provider) {
(Some(endpoint), _) => endpoint.clone(), (Some(endpoint), _) => endpoint.clone(),
(None, Provider::Aws) => format!("https://s3.{}.amazonaws.com", self.region), (None, Provider::Aws) => format!("https://s3.{}.amazonaws.com", self.region),
(None, Provider::Azure) => self
.azure
.as_ref()
.map(|azure| format!("https://{}.{AZURE_BLOB_SUFFIX}", azure.account))
.unwrap_or_default(),
(None, Provider::GcsNative) => GCS_DEFAULT_ENDPOINT.to_string(),
(None, _) => String::new(), (None, _) => String::new(),
} }
} }
@@ -448,6 +561,8 @@ impl SourceConfig {
} }
fn validate(&self) -> Result<(), OnDemandMigrationConfigError> { fn validate(&self) -> Result<(), OnDemandMigrationConfigError> {
self.validate_provider_block()?;
if self.region.is_empty() { if self.region.is_empty() {
return Err(OnDemandMigrationConfigError::EmptyRegion); return Err(OnDemandMigrationConfigError::EmptyRegion);
} }
@@ -466,6 +581,9 @@ impl SourceConfig {
)); ));
} }
} }
// Both native providers derive a fixed endpoint; Azure's is built
// from the account name, already checked by `validate_provider_block`.
None if self.provider.is_native() => {}
None => return Err(OnDemandMigrationConfigError::MissingEndpoint(self.provider)), None => return Err(OnDemandMigrationConfigError::MissingEndpoint(self.provider)),
} }
@@ -496,6 +614,84 @@ impl SourceConfig {
Ok(()) Ok(())
} }
/// The provider-specific block must be present for exactly its own
/// provider: a stray `azure` block on an `s3` source would otherwise be
/// accepted, stored, and silently ignored by the client builder.
fn validate_provider_block(&self) -> Result<(), OnDemandMigrationConfigError> {
let missing = OnDemandMigrationConfigError::MissingProviderBlock;
let unexpected = OnDemandMigrationConfigError::UnexpectedProviderBlock;
let invalid = OnDemandMigrationConfigError::InvalidProviderBlock;
if self.provider != Provider::Azure && self.azure.is_some() {
return Err(unexpected("azure", self.provider));
}
if self.provider != Provider::GcsNative && self.gcs.is_some() {
return Err(unexpected("gcs", self.provider));
}
match self.provider {
Provider::Azure => {
let azure = self.azure.as_ref().ok_or(missing("azure", self.provider))?;
if azure.account.is_empty() {
return Err(invalid("azure", "account must not be empty"));
}
// The account feeds a hostname when the endpoint is derived:
// keep it to label characters so it cannot rewrite the host.
if !azure.account.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-') {
return Err(invalid("azure", "account contains characters outside [A-Za-z0-9-]"));
}
match (azure.account_key.as_deref(), azure.sas_token.as_deref()) {
(Some(_), Some(_)) => return Err(invalid("azure", "account_key and sas_token are mutually exclusive")),
(None, None) => return Err(invalid("azure", "one of account_key and sas_token is required")),
(Some(key), None) => {
if key.is_empty() {
return Err(invalid("azure", "account_key must not be empty"));
}
// Decoded here so a mistyped key fails at the admin
// boundary instead of on the first source request.
if base64_simd::STANDARD.decode_to_vec(key.as_bytes()).is_err() {
return Err(invalid("azure", "account_key is not base64"));
}
}
(None, Some(sas)) => {
if sas.is_empty() {
return Err(invalid("azure", "sas_token must not be empty"));
}
if sas.starts_with('?') {
return Err(invalid("azure", "sas_token must not start with '?'"));
}
if sas.chars().any(char::is_whitespace) {
return Err(invalid("azure", "sas_token must not contain whitespace"));
}
}
}
}
Provider::GcsNative => {
let gcs = self.gcs.as_ref().ok_or(missing("gcs", self.provider))?;
let key: serde_json::Value = serde_json::from_str(&gcs.service_account_json)
.map_err(|_| invalid("gcs", "service_account_json is not valid JSON"))?;
let Some(object) = key.as_object() else {
return Err(invalid("gcs", "service_account_json is not a JSON object"));
};
if object.get("type").and_then(serde_json::Value::as_str) != Some("service_account") {
return Err(invalid("gcs", "service_account_json is not a service_account key"));
}
for field in ["client_email", "private_key"] {
if object
.get(field)
.and_then(serde_json::Value::as_str)
.is_none_or(str::is_empty)
{
return Err(invalid("gcs", "service_account_json is missing client_email or private_key"));
}
}
}
Provider::S3 | Provider::Aws | Provider::Minio | Provider::Rustfs | Provider::R2 | Provider::Gcs => {}
}
Ok(())
}
} }
fn validate_endpoint(endpoint: &str) -> Result<(), OnDemandMigrationConfigError> { fn validate_endpoint(endpoint: &str) -> Result<(), OnDemandMigrationConfigError> {
@@ -699,7 +895,15 @@ mod tests {
), ),
( (
"provider enum", "provider enum",
r#"{"source":{"provider":"azure","endpoint":"https://h","region":"r","bucket":"b"}}"#, r#"{"source":{"provider":"swift","endpoint":"https://h","region":"r","bucket":"b"}}"#,
),
(
"azure block",
r#"{"source":{"provider":"azure","region":"auto","bucket":"b","azure":{"account":"acct","account_key":"a2V5","extra":1}}}"#,
),
(
"gcs block",
r#"{"source":{"provider":"gcs_native","region":"auto","bucket":"b","gcs":{"service_account_json":"{}","extra":1}}}"#,
), ),
] { ] {
let err = OnDemandMigrationConfig::from_json(json.as_bytes()).expect_err(label); let err = OnDemandMigrationConfig::from_json(json.as_bytes()).expect_err(label);
@@ -820,9 +1024,201 @@ mod tests {
"{provider}" "{provider}"
); );
} }
// The native providers never sign with a region, so "auto" is the
// honest value to write for them.
for cfg in [azure_cfg(), gcs_native_cfg()] {
assert_eq!(cfg.source.region, "auto");
cfg.validate(empty_ctx())
.unwrap_or_else(|err| panic!("{}: {err}", cfg.source.provider));
}
assert_eq!(sample().source.effective_region(), "us-west-1"); assert_eq!(sample().source.effective_region(), "us-west-1");
} }
const SERVICE_ACCOUNT_JSON: &str = r#"{"type":"service_account","project_id":"p","client_email":"a@b.iam.gserviceaccount.com","private_key":"-----BEGIN PRIVATE KEY-----\nsecret\n-----END PRIVATE KEY-----"}"#;
fn azure_cfg() -> OnDemandMigrationConfig {
let mut cfg = sample();
cfg.source.provider = Provider::Azure;
cfg.source.endpoint = None;
cfg.source.region = "auto".to_string();
cfg.source.credentials = None;
cfg.source.azure = Some(AzureSourceConfig {
account: "legacyaccount".to_string(),
account_key: Some("c2VjcmV0LWtleQ==".to_string()),
sas_token: None,
});
cfg
}
fn gcs_native_cfg() -> OnDemandMigrationConfig {
let mut cfg = sample();
cfg.source.provider = Provider::GcsNative;
cfg.source.endpoint = None;
cfg.source.region = "auto".to_string();
cfg.source.credentials = None;
cfg.source.gcs = Some(GcsSourceConfig {
service_account_json: SERVICE_ACCOUNT_JSON.to_string(),
});
cfg
}
#[test]
fn native_providers_derive_their_endpoint_and_round_trip_on_the_wire() {
let azure = azure_cfg();
assert_eq!(azure.source.effective_endpoint(), "https://legacyaccount.blob.core.windows.net");
let gcs = gcs_native_cfg();
assert_eq!(gcs.source.effective_endpoint(), "https://storage.googleapis.com");
for cfg in [azure_cfg(), gcs_native_cfg()] {
let json = cfg.to_json().expect("config must serialize");
assert_eq!(OnDemandMigrationConfig::from_json(&json).expect("config must parse"), cfg);
}
// The wire labels are part of the admin contract.
assert!(
String::from_utf8(azure_cfg().to_json().expect("json"))
.expect("utf8")
.contains(r#""provider":"azure""#)
);
assert!(
String::from_utf8(gcs_native_cfg().to_json().expect("json"))
.expect("utf8")
.contains(r#""provider":"gcs_native""#)
);
}
#[test]
fn an_explicit_endpoint_overrides_the_derived_native_one() {
// Azurite and fake-gcs-server are addressed this way.
let mut cfg = azure_cfg();
cfg.source.endpoint = Some("http://azurite.example.com:10000".to_string());
cfg.validate(empty_ctx()).expect("an explicit native endpoint is allowed");
assert_eq!(cfg.source.effective_endpoint(), "http://azurite.example.com:10000");
cfg.source.endpoint = Some("http://azurite.example.com:10000/devstoreaccount1".to_string());
assert!(
matches!(cfg.validate(empty_ctx()), Err(OnDemandMigrationConfigError::InvalidEndpoint(_))),
"a native endpoint is still an origin"
);
}
#[test]
fn a_provider_block_belongs_to_exactly_its_own_provider() {
let mut cfg = sample();
cfg.source.azure = azure_cfg().source.azure;
assert_eq!(
cfg.validate(empty_ctx()),
Err(OnDemandMigrationConfigError::UnexpectedProviderBlock("azure", Provider::S3))
);
let mut cfg = sample();
cfg.source.gcs = gcs_native_cfg().source.gcs;
assert_eq!(
cfg.validate(empty_ctx()),
Err(OnDemandMigrationConfigError::UnexpectedProviderBlock("gcs", Provider::S3))
);
let mut cfg = azure_cfg();
cfg.source.azure = None;
assert_eq!(
cfg.validate(empty_ctx()),
Err(OnDemandMigrationConfigError::MissingProviderBlock("azure", Provider::Azure))
);
let mut cfg = gcs_native_cfg();
cfg.source.gcs = None;
assert_eq!(
cfg.validate(empty_ctx()),
Err(OnDemandMigrationConfigError::MissingProviderBlock("gcs", Provider::GcsNative))
);
}
#[test]
fn azure_block_rules() {
let with = |account: &str, key: Option<&str>, sas: Option<&str>| {
let mut cfg = azure_cfg();
cfg.source.azure = Some(AzureSourceConfig {
account: account.to_string(),
account_key: key.map(str::to_string),
sas_token: sas.map(str::to_string),
});
cfg.validate(empty_ctx())
};
with("legacyaccount", None, Some("sv=2021-08-06&sig=abc%3D")).expect("a SAS token is a complete credential");
with("legacyaccount", Some("c2VjcmV0LWtleQ=="), None).expect("an account key is a complete credential");
for (label, result) in [
("empty account", with("", Some("c2VjcmV0LWtleQ=="), None)),
// The account becomes the first label of the derived hostname.
("account with a dot", with("legacy.account", Some("c2VjcmV0LWtleQ=="), None)),
("account with a slash", with("legacy/account", Some("c2VjcmV0LWtleQ=="), None)),
("no credential", with("legacyaccount", None, None)),
("both credentials", with("legacyaccount", Some("c2VjcmV0LWtleQ=="), Some("sv=1"))),
("empty key", with("legacyaccount", Some(""), None)),
("key that is not base64", with("legacyaccount", Some("not base64!"), None)),
("empty sas", with("legacyaccount", None, Some(""))),
("sas with a leading question mark", with("legacyaccount", None, Some("?sv=1"))),
("sas with whitespace", with("legacyaccount", None, Some("sv=1 &sig=a"))),
] {
assert!(
matches!(result, Err(OnDemandMigrationConfigError::InvalidProviderBlock("azure", _))),
"{label}: {result:?}"
);
}
}
#[test]
fn gcs_native_block_requires_a_usable_service_account_key() {
let with = |json: &str| {
let mut cfg = gcs_native_cfg();
cfg.source.gcs = Some(GcsSourceConfig {
service_account_json: json.to_string(),
});
cfg.validate(empty_ctx())
};
with(SERVICE_ACCOUNT_JSON).expect("a service-account key is accepted");
for (label, json) in [
("empty", ""),
("not json", "not json"),
("not an object", "[]"),
("wrong type", r#"{"type":"authorized_user","client_email":"a@b","private_key":"k"}"#),
("no private key", r#"{"type":"service_account","client_email":"a@b"}"#),
("empty client email", r#"{"type":"service_account","client_email":"","private_key":"k"}"#),
] {
let result = with(json);
assert!(
matches!(result, Err(OnDemandMigrationConfigError::InvalidProviderBlock("gcs", _))),
"{label}: {result:?}"
);
}
}
#[test]
fn native_secrets_never_survive_redaction_or_debug() {
let mut azure = azure_cfg();
azure.source.azure.as_mut().expect("block").sas_token = Some("sv=2021-08-06&sig=top-secret".to_string());
azure.source.azure.as_mut().expect("block").account_key = None;
let gcs = gcs_native_cfg();
for rendered in [
format!("{:?}", azure.redacted()),
format!("{azure:?}"),
String::from_utf8(azure.redacted().to_json().expect("json")).expect("utf8"),
] {
assert!(!rendered.contains("top-secret"), "{rendered}");
assert!(rendered.contains("legacyaccount"), "the account name is not a secret: {rendered}");
}
for rendered in [
format!("{:?}", gcs.redacted()),
format!("{gcs:?}"),
String::from_utf8(gcs.redacted().to_json().expect("json")).expect("utf8"),
] {
assert!(!rendered.contains("PRIVATE KEY-----"), "{rendered}");
assert!(!rendered.contains("gserviceaccount"), "{rendered}");
}
}
#[test] #[test]
fn bucket_rules() { fn bucket_rules() {
let mut cfg = sample(); let mut cfg = sample();
@@ -0,0 +1,506 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Native Google Cloud Storage source backend.
//!
//! The `gcs` provider already reaches GCS through its S3 interoperability API,
//! which needs an HMAC key pair. This backend is the other half: it authorizes
//! with a service-account key, the credential most GCS projects actually issue,
//! by minting OAuth tokens through the shared `google-cloud-auth` credential
//! machinery the tier layer already uses.
//!
//! Two GCS surfaces are involved, each for the half it describes best. The read
//! path uses the XML API (`/{bucket}/{object}`), whose responses carry
//! `x-goog-meta-*` user metadata and the `x-goog-hash` digest in one round trip.
//! Listing uses the JSON API (`objects.list`), whose `pageToken` maps directly
//! onto the shared page cursor and whose `prefixes` are the delimiter roll-up.
//! Both accept the same bearer token.
//!
//! Every call this backend makes needs only `storage.objects.get` and
//! `storage.objects.list`, the two permissions of the `objectViewer` role, so a
//! key scoped to exactly the migration's needs works.
//!
//! `x-goog-hash` carries a base64 MD5 for every non-composite object; it is
//! converted to hex and becomes the head's ETag, so a pulled object is checked
//! against the digest GCS itself computed. A composite object has no MD5, and
//! its ETag is then marked opaque rather than checked.
use super::native_http::{
NativeHeadFields, NativeHttp, base64_md5_to_hex, header, native_source_head, parse_http_timestamp, read_text, response_body,
};
use super::source_client::{
GcsSourceSpec, SourceBackend, SourceError, SourceGet, SourceHead, SourceListRequest, SourceObject, SourcePage,
SourceTimeouts, range_header_value,
};
use crate::bucket::remote_s3_client::RemoteS3ClientError;
use crate::storage_api_contracts::range::HTTPRangeSpec;
use google_cloud_auth::credentials::service_account::{AccessSpecifier, Builder as ServiceAccountBuilder};
use google_cloud_auth::credentials::{CacheableResource, Credentials};
use http::{HeaderMap, HeaderValue, Method};
use serde::Deserialize;
use std::collections::HashMap;
use url::Url;
/// Read-only object scope: this backend never writes to the source.
const READ_ONLY_SCOPE: &str = "https://www.googleapis.com/auth/devstorage.read_only";
const METADATA_PREFIX: &str = "x-goog-meta-";
/// GCS reports its error code in the response body, not a header; the shared
/// transport takes a header name, so it is given one that never matches and
/// classification falls back to the status.
const NO_ERROR_CODE_HEADER: &str = "x-goog-unused-error-code";
/// One `objects.list` page is small; refuse an unbounded document.
const MAX_JSON_BYTES: usize = 8 * 1024 * 1024;
pub struct GcsNativeSourceBackend {
http: NativeHttp,
bucket: String,
credentials: Credentials,
}
impl GcsNativeSourceBackend {
pub fn new(
endpoint: &str,
bucket: &str,
spec: &GcsSourceSpec,
timeouts: SourceTimeouts,
skip_tls_verify: bool,
ca_cert_pem: Option<&str>,
) -> Result<Self, RemoteS3ClientError> {
let key: serde_json::Value = serde_json::from_str(&spec.service_account_json)
.map_err(|_| RemoteS3ClientError::Credentials("gcs service account key is not valid JSON"))?;
let credentials = ServiceAccountBuilder::new(key)
.with_access_specifier(AccessSpecifier::from_scopes([READ_ONLY_SCOPE]))
.build()
.map_err(|_| RemoteS3ClientError::Credentials("gcs service account key is not usable"))?;
Ok(Self {
http: NativeHttp::new(endpoint, timeouts, skip_tls_verify, ca_cert_pem)?,
bucket: bucket.to_string(),
credentials,
})
}
/// Authorization headers for one request. A credential failure is reported
/// as `AccessDenied` with no message: the renderer of a credential error
/// has the key material in scope, and the class is what callers act on.
async fn auth_headers(&self) -> Result<HeaderMap, SourceError> {
match self.credentials.headers(http::Extensions::new()).await {
Ok(CacheableResource::New { data, .. }) => Ok(data),
// Only returned when the caller passes an entity tag, which this
// backend never does; an empty set is still the honest answer.
Ok(CacheableResource::NotModified) => Ok(HeaderMap::new()),
Err(_) => Err(SourceError::AccessDenied),
}
}
/// XML API URL of one object; `/` in the key stay path separators.
fn object_url(&self, key: &str) -> Result<Url, SourceError> {
self.http.url(std::iter::once(self.bucket.as_str()).chain(key.split('/')))
}
/// JSON API URL of the bucket's object collection.
fn objects_url(&self) -> Result<Url, SourceError> {
self.http.url(["storage", "v1", "b", self.bucket.as_str(), "o"])
}
async fn request(&self, method: Method, url: Url, mut headers: HeaderMap) -> Result<reqwest::Request, SourceError> {
for (name, value) in self.auth_headers().await? {
if let Some(name) = name {
headers.insert(name, value);
}
}
let mut request = reqwest::Request::new(method, url);
*request.headers_mut() = headers;
Ok(request)
}
/// Shared mapping for the XML API's HEAD and GET responses.
fn head_from_response(headers: &HeaderMap) -> Result<SourceHead, SourceError> {
if header(headers, "x-goog-encryption-key-sha256").is_some() {
return Err(SourceError::Unsupported(
"source object uses a customer-supplied encryption key; customer-key sources are not supported".to_string(),
));
}
// `x-goog-hash` lists digests as `name=base64`, comma separated, and may
// repeat across header lines. Only the MD5 describes the whole object.
let md5 = headers
.get_all("x-goog-hash")
.iter()
.filter_map(|value| value.to_str().ok())
.flat_map(|value| value.split(','))
.filter_map(|digest| digest.trim().strip_prefix("md5="))
.find_map(base64_md5_to_hex);
let (etag, etag_is_opaque) = match md5 {
Some(md5) => (Some(md5), false),
// A composite object has no MD5; its ETag describes the composition
// rather than the bytes, so it is provenance only.
None => (header(headers, "etag").map(str::to_string), true),
};
native_source_head(
headers,
METADATA_PREFIX,
NativeHeadFields {
etag,
etag_is_opaque,
version_id: header(headers, "x-goog-generation").map(str::to_string),
storage_class: header(headers, "x-goog-storage-class").map(str::to_string),
},
)
}
}
#[async_trait::async_trait]
impl SourceBackend for GcsNativeSourceBackend {
async fn head(&self, key: &str) -> Result<SourceHead, SourceError> {
let request = self.request(Method::HEAD, self.object_url(key)?, HeaderMap::new()).await?;
let response = self.http.send(request, NO_ERROR_CODE_HEADER).await?;
Self::head_from_response(response.headers())
}
async fn get(&self, key: &str, range: Option<&HTTPRangeSpec>) -> Result<SourceGet, SourceError> {
let mut headers = HeaderMap::new();
if let Some(range) = range.map(range_header_value).transpose()? {
headers.insert(
http::header::RANGE,
HeaderValue::from_str(&range).map_err(|_| SourceError::Other("invalid range header".to_string()))?,
);
}
let request = self.request(Method::GET, self.object_url(key)?, headers).await?;
let response = self.http.send(request, NO_ERROR_CODE_HEADER).await?;
let head = Self::head_from_response(response.headers())?;
let content_range = header(response.headers(), "content-range").map(str::to_string);
Ok(SourceGet {
head,
body: response_body(response),
content_range,
})
}
async fn list(&self, request: &SourceListRequest<'_>) -> Result<SourcePage, SourceError> {
// `objects.list` offers `startOffset`, which is inclusive, so it cannot
// express "resume after this key" without silently repeating it.
if request.start_after.is_some() {
return Err(SourceError::Unsupported(
"gcs sources cannot resume a listing from a key; use the continuation token".to_string(),
));
}
let mut url = self.objects_url()?;
{
let mut query = url.query_pairs_mut();
if let Some(prefix) = request.prefix.filter(|prefix| !prefix.is_empty()) {
query.append_pair("prefix", prefix);
}
if let Some(delimiter) = request.delimiter.filter(|delimiter| !delimiter.is_empty()) {
query.append_pair("delimiter", delimiter);
}
if let Some(token) = request.continuation_token.filter(|token| !token.is_empty()) {
query.append_pair("pageToken", token);
}
if request.max_keys > 0 {
query.append_pair("maxResults", &request.max_keys.to_string());
}
}
let request = self.request(Method::GET, url, HeaderMap::new()).await?;
let response = self.http.send(request, NO_ERROR_CODE_HEADER).await?;
let body = read_text(response, MAX_JSON_BYTES).await?;
parse_objects_list(&body)
}
/// GCS has no object tagging API; user metadata is already carried by the
/// head mapping. An empty map keeps `policy.copy_tags` from failing a pull
/// over a concept the provider does not have.
async fn tagging(&self, _key: &str) -> Result<HashMap<String, String>, SourceError> {
Ok(HashMap::new())
}
/// A one-object listing, not `buckets.get`: the migration pipeline only
/// ever needs `storage.objects.list` and `storage.objects.get`, and a key
/// scoped to exactly those (the `objectViewer` role) cannot read the bucket
/// resource. Probing with `buckets.get` would reject a correct key.
async fn probe(&self) -> Result<(), SourceError> {
let mut url = self.objects_url()?;
url.query_pairs_mut().append_pair("maxResults", "1");
let request = self.request(Method::GET, url, HeaderMap::new()).await?;
let response = self.http.send(request, NO_ERROR_CODE_HEADER).await?;
read_text(response, MAX_JSON_BYTES)
.await
.and_then(|body| parse_objects_list(&body))?;
Ok(())
}
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
struct ObjectsList {
#[serde(default)]
items: Vec<ListedObject>,
#[serde(default)]
prefixes: Vec<String>,
#[serde(default)]
next_page_token: Option<String>,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
struct ListedObject {
name: String,
/// GCS renders the size as a decimal string, not a JSON number.
#[serde(default)]
size: Option<String>,
#[serde(default)]
updated: Option<String>,
#[serde(default)]
md5_hash: Option<String>,
#[serde(default)]
etag: Option<String>,
#[serde(default)]
storage_class: Option<String>,
}
fn parse_objects_list(body: &str) -> Result<SourcePage, SourceError> {
let listing: ObjectsList =
serde_json::from_str(body).map_err(|err| SourceError::Other(format!("source listing is not valid JSON: {err}")))?;
let next_continuation_token = listing.next_page_token.filter(|token| !token.is_empty());
let objects = listing
.items
.into_iter()
.map(|item| {
let etag = item
.md5_hash
.as_deref()
.and_then(base64_md5_to_hex)
.or_else(|| item.etag.map(|etag| etag.trim_matches('"').to_string()))
.filter(|etag| !etag.is_empty());
SourceObject {
key: item.name,
etag,
size: item.size.and_then(|size| size.parse().ok()).unwrap_or(0),
last_modified: item.updated.as_deref().and_then(parse_http_timestamp),
storage_class: item.storage_class,
// GCS never encodes a part count in a digest or an ETag.
is_multipart_etag: false,
}
})
.collect();
Ok(SourcePage {
objects,
common_prefixes: listing.prefixes,
is_truncated: next_continuation_token.is_some(),
next_continuation_token,
})
}
#[cfg(test)]
mod tests {
use super::*;
use crate::bucket::on_demand_migration::backend_contract::{BackendCapabilities, assert_backend_contract};
use crate::bucket::on_demand_migration::test_http_fixture::{ScriptedResponse, scripted_server};
use google_cloud_auth::credentials::anonymous::Builder as AnonymousBuilder;
const LIST_PAGE_ONE: &str = r#"{
"kind": "storage#objects",
"nextPageToken": "cursor-1",
"prefixes": ["dir/sub/"],
"items": [
{
"name": "dir/a.txt",
"size": "5",
"updated": "2015-10-21T07:28:00.000Z",
"md5Hash": "XUFAKrxLKna5cZ2REBfFkg==",
"etag": "CJizy9Wq0McCEAE=",
"storageClass": "STANDARD"
}
]
}"#;
const LIST_PAGE_TWO: &str = r#"{
"kind": "storage#objects",
"items": [
{
"name": "dir/b.txt",
"size": "7",
"updated": "2015-10-21T07:28:00.000Z",
"etag": "\"CJizy9Wq0McCEAI=\""
}
]
}"#;
fn backend(endpoint: &Url) -> GcsNativeSourceBackend {
GcsNativeSourceBackend {
http: NativeHttp::for_test(endpoint.clone()),
bucket: "legacy".to_string(),
// Anonymous credentials add no headers, so the fixture sees exactly
// the request this backend builds.
credentials: AnonymousBuilder::new().build(),
}
}
fn object_headers() -> Vec<(&'static str, String)> {
vec![
("Content-Type", "text/plain".to_string()),
("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT".to_string()),
("ETag", "\"CJizy9Wq0McCEAE=\"".to_string()),
("x-goog-hash", "crc32c=AAAAAA==,md5=XUFAKrxLKna5cZ2REBfFkg==".to_string()),
("x-goog-meta-owner", "alice".to_string()),
("x-goog-storage-class", "STANDARD".to_string()),
("x-goog-generation", "1445412480000000".to_string()),
]
}
#[test]
fn objects_list_maps_items_prefixes_and_the_page_token() {
let page = parse_objects_list(LIST_PAGE_ONE).expect("page should parse");
assert_eq!(page.common_prefixes, vec!["dir/sub/"]);
assert!(page.is_truncated);
assert_eq!(page.next_continuation_token.as_deref(), Some("cursor-1"));
assert_eq!(page.objects.len(), 1);
assert_eq!(page.objects[0].key, "dir/a.txt");
assert_eq!(page.objects[0].size, 5, "the string size is parsed");
assert_eq!(
page.objects[0].etag.as_deref(),
Some("5d41402abc4b2a76b9719d911017c592"),
"the base64 md5Hash becomes a hex ETag"
);
assert_eq!(page.objects[0].storage_class.as_deref(), Some("STANDARD"));
assert!(page.objects[0].last_modified.is_some(), "RFC 3339 `updated` is parsed");
let page = parse_objects_list(LIST_PAGE_TWO).expect("page should parse");
assert!(!page.is_truncated);
assert!(page.next_continuation_token.is_none());
assert_eq!(
page.objects[0].etag.as_deref(),
Some("CJizy9Wq0McCEAI="),
"without md5Hash the raw etag is carried"
);
assert!(parse_objects_list("not json").is_err());
}
#[tokio::test]
async fn head_prefers_the_goog_hash_md5_over_the_etag() {
let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, object_headers(), String::new())]).await;
let head = backend(&endpoint).head("dir/a b.txt").await.expect("HEAD should map");
let recorded = recorded.lock().expect("recorder lock").clone();
assert_eq!(recorded[0].method, "HEAD");
assert_eq!(recorded[0].target, "/legacy/dir/a%20b.txt", "the XML API addresses the object by path");
assert_eq!(
head.etag.as_deref(),
Some("5d41402abc4b2a76b9719d911017c592"),
"the x-goog-hash md5 is the content digest"
);
assert!(!head.etag_is_opaque, "a GCS md5 may be checked against the pulled bytes");
assert_eq!(head.user_metadata, HashMap::from([("owner".to_string(), "alice".to_string())]));
assert_eq!(head.version_id.as_deref(), Some("1445412480000000"));
assert_eq!(head.storage_class.as_deref(), Some("STANDARD"));
}
#[tokio::test]
async fn a_composite_object_without_an_md5_keeps_an_opaque_etag() {
let headers = object_headers()
.into_iter()
.map(|(name, value)| {
if name == "x-goog-hash" {
(name, "crc32c=AAAAAA==".to_string())
} else {
(name, value)
}
})
.collect();
let (endpoint, _) = scripted_server(vec![ScriptedResponse::new(200, headers, String::new())]).await;
let head = backend(&endpoint).head("composed").await.expect("HEAD should map");
assert_eq!(head.etag.as_deref(), Some("CJizy9Wq0McCEAE="));
assert!(head.etag_is_opaque, "a composite ETag describes the composition, not the bytes");
}
#[tokio::test]
async fn customer_supplied_key_objects_are_refused() {
let mut headers = object_headers();
headers.push(("x-goog-encryption-key-sha256", "abc".to_string()));
let (endpoint, _) = scripted_server(vec![ScriptedResponse::new(200, headers, String::new())]).await;
let err = backend(&endpoint)
.head("a.txt")
.await
.expect_err("CSEK objects are unsupported");
assert!(matches!(err, SourceError::Unsupported(_)), "{err:?}");
}
#[tokio::test]
async fn list_and_probe_address_the_json_api() {
let (endpoint, recorded) = scripted_server(vec![
ScriptedResponse::new(200, Vec::new(), LIST_PAGE_ONE.to_string()),
ScriptedResponse::new(200, Vec::new(), "{}".to_string()),
])
.await;
let backend = backend(&endpoint);
backend
.list(&SourceListRequest {
prefix: Some("dir/"),
delimiter: Some("/"),
continuation_token: Some("cursor-0"),
max_keys: 2,
..Default::default()
})
.await
.expect("listing should succeed");
backend.probe().await.expect("probe should succeed");
let recorded = recorded.lock().expect("recorder lock").clone();
assert!(recorded[0].target.starts_with("/storage/v1/b/legacy/o?"), "{}", recorded[0].target);
assert!(recorded[0].target.contains("prefix=dir%2F"), "{}", recorded[0].target);
assert!(recorded[0].target.contains("delimiter=%2F"), "{}", recorded[0].target);
assert!(recorded[0].target.contains("pageToken=cursor-0"), "{}", recorded[0].target);
assert!(recorded[0].target.contains("maxResults=2"), "{}", recorded[0].target);
assert_eq!(
recorded[1].target, "/storage/v1/b/legacy/o?maxResults=1",
"the probe uses the listing permission the pipeline already needs"
);
}
#[tokio::test]
async fn gcs_native_backend_satisfies_the_shared_backend_contract() {
let mut ranged = object_headers();
ranged.push(("Content-Range", "bytes 1-3/5".to_string()));
// A HEAD reports the object size with no body, exactly as GCS does.
let mut head_only = object_headers();
head_only.push(("Content-Length", "5".to_string()));
let (endpoint, _) = scripted_server(vec![
ScriptedResponse::new(200, head_only, String::new()),
ScriptedResponse::new(200, object_headers(), "hello".to_string()),
ScriptedResponse::new(206, ranged, "ell".to_string()),
ScriptedResponse::new(200, Vec::new(), LIST_PAGE_ONE.to_string()),
ScriptedResponse::new(200, Vec::new(), LIST_PAGE_TWO.to_string()),
// GCS has no tagging call, so the contract's tag step issues no
// request; the probe is the next one on the wire.
ScriptedResponse::new(200, Vec::new(), "{}".to_string()),
ScriptedResponse::new(404, Vec::new(), String::new()),
ScriptedResponse::new(403, Vec::new(), String::new()),
])
.await;
assert_backend_contract(
&backend(&endpoint),
BackendCapabilities {
etag_is_opaque: false,
supports_start_after: false,
// GCS objects have no tags; the contract's tag step is skipped.
supports_tagging: false,
},
)
.await;
}
}
@@ -25,13 +25,21 @@ use parking_lot::Mutex;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::time::{Duration, Instant}; use std::time::{Duration, Instant};
/// The only continuation-token envelope version this build reads and writes. /// The continuation-token version used by ordinary progressing pages.
pub const LIST_THROUGH_TOKEN_VERSION: u32 = 1; pub const LIST_THROUGH_TOKEN_VERSION: u32 = 1;
const LIST_THROUGH_PROGRESS_TOKEN_VERSION: u32 = 2;
/// The sixteenth consecutive merged page without a key or new EOF fails.
/// This also bounds legitimate sparse listings; it is not a cycle detector.
pub const MAX_LIST_NO_PROGRESS_PAGES: u8 = 16;
/// Envelope marker. A bucket that is *not* merging hands out the local /// Envelope marker. A bucket that is *not* merging hands out the local
/// listing's own marker, so the decoder needs a positive signal before it /// listing's own marker, so the decoder needs a positive signal before it
/// treats an opaque token as a merged one. /// treats an opaque token as a merged one.
const LIST_THROUGH_TOKEN_TAG: &str = "odm-list"; const LIST_THROUGH_TOKEN_TAG: &str = "odm-list";
// Object keys cannot contain NUL (bucket::utils::is_valid_object_prefix),
// so this framing cannot collide with a local key used as an opaque marker.
const LIST_THROUGH_TOKEN_PREFIX: &str = "\0odm-list:";
/// Pages fetched per side per request: the first page, plus at most one refill /// Pages fetched per side per request: the first page, plus at most one refill
/// when the first one was mostly consumed by the previous page. Two pages of /// when the first one was mostly consumed by the previous page. Two pages of
@@ -86,8 +94,7 @@ pub struct MergePick {
} }
/// The continuation-token envelope. Opaque to clients: it is serialized as /// The continuation-token envelope. Opaque to clients: it is serialized as
/// JSON and then base64-encoded by the same helper that encodes a plain local /// framed JSON and then base64-encoded by the same helper as a local marker.
/// marker, so the wire shape is `base64(json)`.
/// ///
/// A `null` cursor with `done = false` means "list that side from the start"; /// A `null` cursor with `done = false` means "list that side from the start";
/// `done = true` means the side is finished and must not be listed again. /// `done = true` means the side is finished and must not be listed again.
@@ -111,6 +118,10 @@ pub struct ListThroughToken {
/// common prefix compares as itself, never as its members. /// common prefix compares as itself, never as its members.
#[serde(default)] #[serde(default)]
pub last_key: Option<String>, pub last_key: Option<String>,
/// Consecutive empty truncated merged pages, present only in v2 tokens.
/// Ordinary v1 tokens retain their original serialized shape.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub no_progress: Option<u8>,
} }
impl ListThroughToken { impl ListThroughToken {
@@ -123,13 +134,14 @@ impl ListThroughToken {
source: source.token, source: source.token,
source_done: source.done, source_done: source.done,
last_key, last_key,
no_progress: None,
} }
} }
pub fn encode(&self) -> String { pub fn encode(&self) -> String {
// The envelope is built here from owned strings, so serialization // The envelope is built here from owned strings, so serialization
// cannot fail; the fallback keeps the signature infallible. // cannot fail; the fallback keeps the signature infallible.
serde_json::to_string(self).unwrap_or_default() format!("{LIST_THROUGH_TOKEN_PREFIX}{}", serde_json::to_string(self).unwrap_or_default())
} }
} }
@@ -153,24 +165,35 @@ pub enum ListThroughTokenError {
/// Classifies an already base64-decoded continuation token. /// Classifies an already base64-decoded continuation token.
/// ///
/// Only a JSON object carrying the envelope marker is read as a merged token; /// Only a framed JSON object is read as a merged token;
/// anything else is a local marker, so a bucket that turns `list_through` off /// anything else is a local marker, so a bucket that turns `list_through` off
/// keeps paginating with the tokens it handed out. A token that *is* an /// keeps paginating with the tokens it handed out. A token that *is* an
/// envelope but was tampered with (unknown version, unknown field, truncated /// envelope but was tampered with (unknown version, unknown field, truncated
/// JSON) is an error, never a silent fallback. /// JSON) is an error, never a silent fallback.
pub fn decode_continuation_token(decoded: &str) -> Result<ListThroughCursor, ListThroughTokenError> { pub fn decode_continuation_token(decoded: &str) -> Result<ListThroughCursor, ListThroughTokenError> {
if !decoded.starts_with('{') { let Some(payload) = decoded.strip_prefix(LIST_THROUGH_TOKEN_PREFIX) else {
return Ok(ListThroughCursor::Local(decoded.to_string()));
}
let Ok(value) = serde_json::from_str::<serde_json::Value>(decoded) else {
// Not JSON at all: an object key may legitimately start with '{'.
return Ok(ListThroughCursor::Local(decoded.to_string())); return Ok(ListThroughCursor::Local(decoded.to_string()));
}; };
let value = serde_json::from_str::<serde_json::Value>(payload).map_err(|_| ListThroughTokenError::Malformed)?;
if value.get("t").and_then(serde_json::Value::as_str) != Some(LIST_THROUGH_TOKEN_TAG) { if value.get("t").and_then(serde_json::Value::as_str) != Some(LIST_THROUGH_TOKEN_TAG) {
return Ok(ListThroughCursor::Local(decoded.to_string())); return Err(ListThroughTokenError::Malformed);
} }
match value.get("v").and_then(serde_json::Value::as_u64) { match value.get("v").and_then(serde_json::Value::as_u64) {
Some(version) if version == u64::from(LIST_THROUGH_TOKEN_VERSION) => {} Some(version) if version == u64::from(LIST_THROUGH_TOKEN_VERSION) => {
// v1 readers reject this field even when it is null or zero.
if value.get("no_progress").is_some() {
return Err(ListThroughTokenError::Malformed);
}
}
Some(version) if version == u64::from(LIST_THROUGH_PROGRESS_TOKEN_VERSION) => {
if !value
.get("no_progress")
.and_then(serde_json::Value::as_u64)
.is_some_and(|count| (1..u64::from(MAX_LIST_NO_PROGRESS_PAGES)).contains(&count))
{
return Err(ListThroughTokenError::Malformed);
}
}
Some(version) => return Err(ListThroughTokenError::UnsupportedVersion(version.min(u64::from(u32::MAX)) as u32)), Some(version) => return Err(ListThroughTokenError::UnsupportedVersion(version.min(u64::from(u32::MAX)) as u32)),
None => return Err(ListThroughTokenError::Malformed), None => return Err(ListThroughTokenError::Malformed),
} }
@@ -189,8 +212,8 @@ pub enum SourceListPlan {
/// delimiter — the source's own roll-up boundary matches the request's. /// delimiter — the source's own roll-up boundary matches the request's.
Page { prefix: String }, Page { prefix: String },
/// `filter.prefix` reaches past a delimiter, so every key the source could /// `filter.prefix` reaches past a delimiter, so every key the source could
/// contribute rolls into this one common prefix. One bounded probe listing /// contribute rolls into this one common prefix. Bounded probes follow
/// decides whether it exists; there is nothing to paginate. /// empty progressing pages until a key proves existence or the source ends.
Folded { probe_prefix: String, common_prefix: String }, Folded { probe_prefix: String, common_prefix: String },
} }
@@ -279,6 +302,31 @@ pub struct FetchRequest {
pub token: Option<String>, pub token: Option<String>,
} }
/// Invalid pagination metadata. Opaque cursor values are never included in errors.
#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)]
pub enum ListPageError {
#[error("truncated listing has no continuation token")]
Missing,
#[error("truncated listing has an empty continuation token")]
Empty,
#[error("truncated listing repeats a continuation token")]
Repeated,
#[error("listing exhausted its consecutive no-progress page budget")]
NoProgress(MergeSide),
}
pub(crate) fn validate_list_page(is_truncated: bool, token: Option<&str>, next_token: Option<&str>) -> Result<(), ListPageError> {
if is_truncated {
match next_token {
None => return Err(ListPageError::Missing),
Some("") => return Err(ListPageError::Empty),
Some(next) if Some(next) == token => return Err(ListPageError::Repeated),
Some(_) => {}
}
}
Ok(())
}
#[derive(Debug, Default)] #[derive(Debug, Default)]
struct SideState { struct SideState {
start: SideCursor, start: SideCursor,
@@ -329,6 +377,7 @@ pub struct MergeOutcome {
#[derive(Debug)] #[derive(Debug)]
pub struct ListThroughMerger { pub struct ListThroughMerger {
max_keys: usize, max_keys: usize,
no_progress: Option<u8>,
last_key: Option<String>, last_key: Option<String>,
local: SideState, local: SideState,
source: SideState, source: SideState,
@@ -348,6 +397,7 @@ impl ListThroughMerger {
}; };
Self { Self {
max_keys, max_keys,
no_progress: token.and_then(|token| token.no_progress),
last_key, last_key,
local, local,
source, source,
@@ -364,6 +414,11 @@ impl ListThroughMerger {
/// or `filter.prefix` excludes it. /// or `filter.prefix` excludes it.
pub fn disable_source(&mut self) { pub fn disable_source(&mut self) {
self.source.disabled = true; self.source.disabled = true;
// A refill can fail after a valid first page. A local-only response
// must discard both that source payload and its ordering horizon.
self.source.entries.clear();
self.source.pages.clear();
self.source.more = false;
} }
pub fn next_fetch(&self) -> Option<FetchRequest> { pub fn next_fetch(&self) -> Option<FetchRequest> {
@@ -378,7 +433,13 @@ impl ListThroughMerger {
/// Records one fetched page. `entries` must be sorted by `name` and already /// Records one fetched page. `entries` must be sorted by `name` and already
/// filtered with [`Self::accepts`]; the caller keeps the matching payloads /// filtered with [`Self::accepts`]; the caller keeps the matching payloads
/// in the same order. /// in the same order.
pub fn push_page(&mut self, side: MergeSide, entries: Vec<ListEntryKey>, is_truncated: bool, next_token: Option<String>) { pub fn push_page(
&mut self,
side: MergeSide,
entries: Vec<ListEntryKey>,
is_truncated: bool,
next_token: Option<String>,
) -> Result<(), ListPageError> {
let state = match side { let state = match side {
MergeSide::Local => &mut self.local, MergeSide::Local => &mut self.local,
MergeSide::Source => &mut self.source, MergeSide::Source => &mut self.source,
@@ -387,24 +448,33 @@ impl ListThroughMerger {
Some(last) => last.next_token.clone(), Some(last) => last.next_token.clone(),
None => state.start.token.clone(), None => state.start.token.clone(),
}; };
// A truncated page without a cursor cannot be continued; treating the validate_list_page(is_truncated, token.as_deref(), next_token.as_deref())?;
// side as finished is the only alternative to looping on it forever. // Also reject a cycle through an earlier page in this bounded fetch.
state.more = is_truncated && next_token.is_some(); if is_truncated && state.pages.iter().any(|page| page.token == next_token) {
return Err(ListPageError::Repeated);
}
state.more = is_truncated;
state.pages.push(FetchedPage { state.pages.push(FetchedPage {
token, token,
count: entries.len(), count: entries.len(),
next_token: is_truncated.then_some(next_token).flatten(), next_token: is_truncated.then_some(next_token).flatten(),
}); });
state.entries.extend(entries); state.entries.extend(entries);
Ok(())
} }
pub fn finish(self) -> MergeOutcome { /// `issue_progress_tokens` allows a v1 chain to start carrying a budget.
/// An existing v2 budget is always enforced, including on reader-only nodes.
/// Borrowing lets a source failure re-merge the fetched local buffers.
pub fn finish(&self, issue_progress_tokens: bool) -> Result<MergeOutcome, ListPageError> {
let Self { let Self {
max_keys, max_keys,
no_progress,
last_key, last_key,
local, local,
source, source,
} = self; } = self;
let max_keys = *max_keys;
// A side with more pages behind it can only be trusted up to the last // A side with more pages behind it can only be trusted up to the last
// key it handed over: past that horizon the other side's entries could // key it handed over: past that horizon the other side's entries could
@@ -470,12 +540,44 @@ impl ListThroughMerger {
let source_left = !source.disabled && (!source_cursor.done || consumed_source < source.entries.len()); let source_left = !source.disabled && (!source_cursor.done || consumed_source < source.entries.len());
let is_truncated = local_left || source_left; let is_truncated = local_left || source_left;
let last_key = consumed_key.or(last_key); let reached_eof = (!local.start.done && local_cursor.done) || (!source.start.done && source_cursor.done);
MergeOutcome { let next_no_progress = if !is_truncated || !picks.is_empty() || reached_eof {
None
} else if max_keys == 0 {
// A zero-sized request cannot consume entries. Preserve an existing
// budget without spending it or starting a new one.
*no_progress
} else if issue_progress_tokens || no_progress.is_some() {
let count = no_progress.unwrap_or(0).saturating_add(1);
if count >= MAX_LIST_NO_PROGRESS_PAGES {
// An empty truncated side closes the merge horizon. Local
// failure takes precedence; disabling the source cannot fix it.
let side = if local.more && local.entries.is_empty() {
MergeSide::Local
} else if !source.disabled && source.more && source.entries.is_empty() {
MergeSide::Source
} else {
MergeSide::Local
};
return Err(ListPageError::NoProgress(side));
}
Some(count)
} else {
None
};
let last_key = consumed_key.or_else(|| last_key.clone());
Ok(MergeOutcome {
picks, picks,
is_truncated, is_truncated,
next_token: is_truncated.then(|| ListThroughToken::new(local_cursor, source_cursor, last_key)), next_token: is_truncated.then(|| {
} let mut token = ListThroughToken::new(local_cursor, source_cursor, last_key);
if let Some(count) = next_no_progress {
token.v = LIST_THROUGH_PROGRESS_TOKEN_VERSION;
token.no_progress = Some(count);
}
token
}),
})
} }
} }
@@ -599,9 +701,15 @@ mod tests {
let (entries, truncated, next) = reference_page(keys, prefix, delimiter, fetch.token.as_deref(), max_keys); let (entries, truncated, next) = reference_page(keys, prefix, delimiter, fetch.token.as_deref(), max_keys);
let kept: Vec<ListEntryKey> = entries.into_iter().filter(|entry| merger.accepts(&entry.name)).collect(); let kept: Vec<ListEntryKey> = entries.into_iter().filter(|entry| merger.accepts(&entry.name)).collect();
buffers[usize::from(fetch.side == MergeSide::Source)].extend(kept.iter().cloned()); buffers[usize::from(fetch.side == MergeSide::Source)].extend(kept.iter().cloned());
merger.push_page(fetch.side, kept, truncated, next); merger
.push_page(fetch.side, kept, truncated, next)
.expect("reference provider pages must advance");
}
let outcome = merger.finish(false).expect("valid merge outcome");
assert_eq!(outcome.is_truncated, outcome.next_token.is_some());
if outcome.is_truncated {
assert_ne!(outcome.next_token, token, "every truncated merged page must make progress");
} }
let outcome = merger.finish();
page_sizes.push(outcome.picks.len()); page_sizes.push(outcome.picks.len());
for pick in &outcome.picks { for pick in &outcome.picks {
let entry = buffers[usize::from(pick.side == MergeSide::Source)][pick.index].clone(); let entry = buffers[usize::from(pick.side == MergeSide::Source)][pick.index].clone();
@@ -616,11 +724,25 @@ mod tests {
} }
fn expected(local: &[String], source: &[String], prefix: &str, delimiter: Option<&str>) -> Vec<ListEntryKey> { fn expected(local: &[String], source: &[String], prefix: &str, delimiter: Option<&str>) -> Vec<ListEntryKey> {
let mut all: Vec<String> = local.iter().chain(source.iter()).cloned().collect(); // This oracle builds the complete namespace independently of the
all.sort(); // provider's page/marker helper and the production merger.
all.dedup(); let mut namespace = std::collections::BTreeMap::new();
let (entries, _, _) = reference_page(&all, prefix, delimiter, None, usize::MAX); for key in local.iter().chain(source) {
entries let Some(suffix) = key.strip_prefix(prefix) else {
continue;
};
if let Some(delimiter) = delimiter.filter(|delimiter| !delimiter.is_empty())
&& let Some((directory, _)) = suffix.split_once(delimiter)
{
namespace.insert(format!("{prefix}{directory}{delimiter}"), true);
continue;
}
namespace.insert(key.clone(), false);
}
namespace
.into_iter()
.map(|(name, is_prefix)| ListEntryKey { name, is_prefix })
.collect()
} }
#[test] #[test]
@@ -662,9 +784,11 @@ mod tests {
token: None token: None
}) })
); );
merger.push_page(MergeSide::Local, vec![ListEntryKey::object("a")], false, None); merger
.push_page(MergeSide::Local, vec![ListEntryKey::object("a")], false, None)
.expect("local EOF is valid");
assert_eq!(merger.next_fetch(), None); assert_eq!(merger.next_fetch(), None);
let outcome = merger.finish(); let outcome = merger.finish(false).expect("valid merge outcome");
assert_eq!(outcome.picks.len(), 1); assert_eq!(outcome.picks.len(), 1);
assert!(!outcome.is_truncated); assert!(!outcome.is_truncated);
assert!(outcome.next_token.is_none()); assert!(outcome.next_token.is_none());
@@ -680,16 +804,19 @@ mod tests {
source: Some("source-1".to_string()), source: Some("source-1".to_string()),
source_done: false, source_done: false,
last_key: Some("a".to_string()), last_key: Some("a".to_string()),
no_progress: None,
}; };
let mut merger = ListThroughMerger::new(1, Some(&resume)); let mut merger = ListThroughMerger::new(1, Some(&resume));
merger.disable_source(); merger.disable_source();
merger.push_page( merger
MergeSide::Local, .push_page(
vec![ListEntryKey::object("b"), ListEntryKey::object("c")], MergeSide::Local,
true, vec![ListEntryKey::object("b"), ListEntryKey::object("c")],
Some("local-2".to_string()), true,
); Some("local-2".to_string()),
let outcome = merger.finish(); )
.expect("local cursor advances");
let outcome = merger.finish(false).expect("valid merge outcome");
assert!(outcome.is_truncated); assert!(outcome.is_truncated);
let token = outcome.next_token.expect("truncated page carries a token"); let token = outcome.next_token.expect("truncated page carries a token");
assert_eq!(token.source.as_deref(), Some("source-1"), "the source cursor must not move"); assert_eq!(token.source.as_deref(), Some("source-1"), "the source cursor must not move");
@@ -698,6 +825,212 @@ mod tests {
assert_eq!(token.local.as_deref(), Some("local-1"), "a partly read page is re-listed"); assert_eq!(token.local.as_deref(), Some("local-1"), "a partly read page is re-listed");
} }
#[test]
fn truncated_pages_require_a_nonempty_advancing_cursor() {
for side in [MergeSide::Local, MergeSide::Source] {
for entries in [vec![], vec![ListEntryKey::object("a")]] {
for (next, expected) in [
(None, Err(ListPageError::Missing)),
(Some(""), Err(ListPageError::Empty)),
(Some("stuck"), Err(ListPageError::Repeated)),
(Some("advances"), Ok(())),
] {
let resume = ListThroughToken::new(
SideCursor {
token: Some("stuck".into()),
done: false,
},
SideCursor {
token: Some("stuck".into()),
done: false,
},
None,
);
let mut merger = ListThroughMerger::new(2, Some(&resume));
let result = merger.push_page(side, entries.clone(), true, next.map(str::to_string));
assert_eq!(result, expected, "{side:?}, {entries:?}, {next:?}");
let state = if side == MergeSide::Local {
&merger.local
} else {
&merger.source
};
assert_eq!(state.pages.len(), usize::from(result.is_ok()), "invalid page must not be accepted");
}
}
}
}
#[test]
fn repeated_empty_cursor_is_rejected_before_an_identical_page_can_escape() {
let resume = ListThroughToken::new(
SideCursor { token: None, done: true },
SideCursor {
token: Some("stuck".into()),
done: false,
},
None,
);
let mut merger = ListThroughMerger::new(2, Some(&resume));
assert_eq!(
merger.next_fetch(),
Some(FetchRequest {
side: MergeSide::Source,
token: Some("stuck".into())
})
);
assert_eq!(
merger.push_page(MergeSide::Source, vec![], true, Some("stuck".into())),
Err(ListPageError::Repeated)
);
}
#[test]
fn empty_pages_may_advance_within_the_fetch_budget_until_eof() {
let mut merger = ListThroughMerger::new(2, None);
merger.push_page(MergeSide::Local, vec![], false, None).expect("local EOF");
for next in ["opaque-z", "opaque-a"] {
assert_eq!(merger.next_fetch().expect("bounded source fetch").side, MergeSide::Source);
merger
.push_page(MergeSide::Source, vec![], true, Some(next.into()))
.expect("opaque cursor advances regardless of sort order");
}
assert!(merger.next_fetch().is_none(), "two source fetches exhaust the request budget");
let outcome = merger.finish(false).expect("valid merge outcome");
assert!(outcome.picks.is_empty());
assert!(outcome.is_truncated);
let token = outcome.next_token.expect("empty progressing page has a cursor");
assert_eq!(token.source.as_deref(), Some("opaque-a"));
let mut merger = ListThroughMerger::new(2, Some(&token));
assert_eq!(merger.next_fetch().expect("source resumes").token.as_deref(), Some("opaque-a"));
merger
.push_page(MergeSide::Source, vec![ListEntryKey::object("result")], false, None)
.expect("source EOF");
let outcome = merger.finish(false).expect("valid merge outcome");
assert_eq!(
outcome.picks,
vec![MergePick {
side: MergeSide::Source,
index: 0
}]
);
assert!(!outcome.is_truncated);
assert!(outcome.next_token.is_none());
}
#[test]
fn a_cursor_cycle_inside_the_fetch_budget_is_rejected() {
let resume = ListThroughToken::new(
SideCursor { token: None, done: true },
SideCursor {
token: Some("first".into()),
done: false,
},
None,
);
let mut merger = ListThroughMerger::new(2, Some(&resume));
merger
.push_page(MergeSide::Source, vec![], true, Some("second".into()))
.expect("first page advances");
assert_eq!(
merger.push_page(MergeSide::Source, vec![], true, Some("first".into())),
Err(ListPageError::Repeated)
);
}
#[test]
fn source_refill_failure_discards_buffered_source_entries_and_horizon() {
let mut merger = ListThroughMerger::new(2, None);
merger
.push_page(MergeSide::Local, vec![ListEntryKey::object("z")], false, None)
.expect("local EOF");
merger
.push_page(MergeSide::Source, vec![ListEntryKey::object("a")], true, Some("stuck".into()))
.expect("first source page advances");
assert_eq!(merger.next_fetch().expect("source refill is required").token.as_deref(), Some("stuck"));
assert_eq!(
merger.push_page(MergeSide::Source, vec![], true, Some("stuck".into())),
Err(ListPageError::Repeated)
);
merger.disable_source();
let outcome = merger.finish(false).expect("valid merge outcome");
assert_eq!(
outcome.picks,
vec![MergePick {
side: MergeSide::Local,
index: 0
}]
);
assert!(!outcome.is_truncated);
assert!(outcome.next_token.is_none());
}
#[test]
fn list_through_static_namespace_boundary_matrix() {
let corpus = [
"a",
"a/",
"a/b",
"a/b/child",
"a0",
"b",
"b/leaf",
"quote\"&<",
"space key",
"z",
"é",
"中/文",
];
for count in [0, 1, 3, 4, corpus.len()] {
let keys: Vec<String> = corpus[..count].iter().map(|key| (*key).to_string()).collect();
for placement in 0..3 {
let (local, source): (Vec<_>, Vec<_>) =
keys.iter()
.enumerate()
.fold((vec![], vec![]), |(mut local, mut source), (index, key)| {
if placement != 1 || index % 2 == 0 {
local.push(key.clone());
}
if placement != 0 || index % 2 == 0 {
source.push(key.clone());
}
(local, source)
});
for prefix in ["", "a", "a/", "中/"] {
for delimiter in [None, Some("/")] {
for max_keys in [1, 3, 4] {
let oracle = expected(&local, &source, prefix, delimiter);
let (emitted, sizes) = walk(&local, &source, prefix, delimiter, max_keys);
assert_eq!(
emitted.iter().map(|(entry, _)| entry.clone()).collect::<Vec<_>>(),
oracle,
"count={count}, placement={placement}, prefix={prefix}, delimiter={delimiter:?}, max={max_keys}"
);
let expected_sizes: Vec<_> = if oracle.is_empty() {
vec![0]
} else {
oracle.chunks(max_keys).map(<[ListEntryKey]>::len).collect()
};
assert_eq!(sizes, expected_sizes, "exact max and max+1 boundaries must agree");
}
}
}
}
}
}
#[test]
fn list_through_large_overlap_walk_keeps_all_5300_keys() {
let source: Vec<_> = (0..5000).map(|index| format!("k{index:05}")).collect();
let local: Vec<_> = (4800..5300).map(|index| format!("k{index:05}")).collect();
let (emitted, sizes) = walk(&local, &source, "", None, 333);
assert_eq!(emitted.len(), 5300);
for (index, (entry, side)) in emitted.iter().enumerate() {
assert_eq!(entry.name, format!("k{index:05}"));
assert_eq!(*side, if index >= 4800 { MergeSide::Local } else { MergeSide::Source });
}
assert_eq!(sizes, [vec![333; 15], vec![305]].concat());
}
#[test] #[test]
fn token_round_trips_and_rejects_tampering() { fn token_round_trips_and_rejects_tampering() {
let token = ListThroughToken::new( let token = ListThroughToken::new(
@@ -711,21 +1044,279 @@ mod tests {
let encoded = token.encode(); let encoded = token.encode();
assert_eq!(decode_continuation_token(&encoded), Ok(ListThroughCursor::Merged(Box::new(token)))); assert_eq!(decode_continuation_token(&encoded), Ok(ListThroughCursor::Merged(Box::new(token))));
let bumped = encoded.replace("\"v\":1", "\"v\":2"); let bumped = encoded.replace("\"v\":1", "\"v\":3");
assert_eq!(decode_continuation_token(&bumped), Err(ListThroughTokenError::UnsupportedVersion(2))); assert_eq!(decode_continuation_token(&bumped), Err(ListThroughTokenError::UnsupportedVersion(3)));
let extra = encoded.replace("{", "{\"x\":1,"); let extra = encoded.replace("{", "{\"x\":1,");
assert_eq!(decode_continuation_token(&extra), Err(ListThroughTokenError::Malformed)); assert_eq!(decode_continuation_token(&extra), Err(ListThroughTokenError::Malformed));
let truncated = &encoded[..encoded.len() - 3]; let truncated = &encoded[..encoded.len() - 3];
assert_eq!(decode_continuation_token(truncated), Ok(ListThroughCursor::Local(truncated.to_string()))); assert_eq!(decode_continuation_token(truncated), Err(ListThroughTokenError::Malformed));
let no_version = "{\"t\":\"odm-list\"}"; let no_version = "\0odm-list:{\"t\":\"odm-list\"}";
assert_eq!(decode_continuation_token(no_version), Err(ListThroughTokenError::Malformed)); assert_eq!(decode_continuation_token(no_version), Err(ListThroughTokenError::Malformed));
} }
fn progress_token(count: Option<u8>, local_done: bool, source_done: bool) -> ListThroughToken {
let mut token = ListThroughToken::new(
SideCursor {
token: None,
done: local_done,
},
SideCursor {
token: Some("A".into()),
done: source_done,
},
Some("last-key".into()),
);
if let Some(count) = count {
token.v = LIST_THROUGH_PROGRESS_TOKEN_VERSION;
token.no_progress = Some(count);
}
token
}
fn push_empty_pages(merger: &mut ListThroughMerger, side: MergeSide) {
for _ in 0..MAX_LIST_FETCHES_PER_SIDE {
let fetch = merger.next_fetch().expect("empty truncated side must be fetched");
assert_eq!(fetch.side, side);
let next = format!("{}:next", fetch.token.unwrap_or_default());
merger
.push_page(side, vec![], true, Some(next))
.expect("opaque cursor advances");
}
}
#[test]
fn progress_tokens_preserve_v1_bytes_and_validate_v2_counts() {
let token = progress_token(None, true, false);
assert_eq!(
token.encode(),
r#"{"t":"odm-list","v":1,"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key"}"#
);
for count in 1..MAX_LIST_NO_PROGRESS_PAGES {
let token = progress_token(Some(count), true, false);
assert_eq!(decode_continuation_token(&token.encode()), Ok(ListThroughCursor::Merged(Box::new(token))));
}
for version in [1, 2] {
for value in ["null", "0", "16", "-1", "1.5", "256", "18446744073709551616", "\"1\""] {
let encoded = format!(r#"{{"t":"odm-list","v":{version},"no_progress":{value}}}"#);
assert_eq!(decode_continuation_token(&encoded), Err(ListThroughTokenError::Malformed), "{encoded}");
}
}
for encoded in [
r#"{"t":"odm-list","v":1,"no_progress":1}"#,
r#"{"t":"odm-list","v":2}"#,
r#"{"t":"odm-list","v":2,"no_progress":1,"extra":true}"#,
] {
assert_eq!(decode_continuation_token(encoded), Err(ListThroughTokenError::Malformed), "{encoded}");
}
}
#[test]
fn reader_only_nodes_do_not_start_a_budget_but_mixed_readers_preserve_one() {
let mut token = progress_token(None, true, false);
for _ in 0..MAX_LIST_NO_PROGRESS_PAGES {
let mut merger = ListThroughMerger::new(2, Some(&token));
push_empty_pages(&mut merger, MergeSide::Source);
token = merger
.finish(false)
.expect("reader-only v1 behavior")
.next_token
.expect("truncated cursor");
assert_eq!(token.v, 1);
assert_eq!(token.no_progress, None);
}
for count in 1..=MAX_LIST_NO_PROGRESS_PAGES {
let mut merger = ListThroughMerger::new(2, Some(&token));
push_empty_pages(&mut merger, MergeSide::Source);
assert!(merger.next_fetch().is_none(), "the per-request two-fetch limit stays intact");
let outcome = merger.finish(count % 2 == 1);
if count == MAX_LIST_NO_PROGRESS_PAGES {
assert_eq!(outcome, Err(ListPageError::NoProgress(MergeSide::Source)));
break;
}
token = outcome.expect("budget not exhausted").next_token.expect("truncated cursor");
assert_eq!(token.no_progress, Some(count));
let ListThroughCursor::Merged(decoded) = decode_continuation_token(&token.encode()).expect("round-trip v2") else {
panic!("merged cursor expected");
};
token = *decoded;
}
}
#[test]
fn objects_and_common_prefixes_reset_a_budget_at_the_boundary() {
for entry in [ListEntryKey::object("result"), ListEntryKey::prefix("result/")] {
for issue_tokens in [false, true] {
let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), true, false);
let mut merger = ListThroughMerger::new(2, Some(&resume));
merger
.push_page(MergeSide::Source, vec![], true, Some("B".into()))
.expect("empty advancing page");
merger
.push_page(MergeSide::Source, vec![entry.clone()], true, Some("C".into()))
.expect("real progress");
let outcome = merger
.finish(issue_tokens)
.expect("real progress does not exhaust the budget");
assert_eq!(
outcome.picks,
vec![MergePick {
side: MergeSide::Source,
index: 0
}]
);
let next = outcome.next_token.expect("source remains truncated");
assert_eq!(next.last_key.as_deref(), Some(entry.name.as_str()));
assert_eq!(next.v, 1);
assert_eq!(next.no_progress, None);
assert!(!next.encode().contains("no_progress"));
}
}
}
#[test]
fn only_a_new_eof_transition_resets_the_empty_page_budget() {
for finished_side in [MergeSide::Local, MergeSide::Source] {
let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), false, false);
let mut merger = ListThroughMerger::new(2, Some(&resume));
if finished_side == MergeSide::Local {
merger
.push_page(MergeSide::Local, vec![], false, None)
.expect("new local EOF");
push_empty_pages(&mut merger, MergeSide::Source);
} else {
push_empty_pages(&mut merger, MergeSide::Local);
merger
.push_page(MergeSide::Source, vec![], false, None)
.expect("new source EOF");
}
let next = merger
.finish(false)
.expect("new EOF is progress")
.next_token
.expect("other side truncated");
assert_eq!(next.no_progress, None);
assert_eq!(next.v, 1);
assert_eq!(next.local_done, finished_side == MergeSide::Local);
assert_eq!(next.source_done, finished_side == MergeSide::Source);
let mut merger = ListThroughMerger::new(2, Some(&next));
let remaining = if finished_side == MergeSide::Local {
MergeSide::Source
} else {
MergeSide::Local
};
push_empty_pages(&mut merger, remaining);
let next = merger
.finish(true)
.expect("a new budget starts")
.next_token
.expect("truncated");
assert_eq!(next.no_progress, Some(1), "an already-done side cannot reset every page");
}
let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), true, false);
let mut merger = ListThroughMerger::new(2, Some(&resume));
merger.push_page(MergeSide::Source, vec![], false, None).expect("final EOF");
let outcome = merger.finish(false).expect("EOF succeeds at the budget boundary");
assert!(!outcome.is_truncated);
assert!(outcome.next_token.is_none());
}
#[test]
fn filtered_duplicates_cannot_reset_the_no_progress_budget() {
let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), true, false);
let mut merger = ListThroughMerger::new(2, Some(&resume));
for next in ["B", "C"] {
let entries = [ListEntryKey::object("last-key"), ListEntryKey::object("earlier")]
.into_iter()
.filter(|entry| merger.accepts(&entry.name))
.collect::<Vec<_>>();
assert!(entries.is_empty(), "both provider entries were already consumed");
merger
.push_page(MergeSide::Source, entries, true, Some(next.into()))
.expect("advancing cursor");
}
assert_eq!(merger.finish(false), Err(ListPageError::NoProgress(MergeSide::Source)));
}
#[test]
fn no_progress_is_attributed_to_local_when_source_cannot_unblock_it() {
for source_mode in ["disabled", "done", "empty", "data"] {
let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), false, source_mode == "done");
let mut merger = ListThroughMerger::new(2, Some(&resume));
if source_mode == "disabled" {
merger.disable_source();
}
push_empty_pages(&mut merger, MergeSide::Local);
match source_mode {
"empty" => push_empty_pages(&mut merger, MergeSide::Source),
"data" => merger
.push_page(MergeSide::Source, vec![ListEntryKey::object("source")], false, None)
.expect("source data"),
_ => {}
}
assert_eq!(merger.finish(false), Err(ListPageError::NoProgress(MergeSide::Local)), "{source_mode}");
}
}
#[test]
fn source_budget_failure_remerges_local_objects_and_prefixes_without_refetching() {
let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), false, false);
let mut merger = ListThroughMerger::new(2, Some(&resume));
merger
.push_page(MergeSide::Local, vec![ListEntryKey::object("local")], true, Some("L1".into()))
.expect("local object");
merger
.push_page(MergeSide::Local, vec![ListEntryKey::prefix("prefix/")], true, Some("L2".into()))
.expect("local prefix");
push_empty_pages(&mut merger, MergeSide::Source);
assert_eq!(merger.finish(false), Err(ListPageError::NoProgress(MergeSide::Source)));
merger.disable_source();
assert!(merger.next_fetch().is_none(), "fallback does not perform another fetch");
let outcome = merger.finish(false).expect("local data makes progress");
assert_eq!(
outcome.picks,
vec![
MergePick {
side: MergeSide::Local,
index: 0
},
MergePick {
side: MergeSide::Local,
index: 1
}
]
);
let token = outcome.next_token.expect("remaining local page");
assert_eq!(token.local.as_deref(), Some("L2"));
assert_eq!(token.source.as_deref(), Some("A"));
assert_eq!(token.last_key.as_deref(), Some("prefix/"));
assert_eq!(token.no_progress, None);
assert_eq!(token.v, 1);
}
#[test]
fn a_zero_sized_merge_preserves_an_existing_budget() {
let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), true, false);
let mut merger = ListThroughMerger::new(0, Some(&resume));
merger
.push_page(MergeSide::Source, vec![ListEntryKey::object("result")], true, Some("B".into()))
.expect("source page");
let outcome = merger.finish(false).expect("a zero-sized request cannot consume entries");
assert!(outcome.picks.is_empty());
assert_eq!(outcome.next_token.expect("unconsumed source").no_progress, resume.no_progress);
}
#[test] #[test]
fn a_plain_local_marker_stays_local() { fn a_plain_local_marker_stays_local() {
for marker in [
r#"{"t":"odm-list","v":1}"#,
r#"{"t":"odm-list","v":2,"local_done":true}"#,
r#"{"t":"odm-list"}"#,
] {
assert_eq!(decode_continuation_token(marker), Ok(ListThroughCursor::Local(marker.to_string())));
}
assert_eq!( assert_eq!(
decode_continuation_token("photos/2024/01.jpg"), decode_continuation_token("photos/2024/01.jpg"),
Ok(ListThroughCursor::Local("photos/2024/01.jpg".to_string())) Ok(ListThroughCursor::Local("photos/2024/01.jpg".to_string()))
@@ -796,7 +1387,10 @@ mod tests {
} }
proptest! { proptest! {
#![proptest_config(ProptestConfig::with_cases(256))] #![proptest_config(ProptestConfig {
rng_seed: proptest::test_runner::RngSeed::Fixed(0xec5706),
..ProptestConfig::with_cases(256)
})]
/// Full pagination of a merged listing equals the sorted, deduplicated /// Full pagination of a merged listing equals the sorted, deduplicated
/// union of both sides, with every shared key served by local, and no /// union of both sides, with every shared key served by local, and no
@@ -19,30 +19,42 @@
//! client, and the per-node runtime (`sys`) that turns configs into live //! client, and the per-node runtime (`sys`) that turns configs into live
//! clients guarded by a breaker, a negative cache, singleflight and a pull //! clients guarded by a breaker, a negative cache, singleflight and a pull
//! concurrency limit (rustfs/backlog#2147). //! concurrency limit (rustfs/backlog#2147).
//!
//! A source is reached through one `SourceBackend`: the S3 dialect for every
//! S3-compatible provider, and a native backend for the providers that have no
//! S3 API (`azure`, `gcs_native`).
pub mod azure;
#[cfg(test)]
mod backend_contract;
pub mod backfill; pub mod backfill;
pub mod breaker; pub mod breaker;
pub mod config; pub mod config;
pub mod gcs;
pub mod list_through; pub mod list_through;
mod native_http;
pub mod negative_cache; pub mod negative_cache;
pub mod pull; pub mod pull;
pub mod source_client; pub mod source_client;
pub mod stats; pub mod stats;
pub mod sys; pub mod sys;
#[cfg(test)]
mod test_http_fixture;
pub use breaker::{ pub use breaker::{
BREAKER_FAILURE_THRESHOLD, BREAKER_FAILURE_WINDOW, BREAKER_HALF_OPEN_MAX_PROBES, BREAKER_OPEN_DURATION, Breaker, BREAKER_FAILURE_THRESHOLD, BREAKER_FAILURE_WINDOW, BREAKER_HALF_OPEN_MAX_PROBES, BREAKER_OPEN_DURATION, Breaker,
BreakerState, BreakerTransition, BreakerVerdict, BreakerState, BreakerTransition, BreakerVerdict,
}; };
pub use config::{ pub use config::{
ConfigPublishHook, FilterConfig, HeadPolicy, ON_DEMAND_MIGRATION_CONFIG_HOOK, ON_DEMAND_MIGRATION_CONFIG_VERSION, AzureSourceConfig, ConfigPublishHook, FilterConfig, GcsSourceConfig, HeadPolicy, ON_DEMAND_MIGRATION_CONFIG_HOOK,
OnDemandMigrationConfig, OnDemandMigrationConfigError, PathStyle, PolicyConfig, Provider, RangeGetPolicy, SourceConfig, ON_DEMAND_MIGRATION_CONFIG_VERSION, OnDemandMigrationConfig, OnDemandMigrationConfigError, PathStyle, PolicyConfig, Provider,
SourceCredentials, SourceErrorPolicy, SourceTimeout, TlsConfig, ValidationContext, RangeGetPolicy, SourceConfig, SourceCredentials, SourceErrorPolicy, SourceTimeout, TlsConfig, ValidationContext,
}; };
pub use list_through::{ pub use list_through::{
FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListThroughCursor, ListThroughMerger, ListThroughToken, FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger,
ListThroughTokenError, MAX_LIST_FETCHES_PER_SIDE, MergeOutcome, MergePick, MergeSide, SOURCE_LIST_MAX_RATE_WAIT, ListThroughToken, ListThroughTokenError, MAX_LIST_FETCHES_PER_SIDE, MAX_LIST_NO_PROGRESS_PAGES, MergeOutcome, MergePick,
SOURCE_LIST_RATE_PER_SEC, SourceListPlan, SourceListRateLimiter, decode_continuation_token, source_list_plan, MergeSide, SOURCE_LIST_MAX_RATE_WAIT, SOURCE_LIST_RATE_PER_SEC, SourceListPlan, SourceListRateLimiter,
decode_continuation_token, source_list_plan,
}; };
pub use negative_cache::{NEGATIVE_CACHE_MAX_ENTRIES, NegativeCache}; pub use negative_cache::{NEGATIVE_CACHE_MAX_ENTRIES, NegativeCache};
pub use pull::{ pub use pull::{
@@ -56,5 +68,6 @@ pub use stats::{
}; };
pub use sys::{ pub use sys::{
ApplyOutcome, BucketOdmState, GLOBAL_ON_DEMAND_MIGRATION_SYS, OdmBucketSnapshot, OdmLookup, OdmStateError, ApplyOutcome, BucketOdmState, GLOBAL_ON_DEMAND_MIGRATION_SYS, OdmBucketSnapshot, OdmLookup, OdmStateError,
OnDemandMigrationSys, PullError, PullFollower, PullLeader, PullOutcome, PullResult, PullSlot, source_client_spec, OnDemandMigrationSys, PullError, PullFollower, PullLeader, PullOutcome, PullResult, PullSlot, source_backend_spec,
source_client_spec,
}; };
@@ -0,0 +1,415 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Shared HTTP transport for the on-demand migration source backends that do
//! not speak S3 (Azure Blob, native GCS).
//!
//! The S3 backend rides the AWS SDK; these providers have no SigV4 dialect, so
//! they talk plain HTTP through one `reqwest` client that carries the same
//! connect/read timeouts and TLS policy the operator configured for the source.
//! Redirects are refused: the endpoint passed the outbound policy gate once, and
//! following a source-chosen `Location` would leave that gate behind.
//!
//! Errors never render the request URL. A SAS token lives in the query string,
//! so a `reqwest` error rendered with its URL would print the credential into
//! the log line and the admin response.
use super::source_client::{SourceError, SourceHead, SourceTimeouts, USER_AGENT_SUFFIX, classify_status, is_multipart_etag};
use crate::bucket::remote_s3_client::{RemoteS3ClientError, validate_remote_endpoint, validate_target_ca_pem};
use aws_sdk_s3::primitives::ByteStream;
use aws_smithy_types::body::SdkBody;
use futures::StreamExt;
use http::HeaderMap;
use std::collections::HashMap;
use std::time::SystemTime;
use time::OffsetDateTime;
use time::format_description::well_known::{Rfc2822, Rfc3339};
use url::Url;
/// Origin the native backends are allowed to address, plus the HTTP client
/// that reaches it.
pub(super) struct NativeHttp {
client: reqwest::Client,
endpoint: Url,
}
impl NativeHttp {
/// `endpoint` must be a bare `scheme://host[:port]` origin; it is checked
/// against the outbound policy exactly like an S3 source endpoint.
pub(super) fn new(
endpoint: &str,
timeouts: SourceTimeouts,
skip_tls_verify: bool,
ca_cert_pem: Option<&str>,
) -> Result<Self, RemoteS3ClientError> {
let endpoint = Url::parse(endpoint.trim()).map_err(|err| RemoteS3ClientError::InvalidEndpoint(err.to_string()))?;
if !matches!(endpoint.scheme(), "http" | "https") {
return Err(RemoteS3ClientError::InvalidEndpoint(format!(
"unsupported scheme {}; expected http or https",
endpoint.scheme()
)));
}
if endpoint.host_str().is_none_or(str::is_empty) {
return Err(RemoteS3ClientError::InvalidEndpoint("endpoint has no host".to_string()));
}
if !endpoint.username().is_empty() || endpoint.password().is_some() {
return Err(RemoteS3ClientError::InvalidEndpoint("endpoint must not carry userinfo".to_string()));
}
if !matches!(endpoint.path(), "" | "/") || endpoint.query().is_some() || endpoint.fragment().is_some() {
return Err(RemoteS3ClientError::InvalidEndpoint(
"endpoint must be an origin without path, query or fragment".to_string(),
));
}
validate_remote_endpoint(&endpoint).map_err(RemoteS3ClientError::EndpointNotAllowed)?;
let mut builder = reqwest::Client::builder()
.connect_timeout(timeouts.connect)
.read_timeout(timeouts.read)
.redirect(reqwest::redirect::Policy::none())
.user_agent(USER_AGENT_SUFFIX);
if skip_tls_verify {
builder = builder.danger_accept_invalid_certs(true);
} else if let Some(pem) = ca_cert_pem.map(str::trim).filter(|pem| !pem.is_empty()) {
// Reject a malformed bundle the same way the S3 path does, so the
// operator sees "invalid CA PEM" instead of a TLS handshake failure.
validate_target_ca_pem(pem)?;
let certificate = reqwest::Certificate::from_pem(pem.as_bytes())
.map_err(|err| RemoteS3ClientError::InvalidCaPem(err.to_string()))?;
builder = builder.add_root_certificate(certificate);
}
let client = builder
.build()
.map_err(|err| RemoteS3ClientError::InvalidEndpoint(format!("http client cannot be built: {err}")))?;
Ok(Self { client, endpoint })
}
#[cfg(test)]
pub(super) fn for_test(endpoint: Url) -> Self {
Self {
client: reqwest::Client::builder()
.redirect(reqwest::redirect::Policy::none())
.build()
.expect("test http client should build"),
endpoint,
}
}
/// A URL under the endpoint origin. `segments` are percent-encoded as
/// path segments, so a key containing `?`, `#` or a space cannot rewrite
/// the request target.
pub(super) fn url<'a>(&self, segments: impl IntoIterator<Item = &'a str>) -> Result<Url, SourceError> {
let mut url = self.endpoint.clone();
{
let mut path = url
.path_segments_mut()
.map_err(|_| SourceError::Other("source endpoint cannot carry a path".to_string()))?;
path.clear();
path.extend(segments);
}
Ok(url)
}
/// Sends the request and returns the response only for a 2xx status.
/// Non-2xx statuses are classified from the status and the provider's own
/// error-code header; response bodies are not read, so no provider message
/// can smuggle credentials or markup into a log line.
pub(super) async fn send(
&self,
request: reqwest::Request,
error_code_header: &str,
) -> Result<reqwest::Response, SourceError> {
let response = self.client.execute(request).await.map_err(classify_transport_error)?;
let status = response.status();
if status.is_success() {
return Ok(response);
}
let code = response
.headers()
.get(error_code_header)
.and_then(|value| value.to_str().ok())
.map(str::to_string);
Err(classify_status(
status.as_u16(),
None,
match &code {
Some(code) => format!("source returned HTTP {status} ({code})"),
None => format!("source returned HTTP {status}"),
},
))
}
}
/// Renders a transport failure without the request URL: a SAS token or a
/// signed query would otherwise reach logs and admin responses.
pub(super) fn classify_transport_error(err: reqwest::Error) -> SourceError {
let is_timeout = err.is_timeout();
let is_connect = err.is_connect();
let message = err.without_url().to_string();
if is_timeout {
SourceError::Timeout
} else if is_connect {
SourceError::Connect(message)
} else {
SourceError::Other(message)
}
}
/// Streams the response body without buffering it.
pub(super) fn response_body(response: reqwest::Response) -> ByteStream {
let stream = response.bytes_stream().map(|chunk| {
chunk
.map(http_body::Frame::data)
.map_err(|err| std::io::Error::other(err.without_url().to_string()))
});
ByteStream::new(SdkBody::from_body_1_x(http_body_util::StreamBody::new(stream)))
}
/// Reads a bounded response body as UTF-8, for the XML and JSON listings.
pub(super) async fn read_text(response: reqwest::Response, max_bytes: usize) -> Result<String, SourceError> {
let mut body = Vec::new();
let mut stream = response.bytes_stream();
while let Some(chunk) = stream.next().await {
let chunk = chunk.map_err(classify_transport_error)?;
if body.len().saturating_add(chunk.len()) > max_bytes {
return Err(SourceError::Other("source listing response exceeded the size limit".to_string()));
}
body.extend_from_slice(&chunk);
}
String::from_utf8(body).map_err(|_| SourceError::Other("source listing response is not valid UTF-8".to_string()))
}
/// Base64 digest (`Content-MD5`, `md5Hash`, `x-goog-hash`) as lowercase hex.
/// `None` when the value is not a 16-byte digest, so a CRC32C never passes as
/// an MD5.
pub(super) fn base64_md5_to_hex(value: &str) -> Option<String> {
let raw = base64_simd::STANDARD.decode_to_vec(value.trim().as_bytes()).ok()?;
(raw.len() == 16).then(|| faster_hex::hex_string(&raw))
}
pub(super) fn header<'a>(headers: &'a HeaderMap, name: &str) -> Option<&'a str> {
headers.get(name).and_then(|value| value.to_str().ok()).map(str::trim)
}
fn header_string(headers: &HeaderMap, name: &str) -> Option<String> {
header(headers, name).filter(|value| !value.is_empty()).map(str::to_string)
}
/// `Last-Modified` and friends arrive as an HTTP date; the JSON dialects use
/// RFC 3339 for the same field, so both are accepted.
pub(super) fn parse_http_timestamp(value: &str) -> Option<SystemTime> {
OffsetDateTime::parse(value, &Rfc2822)
.or_else(|_| OffsetDateTime::parse(value, &Rfc3339))
.ok()
.map(SystemTime::from)
}
/// Provider-specific fields the shared header mapping cannot infer.
pub(super) struct NativeHeadFields {
pub(super) etag: Option<String>,
/// The ETag is an opaque token rather than a digest of the bytes.
pub(super) etag_is_opaque: bool,
pub(super) version_id: Option<String>,
pub(super) storage_class: Option<String>,
}
/// Maps a HEAD or GET response onto [`SourceHead`]. `metadata_prefix` is the
/// provider's user-metadata header prefix (`x-ms-meta-`, `x-goog-meta-`); the
/// stored shape drops it, matching the `x-amz-meta-` handling of the S3 path.
pub(super) fn native_source_head(
headers: &HeaderMap,
metadata_prefix: &str,
fields: NativeHeadFields,
) -> Result<SourceHead, SourceError> {
let size = header(headers, "content-length")
.and_then(|value| value.parse::<u64>().ok())
.ok_or_else(|| SourceError::Other("source response has no valid content-length".to_string()))?;
let mut user_metadata = HashMap::new();
for (name, value) in headers {
let name = name.as_str();
if let Some(key) = name.strip_prefix(metadata_prefix)
&& !key.is_empty()
&& let Ok(value) = value.to_str()
{
user_metadata.insert(key.to_string(), value.to_string());
}
}
let etag = fields
.etag
.map(|etag| etag.trim().trim_matches('"').to_string())
.filter(|etag| !etag.is_empty());
// An opaque ETag never encodes a part count, so the multipart flag stays
// false for it however the provider happens to spell the token.
let is_multipart_etag = !fields.etag_is_opaque && etag.as_deref().is_some_and(is_multipart_etag);
Ok(SourceHead {
etag,
size,
last_modified: header(headers, "last-modified").and_then(parse_http_timestamp),
content_type: header_string(headers, "content-type"),
content_encoding: header_string(headers, "content-encoding"),
content_disposition: header_string(headers, "content-disposition"),
content_language: header_string(headers, "content-language"),
cache_control: header_string(headers, "cache-control"),
expires: header_string(headers, "expires"),
user_metadata,
version_id: fields.version_id,
storage_class: fields.storage_class,
// Neither native provider hands back ciphertext: a customer-key object
// is refused by the backend before it reaches this mapping, and the
// service-managed encryption is transparent to the reader.
sse: None,
is_multipart_etag,
etag_is_opaque: fields.etag_is_opaque,
})
}
#[cfg(test)]
mod tests {
use super::*;
use http::HeaderValue;
fn headers(pairs: &[(&str, &str)]) -> HeaderMap {
let mut headers = HeaderMap::new();
for (name, value) in pairs {
headers.insert(
http::HeaderName::from_bytes(name.as_bytes()).expect("test header name"),
HeaderValue::from_str(value).expect("test header value"),
);
}
headers
}
fn fields() -> NativeHeadFields {
NativeHeadFields {
etag: None,
etag_is_opaque: false,
version_id: None,
storage_class: None,
}
}
#[test]
fn native_source_head_maps_content_headers_and_prefixed_metadata() {
let headers = headers(&[
("content-length", "1234"),
("content-type", "text/plain"),
("content-encoding", "gzip"),
("content-language", "en"),
("content-disposition", "attachment"),
("cache-control", "max-age=60"),
("expires", "Thu, 01 Jan 2026 00:00:00 GMT"),
("last-modified", "Wed, 21 Oct 2015 07:28:00 GMT"),
("x-ms-meta-owner", "alice"),
("x-goog-meta-owner", "not-mine"),
]);
let head = native_source_head(
&headers,
"x-ms-meta-",
NativeHeadFields {
etag: Some("\"0x8DCE1D2\"".to_string()),
etag_is_opaque: true,
version_id: Some("2026-01-01T00:00:00.0000000Z".to_string()),
storage_class: Some("Hot".to_string()),
},
)
.expect("head should map");
assert_eq!(head.size, 1234);
assert_eq!(head.content_type.as_deref(), Some("text/plain"));
assert_eq!(head.content_encoding.as_deref(), Some("gzip"));
assert_eq!(head.content_language.as_deref(), Some("en"));
assert_eq!(head.content_disposition.as_deref(), Some("attachment"));
assert_eq!(head.cache_control.as_deref(), Some("max-age=60"));
assert_eq!(head.expires.as_deref(), Some("Thu, 01 Jan 2026 00:00:00 GMT"));
assert_eq!(
head.last_modified,
Some(SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1_445_412_480)),
"HTTP-date Last-Modified must parse"
);
assert_eq!(
head.user_metadata,
HashMap::from([("owner".to_string(), "alice".to_string())]),
"only the provider's own metadata prefix is read"
);
assert_eq!(head.etag.as_deref(), Some("0x8DCE1D2"), "quotes are stripped, the token is kept");
assert!(head.etag_is_opaque);
assert!(!head.is_multipart_etag);
assert_eq!(head.storage_class.as_deref(), Some("Hot"));
assert!(head.sse.is_none());
}
#[test]
fn native_source_head_requires_a_content_length() {
let err = native_source_head(&headers(&[("content-type", "text/plain")]), "x-ms-meta-", fields())
.expect_err("a response without content-length is unusable");
assert!(matches!(err, SourceError::Other(_)), "{err:?}");
}
#[test]
fn opaque_etag_never_reads_as_a_multipart_etag() {
// A digest-shaped ETag keeps the S3 reading; the same string marked
// opaque must not be split into "digest-partcount".
for (opaque, expected) in [(false, true), (true, false)] {
let head = native_source_head(
&headers(&[("content-length", "1")]),
"x-ms-meta-",
NativeHeadFields {
etag: Some("d41d8cd98f00b204e9800998ecf8427e-3".to_string()),
etag_is_opaque: opaque,
..fields()
},
)
.expect("head should map");
assert_eq!(head.is_multipart_etag, expected, "opaque = {opaque}");
}
}
#[test]
fn base64_md5_converts_only_sixteen_byte_digests() {
assert_eq!(
base64_md5_to_hex("1B2M2Y8AsgTpgAmY7PhCfg==").as_deref(),
Some("d41d8cd98f00b204e9800998ecf8427e")
);
assert_eq!(base64_md5_to_hex("not base64!").as_deref(), None);
// A CRC32C digest is four bytes: it must not pass as an MD5.
assert_eq!(base64_md5_to_hex("AAAAAA==").as_deref(), None);
}
#[test]
fn native_http_rejects_endpoints_that_are_not_bare_origins() {
for bad in [
"ftp://source.example.com",
"https://user:pw@source.example.com",
"https://source.example.com/container",
"https://source.example.com/?x=1",
"not a url",
] {
assert!(
NativeHttp::new(bad, SourceTimeouts::default(), false, None).is_err(),
"{bad} must be rejected"
);
}
}
#[test]
fn native_http_percent_encodes_every_path_segment() {
let http = NativeHttp::for_test(Url::parse("https://acct.blob.core.windows.net").expect("origin"));
let url = http.url(["container", "dir", "a b?c#d.txt"]).expect("url should build");
assert_eq!(url.as_str(), "https://acct.blob.core.windows.net/container/dir/a%20b%3Fc%23d.txt");
assert_eq!(url.query(), None, "a key with '?' must not become a query");
}
}
@@ -46,10 +46,10 @@ use super::stats::{PullFailureReason, PullPath};
use super::sys::{BucketOdmState, OnDemandMigrationSys, PullError, PullOutcome, PullSlot}; use super::sys::{BucketOdmState, OnDemandMigrationSys, PullError, PullOutcome, PullSlot};
use async_trait::async_trait; use async_trait::async_trait;
use bytes::Bytes; use bytes::Bytes;
use futures::{Stream, StreamExt}; use futures::{FutureExt, Stream, StreamExt, future::Shared};
use parking_lot::Mutex; use parking_lot::Mutex;
use rand::RngExt; use rand::RngExt;
use std::collections::{HashMap, HashSet}; use std::collections::HashMap;
use std::fmt; use std::fmt;
use std::io; use std::io;
use std::pin::Pin; use std::pin::Pin;
@@ -133,6 +133,8 @@ pub enum QueuedPullOutcome {
Failed(PullError), Failed(PullError),
} }
pub type QueuedPullReport = Shared<oneshot::Receiver<QueuedPullOutcome>>;
/// Result of [`PullQueue::enqueue`]. /// Result of [`PullQueue::enqueue`].
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum EnqueueOutcome { pub enum EnqueueOutcome {
@@ -251,6 +253,7 @@ pub struct WriteBackRequest {
pub preserve_etag: bool, pub preserve_etag: bool,
/// `policy.emit_events`. /// `policy.emit_events`.
pub emit_events: bool, pub emit_events: bool,
pub respect_delete_marker: bool,
/// Source tags to copy (`policy.copy_tags`), `None` to skip. /// Source tags to copy (`policy.copy_tags`), `None` to skip.
pub tags: Option<HashMap<String, String>>, pub tags: Option<HashMap<String, String>>,
} }
@@ -266,6 +269,7 @@ impl WriteBackRequest {
pulled_at: OffsetDateTime::now_utc(), pulled_at: OffsetDateTime::now_utc(),
preserve_etag: config.policy.preserve_etag, preserve_etag: config.policy.preserve_etag,
emit_events: config.policy.emit_events, emit_events: config.policy.emit_events,
respect_delete_marker: config.policy.respect_local_delete_marker,
tags, tags,
} }
} }
@@ -830,7 +834,7 @@ pub struct PullQueue {
bucket: String, bucket: String,
tx: mpsc::Sender<PullJob>, tx: mpsc::Sender<PullJob>,
/// Keys queued or running; the job removes its key when it ends. /// Keys queued or running; the job removes its key when it ends.
pending: Mutex<HashSet<String>>, pending: Mutex<HashMap<String, QueuedPullReport>>,
capacity: usize, capacity: usize,
cancel: CancellationToken, cancel: CancellationToken,
stats: Arc<super::stats::OdmStats>, stats: Arc<super::stats::OdmStats>,
@@ -869,7 +873,7 @@ impl PullQueue {
let queue = Arc::new(Self { let queue = Arc::new(Self {
bucket: state.bucket().to_string(), bucket: state.bucket().to_string(),
tx, tx,
pending: Mutex::new(HashSet::new()), pending: Mutex::new(HashMap::new()),
capacity, capacity,
cancel: state.cancel_token(), cancel: state.cancel_token(),
stats: Arc::clone(state.stats()), stats: Arc::clone(state.stats()),
@@ -903,29 +907,24 @@ impl PullQueue {
self.enqueue_with_report(key, reason).0 self.enqueue_with_report(key, reason).0
} }
/// [`Self::enqueue`] that also hands back the job's report channel when /// [`Self::enqueue`] with a shared report, including for coalesced pulls.
/// a new job was queued (`Coalesced` pulls report to their first pub fn enqueue_with_report(&self, key: &str, reason: PullReason) -> (EnqueueOutcome, Option<QueuedPullReport>) {
/// requester only).
pub fn enqueue_with_report(
&self,
key: &str,
reason: PullReason,
) -> (EnqueueOutcome, Option<oneshot::Receiver<QueuedPullOutcome>>) {
if self.cancel.is_cancelled() { if self.cancel.is_cancelled() {
return (EnqueueOutcome::Unavailable, None); return (EnqueueOutcome::Unavailable, None);
} }
let mut pending = self.pending.lock(); let mut pending = self.pending.lock();
if pending.contains(key) { if let Some(report) = pending.get(key) {
return (EnqueueOutcome::Coalesced, None); return (EnqueueOutcome::Coalesced, Some(report.clone()));
} }
let (report_tx, report_rx) = oneshot::channel(); let (report_tx, report_rx) = oneshot::channel();
let report_rx = report_rx.shared();
match self.tx.try_send(PullJob { match self.tx.try_send(PullJob {
key: key.to_string(), key: key.to_string(),
reason, reason,
report: Some(report_tx), report: Some(report_tx),
}) { }) {
Ok(()) => { Ok(()) => {
pending.insert(key.to_string()); pending.insert(key.to_string(), report_rx.clone());
(EnqueueOutcome::Enqueued, Some(report_rx)) (EnqueueOutcome::Enqueued, Some(report_rx))
} }
Err(TrySendError::Full(_)) => { Err(TrySendError::Full(_)) => {
@@ -1072,7 +1071,7 @@ impl BucketOdmState {
self: &Arc<Self>, self: &Arc<Self>,
key: &str, key: &str,
reason: PullReason, reason: PullReason,
) -> (EnqueueOutcome, Option<oneshot::Receiver<QueuedPullOutcome>>) { ) -> (EnqueueOutcome, Option<QueuedPullReport>) {
match self.pull_queue() { match self.pull_queue() {
Some(queue) => queue.enqueue_with_report(key, reason), Some(queue) => queue.enqueue_with_report(key, reason),
None => (EnqueueOutcome::Unavailable, None), None => (EnqueueOutcome::Unavailable, None),
@@ -1119,6 +1118,8 @@ mod tests {
session_token: None, session_token: None,
}), }),
tls: TlsConfig::default(), tls: TlsConfig::default(),
azure: None,
gcs: None,
}, },
filter: FilterConfig::default(), filter: FilterConfig::default(),
policy: PolicyConfig::default(), policy: PolicyConfig::default(),
@@ -1399,13 +1400,21 @@ mod tests {
assert_eq!(queue.capacity(), 1024); assert_eq!(queue.capacity(), 1024);
let mut outcomes = HashMap::new(); let mut outcomes = HashMap::new();
let mut shared_report = None;
for _ in 0..100 { for _ in 0..100 {
*outcomes.entry(queue.enqueue("a", PullReason::RangeGet)).or_insert(0) += 1; let (outcome, report) = queue.enqueue_with_report("a", PullReason::RangeGet);
*outcomes.entry(outcome).or_insert(0) += 1;
shared_report = report;
} }
assert_eq!(outcomes.get(&EnqueueOutcome::Enqueued), Some(&1)); assert_eq!(outcomes.get(&EnqueueOutcome::Enqueued), Some(&1));
assert_eq!(outcomes.get(&EnqueueOutcome::Coalesced), Some(&99)); assert_eq!(outcomes.get(&EnqueueOutcome::Coalesced), Some(&99));
assert_eq!(queue.pending_keys(), 1); assert_eq!(queue.pending_keys(), 1);
assert_eq!(
shared_report.expect("coalesced report").await,
Ok(QueuedPullOutcome::Stored { size: 1000 })
);
wait_until("first pull to finish", || queue.pending_keys() == 0).await; wait_until("first pull to finish", || queue.pending_keys() == 0).await;
assert_eq!(source.head_calls.load(Ordering::SeqCst), 1); assert_eq!(source.head_calls.load(Ordering::SeqCst), 1);
assert_eq!(source.get_calls.load(Ordering::SeqCst), 1); assert_eq!(source.get_calls.load(Ordering::SeqCst), 1);
@@ -1438,6 +1447,23 @@ mod tests {
assert_eq!(queue.enqueue("a", PullReason::RangeGet), EnqueueOutcome::Unavailable); assert_eq!(queue.enqueue("a", PullReason::RangeGet), EnqueueOutcome::Unavailable);
} }
#[tokio::test]
async fn coalesced_enqueues_share_failure_reports() {
let sys = OnDemandMigrationSys::new();
let state = enabled_state(&sys, &config()).await;
let source = MockSource::with_object("missing", 1000, BodyKind::Bytes(body_bytes(1000)));
let queue = PullQueue::start(Arc::clone(&state), source, Arc::new(MockWriteBack::default()));
let (first, first_report) = queue.enqueue_with_report("absent", PullReason::RangeGet);
let (second, second_report) = queue.enqueue_with_report("absent", PullReason::Backfill);
assert_eq!(first, EnqueueOutcome::Enqueued);
assert_eq!(second, EnqueueOutcome::Coalesced);
let (first, second) = tokio::join!(first_report.expect("leader report"), second_report.expect("coalesced report"));
assert_eq!(first, second);
assert!(matches!(first, Ok(QueuedPullOutcome::Failed(_))));
sys.remove(BUCKET);
queue.wait_until_stopped().await;
}
#[tokio::test] #[tokio::test]
async fn queue_full_is_reported_and_cancel_drains_without_leaking_tasks() { async fn queue_full_is_reported_and_cancel_drains_without_leaking_tasks() {
let sys = OnDemandMigrationSys::new(); let sys = OnDemandMigrationSys::new();
@@ -1467,7 +1493,8 @@ mod tests {
wait_until("dispatcher to wait for a slot", || state.stats().queue_depth() == 1).await; wait_until("dispatcher to wait for a slot", || state.stats().queue_depth() == 1).await;
assert_eq!(queue.enqueue("c", PullReason::LargeObject), EnqueueOutcome::Enqueued); assert_eq!(queue.enqueue("c", PullReason::LargeObject), EnqueueOutcome::Enqueued);
assert_eq!(queue.enqueue("d", PullReason::LargeObject), EnqueueOutcome::QueueFull); assert_eq!(queue.enqueue("d", PullReason::LargeObject), EnqueueOutcome::QueueFull);
assert_eq!(queue.enqueue("c", PullReason::LargeObject), EnqueueOutcome::Coalesced); let (coalesced, canceled_report) = queue.enqueue_with_report("c", PullReason::LargeObject);
assert_eq!(coalesced, EnqueueOutcome::Coalesced);
assert_eq!(queue.pending_keys(), 3); assert_eq!(queue.pending_keys(), 3);
assert_eq!(failures(&state).get("queue_full"), Some(&1)); assert_eq!(failures(&state).get("queue_full"), Some(&1));
assert!(!queue.is_stopped()); assert!(!queue.is_stopped());
@@ -1477,6 +1504,12 @@ mod tests {
.await .await
.expect("dispatcher and in-flight job must exit after cancel"); .expect("dispatcher and in-flight job must exit after cancel");
assert!(queue.is_stopped()); assert!(queue.is_stopped());
assert!(
tokio::time::timeout(Duration::from_secs(5), canceled_report.expect("coalesced cancellation report"))
.await
.expect("cancellation closes the report")
.is_err()
);
assert_eq!(queue.pending_keys(), 0); assert_eq!(queue.pending_keys(), 0);
assert_eq!(state.inflight_keys(), 0); assert_eq!(state.inflight_keys(), 0);
assert_eq!(state.stats().inflight_pulls(), 0); assert_eq!(state.stats().inflight_pulls(), 0);
@@ -25,6 +25,9 @@
//! Client-supplied `If-*`, `Authorization`, `Host` and SSE-C headers are never //! Client-supplied `If-*`, `Authorization`, `Host` and SSE-C headers are never
//! forwarded: v1 rejects SSE-C source objects outright. //! forwarded: v1 rejects SSE-C source objects outright.
use super::azure::AzureSourceBackend;
use super::gcs::GcsNativeSourceBackend;
use super::list_through::{ListPageError, validate_list_page};
use crate::bucket::remote_s3_client::{ use crate::bucket::remote_s3_client::{
PathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3EndpointSpec, RemoteS3RetryPolicy, build_remote_s3_config, PathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3EndpointSpec, RemoteS3RetryPolicy, build_remote_s3_config,
}; };
@@ -64,6 +67,10 @@ pub enum SourceProvider {
/// Generic S3-compatible service. /// Generic S3-compatible service.
#[default] #[default]
S3, S3,
/// Native Azure Blob service; not an S3 dialect.
Azure,
/// Native GCS JSON API with a service-account key; not an S3 dialect.
GcsNative,
} }
impl SourceProvider { impl SourceProvider {
@@ -75,6 +82,8 @@ impl SourceProvider {
"minio" => Some(Self::Minio), "minio" => Some(Self::Minio),
"rustfs" => Some(Self::Rustfs), "rustfs" => Some(Self::Rustfs),
"s3" => Some(Self::S3), "s3" => Some(Self::S3),
"azure" => Some(Self::Azure),
"gcs_native" => Some(Self::GcsNative),
_ => None, _ => None,
} }
} }
@@ -87,6 +96,8 @@ impl SourceProvider {
Self::Minio => "minio", Self::Minio => "minio",
Self::Rustfs => "rustfs", Self::Rustfs => "rustfs",
Self::S3 => "s3", Self::S3 => "s3",
Self::Azure => "azure",
Self::GcsNative => "gcs_native",
} }
} }
@@ -152,12 +163,75 @@ pub struct SourceClientSpec {
/// Wire requests one logical source call may cost. The pull pipeline and /// Wire requests one logical source call may cost. The pull pipeline and
/// the backfill job own the retry budget (`pull.rs` `PULL_MAX_RETRIES`, /// the backfill job own the retry budget (`pull.rs` `PULL_MAX_RETRIES`,
/// `backfill.rs` `LIST_MAX_RETRIES`) and the breaker counts logical calls, /// `backfill.rs` `LIST_MAX_RETRIES`) and the breaker counts logical calls,
/// so ODM declares [`RemoteS3RetryPolicy::Disabled`] and keeps one counted /// so ODM declares [`RemoteS3RetryPolicy::Disabled`]. An ambiguous HEAD
/// failure equal to one request against a struggling source. /// 404 additionally probes the bucket before declaring a key absent.
pub retry: RemoteS3RetryPolicy, pub retry: RemoteS3RetryPolicy,
/// Bytes per second the pull pipeline may consume from this source; /// Bytes per second the pull pipeline may consume from this source;
/// `None` means unlimited. Enforced by the consumer, not by this client. /// `None` means unlimited. Enforced by the consumer, not by this client.
pub bandwidth_limit: Option<NonZeroU64>, pub bandwidth_limit: Option<NonZeroU64>,
/// Which [`SourceBackend`] to build. The S3 variant reads `region`,
/// `path_style` and `credentials`; the native variants ignore all three
/// and carry their own credentials.
pub backend: SourceBackendSpec,
}
/// Provider-specific half of [`SourceClientSpec`].
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub enum SourceBackendSpec {
#[default]
S3,
Azure(AzureSourceSpec),
Gcs(GcsSourceSpec),
}
/// Native Azure Blob parameters. The container is [`SourceClientSpec::bucket`].
#[derive(Clone, PartialEq, Eq)]
pub struct AzureSourceSpec {
pub account: String,
pub auth: AzureAuth,
}
impl fmt::Debug for AzureSourceSpec {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("AzureSourceSpec")
.field("account", &self.account)
.field("auth", &self.auth)
.finish()
}
}
/// How Azure requests are authorized.
#[derive(Clone, PartialEq, Eq)]
pub enum AzureAuth {
/// Base64 storage-account key, signed per request with Shared Key.
SharedKey(String),
/// SAS query string without the leading `?`, appended to every URL.
Sas(String),
}
impl fmt::Debug for AzureAuth {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
// Both variants are secrets; only the scheme may be rendered.
f.write_str(match self {
Self::SharedKey(_) => "SharedKey(REDACTED)",
Self::Sas(_) => "Sas(REDACTED)",
})
}
}
/// Native GCS parameters. The bucket is [`SourceClientSpec::bucket`].
#[derive(Clone, PartialEq, Eq)]
pub struct GcsSourceSpec {
/// Service-account key JSON.
pub service_account_json: String,
}
impl fmt::Debug for GcsSourceSpec {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("GcsSourceSpec")
.field("service_account_json", &"REDACTED")
.finish()
}
} }
impl SourceClientSpec { impl SourceClientSpec {
@@ -223,6 +297,8 @@ pub enum SourceError {
ServerError(u16), ServerError(u16),
#[error("unsupported source object: {0}")] #[error("unsupported source object: {0}")]
Unsupported(String), Unsupported(String),
#[error("invalid source listing: {0}")]
InvalidPagination(#[from] ListPageError),
#[error("source request failed: {0}")] #[error("source request failed: {0}")]
Other(String), Other(String),
} }
@@ -245,6 +321,7 @@ impl SourceError {
SourceError::Connect(_) => "connect", SourceError::Connect(_) => "connect",
SourceError::ServerError(_) => "server_error", SourceError::ServerError(_) => "server_error",
SourceError::Unsupported(_) => "unsupported", SourceError::Unsupported(_) => "unsupported",
SourceError::InvalidPagination(_) => "invalid_pagination",
SourceError::Other(_) => "other", SourceError::Other(_) => "other",
} }
} }
@@ -258,7 +335,7 @@ const THROTTLE_CODES: &[&str] = &[
"TooManyRequests", "TooManyRequests",
"RequestThrottled", "RequestThrottled",
]; ];
const NOT_FOUND_CODES: &[&str] = &["NoSuchKey", "NotFound", "NoSuchBucket", "NoSuchVersion"]; const NOT_FOUND_CODES: &[&str] = &["NoSuchKey"];
const ACCESS_DENIED_CODES: &[&str] = &[ const ACCESS_DENIED_CODES: &[&str] = &[
"AccessDenied", "AccessDenied",
"InvalidAccessKeyId", "InvalidAccessKeyId",
@@ -268,7 +345,7 @@ const ACCESS_DENIED_CODES: &[&str] = &[
"InvalidToken", "InvalidToken",
]; ];
fn classify_status(status: u16, code: Option<&str>, message: String) -> SourceError { pub(super) fn classify_status(status: u16, code: Option<&str>, message: String) -> SourceError {
if let Some(code) = code { if let Some(code) = code {
if THROTTLE_CODES.contains(&code) { if THROTTLE_CODES.contains(&code) {
return SourceError::Throttled; return SourceError::Throttled;
@@ -281,7 +358,6 @@ fn classify_status(status: u16, code: Option<&str>, message: String) -> SourceEr
} }
} }
match status { match status {
404 => SourceError::NotFound,
401 | 403 => SourceError::AccessDenied, 401 | 403 => SourceError::AccessDenied,
429 | 503 => SourceError::Throttled, 429 | 503 => SourceError::Throttled,
500..=599 => SourceError::ServerError(status), 500..=599 => SourceError::ServerError(status),
@@ -341,6 +417,11 @@ pub struct SourceHead {
pub storage_class: Option<String>, pub storage_class: Option<String>,
pub sse: Option<SourceSse>, pub sse: Option<SourceSse>,
pub is_multipart_etag: bool, pub is_multipart_etag: bool,
/// The provider's ETag is not derived from the object bytes (Azure
/// stamps an opaque concurrency token). Such an ETag is recorded for
/// provenance but must never be read as a content digest, so the
/// write-back path refuses to use it as the expected MD5.
pub etag_is_opaque: bool,
} }
/// Per-operation fields shared by HEAD and GET outputs. /// Per-operation fields shared by HEAD and GET outputs.
@@ -362,7 +443,7 @@ struct HeadParts {
sse_customer_algorithm: Option<String>, sse_customer_algorithm: Option<String>,
} }
fn normalize_etag(etag: Option<String>) -> Option<String> { pub(super) fn normalize_etag(etag: Option<String>) -> Option<String> {
etag.map(|etag| etag.trim().trim_matches('"').to_string()) etag.map(|etag| etag.trim().trim_matches('"').to_string())
.filter(|etag| !etag.is_empty()) .filter(|etag| !etag.is_empty())
} }
@@ -411,6 +492,7 @@ fn source_head(parts: HeadParts) -> Result<SourceHead, SourceError> {
storage_class: parts.storage_class, storage_class: parts.storage_class,
sse, sse,
is_multipart_etag, is_multipart_etag,
etag_is_opaque: false,
}) })
} }
@@ -621,14 +703,52 @@ impl fmt::Debug for SourceClient {
impl SourceClient { impl SourceClient {
pub async fn new(spec: &SourceClientSpec) -> Result<Self, RemoteS3ClientError> { pub async fn new(spec: &SourceClientSpec) -> Result<Self, RemoteS3ClientError> {
let endpoint = spec.endpoint_spec()?; match &spec.backend {
let config = build_remote_s3_config(&endpoint).await?; SourceBackendSpec::S3 => {
Ok(Self::from_config_builder(config, endpoint.endpoint_url(), spec)) let endpoint = spec.endpoint_spec()?;
let config = build_remote_s3_config(&endpoint).await?;
Ok(Self::from_config_builder(config, endpoint.endpoint_url(), spec))
}
SourceBackendSpec::Azure(azure) => {
let backend = AzureSourceBackend::new(
&spec.endpoint,
&spec.bucket,
azure,
spec.timeouts,
spec.skip_tls_verify,
spec.ca_cert_pem.as_deref(),
)?;
Ok(Self::from_backend(Box::new(backend), spec))
}
SourceBackendSpec::Gcs(gcs) => {
let backend = GcsNativeSourceBackend::new(
&spec.endpoint,
&spec.bucket,
gcs,
spec.timeouts,
spec.skip_tls_verify,
spec.ca_cert_pem.as_deref(),
)?;
Ok(Self::from_backend(Box::new(backend), spec))
}
}
}
/// Wraps a ready backend in the prefix-mapping client. The endpoint is
/// kept only for `Debug` and admin status.
fn from_backend(backend: Box<dyn SourceBackend>, spec: &SourceClientSpec) -> Self {
Self {
backend,
endpoint: spec.endpoint.clone(),
bucket: spec.bucket.clone(),
source_prefix: spec.source_prefix.clone().filter(|prefix| !prefix.is_empty()),
timeouts: spec.timeouts,
bandwidth_limit: spec.bandwidth_limit,
}
} }
/// `config` must come from [`SourceClientSpec::endpoint_spec`], which is /// `config` must come from [`SourceClientSpec::endpoint_spec`], which is
/// where the retry policy that keeps one logical call equal to one wire /// where the policy disabling SDK-level retries is declared.
/// request is declared.
fn from_config_builder(config: aws_sdk_s3::config::Builder, endpoint: String, spec: &SourceClientSpec) -> Self { fn from_config_builder(config: aws_sdk_s3::config::Builder, endpoint: String, spec: &SourceClientSpec) -> Self {
let client = S3Client::from_conf(config.interceptor(SourceProxyMarkerInterceptor::new()).build()); let client = S3Client::from_conf(config.interceptor(SourceProxyMarkerInterceptor::new()).build());
Self { Self {
@@ -714,6 +834,7 @@ impl SourceClient {
..*request ..*request
}) })
.await?; .await?;
validate_list_page(page.is_truncated, request.continuation_token, page.next_continuation_token.as_deref())?;
page.objects = page page.objects = page
.objects .objects
.into_iter() .into_iter()
@@ -749,15 +870,16 @@ impl SourceClient {
#[async_trait::async_trait] #[async_trait::async_trait]
impl SourceBackend for S3SourceBackend { impl SourceBackend for S3SourceBackend {
async fn head(&self, key: &str) -> Result<SourceHead, SourceError> { async fn head(&self, key: &str) -> Result<SourceHead, SourceError> {
let output = self match self.client.head_object().bucket(&self.bucket).key(key).send().await {
.client Ok(output) => source_head_from_head_output(output),
.head_object() Err(err) if err.raw_response().is_some_and(|response| response.status().as_u16() == 404) => {
.bucket(&self.bucket) // HEAD has no error body: a missing bucket must not poison
.key(key) // the per-key negative cache as though only the key was absent.
.send() self.probe().await?;
.await Err(SourceError::NotFound)
.map_err(classify_sdk_error)?; }
source_head_from_head_output(output) Err(err) => Err(classify_sdk_error(err)),
}
} }
/// Streams the object; `range` is passed through as an HTTP `Range` /// Streams the object; `range` is passed through as an HTTP `Range`
@@ -800,17 +922,12 @@ impl SourceBackend for S3SourceBackend {
let is_truncated = output.is_truncated.unwrap_or(false); let is_truncated = output.is_truncated.unwrap_or(false);
let next_continuation_token = output.next_continuation_token; let next_continuation_token = output.next_continuation_token;
if is_truncated && next_continuation_token.is_none() {
return Err(SourceError::Other(
"source reported a truncated listing without a continuation token".to_string(),
));
}
let objects = output let objects = output
.contents .contents
.unwrap_or_default() .unwrap_or_default()
.into_iter() .into_iter()
.filter_map(s3_source_object) .map(s3_source_object)
.collect(); .collect::<Result<Vec<_>, _>>()?;
let common_prefixes = output let common_prefixes = output
.common_prefixes .common_prefixes
.unwrap_or_default() .unwrap_or_default()
@@ -849,14 +966,20 @@ impl SourceBackend for S3SourceBackend {
} }
} }
fn s3_source_object(object: SdkObject) -> Option<SourceObject> { fn s3_source_object(object: SdkObject) -> Result<SourceObject, SourceError> {
let key = object.key?; let key = object
.key
.ok_or_else(|| SourceError::Other("source listing object has no key".to_string()))?;
let size = object
.size
.and_then(|size| u64::try_from(size).ok())
.ok_or_else(|| SourceError::Other("source listing object has no valid size".to_string()))?;
let etag = normalize_etag(object.e_tag); let etag = normalize_etag(object.e_tag);
let is_multipart_etag = etag.as_deref().is_some_and(is_multipart_etag); let is_multipart_etag = etag.as_deref().is_some_and(is_multipart_etag);
Some(SourceObject { Ok(SourceObject {
key, key,
etag, etag,
size: object.size.and_then(|size| u64::try_from(size).ok()).unwrap_or(0), size,
last_modified: system_time(object.last_modified), last_modified: system_time(object.last_modified),
storage_class: object.storage_class.map(|class| class.as_str().to_string()), storage_class: object.storage_class.map(|class| class.as_str().to_string()),
is_multipart_etag, is_multipart_etag,
@@ -866,6 +989,7 @@ fn s3_source_object(object: SdkObject) -> Option<SourceObject> {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use crate::bucket::on_demand_migration::backend_contract::{BackendCapabilities, OBJECT_MD5, assert_backend_contract};
use aws_smithy_runtime_api::client::http::{HttpConnector, HttpConnectorFuture, SharedHttpConnector, http_client_fn}; use aws_smithy_runtime_api::client::http::{HttpConnector, HttpConnectorFuture, SharedHttpConnector, http_client_fn};
use aws_smithy_runtime_api::client::orchestrator::HttpRequest; use aws_smithy_runtime_api::client::orchestrator::HttpRequest;
use aws_smithy_runtime_api::client::result::ConnectorError; use aws_smithy_runtime_api::client::result::ConnectorError;
@@ -983,6 +1107,7 @@ mod tests {
retry: RemoteS3RetryPolicy::Disabled, retry: RemoteS3RetryPolicy::Disabled,
timeouts: SourceTimeouts::default(), timeouts: SourceTimeouts::default(),
bandwidth_limit: NonZeroU64::new(1_000_000), bandwidth_limit: NonZeroU64::new(1_000_000),
backend: SourceBackendSpec::S3,
} }
} }
@@ -1274,7 +1399,9 @@ mod tests {
<CommonPrefixes><Prefix>data/photos/</Prefix></CommonPrefixes> <CommonPrefixes><Prefix>data/photos/</Prefix></CommonPrefixes>
<CommonPrefixes><Prefix>outside/</Prefix></CommonPrefixes> <CommonPrefixes><Prefix>outside/</Prefix></CommonPrefixes>
</ListBucketResult>"#; </ListBucketResult>"#;
let (client, requests) = scripted_client(&spec(Some("data/")), vec![ok(Vec::new(), body), ok(Vec::new(), body)]).await; let next_body = body.replace("data/opaque", "data/next");
let (client, requests) =
scripted_client(&spec(Some("data/")), vec![ok(Vec::new(), body), ok(Vec::new(), &next_body)]).await;
let first = client let first = client
.list_page(&SourceListRequest { .list_page(&SourceListRequest {
prefix: Some("photos/"), prefix: Some("photos/"),
@@ -1336,7 +1463,104 @@ mod tests {
.list_objects_v2(None, None, 10) .list_objects_v2(None, None, 10)
.await .await
.expect_err("truncated page without token is corrupt"); .expect_err("truncated page without token is corrupt");
assert!(matches!(err, SourceError::Other(_)), "{err:?}"); assert!(matches!(err, SourceError::InvalidPagination(ListPageError::Missing)), "{err:?}");
}
#[tokio::test]
async fn list_page_validates_s3_cursor_progress_before_mapping_entries() {
for contents in ["", "<Contents><Key>data/a</Key><Size>1</Size></Contents>"] {
for (truncated, next, expected) in [
(true, None, Some(ListPageError::Missing)),
(true, Some(""), Some(ListPageError::Empty)),
(true, Some("stuck"), Some(ListPageError::Repeated)),
(true, Some("opaque-next"), None),
(false, None, None),
(false, Some("stuck"), None),
] {
let next_xml = next
.map(|next| format!("<NextContinuationToken>{next}</NextContinuationToken>"))
.unwrap_or_default();
let body = format!(
"<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><IsTruncated>{truncated}</IsTruncated>{next_xml}{contents}</ListBucketResult>"
);
let (client, requests) = scripted_client(&spec(Some("data/")), vec![ok(Vec::new(), &body)]).await;
let result = client
.list_page(&SourceListRequest {
continuation_token: Some("stuck"),
max_keys: 2,
..Default::default()
})
.await;
match expected {
Some(expected) => {
let error = result.expect_err("malformed pagination must fail at the provider boundary");
assert!(
matches!(&error, SourceError::InvalidPagination(actual) if *actual == expected),
"{error:?}"
);
assert_eq!(error.class_label(), "invalid_pagination");
assert!(!error.is_retryable());
assert!(!error.to_string().contains("stuck"), "errors must not echo opaque tokens");
}
None => {
let page = result.expect("progressing empty/nonempty pages and EOF are valid");
assert_eq!(page.is_truncated, truncated);
assert_eq!(page.next_continuation_token.as_deref(), next);
assert_eq!(page.objects.len(), usize::from(!contents.is_empty()));
if let Some(object) = page.objects.first() {
assert_eq!(object.key, "a");
}
}
}
let requests = recorded(&requests);
assert_eq!(requests.len(), 1, "invalid pagination must not be retried");
assert!(requests[0].uri.contains("continuation-token=stuck"));
}
}
}
struct ListOnlyBackend(SourcePage);
#[async_trait::async_trait]
impl SourceBackend for ListOnlyBackend {
async fn list(&self, request: &SourceListRequest<'_>) -> Result<SourcePage, SourceError> {
assert_eq!(request.continuation_token, Some("stuck"), "opaque cursors reach every provider unchanged");
Ok(self.0.clone())
}
async fn head(&self, _key: &str) -> Result<SourceHead, SourceError> {
panic!("unexpected HEAD in list test")
}
async fn get(&self, _key: &str, _range: Option<&HTTPRangeSpec>) -> Result<SourceGet, SourceError> {
panic!("unexpected GET in list test")
}
async fn tagging(&self, _key: &str) -> Result<HashMap<String, String>, SourceError> {
panic!("unexpected tagging in list test")
}
async fn probe(&self) -> Result<(), SourceError> {
panic!("unexpected probe in list test")
}
}
#[tokio::test]
async fn list_page_validates_non_s3_provider_cursors_at_the_common_boundary() {
for (next, expected) in [
(None, ListPageError::Missing),
(Some(""), ListPageError::Empty),
(Some("stuck"), ListPageError::Repeated),
] {
let mut client = prefix_client(Some("data/".into()));
client.backend = Box::new(ListOnlyBackend(SourcePage {
is_truncated: true,
next_continuation_token: next.map(str::to_string),
..Default::default()
}));
let error = client
.list_objects_v2(None, Some("stuck"), 2)
.await
.expect_err("all providers must advance pagination");
assert!(matches!(error, SourceError::InvalidPagination(actual) if actual == expected));
}
} }
const TAGGING_BODY: &str = r#"<?xml version="1.0" encoding="UTF-8"?> const TAGGING_BODY: &str = r#"<?xml version="1.0" encoding="UTF-8"?>
@@ -1390,7 +1614,10 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn source_error_classification_covers_every_class() { async fn source_error_classification_covers_every_class() {
let cases: Vec<(Scripted, &str, bool)> = vec![ let cases: Vec<(Scripted, &str, bool)> = vec![
(status(404, ""), "not_found", false), (status(404, ""), "other", false),
(status(404, "<Error><Code>NoSuchKey</Code></Error>"), "not_found", false),
(status(404, "<Error><Code>NoSuchBucket</Code></Error>"), "other", false),
(status(404, "<Error><Code>NoSuchVersion</Code></Error>"), "other", false),
(status(403, ACCESS_DENIED_BODY), "access_denied", false), (status(403, ACCESS_DENIED_BODY), "access_denied", false),
(status(401, ""), "access_denied", false), (status(401, ""), "access_denied", false),
(status(429, ""), "throttled", true), (status(429, ""), "throttled", true),
@@ -1413,14 +1640,35 @@ mod tests {
} }
} }
// HEAD carries no error body, so the classification must work from the let (client, requests) = scripted_client(&spec(None), vec![status(404, ""), status(200, "")]).await;
// status alone as well.
let (client, _) = scripted_client(&spec(None), vec![status(404, "")]).await;
assert!(matches!(client.head_object("missing").await, Err(SourceError::NotFound))); assert!(matches!(client.head_object("missing").await, Err(SourceError::NotFound)));
assert_eq!(recorded(&requests).len(), 2, "ambiguous HEAD 404 must check the bucket");
let (client, _) = scripted_client(&spec(None), vec![status(404, ""), status(404, "")]).await;
assert!(matches!(client.head_object("missing").await, Err(SourceError::Other(_))));
let (client, _) = scripted_client(&spec(None), vec![status(404, ""), status(403, "")]).await;
assert!(matches!(client.head_object("missing").await, Err(SourceError::AccessDenied)));
let (client, _) = scripted_client(&spec(None), vec![status(403, "")]).await; let (client, _) = scripted_client(&spec(None), vec![status(403, "")]).await;
assert!(matches!(client.head_object("secret").await, Err(SourceError::AccessDenied))); assert!(matches!(client.head_object("secret").await, Err(SourceError::AccessDenied)));
} }
#[test]
fn source_listing_rejects_missing_and_negative_sizes() {
for size in [None, Some(-1)] {
let object = SdkObject::builder().key("key").set_size(size).build();
assert!(matches!(s3_source_object(object), Err(SourceError::Other(_))));
}
assert!(matches!(
s3_source_object(SdkObject::builder().size(0).build()),
Err(SourceError::Other(_))
));
assert_eq!(
s3_source_object(SdkObject::builder().key("empty").size(0).build())
.expect("empty object")
.size,
0
);
}
#[tokio::test] #[tokio::test]
async fn source_client_debug_redacts_credentials() { async fn source_client_debug_redacts_credentials() {
let (client, _) = scripted_client(&spec(Some("data/")), Vec::new()).await; let (client, _) = scripted_client(&spec(Some("data/")), Vec::new()).await;
@@ -1487,7 +1735,101 @@ mod tests {
assert_eq!(resolve_path_style(PathStyle::VirtualHost, Minio, "10.0.0.1"), PathStyle::VirtualHost); assert_eq!(resolve_path_style(PathStyle::VirtualHost, Minio, "10.0.0.1"), PathStyle::VirtualHost);
assert_eq!(resolve_path_style(PathStyle::Path, Aws, "s3.amazonaws.com"), PathStyle::Path); assert_eq!(resolve_path_style(PathStyle::Path, Aws, "s3.amazonaws.com"), PathStyle::Path);
assert_eq!(SourceProvider::from_label(" AWS "), Some(Aws)); assert_eq!(SourceProvider::from_label(" AWS "), Some(Aws));
assert_eq!(SourceProvider::from_label("azure"), None); assert_eq!(SourceProvider::from_label(" Azure "), Some(Azure));
assert_eq!(SourceProvider::from_label("gcs_native"), Some(GcsNative));
assert_eq!(SourceProvider::from_label("swift"), None);
}
const CONTRACT_LIST_PAGE_ONE: &str = r#"<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Name>source-bucket</Name>
<IsTruncated>true</IsTruncated>
<NextContinuationToken>cursor-1</NextContinuationToken>
<Contents>
<Key>dir/a.txt</Key>
<LastModified>2015-10-21T07:28:00.000Z</LastModified>
<ETag>&quot;5d41402abc4b2a76b9719d911017c592&quot;</ETag>
<Size>5</Size>
<StorageClass>STANDARD</StorageClass>
</Contents>
<CommonPrefixes><Prefix>dir/sub/</Prefix></CommonPrefixes>
</ListBucketResult>"#;
const CONTRACT_LIST_PAGE_TWO: &str = r#"<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Name>source-bucket</Name>
<IsTruncated>false</IsTruncated>
<Contents>
<Key>dir/b.txt</Key>
<LastModified>2015-10-21T07:28:00.000Z</LastModified>
<ETag>&quot;7d41402abc4b2a76b9719d911017c592&quot;</ETag>
<Size>7</Size>
</Contents>
</ListBucketResult>"#;
const CONTRACT_TAGGING: &str = r#"<?xml version="1.0" encoding="UTF-8"?>
<Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><TagSet>
<Tag><Key>env</Key><Value>prod</Value></Tag>
</TagSet></Tagging>"#;
fn contract_object_headers(content_length: u64) -> Vec<(&'static str, String)> {
vec![
("etag", format!("\"{OBJECT_MD5}\"")),
("content-length", content_length.to_string()),
("content-type", "text/plain".to_string()),
("last-modified", "Wed, 21 Oct 2015 07:28:00 GMT".to_string()),
("x-amz-meta-owner", "alice".to_string()),
("x-amz-storage-class", "STANDARD".to_string()),
]
}
/// The S3 backend behind the scripted connector, without the prefix-mapping
/// client on top: the contract is a property of the backend itself.
async fn scripted_s3_backend(responses: Vec<Scripted>) -> S3SourceBackend {
let spec = spec(None);
let connector = SharedHttpConnector::new(ScriptedConnector {
requests: Arc::new(Mutex::new(Vec::new())),
responses: Arc::new(Mutex::new(responses.into_iter().collect())),
});
let http_client = http_client_fn(move |_settings, _components| connector.clone());
let endpoint = spec.endpoint_spec().expect("test spec endpoint should parse");
let config = build_remote_s3_config(&endpoint)
.await
.expect("test spec should build")
.http_client(http_client)
.interceptor(SourceProxyMarkerInterceptor::new());
S3SourceBackend {
client: S3Client::from_conf(config.build()),
bucket: spec.bucket.clone(),
}
}
#[tokio::test]
async fn s3_backend_satisfies_the_shared_backend_contract() {
let mut ranged = contract_object_headers(3);
ranged.push(("content-range", "bytes 1-3/5".to_string()));
let backend = scripted_s3_backend(vec![
ok(contract_object_headers(5), ""),
ok(contract_object_headers(5), "hello"),
ok(ranged, "ell"),
ok(Vec::new(), CONTRACT_LIST_PAGE_ONE),
ok(Vec::new(), CONTRACT_LIST_PAGE_TWO),
ok(Vec::new(), CONTRACT_TAGGING),
ok(Vec::new(), ""),
status(404, ""),
status(403, ACCESS_DENIED_BODY),
])
.await;
assert_backend_contract(
&backend,
BackendCapabilities {
etag_is_opaque: false,
supports_start_after: true,
supports_tagging: true,
},
)
.await;
} }
fn prefix_client(prefix: Option<String>) -> SourceClient { fn prefix_client(prefix: Option<String>) -> SourceClient {
@@ -177,7 +177,7 @@ impl From<&SourceError> for PullFailureReason {
SourceError::Connect(_) => PullFailureReason::SourceConnect, SourceError::Connect(_) => PullFailureReason::SourceConnect,
SourceError::ServerError(_) => PullFailureReason::SourceServerError, SourceError::ServerError(_) => PullFailureReason::SourceServerError,
SourceError::Unsupported(_) => PullFailureReason::SourceUnsupported, SourceError::Unsupported(_) => PullFailureReason::SourceUnsupported,
SourceError::Other(_) => PullFailureReason::SourceOther, SourceError::InvalidPagination(_) | SourceError::Other(_) => PullFailureReason::SourceOther,
} }
} }
} }
@@ -47,7 +47,10 @@ use super::config::{
use super::list_through::{SOURCE_LIST_RATE_PER_SEC, SourceListRateLimiter}; use super::list_through::{SOURCE_LIST_RATE_PER_SEC, SourceListRateLimiter};
use super::negative_cache::NegativeCache; use super::negative_cache::NegativeCache;
use super::pull::{OdmWriteBack, PullQueue}; use super::pull::{OdmWriteBack, PullQueue};
use super::source_client::{SourceClient, SourceClientSpec, SourceError, SourceProvider, SourceTimeouts}; use super::source_client::{
AzureAuth, AzureSourceSpec, GcsSourceSpec, SourceBackendSpec, SourceClient, SourceClientSpec, SourceError, SourceProvider,
SourceTimeouts,
};
use super::stats::{GaugeGuard, OdmStats, OdmStatsSnapshot, PullFailureReason}; use super::stats::{GaugeGuard, OdmStats, OdmStatsSnapshot, PullFailureReason};
use crate::bucket::remote_s3_client::{ use crate::bucket::remote_s3_client::{
PathStyle as ClientPathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3RetryPolicy, PathStyle as ClientPathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3RetryPolicy,
@@ -619,6 +622,7 @@ pub fn source_client_spec(config: &OnDemandMigrationConfig) -> SourceClientSpec
// load on a source that is already failing. // load on a source that is already failing.
retry: RemoteS3RetryPolicy::Disabled, retry: RemoteS3RetryPolicy::Disabled,
bandwidth_limit: policy.bandwidth_limit_bytes_per_sec.and_then(NonZeroU64::new), bandwidth_limit: policy.bandwidth_limit_bytes_per_sec.and_then(NonZeroU64::new),
backend: source_backend_spec(source),
} }
} }
@@ -630,6 +634,31 @@ fn source_provider(provider: Provider) -> SourceProvider {
Provider::Rustfs => SourceProvider::Rustfs, Provider::Rustfs => SourceProvider::Rustfs,
Provider::R2 => SourceProvider::R2, Provider::R2 => SourceProvider::R2,
Provider::Gcs => SourceProvider::Gcs, Provider::Gcs => SourceProvider::Gcs,
Provider::Azure => SourceProvider::Azure,
Provider::GcsNative => SourceProvider::GcsNative,
}
}
/// Which backend the client builds. A native provider whose block is missing
/// falls back to the S3 spec, where the builder reports the missing
/// credentials: the config layer already refuses to store that shape, so this
/// only covers a config written by an older or hand-edited build.
pub fn source_backend_spec(source: &SourceConfig) -> SourceBackendSpec {
match (source.provider, source.azure.as_ref(), source.gcs.as_ref()) {
(Provider::Azure, Some(azure), _) => SourceBackendSpec::Azure(AzureSourceSpec {
account: azure.account.clone(),
auth: match (&azure.account_key, &azure.sas_token) {
(Some(key), _) => AzureAuth::SharedKey(key.clone()),
(None, Some(sas)) => AzureAuth::Sas(sas.clone()),
// Refused by `SourceConfig::validate`; an empty shared key
// fails closed at the builder rather than signing with none.
(None, None) => AzureAuth::SharedKey(String::new()),
},
}),
(Provider::GcsNative, _, Some(gcs)) => SourceBackendSpec::Gcs(GcsSourceSpec {
service_account_json: gcs.service_account_json.clone(),
}),
_ => SourceBackendSpec::S3,
} }
} }
@@ -929,6 +958,8 @@ mod tests {
session_token: None, session_token: None,
}), }),
tls: TlsConfig::default(), tls: TlsConfig::default(),
azure: None,
gcs: None,
}, },
filter: FilterConfig { filter: FilterConfig {
prefix: prefix.map(str::to_string), prefix: prefix.map(str::to_string),
@@ -0,0 +1,120 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Scripted HTTP server for the native source backends' tests.
//!
//! The S3 backend can be driven through the SDK's own connector; the native
//! backends talk to a real socket, so their tests need a server that answers a
//! fixed script and records what it was asked. Every response closes its
//! connection, which keeps one request on one socket and makes the script order
//! exactly the request order.
use std::sync::{Arc, Mutex};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpListener;
use url::Url;
pub(super) struct ScriptedResponse {
status: u16,
headers: Vec<(&'static str, String)>,
body: String,
}
impl ScriptedResponse {
pub(super) fn new(status: u16, headers: Vec<(&'static str, String)>, body: String) -> Self {
Self { status, headers, body }
}
}
#[derive(Clone, Debug)]
pub(super) struct RecordedRequest {
pub(super) method: String,
/// Request target as it appeared on the wire: path plus query.
pub(super) target: String,
pub(super) headers: Vec<(String, String)>,
}
impl RecordedRequest {
pub(super) fn header(&self, name: &str) -> Option<&str> {
self.headers
.iter()
.find(|(key, _)| key.eq_ignore_ascii_case(name))
.map(|(_, value)| value.as_str())
}
}
pub(super) type Recorder = Arc<Mutex<Vec<RecordedRequest>>>;
/// Binds a loopback listener that answers `responses` in order and returns its
/// origin plus the recorder. The task ends once the script is exhausted.
pub(super) async fn scripted_server(responses: Vec<ScriptedResponse>) -> (Url, Recorder) {
let listener = TcpListener::bind("127.0.0.1:0").await.expect("fixture listener should bind");
let port = listener.local_addr().expect("fixture address").port();
let recorder: Recorder = Arc::new(Mutex::new(Vec::new()));
let sink = Arc::clone(&recorder);
tokio::spawn(async move {
for response in responses {
let Ok((mut stream, _)) = listener.accept().await else {
return;
};
let mut request = Vec::new();
let mut buffer = [0_u8; 2048];
while !request.windows(4).any(|window| window == b"\r\n\r\n") {
match stream.read(&mut buffer).await {
Ok(0) | Err(_) => break,
Ok(read) => request.extend_from_slice(&buffer[..read]),
}
}
let text = String::from_utf8_lossy(&request).into_owned();
let mut lines = text.lines();
let start = lines.next().unwrap_or_default().to_string();
let mut parts = start.split_whitespace();
sink.lock().expect("recorder lock").push(RecordedRequest {
method: parts.next().unwrap_or_default().to_string(),
target: parts.next().unwrap_or_default().to_string(),
headers: lines
.take_while(|line| !line.is_empty())
.filter_map(|line| line.split_once(':'))
.map(|(name, value)| (name.trim().to_string(), value.trim().to_string()))
.collect(),
});
// A scripted HEAD declares the object size in its own headers while
// carrying no body, so an explicit `Content-Length` wins over the
// body length.
let declares_length = response
.headers
.iter()
.any(|(name, _)| name.eq_ignore_ascii_case("content-length"));
let mut rendered = match declares_length {
true => format!("HTTP/1.1 {} Scripted\r\nConnection: close\r\n", response.status),
false => format!(
"HTTP/1.1 {} Scripted\r\nContent-Length: {}\r\nConnection: close\r\n",
response.status,
response.body.len()
),
};
for (name, value) in response.headers {
rendered.push_str(&format!("{name}: {value}\r\n"));
}
rendered.push_str("\r\n");
rendered.push_str(&response.body);
let _ = stream.write_all(rendered.as_bytes()).await;
let _ = stream.flush().await;
}
});
(Url::parse(&format!("http://127.0.0.1:{port}")).expect("fixture endpoint"), recorder)
}
+170 -1
View File
@@ -652,9 +652,10 @@ async fn build_aws_s3_http_client_from_tls_path() -> Option<SharedHttpClient> {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use aws_smithy_async::time::TimeSource;
use aws_smithy_runtime_api::http::StatusCode as SmithyStatusCode; use aws_smithy_runtime_api::http::StatusCode as SmithyStatusCode;
use std::sync::Mutex; use std::sync::Mutex;
use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
fn spec(endpoint: &str, secure: bool) -> RemoteS3EndpointSpec { fn spec(endpoint: &str, secure: bool) -> RemoteS3EndpointSpec {
RemoteS3EndpointSpec { RemoteS3EndpointSpec {
@@ -824,6 +825,174 @@ mod tests {
); );
} }
#[derive(Clone, Debug)]
struct ClockSkewTimeSource(Arc<AtomicU64>);
impl TimeSource for ClockSkewTimeSource {
fn now(&self) -> SystemTime {
SystemTime::UNIX_EPOCH + Duration::from_secs(self.0.load(Ordering::SeqCst))
}
}
#[derive(Clone, Debug)]
struct ClockSkewConnector {
request_headers: RecordedHeaders,
error_code: &'static str,
skew_seconds: i64,
clock: ClockSkewTimeSource,
}
fn recorded_header<'a>(headers: &'a [(String, String)], name: &str) -> &'a str {
headers
.iter()
.find(|(key, _)| key.eq_ignore_ascii_case(name))
.map(|(_, value)| value.as_str())
.unwrap_or_else(|| panic!("signed request must contain {name}"))
}
fn signing_time(headers: &[(String, String)]) -> chrono::NaiveDateTime {
chrono::NaiveDateTime::parse_from_str(recorded_header(headers, "x-amz-date"), "%Y%m%dT%H%M%SZ")
.expect("SDK signing timestamp must use the SigV4 format")
}
impl SmithyHttpConnector for ClockSkewConnector {
fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
let mut headers = self.request_headers.lock().expect("clock skew request capture lock");
assert!(headers.len() < 3, "clock skew fixture must not exceed two GET attempts and one HEAD");
headers.push(
request
.headers()
.iter()
.map(|(key, value)| (key.to_string(), value.to_string()))
.collect(),
);
let server_time = chrono::DateTime::<chrono::Utc>::from(self.clock.now()).naive_utc()
+ chrono::Duration::seconds(self.skew_seconds);
let (status, body) = if headers.len() == 1 {
(
403,
format!("<Error><Code>{}</Code><Message>Clock skew fixture</Message></Error>", self.error_code),
)
} else {
(200, String::new())
};
let response = http::Response::builder()
.status(status)
.header("date", server_time.format("%a, %d %b %Y %H:%M:%S GMT").to_string())
.header("content-type", "application/xml")
.header("content-length", body.len())
.body(SdkBody::from(body))
.expect("clock skew fixture response");
HttpConnectorFuture::ready(Ok(HttpResponse::try_from(response).expect("Smithy fixture response")))
}
}
async fn clock_skew_client(
error_code: &'static str,
skew_seconds: i64,
retry: RemoteS3RetryPolicy,
) -> (S3Client, RecordedHeaders, ClockSkewTimeSource) {
let headers: RecordedHeaders = Arc::new(Mutex::new(Vec::new()));
let clock = ClockSkewTimeSource(Arc::new(AtomicU64::new(1_700_000_000)));
let connector = SharedHttpConnector::new(ClockSkewConnector {
request_headers: Arc::clone(&headers),
error_code,
skew_seconds,
clock: clock.clone(),
});
let mut spec = spec("s3.example.com", true);
spec.retry = retry;
let config = build_remote_s3_config(&spec)
.await
.expect("clock skew fixture uses the production outbound configuration")
.http_client(http_client_fn(move |_settings, _components| connector.clone()))
.time_source(clock.clone())
.build();
(S3Client::from_conf(config), headers, clock)
}
#[tokio::test(start_paused = true)]
async fn remote_s3_clock_skew_retries_resign_and_seed_next_operation() {
for error_code in ["RequestTimeTooSkewed", "SignatureDoesNotMatch"] {
for skew_seconds in [-600, 600] {
let (client, headers, clock) = clock_skew_client(error_code, skew_seconds, REPLICATION_TARGET_RETRY_POLICY).await;
let initial = chrono::DateTime::<chrono::Utc>::from(clock.now()).naive_utc();
client
.get_object()
.bucket("bucket")
.key("object")
.send()
.await
.expect("clock skew GET must retry successfully");
assert_eq!(
headers.lock().expect("captured requests").len(),
2,
"{error_code}: GET needs exactly one retry"
);
clock.0.fetch_add(17, Ordering::SeqCst);
// SDK signing time is independent of Tokio's retry/scheduler clock.
tokio::time::advance(Duration::from_secs(61)).await;
client
.head_bucket()
.bucket("bucket")
.send()
.await
.expect("subsequent HEAD must use the client's cached skew");
let headers = headers.lock().expect("captured signed requests");
assert_eq!(headers.len(), 3, "subsequent operation must succeed on its first attempt");
assert_eq!(signing_time(&headers[0]), initial, "the first attempt must use the injected clock");
assert_eq!(
signing_time(&headers[1]),
initial + chrono::Duration::seconds(skew_seconds),
"{error_code}: retry must apply the measured offset exactly"
);
assert_eq!(
signing_time(&headers[2]),
initial + chrono::Duration::seconds(skew_seconds + 17),
"{error_code}: the next operation must apply cached skew to the advanced signing clock"
);
let signature = |index: usize| {
recorded_header(&headers[index], "authorization")
.rsplit_once("Signature=")
.expect("SigV4 authorization contains a signature")
.1
};
assert_ne!(
signature(0),
signature(1),
"{error_code}: retry must be signed again after adjusting its date"
);
}
}
}
#[tokio::test(start_paused = true)]
async fn remote_s3_clock_skew_respects_one_attempt_policy() {
use aws_smithy_types::error::metadata::ProvideErrorMetadata;
for error_code in ["RequestTimeTooSkewed", "SignatureDoesNotMatch"] {
for retry in [
RemoteS3RetryPolicy::Disabled,
RemoteS3RetryPolicy::Standard { max_attempts: 1 },
] {
let (client, headers, _clock) = clock_skew_client(error_code, 600, retry).await;
let error = client
.get_object()
.bucket("bucket")
.key("object")
.send()
.await
.expect_err("clock skew must not override the caller's one-attempt budget");
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some(error_code));
assert_eq!(
headers.lock().expect("captured requests").len(),
1,
"{error_code}: {retry:?} must send exactly one request"
);
}
}
}
#[test] #[test]
fn path_style_auto_and_path_force_path_style() { fn path_style_auto_and_path_force_path_style() {
assert!(PathStyle::Auto.force_path_style()); assert!(PathStyle::Auto.force_path_style());
@@ -46,7 +46,7 @@ use super::replication_storage_boundary::{
HTTPPreconditions, ObjectInfo, ObjectOptions, ObjectToDelete, ReplicationDeletedObject, ReplicationObjectIO, HTTPPreconditions, ObjectInfo, ObjectOptions, ObjectToDelete, ReplicationDeletedObject, ReplicationObjectIO,
ReplicationStorage, ReplicationStorage,
}; };
use super::replication_target_boundary::{ReplicationTargetStore, replication_object_is_ssec_encrypted}; use super::replication_target_boundary::{BucketTargetError, ReplicationTargetStore, replication_object_is_ssec_encrypted};
use super::replication_versioning_boundary::ReplicationVersioningStore; use super::replication_versioning_boundary::ReplicationVersioningStore;
use super::runtime_boundary as runtime_sources; use super::runtime_boundary as runtime_sources;
use futures_util::stream::{self, StreamExt}; use futures_util::stream::{self, StreamExt};
@@ -3084,6 +3084,23 @@ pub async fn queue_replication_heal(bucket: &str, oi: ObjectInfo, retry_count: u
let tgts = match ReplicationTargetStore::list_bucket_targets(bucket).await { let tgts = match ReplicationTargetStore::list_bucket_targets(bucket).await {
Ok(targets) => Some(targets), Ok(targets) => Some(targets),
// A bucket whose persisted target configuration cannot be decoded has
// an unknown target set, not an empty one: scheduling against `None`
// here would drop every heal for it without a trace
// (rustfs/backlog#2282). Report it missed so the object is retried
// once the configuration is readable again.
Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. }) => {
warn!(
event = EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
bucket,
reason = "target_config_unreadable",
"Bucket replication targets are unreadable; replication heal queue fails closed"
);
return ReplicationQueueAdmission::Missed;
}
Err(err) => { Err(err) => {
debug!( debug!(
event = EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED, event = EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED,
@@ -15,7 +15,8 @@
use std::collections::HashMap; use std::collections::HashMap;
use std::sync::Arc; use std::sync::Arc;
use crate::bucket::bucket_target_sys::{BucketTargetError, BucketTargetSys}; pub(crate) use crate::bucket::bucket_target_sys::BucketTargetError;
use crate::bucket::bucket_target_sys::BucketTargetSys;
use aws_sdk_s3::operation::head_object::HeadObjectOutput; use aws_sdk_s3::operation::head_object::HeadObjectOutput;
use aws_sdk_s3::types::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode}; use aws_sdk_s3::types::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode};
use http::HeaderMap; use http::HeaderMap;
+19
View File
@@ -30,6 +30,7 @@ use rustfs_protos::{
ChannelClass, create_new_channel, get_channel_for_class, ChannelClass, create_new_channel, get_channel_for_class,
proto_gen::node_service::{ proto_gen::node_service::{
heal_control_service_client::HealControlServiceClient, node_service_client::NodeServiceClient, heal_control_service_client::HealControlServiceClient, node_service_client::NodeServiceClient,
scanner_control_service_client::ScannerControlServiceClient,
tier_mutation_control_service_client::TierMutationControlServiceClient, tier_mutation_control_service_client::TierMutationControlServiceClient,
}, },
}; };
@@ -60,6 +61,24 @@ pub async fn node_service_time_out_client(
node_service_time_out_client_for_class(addr, interceptor, ChannelClass::Control).await node_service_time_out_client_for_class(addr, interceptor, ChannelClass::Control).await
} }
pub(crate) async fn scanner_control_time_out_client(
addr: &str,
interceptor: TonicInterceptor,
) -> crate::error::Result<ScannerControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
let interceptor = interceptor.with_rpc_audience(addr)?;
let channel = match runtime_sources::cached_node_channel(addr).await {
Some(channel) => channel,
None => create_new_channel(addr)
.await
.map_err(|err| crate::error::Error::other(err.to_string()))?,
};
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
let limit = rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize;
Ok(ScannerControlServiceClient::with_interceptor(channel, interceptor)
.max_decoding_message_size(limit)
.max_encoding_message_size(limit))
}
pub async fn heal_control_time_out_client( pub async fn heal_control_time_out_client(
addr: &str, addr: &str,
interceptor: TonicInterceptor, interceptor: TonicInterceptor,
@@ -2050,6 +2050,53 @@ impl PeerRestClient {
.await .await
} }
/// Probe only: scoped ACK production requires a durable per-bucket proof.
pub async fn scanner_scoped_dirty_usage_capability(
&self,
owner_id: String,
instance_id: String,
entries: Vec<rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry>,
) -> Result<bool> {
use rustfs_protos::scoped_dirty_usage::*;
let payload = rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageAckRequest {
challenge: Uuid::new_v4().as_bytes().to_vec().into(),
protocol_version: SCOPED_DIRTY_USAGE_PROTOCOL_VERSION,
owner_id,
instance_id,
scope: SCOPED_DIRTY_USAGE_BUCKET_SCOPE,
probe_only: true,
entries,
};
let canonical = canonical_scoped_dirty_usage_request(&payload).map_err(|err| Error::other(err.to_string()))?;
self.finalize_result(
async {
let mut client = super::client::scanner_control_time_out_client(
&self.grid_host,
TonicInterceptor::Signature(gen_tonic_signature_interceptor()),
)
.await?;
let mut request = Request::new(payload.clone());
set_tonic_canonical_body_digest(&mut request, &canonical)?;
let response = client.scanner_scoped_dirty_usage_ack(request).await?.into_inner();
let body = canonical_scoped_dirty_usage_response(&canonical, &response)
.map_err(|_| Error::other("scoped dirty usage capability response is too large"))?;
verify_tonic_rpc_response_proof(&body, response.response_proof.as_ref())?;
if response.protocol_version != SCOPED_DIRTY_USAGE_PROTOCOL_VERSION
|| response.owner_id != payload.owner_id
|| response.instance_id != payload.instance_id
|| response.max_entries != SCOPED_DIRTY_USAGE_MAX_ENTRIES
|| response.max_request_bytes != SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES
|| response.cleared != 0
{
return Err(Error::other("scoped dirty usage capability response does not match request"));
}
Ok(response.supported)
}
.await,
)
.await
}
pub async fn acknowledge_scanner_dirty_usage(&self, instance_id: String, generation: u64) -> Result<ScannerPeerActivity> { pub async fn acknowledge_scanner_dirty_usage(&self, instance_id: String, generation: u64) -> Result<ScannerPeerActivity> {
let result = self let result = self
.scanner_activity_request_with_protocol(instance_id.clone(), generation, SCANNER_ACTIVITY_PROTOCOL_VERSION) .scanner_activity_request_with_protocol(instance_id.clone(), generation, SCANNER_ACTIVITY_PROTOCOL_VERSION)
+4
View File
@@ -5493,6 +5493,7 @@ where
fence.ensure_held()?; fence.ensure_held()?;
let mut opts = ObjectOptions { let mut opts = ObjectOptions {
max_parity: true, max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
no_lock: true, no_lock: true,
http_preconditions: Some(pool_meta_cas_preconditions(token, object)?), http_preconditions: Some(pool_meta_cas_preconditions(token, object)?),
..Default::default() ..Default::default()
@@ -14412,6 +14413,7 @@ impl ECStore {
encoded.clone(), encoded.clone(),
&ObjectOptions { &ObjectOptions {
max_parity: true, max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions { http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("*".to_string()), if_none_match: Some("*".to_string()),
..Default::default() ..Default::default()
@@ -14566,6 +14568,7 @@ impl ECStore {
encoded, encoded,
&ObjectOptions { &ObjectOptions {
max_parity: true, max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(http_preconditions), http_preconditions: Some(http_preconditions),
..Default::default() ..Default::default()
}, },
@@ -14957,6 +14960,7 @@ impl ECStore {
encoded, encoded,
&ObjectOptions { &ObjectOptions {
max_parity: true, max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions { http_preconditions: Some(HTTPPreconditions {
if_match: Some(etag), if_match: Some(etag),
..Default::default() ..Default::default()
+77 -8
View File
@@ -317,6 +317,22 @@ impl DiskStoreRenameDataExt for LocalDiskWrapper {
dst_path: &str, dst_path: &str,
external_guard: Option<Arc<dyn Send + Sync>>, external_guard: Option<Arc<dyn Send + Sync>>,
) -> Result<RenameDataResp> { ) -> Result<RenameDataResp> {
self.rename_data_observed(src_volume, src_path, fi, dst_volume, dst_path, external_guard)
.await
.result
}
}
impl LocalDiskWrapper {
pub(in crate::disk) async fn rename_data_observed(
&self,
src_volume: &str,
src_path: &str,
fi: &FileInfo,
dst_volume: &str,
dst_path: &str,
external_guard: Option<Arc<dyn Send + Sync>>,
) -> super::RenameDataObservation {
let operation = self.clone(); let operation = self.clone();
let src_volume = src_volume.to_owned(); let src_volume = src_volume.to_owned();
let src_path = src_path.to_owned(); let src_path = src_path.to_owned();
@@ -333,22 +349,35 @@ impl DiskStoreRenameDataExt for LocalDiskWrapper {
} else { } else {
get_max_timeout_duration() get_max_timeout_duration()
}; };
run_owned_mutation(external_guard, move || async move { let observed = run_owned_mutation(external_guard, move || async move {
operation let mut preflight_rejection = None;
let result = operation
.track_disk_health_mutation( .track_disk_health_mutation(
"rename_data", "rename_data",
DiskMetricMutation::Write, DiskMetricMutation::Write,
|| async { || async {
operation // Preserve the former DiskAPI future's single boxing boundary.
.disk let observed =
.rename_data_borrowed(&src_volume, &src_path, &fi, &dst_volume, &dst_path) Box::pin(
.await operation
.disk
.rename_data_observed(&src_volume, &src_path, &fi, &dst_volume, &dst_path),
)
.await;
preflight_rejection = observed.preflight_rejection;
observed.result
}, },
timeout_duration, timeout_duration,
) )
.await .await;
// Health tracking must observe the real disk error, not an Ok tuple.
Ok(super::RenameDataObservation {
result,
preflight_rejection,
})
}) })
.await .await;
observed.unwrap_or_else(|error| super::RenameDataObservation::unknown(Err(error)))
} }
} }
@@ -2588,6 +2617,46 @@ mod tests {
assert_eq!(wrapper.metrics_snapshot().api_calls.get("unknown"), Some(&1)); assert_eq!(wrapper.metrics_snapshot().api_calls.get("unknown"), Some(&1));
} }
#[tokio::test]
async fn rename_preflight_evidence_preserves_health_errors_and_owned_reply() {
for source_exists in [false, true] {
for guarded in [false, true] {
let dir = tempfile::tempdir().expect("temp dir should be created");
let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be valid UTF-8"))
.expect("endpoint should parse");
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created"));
if source_exists {
disk.make_volume("source").await.expect("source volume should exist");
}
let wrapper = LocalDiskWrapper::new(disk, false);
let drops = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let external_guard = guarded.then(|| Arc::new(DropProbe(Arc::clone(&drops))) as Arc<dyn Send + Sync>);
let mut file_info = FileInfo::new("object", 1, 0);
file_info.mod_time = Some(::time::OffsetDateTime::now_utc());
file_info.erasure.index = 1;
let observed = wrapper
.rename_data_observed("source", "object", &file_info, "missing-destination", "object", external_guard)
.await;
assert!(observed.rejected_before_publication(), "normal access rejection must carry proof");
assert!(matches!(observed.result, Err(DiskError::VolumeNotFound)));
let snapshot = wrapper.metrics_snapshot();
assert_eq!(snapshot.api_calls.get("rename_data"), Some(&1));
assert_eq!(snapshot.total_writes, 0, "health tracking must not observe the rejection as Ok");
assert_eq!(drops.load(Ordering::SeqCst), usize::from(guarded));
wrapper.health.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
let observed = wrapper
.rename_data_observed("source", "object", &file_info, "missing-destination", "object", None)
.await;
assert!(!observed.rejected_before_publication(), "wrapper errors carry no local preflight proof");
assert!(matches!(observed.result, Err(DiskError::FaultyDisk)));
let snapshot = wrapper.metrics_snapshot();
assert_eq!(snapshot.total_errors_availability, 1);
assert_eq!(snapshot.total_writes, 0);
}
}
}
#[tokio::test] #[tokio::test]
async fn local_disk_health_wrapper_counts_returned_availability_errors() { async fn local_disk_health_wrapper_counts_returned_availability_errors() {
let dir = tempfile::tempdir().expect("temp dir should be created"); let dir = tempfile::tempdir().expect("temp dir should be created");
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+49
View File
@@ -75,6 +75,25 @@ use time::OffsetDateTime;
use tokio::io::{AsyncRead, AsyncWrite}; use tokio::io::{AsyncRead, AsyncWrite};
use uuid::Uuid; use uuid::Uuid;
/// Local preflight evidence stays outside DiskAPI and the RPC response format.
pub(crate) struct RenameDataObservation {
pub(crate) result: Result<RenameDataResp>,
preflight_rejection: Option<local::LocalRenamePreflightRejection>,
}
impl RenameDataObservation {
fn unknown(result: Result<RenameDataResp>) -> Self {
Self {
result,
preflight_rejection: None,
}
}
pub(crate) fn rejected_before_publication(&self) -> bool {
self.result.is_err() && self.preflight_rejection.is_some()
}
}
const QUOTA_MUTATION_FENCE_PREFIX: &str = "tmp/quota-mutation-fences/"; const QUOTA_MUTATION_FENCE_PREFIX: &str = "tmp/quota-mutation-fences/";
pub(crate) const QUOTA_MUTATION_FENCE_METADATA_SUFFIX: &str = "quota-mutation-fence-token"; pub(crate) const QUOTA_MUTATION_FENCE_METADATA_SUFFIX: &str = "quota-mutation-fence-token";
@@ -711,6 +730,36 @@ impl Disk {
.await .await
} }
pub(crate) async fn rename_data_borrowed_with_fence_observed(
&self,
src_volume: &str,
src_path: &str,
fi: &FileInfo,
dst_volume: &str,
dst_path: &str,
scanner_publication_lease_token: Option<Uuid>,
) -> RenameDataObservation {
match self {
Disk::Local(local_disk) => {
local_disk
.rename_data_observed(src_volume, src_path, fi, dst_volume, dst_path, None)
.await
}
Disk::Remote(remote_disk) => RenameDataObservation::unknown(
remote_disk
.rename_data_borrowed_with_fence(
src_volume,
src_path,
fi,
dst_volume,
dst_path,
scanner_publication_lease_token,
)
.await,
),
}
}
pub(crate) async fn rename_data_borrowed_with_fence( pub(crate) async fn rename_data_borrowed_with_fence(
&self, &self,
src_volume: &str, src_volume: &str,
+19
View File
@@ -870,6 +870,18 @@ impl TierFreeVersionReceiptSink {
} }
} }
/// Internal PUT completion boundary; this does not change fsync or write quorum.
#[doc(hidden)]
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
pub enum WriteCompletion {
/// Return at write quorum when the commit owner can retain its guards.
#[default]
Quorum,
/// Drain the rename fan-out before returning. Minority failures still heal
/// after a successful quorum commit; this does not require every disk to succeed.
TailDrained,
}
#[derive(Default, Clone)] #[derive(Default, Clone)]
pub struct ObjectOptions { pub struct ObjectOptions {
// Use the maximum parity (N/2), used when saving server configuration files // Use the maximum parity (N/2), used when saving server configuration files
@@ -896,6 +908,10 @@ pub struct ObjectOptions {
/// Persisted bucket incarnation observed before authorization. /// Persisted bucket incarnation observed before authorization.
pub expected_bucket_incarnation_id: Option<Uuid>, pub expected_bucket_incarnation_id: Option<Uuid>,
pub no_lock: bool, pub no_lock: bool,
/// Control-plane writers that immediately read or CAS the same namespace
/// key use TailDrained without changing namespace lock ownership.
#[doc(hidden)]
pub write_completion: WriteCompletion,
/// True when an upper layer already holds the object read lock before /// True when an upper layer already holds the object read lock before
/// forwarding a no_lock read to the set layer. /// forwarding a no_lock read to the set layer.
pub metadata_cache_safe: bool, pub metadata_cache_safe: bool,
@@ -940,6 +956,9 @@ pub struct ObjectOptions {
pub preserve_etag: Option<String>, pub preserve_etag: Option<String>,
pub metadata_chg: bool, pub metadata_chg: bool,
pub http_preconditions: Option<HTTPPreconditions>, pub http_preconditions: Option<HTTPPreconditions>,
/// Internal create-only writes may also preserve an acknowledged deletion.
/// Evaluated with `http_preconditions` under the namespace commit lock.
pub preserve_delete_marker: bool,
pub delete_replication: Option<ReplicationState>, pub delete_replication: Option<ReplicationState>,
pub delete_replication_config_snapshot: Option<Arc<DeleteReplicationConfigSnapshot>>, pub delete_replication_config_snapshot: Option<Arc<DeleteReplicationConfigSnapshot>>,
+112
View File
@@ -78,6 +78,21 @@ pub(crate) struct ScannerPublicationLeaseEntry {
pub(crate) _operation_guard: OwnedRwLockReadGuard<()>, pub(crate) _operation_guard: OwnedRwLockReadGuard<()>,
} }
pub(crate) struct NamespaceCommitGuard {
ctx: Arc<InstanceContext>,
counted: bool,
}
impl Drop for NamespaceCommitGuard {
fn drop(&mut self) {
if self.counted {
// Publish the new generation before a zero-pending publication probe.
self.ctx.advance_namespace_commit_generation();
self.ctx.namespace_commits.fetch_sub(1, Ordering::AcqRel);
}
}
}
/// Runtime state owned by a single `ECStore` instance. /// Runtime state owned by a single `ECStore` instance.
/// ///
/// This is intentionally minimal in the first migration slice; subsequent /// This is intentionally minimal in the first migration slice; subsequent
@@ -209,9 +224,13 @@ pub struct InstanceContext {
/// Last storage-owned movement snapshot observed under the operation /// Last storage-owned movement snapshot observed under the operation
/// gate. SetDisks cache writers fail closed until ECStore refreshes it. /// gate. SetDisks cache writers fail closed until ECStore refreshes it.
scanner_publication_state: AtomicU8, scanner_publication_state: AtomicU8,
namespace_commits: AtomicU64,
namespace_commit_generation: AtomicU64,
/// Resolves object-encryption material at the application boundary. /// Resolves object-encryption material at the application boundary.
object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>, object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>, tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
#[cfg(test)]
suppress_tier_delete_journal_recovery: bool,
transition_transaction_recovery_stores: std::sync::Mutex<HashSet<Uuid>>, transition_transaction_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
tier_delete_journal_recovery_wakeup: tokio::sync::Notify, tier_delete_journal_recovery_wakeup: tokio::sync::Notify,
} }
@@ -256,8 +275,12 @@ impl InstanceContext {
data_movement_generation_exhausted: AtomicBool::new(false), data_movement_generation_exhausted: AtomicBool::new(false),
data_movement_generation_notify: Arc::new(Notify::new()), data_movement_generation_notify: Arc::new(Notify::new()),
scanner_publication_state: AtomicU8::new(SCANNER_PUBLICATION_STATE_UNKNOWN), scanner_publication_state: AtomicU8::new(SCANNER_PUBLICATION_STATE_UNKNOWN),
namespace_commits: AtomicU64::new(0),
namespace_commit_generation: AtomicU64::new(0),
object_encryption_resolver: OnceLock::new(), object_encryption_resolver: OnceLock::new(),
tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()), tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
#[cfg(test)]
suppress_tier_delete_journal_recovery: false,
transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()), transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()),
tier_delete_journal_recovery_wakeup: tokio::sync::Notify::new(), tier_delete_journal_recovery_wakeup: tokio::sync::Notify::new(),
} }
@@ -385,6 +408,36 @@ impl InstanceContext {
&& self.scanner_publication_state.load(Ordering::Acquire) == SCANNER_PUBLICATION_STATE_ALLOWED && self.scanner_publication_state.load(Ordering::Acquire) == SCANNER_PUBLICATION_STATE_ALLOWED
} }
pub(crate) fn begin_namespace_commit(self: &Arc<Self>) -> Arc<NamespaceCommitGuard> {
let counted = self
.namespace_commits
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| count.checked_add(1))
.is_ok();
if counted {
self.advance_namespace_commit_generation();
} else {
self.namespace_commit_generation.store(u64::MAX, Ordering::Release);
}
Arc::new(NamespaceCommitGuard {
ctx: Arc::clone(self),
counted,
})
}
fn advance_namespace_commit_generation(&self) {
let _ = self
.namespace_commit_generation
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |generation| Some(generation.saturating_add(1)));
}
pub(crate) fn namespace_commit_generation(&self) -> u64 {
self.namespace_commit_generation.load(Ordering::Acquire)
}
pub(crate) fn namespace_commits_pending(&self) -> bool {
self.namespace_commits.load(Ordering::Acquire) != 0 || self.namespace_commit_generation() == u64::MAX
}
pub(crate) fn set_scanner_publication_state(&self, blocked: bool) { pub(crate) fn set_scanner_publication_state(&self, blocked: bool) {
self.scanner_publication_state.store( self.scanner_publication_state.store(
if blocked { if blocked {
@@ -640,12 +693,21 @@ impl InstanceContext {
} }
pub(crate) fn mark_tier_delete_journal_recovery_started(&self, store_id: Uuid) -> bool { pub(crate) fn mark_tier_delete_journal_recovery_started(&self, store_id: Uuid) -> bool {
#[cfg(test)]
if self.suppress_tier_delete_journal_recovery {
return false;
}
self.tier_delete_journal_recovery_stores self.tier_delete_journal_recovery_stores
.lock() .lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) .unwrap_or_else(std::sync::PoisonError::into_inner)
.insert(store_id) .insert(store_id)
} }
#[cfg(test)]
pub(crate) fn suppress_tier_delete_journal_recovery_for_test(&mut self) {
self.suppress_tier_delete_journal_recovery = true;
}
pub(crate) fn mark_transition_transaction_recovery_started(&self, store_id: Uuid) -> bool { pub(crate) fn mark_transition_transaction_recovery_started(&self, store_id: Uuid) -> bool {
self.transition_transaction_recovery_stores self.transition_transaction_recovery_stores
.lock() .lock()
@@ -756,6 +818,50 @@ pub fn bootstrap_ctx() -> Arc<InstanceContext> {
mod tests { mod tests {
use super::*; use super::*;
#[test]
fn namespace_commit_guards_are_instance_local_and_count_until_last_owner() {
let first = Arc::new(InstanceContext::new());
let other = Arc::new(InstanceContext::new());
first.set_scanner_publication_state(false);
other.set_scanner_publication_state(false);
assert!(first.scanner_publication_state_allowed());
let one = first.begin_namespace_commit();
let shared_owner = Arc::clone(&one);
let two = first.begin_namespace_commit();
assert!(first.namespace_commits_pending());
assert!(first.scanner_publication_state_allowed(), "pending writes must not block scan admission");
assert_eq!(first.namespace_commit_generation(), 2);
assert!(!other.namespace_commits_pending());
assert_eq!(other.namespace_commit_generation(), 0);
assert!(other.scanner_publication_state_allowed());
drop(one);
assert_eq!(first.namespace_commit_generation(), 2);
drop(shared_owner);
assert!(first.namespace_commits_pending());
assert_eq!(first.namespace_commit_generation(), 3);
drop(two);
assert!(!first.namespace_commits_pending());
assert_eq!(first.namespace_commit_generation(), 4);
assert!(first.scanner_publication_state_allowed());
}
#[test]
fn namespace_commit_counter_exhaustion_keeps_publication_blocked() {
for (count, generation) in [(0, u64::MAX - 1), (u64::MAX, 0)] {
let ctx = Arc::new(InstanceContext::new());
ctx.set_scanner_publication_state(false);
ctx.namespace_commits.store(count, Ordering::Release);
ctx.namespace_commit_generation.store(generation, Ordering::Release);
let guard = ctx.begin_namespace_commit();
assert!(ctx.namespace_commits_pending());
assert_eq!(ctx.namespace_commit_generation(), u64::MAX);
drop(guard);
assert!(ctx.namespace_commits_pending());
assert_eq!(ctx.namespace_commit_generation(), u64::MAX);
assert_eq!(ctx.namespace_commits.load(Ordering::Acquire), count);
}
}
// The SetupType inputs must derive the exact (is_erasure, // The SetupType inputs must derive the exact (is_erasure,
// is_dist_erasure, is_erasure_sd) triples that the original three // is_dist_erasure, is_erasure_sd) triples that the original three
// process-global erasure bools produced via update_erasure_type(). // process-global erasure bools produced via update_erasure_type().
@@ -1073,6 +1179,12 @@ mod tests {
assert!(!ctx_a.mark_tier_delete_journal_recovery_started(store_a)); assert!(!ctx_a.mark_tier_delete_journal_recovery_started(store_a));
assert!(ctx_a.mark_tier_delete_journal_recovery_started(store_b)); assert!(ctx_a.mark_tier_delete_journal_recovery_started(store_b));
assert!(ctx_b.mark_tier_delete_journal_recovery_started(store_a)); assert!(ctx_b.mark_tier_delete_journal_recovery_started(store_a));
let mut manual_ctx = InstanceContext::new();
manual_ctx.suppress_tier_delete_journal_recovery_for_test();
assert!(!manual_ctx.mark_tier_delete_journal_recovery_started(store_a));
assert!(!manual_ctx.mark_tier_delete_journal_recovery_started(store_b));
assert!(ctx_b.mark_tier_delete_journal_recovery_started(store_b));
} }
#[test] #[test]
@@ -701,7 +701,7 @@ impl WarmBackend for MockWarmBackend {
Ok(version) Ok(version)
} }
async fn get(&self, object: &str, _rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> { async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
self.precondition().await?; self.precondition().await?;
let barrier = self.inner.get_barrier.lock().await.take(); let barrier = self.inner.get_barrier.lock().await.take();
if let Some(barrier) = barrier { if let Some(barrier) = barrier {
@@ -719,6 +719,9 @@ impl WarmBackend for MockWarmBackend {
let Some(stored) = objects.get(object) else { let Some(stored) = objects.get(object) else {
return Err(std::io::Error::new(std::io::ErrorKind::NotFound, "mock object not found")); return Err(std::io::Error::new(std::io::ErrorKind::NotFound, "mock object not found"));
}; };
if !rv.is_empty() && stored.remote_version_id != rv {
return Err(std::io::Error::new(std::io::ErrorKind::NotFound, "NoSuchVersion"));
}
let bytes = &stored.bytes; let bytes = &stored.bytes;
let start = opts.start_offset.max(0) as usize; let start = opts.start_offset.max(0) as usize;
+181 -19
View File
@@ -2346,6 +2346,10 @@ impl WarmBackend for SharedWarmBackendProxy {
self.0.probe_transition_candidate(object).await self.0.probe_transition_candidate(object).await
} }
async fn probe_transition_version(&self, object: &str, remote_version_id: &str) -> io::Result<TransitionCandidateProbe> {
self.0.probe_transition_version(object, remote_version_id).await
}
async fn in_use(&self) -> io::Result<bool> { async fn in_use(&self) -> io::Result<bool> {
self.0.in_use().await self.0.in_use().await
} }
@@ -2458,6 +2462,15 @@ impl TierOperationLease {
Ok(()) Ok(())
} }
pub(crate) async fn probe_transition_version(
&self,
object: &str,
remote_version_id: &str,
) -> io::Result<TransitionCandidateProbe> {
self.validate_remote_version_id(remote_version_id)?;
self.inner.driver.probe_transition_version(object, remote_version_id).await
}
pub(crate) fn is_current_generation(&self) -> bool { pub(crate) fn is_current_generation(&self) -> bool {
lock_unpoisoned(&self.runtime) lock_unpoisoned(&self.runtime)
.generations .generations
@@ -3528,7 +3541,7 @@ impl TierConfigMgr {
// Get tier configuration and create new driver // Get tier configuration and create new driver
let tier_config = self.tiers.get(tier_name).ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?; let tier_config = self.tiers.get(tier_name).ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?;
let driver = new_warm_backend(tier_config, false).await?; let driver = construct_warm_backend(tier_config).await?;
self.replace_driver(tier_name, driver)?; self.replace_driver(tier_name, driver)?;
Ok(self Ok(self
@@ -4473,6 +4486,11 @@ impl TierConfigMgr {
let committed_coordinator_intent = let committed_coordinator_intent =
committed_tier_mutation_intent(coordinator_intent.as_ref(), &committed_config_etag) committed_tier_mutation_intent(coordinator_intent.as_ref(), &committed_config_etag)
.map_err(TierConfigUpdateError::Save)?; .map_err(TierConfigUpdateError::Save)?;
// Persist Committed before notifying refresh; a Prepared disk record
// would restore the prepared block and invalidate our publish allowance.
let coordinator_commit =
commit_coordinator_tier_mutation_intent(api.clone(), coordinator_intent.as_ref(), &committed_config_etag)
.await;
if let Some(intent) = committed_coordinator_intent.as_ref() { if let Some(intent) = committed_coordinator_intent.as_ref() {
TierConfigMgr::apply_committed_mutation_intent_block(&handle, intent) TierConfigMgr::apply_committed_mutation_intent_block(&handle, intent)
.await .await
@@ -4483,9 +4501,9 @@ impl TierConfigMgr {
.map_err(TierConfigUpdateError::Publish)?, .map_err(TierConfigUpdateError::Publish)?,
); );
} }
commit_coordinator_tier_mutation_intent(api.clone(), coordinator_intent.as_ref(), &committed_config_etag) // Config is already saved: retain the committed fence and wake recovery
.await // even when the coordinator commit failed or its outcome is unknown.
.map_err(TierConfigUpdateError::Save)?; coordinator_commit.map_err(TierConfigUpdateError::Save)?;
if coordinated_config_update { if coordinated_config_update {
drop(update.take()); drop(update.take());
drop(config_lock.take()); drop(config_lock.take());
@@ -10590,6 +10608,11 @@ mod tests {
.expect_err("coordinator committed-state CAS failure must be observable"); .expect_err("coordinator committed-state CAS failure must be observable");
assert!(matches!(err, TierConfigUpdateError::Save(_))); assert!(matches!(err, TierConfigUpdateError::Save(_)));
assert!(manager.read().await.tiers.contains_key("COLD-A")); assert!(manager.read().await.tiers.contains_key("COLD-A"));
assert!(TierConfigMgr::has_committed_mutation_block(&manager).await);
let refresh = TierConfigMgr::mutation_refresh_notifier(&manager).await;
tokio::time::timeout(Duration::from_secs(1), refresh.notified())
.await
.expect("failed coordinator commit must notify recovery after saving config");
let blocked = match TierConfigMgr::acquire_operation_lease(&manager, "COLD-A").await { let blocked = match TierConfigMgr::acquire_operation_lease(&manager, "COLD-A").await {
Ok(_) => panic!("failed coordinator commit CAS must retain the local committed fence"), Ok(_) => panic!("failed coordinator commit CAS must retain the local committed fence"),
Err(err) => err, Err(err) => err,
@@ -14316,6 +14339,12 @@ mod tests {
after_commit: bool, after_commit: bool,
} }
#[derive(Debug, Default)]
struct CasCoordinatorCommitBarrier {
arrived: tokio::sync::Notify,
release: tokio::sync::Notify,
}
#[derive(Debug)] #[derive(Debug)]
struct CasConfigStore { struct CasConfigStore {
objects: tokio::sync::Mutex<HashMap<String, (Vec<u8>, String)>>, objects: tokio::sync::Mutex<HashMap<String, (Vec<u8>, String)>>,
@@ -14328,6 +14357,7 @@ mod tests {
fail_delete_prefix: tokio::sync::Mutex<Option<(String, usize)>>, fail_delete_prefix: tokio::sync::Mutex<Option<(String, usize)>>,
delete_log: tokio::sync::Mutex<Vec<String>>, delete_log: tokio::sync::Mutex<Vec<String>>,
list_barrier: tokio::sync::Mutex<Option<Arc<CasListBarrier>>>, list_barrier: tokio::sync::Mutex<Option<Arc<CasListBarrier>>>,
coordinator_commit_barrier: tokio::sync::Mutex<Option<Arc<CasCoordinatorCommitBarrier>>>,
intent_list_calls: AtomicUsize, intent_list_calls: AtomicUsize,
fail_reference_walk: AtomicBool, fail_reference_walk: AtomicBool,
reference_walk_send_count: AtomicUsize, reference_walk_send_count: AtomicUsize,
@@ -14350,6 +14380,7 @@ mod tests {
fail_delete_prefix: tokio::sync::Mutex::new(None), fail_delete_prefix: tokio::sync::Mutex::new(None),
delete_log: tokio::sync::Mutex::new(Vec::new()), delete_log: tokio::sync::Mutex::new(Vec::new()),
list_barrier: tokio::sync::Mutex::new(None), list_barrier: tokio::sync::Mutex::new(None),
coordinator_commit_barrier: tokio::sync::Mutex::new(None),
intent_list_calls: AtomicUsize::new(0), intent_list_calls: AtomicUsize::new(0),
fail_reference_walk: AtomicBool::new(false), fail_reference_walk: AtomicBool::new(false),
reference_walk_send_count: AtomicUsize::new(0), reference_walk_send_count: AtomicUsize::new(0),
@@ -14541,6 +14572,19 @@ mod tests {
} }
let mut payload = Vec::new(); let mut payload = Vec::new();
tokio::io::AsyncReadExt::read_to_end(&mut data.stream, &mut payload).await?; tokio::io::AsyncReadExt::read_to_end(&mut data.stream, &mut payload).await?;
if object.starts_with(crate::services::tier::tier_mutation_intent::TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX)
&& opts
.http_preconditions
.as_ref()
.and_then(HTTPPreconditions::if_match_value)
.is_some()
{
let barrier = self.coordinator_commit_barrier.lock().await.take();
if let Some(barrier) = barrier {
barrier.arrived.notify_one();
barrier.release.notified().await;
}
}
let race_rewrite = if opts let race_rewrite = if opts
.http_preconditions .http_preconditions
.as_ref() .as_ref()
@@ -15638,14 +15682,7 @@ mod tests {
); );
} }
#[tokio::test] async fn assert_lifecycle_only_reference_obeys_force(clear: bool, force: bool) {
async fn force_remove_and_save_bypasses_lifecycle_only_reference() {
// rustfs/rustfs#6832: reproduces the admin RemoveTier path (not just the lower-level
// reference-proof function) for a tier with zero transitioned objects but a lifecycle
// rule still pointing at it — the exact shape of
// `test_manual_transition_async_tier_failure_reports_terminal_partial` in e2e_test,
// which force-removes a tier a lifecycle rule still references to simulate a
// decommissioned backend.
let store = Arc::new(CasConfigStore::default()); let store = Arc::new(CasConfigStore::default());
let tier = build_rustfs_tier("COLD-A"); let tier = build_rustfs_tier("COLD-A");
let mut persisted = empty_mgr(); let mut persisted = empty_mgr();
@@ -15686,22 +15723,55 @@ mod tests {
let manager = TierConfigMgr::new(); let manager = TierConfigMgr::new();
manager.write().await.tiers.insert("COLD-A".to_string(), tier); manager.write().await.tiers.insert("COLD-A".to_string(), tier);
TierConfigMgr::remove_and_save_with(&manager, store.clone(), "COLD-A", true) let mutation = if clear {
.await TierCandidateMutation::Clear(force)
.expect("force remove must bypass a lifecycle-config-only reference"); } else {
TierCandidateMutation::Remove("COLD-A".to_string(), force)
};
let result = TIER_DRIVER_TEST_FACTORY
.scope(
healthy_driver_factory(),
TierConfigMgr::update_candidate_with_config_lock(&manager, store.clone(), mutation),
)
.await;
if force {
result.expect("force mutation must bypass a lifecycle-config-only reference");
} else {
let err = result.expect_err("non-force mutation must reject a lifecycle-only reference");
let TierConfigUpdateError::Publish(err) = err else {
panic!("non-force mutation must fail during reference proof: {err:?}");
};
assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
assert!(err.message.contains("move-current"), "{err}");
}
assert!(!manager.read().await.tiers.contains_key("COLD-A")); assert_eq!(manager.read().await.tiers.contains_key("COLD-A"), !force);
assert!( assert_eq!(
!load_tier_config_for_update(store) load_tier_config_for_update(store)
.await .await
.expect("config should still reload") .expect("config should still reload")
.0 .0
.tiers .tiers
.contains_key("COLD-A"), .contains_key("COLD-A"),
"force removal must persist the empty candidate" !force,
"persisted state must match the force mutation result"
); );
} }
#[tokio::test]
async fn remove_with_config_lock_obeys_force_for_lifecycle_only_reference() {
for force in [false, true] {
assert_lifecycle_only_reference_obeys_force(false, force).await;
}
}
#[tokio::test]
async fn clear_with_config_lock_obeys_force_for_lifecycle_only_reference() {
for force in [false, true] {
assert_lifecycle_only_reference_obeys_force(true, force).await;
}
}
#[tokio::test] #[tokio::test]
async fn zero_reference_proof_blocks_clear_before_config_save() { async fn zero_reference_proof_blocks_clear_before_config_save() {
let store = Arc::new(CasConfigStore::default()); let store = Arc::new(CasConfigStore::default());
@@ -17242,6 +17312,98 @@ mod tests {
assert_ne!(manager_a.read().await.empty(), manager_b.read().await.empty()); assert_ne!(manager_a.read().await.empty(), manager_b.read().await.empty());
} }
async fn assert_coordinator_commit_refresh_succeeds(mutation: TierCandidateMutation) {
let adding = matches!(mutation, TierCandidateMutation::Add(..));
let manager = TierConfigMgr::new();
let store = Arc::new(CasConfigStore::default());
if !adding {
let mut persisted = empty_mgr();
persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
persisted
.save_tiering_config_if_current(store.clone(), None)
.await
.expect("existing tier fixture should persist");
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
}
let barrier = Arc::new(CasCoordinatorCommitBarrier::default());
*store.coordinator_commit_barrier.lock().await = Some(barrier.clone());
let update_manager = manager.clone();
let update_store = store.clone();
let update = tokio::spawn(async move {
TIER_DRIVER_TEST_FACTORY
.scope(
healthy_driver_factory(),
TIER_MUTATION_TEST_PEERS.scope(
Vec::new(),
TierConfigMgr::update_candidate_with_config_lock(&update_manager, update_store, mutation),
),
)
.await
});
tokio::time::timeout(Duration::from_secs(5), barrier.arrived.notified())
.await
.expect("mutation should reach coordinator commit after saving config");
assert_eq!(
load_tier_config_for_update(store.clone())
.await
.expect("saved config should be readable before coordinator commit")
.0
.tiers
.contains_key("COLD-A"),
adding
);
assert_eq!(
TierConfigMgr::load_coordinator_mutation_intents(store.clone())
.await
.expect("coordinator intent should remain readable")[0]
.state,
TierMutationIntentState::Prepared
);
let lock_requests = lock_unpoisoned(&store.lock_requests).len();
// Also exercise an independently scheduled refresh while the durable
// coordinator record is still Prepared, before its commit notification.
TierConfigMgr::request_committed_mutation_refresh(&manager).await;
TIER_MUTATION_TEST_PEERS
.scope(Vec::new(), async {
let worker = TierConfigMgr::refresh_tier_config_handle_with(manager.clone(), store.clone());
tokio::pin!(worker);
tokio::time::timeout(Duration::from_secs(5), async {
while lock_unpoisoned(&store.lock_requests).len() == lock_requests {
tokio::select! {
_ = &mut worker => panic!("refresh worker must remain available"),
_ = tokio::task::yield_now() => {}
}
}
})
.await
.expect("refresh should reconcile the Prepared record before waiting for the config lock");
barrier.release.notify_one();
let result = tokio::time::timeout(Duration::from_secs(5), async {
tokio::select! {
_ = &mut worker => panic!("refresh worker must remain available"),
result = update => result.expect("tier mutation task should join"),
}
})
.await
.expect("tier mutation should finish with refresh running");
result.expect("saved tier mutation must publish successfully on the first attempt");
})
.await;
assert_eq!(manager.read().await.tiers.contains_key("COLD-A"), adding);
}
#[tokio::test]
async fn tier_add_succeeds_with_refresh_during_coordinator_commit() {
assert_coordinator_commit_refresh_succeeds(TierCandidateMutation::Add(build_rustfs_tier("COLD-A"), true)).await;
}
#[tokio::test]
async fn tier_remove_succeeds_with_refresh_during_coordinator_commit() {
assert_coordinator_commit_refresh_succeeds(TierCandidateMutation::Remove("COLD-A".to_string(), true)).await;
}
async fn committed_refresh_fixture(fail_cleanup: bool) -> (Arc<RwLock<TierConfigMgr>>, Arc<CasConfigStore>, uuid::Uuid) { async fn committed_refresh_fixture(fail_cleanup: bool) -> (Arc<RwLock<TierConfigMgr>>, Arc<CasConfigStore>, uuid::Uuid) {
let manager = TierConfigMgr::new(); let manager = TierConfigMgr::new();
{ {
@@ -15,8 +15,6 @@
#![allow(unused_variables)] #![allow(unused_variables)]
#![allow(unused_mut)] #![allow(unused_mut)]
#![allow(unused_assignments)] #![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use serde::{Deserialize, Deserializer, Serialize, Serializer, de}; use serde::{Deserialize, Deserializer, Serialize, Serializer, de};
@@ -145,7 +143,7 @@ mod tests {
assert_eq!(creds.access_key, "access"); assert_eq!(creds.access_key, "access");
assert_eq!(creds.secret_key, "secret"); assert_eq!(creds.secret_key, "secret");
assert_eq!(creds.creds_json.as_slice(), &service_account[..]); assert_eq!(creds.creds_json.as_slice(), service_account);
let wire = serde_json::to_value(&creds).expect("madmin tier credentials should encode"); let wire = serde_json::to_value(&creds).expect("madmin tier credentials should encode");
assert_eq!(wire["access"], "access"); assert_eq!(wire["access"], "access");
@@ -162,7 +160,7 @@ mod tests {
.expect("the former RustFS field names and byte-array encoding should remain readable"); .expect("the former RustFS field names and byte-array encoding should remain readable");
assert_eq!(legacy.access_key, "legacy-access"); assert_eq!(legacy.access_key, "legacy-access");
assert_eq!(legacy.secret_key, "legacy-secret"); assert_eq!(legacy.secret_key, "legacy-secret");
assert_eq!(legacy.creds_json.as_slice(), &service_account[..]); assert_eq!(legacy.creds_json.as_slice(), service_account);
} }
#[test] #[test]
@@ -460,6 +460,7 @@ where
data, data,
&ObjectOptions { &ObjectOptions {
max_parity: true, max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions { http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("*".to_string()), if_none_match: Some("*".to_string()),
..Default::default() ..Default::default()
@@ -556,6 +557,7 @@ where
data, data,
&ObjectOptions { &ObjectOptions {
max_parity: true, max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions { http_preconditions: Some(HTTPPreconditions {
if_match: Some(current_etag.to_string()), if_match: Some(current_etag.to_string()),
..Default::default() ..Default::default()
@@ -494,6 +494,7 @@ where
data, data,
&ObjectOptions { &ObjectOptions {
max_parity: true, max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions { http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("*".to_string()), if_none_match: Some("*".to_string()),
..Default::default() ..Default::default()
@@ -549,6 +550,7 @@ where
data, data,
&ObjectOptions { &ObjectOptions {
max_parity: true, max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions { http_preconditions: Some(HTTPPreconditions {
if_match: Some(current.record_etag.clone()), if_match: Some(current.record_etag.clone()),
..Default::default() ..Default::default()
@@ -37,9 +37,10 @@ use crate::services::tier::{
use bytes::Bytes; use bytes::Bytes;
use http::StatusCode; use http::StatusCode;
use rustfs_s3_client::credentials::{Credentials, SignatureType, Static, Value}; use rustfs_s3_client::credentials::{Credentials, SignatureType, Static, Value};
use rustfs_s3_client::transition_api::{BucketLookupType, Options, TransitionClient, TransitionCore}; use rustfs_s3_client::transition_api::{BucketLookupType, Options, TransitionClient, TransitionClientTimeouts, TransitionCore};
use rustfs_s3_client::{ use rustfs_s3_client::{
admin_handler_utils::AdminError, admin_handler_utils::AdminError,
api_error_response::to_error_response,
api_put_object::{AdvancedPutOptions, PutObjectOptions}, api_put_object::{AdvancedPutOptions, PutObjectOptions},
transition_api::{ReadCloser, ReaderImpl}, transition_api::{ReadCloser, ReaderImpl},
}; };
@@ -48,11 +49,14 @@ use rustfs_utils::egress::validate_outbound_url;
use rustfs_utils::http::headers::{ use rustfs_utils::http::headers::{
CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, EXPIRES, HeaderExt as _, CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, EXPIRES, HeaderExt as _,
}; };
use s3s::dto::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode, ReplicationStatus};
use s3s::header::{ use s3s::header::{
X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_REPLICATION_STATUS, X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_REPLICATION_STATUS,
X_AMZ_STORAGE_CLASS, X_AMZ_STORAGE_CLASS,
}; };
use s3s::{
S3ErrorCode,
dto::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode, ReplicationStatus},
};
use std::collections::HashMap; use std::collections::HashMap;
use std::sync::Arc; use std::sync::Arc;
use std::time::Duration; use std::time::Duration;
@@ -141,6 +145,42 @@ pub trait WarmBackend {
async fn probe_transition_candidate(&self, _object: &str) -> Result<TransitionCandidateProbe, std::io::Error> { async fn probe_transition_candidate(&self, _object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
Ok(TransitionCandidateProbe::Unsupported) Ok(TransitionCandidateProbe::Unsupported)
} }
async fn probe_transition_version(
&self,
object: &str,
remote_version_id: &str,
) -> Result<TransitionCandidateProbe, std::io::Error> {
if remote_version_id.is_empty() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"an exact tier probe requires a remote version ID",
));
}
self.validate_remote_version_id(remote_version_id)?;
match self
.get(
object,
remote_version_id,
WarmBackendGetOpts {
start_offset: 0,
length: 1,
},
)
.await
{
Ok(_) => Ok(TransitionCandidateProbe::VersionedPresent(remote_version_id.to_string())),
Err(err) if matches!(to_error_response(&err).code, S3ErrorCode::InvalidRange) => {
Ok(TransitionCandidateProbe::VersionedPresent(remote_version_id.to_string()))
}
Err(err)
if err.kind() == std::io::ErrorKind::NotFound
|| matches!(to_error_response(&err).code, S3ErrorCode::NoSuchKey | S3ErrorCode::NoSuchVersion) =>
{
Ok(TransitionCandidateProbe::Missing)
}
Err(err) => Err(err),
}
}
async fn in_use(&self) -> Result<bool, std::io::Error>; async fn in_use(&self) -> Result<bool, std::io::Error>;
} }
@@ -280,6 +320,27 @@ pub(crate) fn endpoint_authority(url: &url::Url) -> Result<String, std::io::Erro
} }
} }
fn transition_timeout_from_env(env_key: &str, default_secs: u64) -> Duration {
Duration::from_secs(rustfs_utils::get_env_u64(env_key, default_secs))
}
pub(crate) fn transition_client_timeouts_from_env() -> TransitionClientTimeouts {
TransitionClientTimeouts::new(
transition_timeout_from_env(
rustfs_config::ENV_TIER_REMOTE_CONNECT_TIMEOUT_SECS,
rustfs_config::DEFAULT_TIER_REMOTE_CONNECT_TIMEOUT_SECS,
),
transition_timeout_from_env(
rustfs_config::ENV_TIER_REMOTE_REQUEST_TIMEOUT_SECS,
rustfs_config::DEFAULT_TIER_REMOTE_REQUEST_TIMEOUT_SECS,
),
transition_timeout_from_env(
rustfs_config::ENV_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS,
rustfs_config::DEFAULT_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS,
),
)
}
/// Build the [`WarmBackendS3`] shared by the S3-compatible warm backend providers. /// Build the [`WarmBackendS3`] shared by the S3-compatible warm backend providers.
/// ///
/// Credential, bucket, and endpoint validation run in this order because the /// Credential, bucket, and endpoint validation run in this order because the
@@ -310,6 +371,7 @@ pub(crate) async fn new_s3_compatible_warm_backend(
signer_type: SignatureType::SignatureV4, signer_type: SignatureType::SignatureV4,
..Default::default() ..Default::default()
})); }));
let timeouts = transition_client_timeouts_from_env();
let opts = Options { let opts = Options {
creds, creds,
secure: u.scheme() == "https", secure: u.scheme() == "https",
@@ -322,7 +384,7 @@ pub(crate) async fn new_s3_compatible_warm_backend(
// Run the SSRF guard after the host-presence check so a host-less endpoint // Run the SSRF guard after the host-presence check so a host-less endpoint
// keeps this constructor's stable error text. // keeps this constructor's stable error text.
(params.validate_endpoint)(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?; (params.validate_endpoint)(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
let client = TransitionClient::new(&endpoint, opts, params.provider_tag).await?; let client = TransitionClient::new_with_timeouts(&endpoint, opts, params.provider_tag, timeouts).await?;
let client = Arc::new(client); let client = Arc::new(client);
let core = TransitionCore(Arc::clone(&client)); let core = TransitionCore(Arc::clone(&client));
@@ -437,6 +499,17 @@ impl WarmBackend for MeteredWarmBackend {
Self::record(TierRequestOperation::Probe, result) Self::record(TierRequestOperation::Probe, result)
} }
async fn probe_transition_version(
&self,
object: &str,
remote_version_id: &str,
) -> Result<TransitionCandidateProbe, std::io::Error> {
Self::record(
TierRequestOperation::Probe,
self.inner.probe_transition_version(object, remote_version_id).await,
)
}
async fn in_use(&self) -> Result<bool, std::io::Error> { async fn in_use(&self) -> Result<bool, std::io::Error> {
Self::record(TierRequestOperation::InUse, self.inner.in_use().await) Self::record(TierRequestOperation::InUse, self.inner.in_use().await)
} }
@@ -15,8 +15,6 @@
#![allow(unused_variables)] #![allow(unused_variables)]
#![allow(unused_mut)] #![allow(unused_mut)]
#![allow(unused_assignments)] #![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use std::collections::HashMap; use std::collections::HashMap;
@@ -15,8 +15,6 @@
#![allow(unused_variables)] #![allow(unused_variables)]
#![allow(unused_mut)] #![allow(unused_mut)]
#![allow(unused_assignments)] #![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use std::collections::HashMap; use std::collections::HashMap;
@@ -15,8 +15,6 @@
#![allow(unused_variables)] #![allow(unused_variables)]
#![allow(unused_mut)] #![allow(unused_mut)]
#![allow(unused_assignments)] #![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use std::collections::{HashMap, HashSet}; use std::collections::{HashMap, HashSet};
use std::future::Future; use std::future::Future;
@@ -146,11 +144,11 @@ pub struct WarmBackendGCS {
impl WarmBackendGCS { impl WarmBackendGCS {
pub async fn new(conf: &TierGCS, tier: &str) -> Result<Self, std::io::Error> { pub async fn new(conf: &TierGCS, tier: &str) -> Result<Self, std::io::Error> {
if conf.creds == "" { if conf.creds.is_empty() {
return Err(std::io::Error::other("both access and secret keys are required")); return Err(std::io::Error::other("both access and secret keys are required"));
} }
if conf.bucket == "" { if conf.bucket.is_empty() {
return Err(std::io::Error::other("no bucket name was provided")); return Err(std::io::Error::other("no bucket name was provided"));
} }
@@ -195,11 +193,11 @@ impl WarmBackendGCS {
} }
pub fn get_dest(&self, object: &str) -> String { pub fn get_dest(&self, object: &str) -> String {
let mut dest_obj = object.to_string(); if self.prefix.is_empty() {
if self.prefix != "" { object.to_string()
dest_obj = format!("{}/{}", &self.prefix, object); } else {
format!("{}/{}", self.prefix, object)
} }
return dest_obj;
} }
} }
@@ -223,7 +221,7 @@ impl WarmBackend for WarmBackendGCS {
let bucket = gcs_bucket_resource_name(&self.bucket); let bucket = gcs_bucket_resource_name(&self.bucket);
let Ok(res) = Box::pin( let Ok(res) = Box::pin(
self.client self.client
.write_object(&bucket, &self.get_dest(object), Bytes::from(d)) .write_object(&bucket, self.get_dest(object), Bytes::from(d))
.send_buffered(), .send_buffered(),
) )
.await .await
@@ -240,7 +238,7 @@ impl WarmBackend for WarmBackendGCS {
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> { async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
let bucket = gcs_bucket_resource_name(&self.bucket); let bucket = gcs_bucket_resource_name(&self.bucket);
let mut req = self.client.read_object(&bucket, &self.get_dest(object)); let mut req = self.client.read_object(&bucket, self.get_dest(object));
let mut max_response_bytes = None; let mut max_response_bytes = None;
if let Some(generation) = parse_generation(rv)? { if let Some(generation) = parse_generation(rv)? {
req = req.set_generation(generation); req = req.set_generation(generation);
@@ -15,8 +15,6 @@
#![allow(unused_variables)] #![allow(unused_variables)]
#![allow(unused_mut)] #![allow(unused_mut)]
#![allow(unused_assignments)] #![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use std::collections::HashMap; use std::collections::HashMap;
@@ -15,8 +15,6 @@
#![allow(unused_variables)] #![allow(unused_variables)]
#![allow(unused_mut)] #![allow(unused_mut)]
#![allow(unused_assignments)] #![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use std::collections::HashMap; use std::collections::HashMap;
@@ -15,8 +15,6 @@
#![allow(unused_variables)] #![allow(unused_variables)]
#![allow(unused_mut)] #![allow(unused_mut)]
#![allow(unused_assignments)] #![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use std::collections::HashMap; use std::collections::HashMap;
@@ -15,8 +15,6 @@
#![allow(unused_variables)] #![allow(unused_variables)]
#![allow(unused_mut)] #![allow(unused_mut)]
#![allow(unused_assignments)] #![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use std::collections::HashMap; use std::collections::HashMap;
@@ -26,7 +26,7 @@ use crate::services::tier::{
tier_config::TierS3, tier_config::TierS3,
warm_backend::{ warm_backend::{
TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts, TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts,
build_transition_put_options, endpoint_authority, build_transition_put_options, endpoint_authority, transition_client_timeouts_from_env,
}, },
}; };
use http::HeaderMap; use http::HeaderMap;
@@ -139,6 +139,7 @@ impl WarmBackendS3 {
} else { } else {
return Err(std::io::Error::other("insufficient parameters for S3 backend authentication")); return Err(std::io::Error::other("insufficient parameters for S3 backend authentication"));
} }
let timeouts = transition_client_timeouts_from_env();
let opts = Options { let opts = Options {
creds, creds,
secure: u.scheme() == "https", secure: u.scheme() == "https",
@@ -147,7 +148,7 @@ impl WarmBackendS3 {
..Default::default() ..Default::default()
}; };
let endpoint = endpoint_authority(&u)?; let endpoint = endpoint_authority(&u)?;
let client = TransitionClient::new(&endpoint, opts, tier_type).await?; let client = TransitionClient::new_with_timeouts(&endpoint, opts, tier_type, timeouts).await?;
let client = Arc::new(client); let client = Arc::new(client);
let core = TransitionCore(Arc::clone(&client)); let core = TransitionCore(Arc::clone(&client));
@@ -529,6 +530,10 @@ mod tests {
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>", "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 66\r\nConnection: close\r\n\r\n<Error><Code>NoSuchObject</Code><Message>missing</Message></Error>", "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 66\r\nConnection: close\r\n\r\n<Error><Code>NoSuchObject</Code><Message>missing</Message></Error>",
"HTTP/1.1 403 Forbidden\r\nContent-Type: application/xml\r\nContent-Length: 65\r\nConnection: close\r\n\r\n<Error><Code>AccessDenied</Code><Message>denied</Message></Error>", "HTTP/1.1 403 Forbidden\r\nContent-Type: application/xml\r\nContent-Length: 65\r\nConnection: close\r\n\r\n<Error><Code>AccessDenied</Code><Message>denied</Message></Error>",
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
"HTTP/1.1 416 Range Not Satisfiable\r\nContent-Type: application/xml\r\nContent-Length: 72\r\nConnection: close\r\n\r\n<Error><Code>InvalidRange</Code><Message>empty version</Message></Error>",
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 67\r\nConnection: close\r\n\r\n<Error><Code>NoSuchVersion</Code><Message>missing</Message></Error>",
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
]; ];
let mut requests = Vec::new(); let mut requests = Vec::new();
for response in responses { for response in responses {
@@ -622,15 +627,52 @@ mod tests {
.await .await
.expect_err("an authorization failure must not be mistaken for a missing key"); .expect_err("an authorization failure must not be mistaken for a missing key");
assert_eq!(to_error_response(&err).code, S3ErrorCode::AccessDenied); assert_eq!(to_error_response(&err).code, S3ErrorCode::AccessDenied);
assert_eq!(
backend
.probe_transition_candidate("delete-marker-hidden")
.await
.expect("a current delete marker should hide the data version"),
TransitionCandidateProbe::Missing
);
assert_eq!(
backend
.probe_transition_version("delete-marker-hidden", "historical-version")
.await
.expect("the stored historical version should be probed exactly"),
TransitionCandidateProbe::VersionedPresent("historical-version".to_string())
);
assert_eq!(
backend
.probe_transition_version("delete-marker-hidden", "missing-version")
.await
.expect("a missing exact version should be classified"),
TransitionCandidateProbe::Missing
);
assert_eq!(
backend
.probe_transition_version("missing-object", "historical-version")
.await
.expect("a missing key for an exact version probe should be classified"),
TransitionCandidateProbe::Missing
);
let requests = fixture.await.expect("candidate fixture should join"); let requests = fixture.await.expect("candidate fixture should join");
for request in requests { for request in &requests[..6] {
let request = request.to_ascii_lowercase(); let request = request.to_ascii_lowercase();
assert!(request.starts_with("get /bucket/"), "candidate discovery must use object GET"); assert!(request.starts_with("get /bucket/"), "candidate discovery must use object GET");
assert!(request.contains("\r\nrange: bytes=0-0\r\n")); assert!(request.contains("\r\nrange: bytes=0-0\r\n"));
assert!(!request.contains("?versioning")); assert!(!request.contains("?versioning"));
assert!(!request.contains("?versions")); assert!(!request.contains("?versions"));
} }
for request in &requests[6..] {
let request = request.to_ascii_lowercase();
assert!(request.starts_with("get /bucket/"), "exact discovery must use object GET");
assert!(request.contains("\r\nrange: bytes=0-0\r\n"));
}
assert!(!requests[5].to_ascii_lowercase().contains("versionid="));
assert!(requests[6].to_ascii_lowercase().contains("?versionid=historical-version"));
assert!(requests[7].to_ascii_lowercase().contains("?versionid=missing-version"));
assert!(requests[8].to_ascii_lowercase().contains("?versionid=historical-version"));
} }
fn list_versions(versions: &[(&str, &str)], delete_markers: &[(&str, &str)], is_truncated: bool) -> ListVersionsResult { fn list_versions(versions: &[(&str, &str)], delete_markers: &[(&str, &str)], is_truncated: bool) -> ListVersionsResult {
@@ -15,8 +15,6 @@
#![allow(unused_variables)] #![allow(unused_variables)]
#![allow(unused_mut)] #![allow(unused_mut)]
#![allow(unused_assignments)] #![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use std::collections::HashMap; use std::collections::HashMap;
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,385 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Pure metadata quorum and early-stop decisions for `SetDisks` reads.
//!
//! Disk scheduling, coalescing, cancellation, and late shard materialization
//! remain with their existing owners; this module only classifies observations.
use crate::diagnostics::get::{
GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER,
GET_METADATA_EARLY_STOP_REASON_ERROR, GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM,
GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST,
GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM,
GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND,
};
use crate::disk::error::DiskError;
use crate::disk::error_reduce::OBJECT_OP_IGNORED_ERRS;
use crate::set_disk::file_info_is_valid_for_metadata;
use rustfs_filemeta::FileInfo;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(in crate::set_disk) struct MetadataEarlyStopDecision {
pub(in crate::set_disk) reason: &'static str,
}
#[derive(Clone, Debug)]
pub(in crate::set_disk) struct MetadataQuorumAccumulator {
pub(in crate::set_disk) total_disks: usize,
pub(in crate::set_disk) default_parity_count: usize,
pub(in crate::set_disk) allow_early_stop: bool,
pub(in crate::set_disk) valid_responses: usize,
pub(in crate::set_disk) not_found_responses: usize,
pub(in crate::set_disk) version_not_found_responses: usize,
pub(in crate::set_disk) ignored_errors: usize,
pub(in crate::set_disk) hard_errors: usize,
pub(in crate::set_disk) candidate: Option<FileInfo>,
pub(in crate::set_disk) candidate_votes: usize,
// Bitset of shard indexes whose metadata matches the candidate. Erasure
// layouts are capped at 16 shards, so this stays allocation-free on the
// GET metadata hot path.
candidate_shard_mask: u16,
pub(in crate::set_disk) conflicting_metadata: bool,
pub(in crate::set_disk) delete_marker_seen: bool,
pub(in crate::set_disk) delete_marker_candidates: Vec<(FileInfo, usize)>,
pub(in crate::set_disk) delete_marker_votes: usize,
pub(in crate::set_disk) requested_version_id: String,
pub(in crate::set_disk) matching_version_votes: usize,
}
impl MetadataQuorumAccumulator {
pub(in crate::set_disk) fn new(total_disks: usize, default_parity_count: usize, allow_early_stop: bool) -> Self {
Self {
total_disks,
default_parity_count,
allow_early_stop,
valid_responses: 0,
not_found_responses: 0,
version_not_found_responses: 0,
ignored_errors: 0,
hard_errors: 0,
candidate: None,
candidate_votes: 0,
candidate_shard_mask: 0,
conflicting_metadata: false,
delete_marker_seen: false,
delete_marker_candidates: Vec::new(),
delete_marker_votes: 0,
requested_version_id: String::new(),
matching_version_votes: 0,
}
}
pub(in crate::set_disk) fn with_requested_version_id(mut self, version_id: &str) -> Self {
self.requested_version_id = version_id.to_string();
self
}
pub(in crate::set_disk) fn observe_file_info(&mut self, file_info: &FileInfo) {
self.observe_file_info_with_index(None, file_info);
}
pub(in crate::set_disk) fn observe_file_info_at(&mut self, disk_index: usize, file_info: &FileInfo) {
self.observe_file_info_with_index(Some(disk_index), file_info);
}
fn observe_file_info_with_index(&mut self, disk_index: Option<usize>, file_info: &FileInfo) {
if !file_info_is_valid_for_metadata(file_info) {
self.hard_errors = self.hard_errors.saturating_add(1);
return;
}
self.valid_responses = self.valid_responses.saturating_add(1);
// Track version match for versioned requests
if !self.requested_version_id.is_empty()
&& let Some(ref vid) = file_info.version_id
&& vid.to_string() == self.requested_version_id
{
self.matching_version_votes = self.matching_version_votes.saturating_add(1);
}
if file_info.is_canonical_delete_marker() {
self.delete_marker_seen = true;
if let Some((_, votes)) = self
.delete_marker_candidates
.iter_mut()
.find(|(candidate, _)| metadata_early_stop_candidate_matches(candidate, file_info))
{
*votes = votes.saturating_add(1);
} else {
self.delete_marker_candidates.push((file_info.clone(), 1));
}
self.delete_marker_votes = self
.delete_marker_candidates
.iter()
.map(|(_, votes)| *votes)
.max()
.unwrap_or_default();
self.conflicting_metadata |= self.delete_marker_candidates.len() > 1;
return;
}
match &self.candidate {
Some(candidate) if metadata_early_stop_candidate_matches(candidate, file_info) => {
self.candidate_votes = self.candidate_votes.saturating_add(1);
if let Some(disk_index) = disk_index
&& let Some(bit) = Self::candidate_shard_bit(candidate, file_info, disk_index)
{
self.candidate_shard_mask |= bit;
}
}
Some(_) => {
self.conflicting_metadata = true;
}
None => {
self.candidate = Some(file_info.clone());
self.candidate_votes = 1;
if let Some(disk_index) = disk_index
&& let Some(bit) = Self::candidate_shard_bit(file_info, file_info, disk_index)
{
self.candidate_shard_mask |= bit;
}
}
}
}
fn candidate_shard_bit(candidate: &FileInfo, file_info: &FileInfo, disk_index: usize) -> Option<u16> {
let &erasure_index = candidate.erasure.distribution.get(disk_index)?;
if erasure_index == 0 || erasure_index > u16::BITS as usize || file_info.erasure.index != erasure_index {
return None;
}
Some(1u16 << (erasure_index - 1))
}
pub(in crate::set_disk) fn candidate_has_read_reserve(&self) -> bool {
self.candidate_read_reserve_target()
.is_some_and(|required| self.candidate_shard_mask.count_ones() as usize >= required)
}
pub(in crate::set_disk) fn candidate_read_reserve_target(&self) -> Option<usize> {
let candidate = self.candidate.as_ref()?;
Some(
candidate
.erasure
.data_blocks
.saturating_add(usize::from(candidate.erasure.parity_blocks > 0)),
)
}
pub(in crate::set_disk) fn observe_error(&mut self, err: &DiskError) {
match err {
DiskError::FileNotFound | DiskError::VolumeNotFound => {
self.not_found_responses = self.not_found_responses.saturating_add(1);
}
DiskError::FileVersionNotFound => {
self.version_not_found_responses = self.version_not_found_responses.saturating_add(1);
}
_ if is_metadata_fanout_ignored_error(err) => {
self.ignored_errors = self.ignored_errors.saturating_add(1);
}
_ => {
self.hard_errors = self.hard_errors.saturating_add(1);
}
}
}
pub(in crate::set_disk) fn early_stop_decision(&self) -> Option<MetadataEarlyStopDecision> {
if !self.allow_early_stop {
return None;
}
if self.delete_marker_votes >= self.default_write_quorum() {
return Some(MetadataEarlyStopDecision {
reason: GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER,
});
}
if self.conflicting_metadata
|| self.delete_marker_seen
|| self.not_found_responses > 0
|| self.version_not_found_responses > 0
|| self.hard_errors > 0
{
return None;
}
if self
.candidate
.as_ref()
.and_then(|candidate| self.candidate_latest_quorum(candidate))
.is_some_and(|latest_quorum| self.candidate_votes >= latest_quorum)
{
return Some(MetadataEarlyStopDecision {
reason: GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM,
});
}
None
}
/// Check if a versioned request can early-stop because the requested
/// version_id has reached quorum across disks.
pub(in crate::set_disk) fn version_early_stop_decision(&self) -> Option<MetadataEarlyStopDecision> {
if !self.allow_early_stop {
return None;
}
if self.requested_version_id.is_empty() {
return None;
}
if self.conflicting_metadata
|| self.delete_marker_seen
|| self.not_found_responses > 0
|| self.version_not_found_responses > 0
|| self.hard_errors > 0
{
return None;
}
if self.matching_version_votes >= self.read_quorum_for_version() {
return Some(MetadataEarlyStopDecision {
reason: GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM,
});
}
None
}
pub(in crate::set_disk) fn can_still_reach_early_stop_with_pending(&self, pending: usize) -> bool {
if !self.allow_early_stop {
return false;
}
if self.delete_marker_votes.saturating_add(pending) >= self.default_write_quorum() {
return true;
}
if self.conflicting_metadata
|| self.delete_marker_seen
|| self.not_found_responses > 0
|| self.version_not_found_responses > 0
|| self.hard_errors > 0
{
return false;
}
if !self.requested_version_id.is_empty()
&& self.matching_version_votes.saturating_add(pending) >= self.read_quorum_for_version()
{
return true;
}
match &self.candidate {
Some(candidate) => self
.candidate_latest_quorum(candidate)
.is_some_and(|latest_quorum| self.candidate_votes.saturating_add(pending) >= latest_quorum),
None => pending >= self.default_write_quorum(),
}
}
/// Compute the read quorum threshold for version-aware early-stop.
/// Uses `total_disks / 2` (like `missing_response_quorum`) when
/// `default_parity_count` is set, otherwise requires all disks.
pub(in crate::set_disk) fn read_quorum_for_version(&self) -> usize {
self.missing_response_quorum()
}
pub(in crate::set_disk) fn final_miss_reason(&self) -> &'static str {
if !self.allow_early_stop {
return GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST;
}
if self.conflicting_metadata {
return GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA;
}
if self.delete_marker_seen {
return GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER;
}
let missing_response_quorum = self.missing_response_quorum();
if self.version_not_found_responses >= missing_response_quorum {
return GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND;
}
if self.not_found_responses >= missing_response_quorum {
return GET_METADATA_EARLY_STOP_REASON_NOT_FOUND;
}
if self.hard_errors > 0 {
return GET_METADATA_EARLY_STOP_REASON_ERROR;
}
if self.ignored_errors > 0 {
return GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM;
}
GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM
}
pub(in crate::set_disk) fn candidate_latest_quorum(&self, candidate: &FileInfo) -> Option<usize> {
if self.default_parity_count == 0 {
return Some(self.total_disks);
}
if candidate.is_canonical_delete_marker() || candidate.size == 0 || candidate.erasure.parity_blocks >= self.total_disks {
return None;
}
let data_blocks = candidate.erasure.data_blocks;
Some(if data_blocks == candidate.erasure.parity_blocks {
data_blocks.saturating_add(1)
} else {
data_blocks
})
}
pub(crate) fn default_write_quorum(&self) -> usize {
if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks {
return self.total_disks;
}
let data_blocks = self.total_disks.saturating_sub(self.default_parity_count);
if data_blocks == self.default_parity_count {
data_blocks.saturating_add(1)
} else {
data_blocks
}
}
pub(in crate::set_disk) fn missing_response_quorum(&self) -> usize {
if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks {
self.total_disks
} else {
self.total_disks / 2
}
}
}
pub(in crate::set_disk) fn metadata_early_stop_candidate_matches(left: &FileInfo, right: &FileInfo) -> bool {
left.volume == right.volume
&& left.name == right.name
&& left.version_id == right.version_id
&& left.is_latest == right.is_latest
&& left.deleted == right.deleted
&& left.mark_deleted == right.mark_deleted
&& left.transition_status == right.transition_status
&& left.transitioned_objname == right.transitioned_objname
&& left.transition_tier == right.transition_tier
&& left.transition_version_id == right.transition_version_id
&& left.transition_version == right.transition_version
&& left.transition_version_state == right.transition_version_state
&& left.expire_restored == right.expire_restored
&& left.size == right.size
&& left.mod_time == right.mod_time
&& left.mode == right.mode
&& left.written_by_version == right.written_by_version
&& left.metadata == right.metadata
&& left.replication_state_internal == right.replication_state_internal
&& left.parts == right.parts
&& left.checksum == right.checksum
&& left.versioned == right.versioned
&& left.num_versions == right.num_versions
&& left.successor_mod_time == right.successor_mod_time
&& left.data_dir == right.data_dir
&& left.erasure.algorithm == right.erasure.algorithm
&& left.erasure.data_blocks == right.erasure.data_blocks
&& left.erasure.parity_blocks == right.erasure.parity_blocks
&& left.erasure.block_size == right.erasure.block_size
&& left.erasure.distribution == right.erasure.distribution
}
pub(in crate::set_disk) fn is_metadata_fanout_ignored_error(err: &DiskError) -> bool {
OBJECT_OP_IGNORED_ERRS.iter().any(|ignored| ignored == err)
}
+1
View File
@@ -18,3 +18,4 @@
//! duplicating read/write/erasure logic. //! duplicating read/write/erasure logic.
pub(crate) mod io_primitives; pub(crate) mod io_primitives;
mod metadata_quorum;
+1 -1
View File
@@ -876,7 +876,7 @@ pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause};
pub(crate) use ops::object::DeleteObjectCommitBarrier; pub(crate) use ops::object::DeleteObjectCommitBarrier;
#[cfg(any(test, feature = "test-util"))] #[cfg(any(test, feature = "test-util"))]
pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier; pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
#[cfg(test)] #[cfg(all(test, feature = "test-util"))]
pub(crate) use ops::object::TransitionUploadedCommitBarrier as SetDiskTransitionUploadedCommitBarrier; pub(crate) use ops::object::TransitionUploadedCommitBarrier as SetDiskTransitionUploadedCommitBarrier;
pub(crate) use ops::object::body_cache_plaintext_len; pub(crate) use ops::object::body_cache_plaintext_len;
#[cfg(all(test, feature = "test-util"))] #[cfg(all(test, feature = "test-util"))]
+363 -3
View File
@@ -2490,9 +2490,9 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks {
return Ok((result, err.map(|e| e.into()))); return Ok((result, err.map(|e| e.into())));
} }
let disks = self.disks.read().await; // The inner heal and missing-object report read the registry again;
// release this snapshot guard before a topology writer can queue between reads.
let disks = disks.clone(); let disks = self.get_disks_internal().await;
let (_, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, false, false, false) let (_, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, false, false, false)
.await .await
.map_err(|e| to_object_err(e.into(), vec![bucket, object]))?; .map_err(|e| to_object_err(e.into(), vec![bucket, object]))?;
@@ -3419,6 +3419,366 @@ mod heal_result_report_tests {
assert_eq!(unformatted, DiskError::UnformattedDisk); assert_eq!(unformatted, DiskError::UnformattedDisk);
} }
#[derive(Clone, Copy)]
enum InventoryWriterHealCase {
Existing,
Missing,
MissingVersion,
}
async fn assert_heal_object_inventory_writer(case: InventoryWriterHealCase) {
use crate::set_disk::core::io_primitives::disk_call_counters;
use std::time::Duration;
use tokio::io::AsyncReadExt;
let (_temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
let bucket = "heal-inventory-writer-bucket";
let object = match case {
InventoryWriterHealCase::Existing => "heal-inventory-writer-existing",
InventoryWriterHealCase::Missing => "heal-inventory-writer-missing",
InventoryWriterHealCase::MissingVersion => "heal-inventory-writer-missing-version",
};
set.make_bucket(
bucket,
&MakeBucketOptions {
versioning_enabled: true,
..Default::default()
},
)
.await
.expect("heal fixture bucket should be created");
let body = vec![0x67; 64 * 1024];
let stored_version = Uuid::new_v4();
let stored_version_string = stored_version.to_string();
let published = if matches!(case, InventoryWriterHealCase::Missing) {
None
} else {
let mut reader = PutObjReader::from_vec(body.clone());
let info = set
.put_object(
bucket,
object,
&mut reader,
&ObjectOptions {
no_lock: true,
versioned: true,
version_id: Some(stored_version_string.clone()),
..Default::default()
},
)
.await
.expect("full-fanout PUT should seed the heal fixture");
for disk in &disks {
let metadata = disk
.read_version("", bucket, object, &stored_version_string, &ReadOptions::default())
.await
.expect("the seeded version must be present on every disk");
assert_eq!(metadata.version_id, Some(stored_version));
assert_eq!(metadata.size, i64::try_from(body.len()).expect("fixture size should fit i64"));
}
Some(info)
};
let requested_version = match case {
InventoryWriterHealCase::Existing => stored_version_string.clone(),
InventoryWriterHealCase::Missing => String::new(),
InventoryWriterHealCase::MissingVersion => Uuid::new_v4().to_string(),
};
let opts = HealOpts {
no_lock: true,
..Default::default()
};
let calls = disk_call_counters::observe(object);
let read_gate = set.disks.read().await;
// UFCS selects the trait's outer precheck, not the same-named inherent heal.
let heal = <SetDisks as crate::storage_api_contracts::heal::HealOperations>::heal_object(
set.as_ref(),
bucket,
object,
&requested_version,
&opts,
);
tokio::pin!(heal);
assert!(matches!(
futures::poll!(tokio::task::unconstrained(heal.as_mut())),
std::task::Poll::Pending
));
// These tests use the current-thread runtime: full-wait metadata tasks
// have been spawned, but cannot run during the single unconstrained poll.
assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), 0);
let writer = set.disks.write();
tokio::pin!(writer);
assert!(matches!(
futures::poll!(tokio::task::unconstrained(writer.as_mut())),
std::task::Poll::Pending
));
assert!(set.disks.try_read().is_err(), "the writer must already block new inventory readers");
tokio::time::timeout(Duration::from_secs(5), async {
while calls.total(disk_call_counters::KIND_READ_VERSION) < 4 {
tokio::task::yield_now().await;
}
})
.await
.expect("the suspended trait heal must have started the real metadata fanout");
for disk_index in 0..4 {
assert_eq!(calls.for_disk(disk_call_counters::KIND_READ_VERSION, disk_index), 1);
}
drop(read_gate);
let (_, outcome) =
tokio::time::timeout(Duration::from_secs(5), async { tokio::join!(async { drop(writer.await) }, heal) })
.await
.expect("trait heal must not deadlock its nested inventory read with the queued writer");
let (result, error) = outcome.expect("heal should report the object's outcome");
match case {
InventoryWriterHealCase::Existing => assert!(error.is_none(), "existing object heal failed: {error:?}"),
InventoryWriterHealCase::Missing => assert!(matches!(error, Some(Error::FileNotFound))),
InventoryWriterHealCase::MissingVersion => assert!(matches!(error, Some(Error::FileVersionNotFound))),
}
assert_eq!(result.bucket, bucket);
assert_eq!(result.object, object);
assert_eq!(result.version_id, requested_version);
assert_eq!(result.disk_count, 4);
assert_eq!(result.before.drives.len(), 4);
assert_eq!(result.after.drives.len(), 4);
for disk_index in 0..4 {
let endpoint = set.set_endpoints[disk_index].to_string();
assert_eq!(result.before.drives[disk_index].endpoint, endpoint);
assert_eq!(result.after.drives[disk_index].endpoint, endpoint);
}
if let Some(published) = published {
tokio::time::timeout(Duration::from_secs(10), async {
let mut reader = set
.get_object_reader(
bucket,
object,
None,
Default::default(),
&ObjectOptions {
versioned: true,
version_id: Some(stored_version_string),
..Default::default()
},
)
.await
.expect("the stored version must remain readable after heal");
assert_eq!(reader.object_info.etag, published.etag);
assert_eq!(reader.object_info.version_id, Some(stored_version));
let mut observed_body = Vec::new();
reader
.stream
.read_to_end(&mut observed_body)
.await
.expect("stored body should stream");
assert_eq!(observed_body, body);
})
.await
.expect("GET must finish after the inventory writer and heal");
}
}
#[tokio::test]
async fn heal_object_inventory_writer_existing() {
assert_heal_object_inventory_writer(InventoryWriterHealCase::Existing).await;
}
#[tokio::test]
async fn heal_object_inventory_writer_missing() {
assert_heal_object_inventory_writer(InventoryWriterHealCase::Missing).await;
}
#[tokio::test]
async fn heal_object_inventory_writer_missing_version() {
assert_heal_object_inventory_writer(InventoryWriterHealCase::MissingVersion).await;
}
#[tokio::test]
#[serial_test::serial]
async fn heal_object_with_queued_disk_renewal() {
use crate::layout::endpoints::SetupType;
use crate::runtime::instance::InstanceContext;
use crate::set_disk::core::io_primitives::disk_call_counters;
use std::collections::HashMap;
use std::future::Future;
use std::task::Poll;
use std::time::Duration;
use tokio::io::AsyncReadExt;
// renew_disk still registers local disks on the ambient context. Match
// the default serial group used by its other setup/registry fixtures,
// and restore only this temporary endpoint, including on a failed join.
struct RenewDiskTestState {
ctx: Arc<InstanceContext>,
was_dist_erasure: bool,
map: Arc<RwLock<HashMap<String, Option<DiskStore>>>>,
endpoint: String,
previous_disk: Option<Option<DiskStore>>,
}
impl Drop for RenewDiskTestState {
fn drop(&mut self) {
let ctx = self.ctx.clone();
let was_dist_erasure = self.was_dist_erasure;
let map = self.map.clone();
let endpoint = self.endpoint.clone();
let previous_disk = self.previous_disk.take();
let handle = tokio::runtime::Handle::current();
std::thread::spawn(move || {
handle.block_on(async move {
let mut map = map.write().await;
match previous_disk {
Some(disk) => {
map.insert(endpoint, disk);
}
None => {
map.remove(&endpoint);
}
}
drop(map);
if was_dist_erasure {
ctx.update_erasure_type(SetupType::DistErasure).await;
}
});
})
.join()
.expect("renew fixture state restoration should finish");
}
}
let (_temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
let endpoint = set.set_endpoints[0].clone();
let ctx = crate::runtime::global::current_ctx();
let map = ctx.local_disk_map();
let restore = RenewDiskTestState {
ctx: ctx.clone(),
was_dist_erasure: ctx.is_dist_erasure().await,
map: map.clone(),
endpoint: endpoint.to_string(),
previous_disk: map.read().await.get(&endpoint.to_string()).cloned(),
};
// Only distributed erasure needs an override to avoid the ambient slot array.
if restore.was_dist_erasure {
ctx.update_erasure_type(SetupType::Erasure).await;
}
let bucket = "heal-disk-renewal-bucket";
let object = "heal-disk-renewal-object";
set.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("renew fixture bucket should be created");
let body = vec![0x73; 64 * 1024];
let mut reader = PutObjReader::from_vec(body.clone());
let published = set
.put_object(
bucket,
object,
&mut reader,
&ObjectOptions {
no_lock: true,
..Default::default()
},
)
.await
.expect("full-fanout PUT should seed the renewal fixture");
for disk in &disks {
let metadata = disk
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.expect("the seeded object must be present on every disk");
assert_eq!(metadata.size, i64::try_from(body.len()).expect("fixture size should fit i64"));
}
let opts = HealOpts {
no_lock: true,
..Default::default()
};
let calls = disk_call_counters::observe(object);
let read_gate = set.disks.read().await;
let heal = <SetDisks as crate::storage_api_contracts::heal::HealOperations>::heal_object(
set.as_ref(),
bucket,
object,
"",
&opts,
);
tokio::pin!(heal);
assert!(matches!(futures::poll!(tokio::task::unconstrained(heal.as_mut())), Poll::Pending));
assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), 0);
let renew = set.renew_disk(&endpoint);
tokio::pin!(renew);
tokio::time::timeout(
Duration::from_secs(5),
futures::future::poll_fn(|cx| {
assert!(
std::pin::pin!(tokio::task::unconstrained(renew.as_mut()))
.poll(cx)
.is_pending(),
"renewal must reach its inventory write before returning"
);
if set.disks.try_read().is_err() {
Poll::Ready(())
} else {
Poll::Pending
}
}),
)
.await
.expect("real renewal must queue its topology writer behind the read gate");
let registered = map
.read()
.await
.get(&endpoint.to_string())
.cloned()
.flatten()
.expect("renewal must register the connected disk before its inventory write");
assert!(!Arc::ptr_eq(&registered, &disks[0]), "renewal must construct a new disk handle");
tokio::time::timeout(Duration::from_secs(5), async {
while calls.total(disk_call_counters::KIND_READ_VERSION) < 4 {
tokio::task::yield_now().await;
}
})
.await
.expect("the suspended trait heal must have started the real metadata fanout");
for disk_index in 0..4 {
assert_eq!(calls.for_disk(disk_call_counters::KIND_READ_VERSION, disk_index), 1);
}
drop(read_gate);
let (_, outcome) = tokio::time::timeout(Duration::from_secs(5), async { tokio::join!(renew, heal) })
.await
.expect("trait heal and real disk renewal must finish without a nested inventory read deadlock");
let (report, error) = outcome.expect("heal should report the existing object");
assert!(error.is_none(), "existing object heal failed after renewal: {error:?}");
assert_eq!(report.bucket, bucket);
assert_eq!(report.object, object);
assert_eq!(report.disk_count, 4);
let renewed = set.get_disks_internal().await[0]
.clone()
.expect("the renewed slot must remain online");
assert!(Arc::ptr_eq(&renewed, &registered), "the set must publish the newly connected handle");
assert_eq!(renewed.endpoint(), endpoint);
let format = load_format_erasure(&renewed, false)
.await
.expect("renewed disk format should remain readable");
assert_eq!(format.erasure.this, set.format.erasure.sets[0][0]);
tokio::time::timeout(Duration::from_secs(10), async {
let mut reader = set
.get_object_reader(bucket, object, None, Default::default(), &ObjectOptions::default())
.await
.expect("the object must remain readable after renewal and heal");
assert_eq!(reader.object_info.etag, published.etag);
let mut observed_body = Vec::new();
reader
.stream
.read_to_end(&mut observed_body)
.await
.expect("stored body should stream");
assert_eq!(observed_body, body);
})
.await
.expect("GET must finish after renewal and heal");
}
// Regression for #955: an offline disk must contribute exactly one drive // Regression for #955: an offline disk must contribute exactly one drive
// record. Before the fix the offline branch fell through and pushed a second // record. Before the fix the offline branch fell through and pushed a second
// (Corrupt) record for the same disk, so `before/after.drives` grew to // (Corrupt) record for the same disk, so `before/after.drives` grew to
+119 -23
View File
@@ -2452,10 +2452,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
let write_quorum = fi.write_quorum(self.default_write_quorum()); let write_quorum = fi.write_quorum(self.default_write_quorum());
let read_quorum = fi.read_quorum(self.default_read_quorum()); let read_quorum = fi.read_quorum(self.default_read_quorum());
let disks = self.disks.read().await; // Release the registry guard before recovery and cleanup read it again:
// a queued topology writer would otherwise deadlock those nested reads.
let disks = disks.clone(); let disks = self.get_disks_internal().await;
// let disks = Self::shuffle_disks(&disks, &fi.erasure.distribution);
let part_path = format!("{}/{}/", upload_id_path, fi.data_dir.unwrap_or(Uuid::nil())); let part_path = format!("{}/{}/", upload_id_path, fi.data_dir.unwrap_or(Uuid::nil()));
self.recover_part_transactions(&part_path, read_quorum, write_quorum) self.recover_part_transactions(&part_path, read_quorum, write_quorum)
@@ -4051,6 +4050,7 @@ mod tests {
let _ = drain_global_dirty_scopes(); let _ = drain_global_dirty_scopes();
let rename_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); let rename_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let rename_tasks = rename_fanout_barrier::observe_tasks(object);
let complete_store = Arc::clone(&set_disks); let complete_store = Arc::clone(&set_disks);
let mut complete = tokio::spawn(async move { let mut complete = tokio::spawn(async move {
let mut opts = ObjectOptions::default(); let mut opts = ObjectOptions::default();
@@ -4062,16 +4062,6 @@ mod tests {
tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused()) tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused())
.await .await
.expect("multipart completion should pause one tail disk during rename"); .expect("multipart completion should pause one tail disk during rename");
assert!(
tokio::time::timeout(Duration::from_millis(100), &mut complete).await.is_err(),
"multipart completion must not publish success while a tail rename is still paused"
);
let initial = drain_global_dirty_scopes().into_iter().collect::<HashSet<_>>();
assert!(
initial.is_empty(),
"capacity must not be marked as committed before the full multipart rename finishes"
);
let abort_store = Arc::clone(&set_disks); let abort_store = Arc::clone(&set_disks);
let abort = tokio::spawn(async move { let abort = tokio::spawn(async move {
@@ -4080,21 +4070,46 @@ mod tests {
.await .await
}); });
signaling.wait_for_attempts(2).await; signaling.wait_for_attempts(2).await;
assert!(!abort.is_finished(), "the in-flight completion must retain the multipart upload guard");
let retained_staging = futures::future::join_all( // A paused rename does not establish that the other disks reached quorum.
disk_stores let retained_staging = tokio::time::timeout(Duration::from_secs(30), async {
.iter() loop {
.map(|disk| disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &staged_part)), let mut retained = 0;
) for result in futures::future::join_all(
disk_stores
.iter()
.map(|disk| disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &staged_part)),
)
.await
{
match result {
Ok(_) => retained += 1,
Err(DiskError::FileNotFound) => {}
Err(error) => panic!("staged rename source lookup failed: {error}"),
}
}
if retained <= 1 && rename_tasks.running() == 1 {
break retained;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await .await
.into_iter() .expect("unpaused multipart renames should finish before the tail is released");
.filter(|result| result.is_ok())
.count();
assert_eq!( assert_eq!(
retained_staging, 1, retained_staging, 1,
"only the paused tail disk should still retain the multipart rename source" "only the paused tail disk should still retain the multipart rename source"
); );
assert!(
tokio::time::timeout(Duration::from_millis(100), &mut complete).await.is_err(),
"multipart completion must not publish success while a tail rename is still paused"
);
let initial = drain_global_dirty_scopes().into_iter().collect::<HashSet<_>>();
assert!(
initial.is_empty(),
"capacity must not be marked as committed before the full multipart rename finishes"
);
assert!(!abort.is_finished(), "the in-flight completion must retain the multipart upload guard");
signaling.set_target(rustfs_lock::ObjectKey::new(bucket, object)); signaling.set_target(rustfs_lock::ObjectKey::new(bucket, object));
let object_attempt = signaling.attempts.load(Ordering::Acquire) + 1; let object_attempt = signaling.attempts.load(Ordering::Acquire) + 1;
@@ -6743,6 +6758,87 @@ mod tests {
.await; .await;
} }
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn complete_multipart_releases_disk_snapshot_before_cleanup() {
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "multipart-topology-lock-bucket";
let object = "object";
let body = vec![0x65; 4096];
make_bucket_on_all(&disk_stores, bucket).await;
let (upload_id, parts) =
stage_upload_with_create_opts(&set_disks, bucket, object, &body, &ObjectOptions::default()).await;
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
for dir in &temp_dirs {
assert!(
dir.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&upload_id_path).exists(),
"the test must create real upload staging on every disk"
);
}
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterObjectPublication);
let complete_store = set_disks.clone();
let complete_upload_id = upload_id.clone();
let complete = tokio::spawn(async move {
complete_store
.complete_multipart_upload(bucket, object, &complete_upload_id, parts, &ObjectOptions::default())
.await
});
barrier.wait_until_paused().await;
// Hold a separate read gate so the real writer queues even when completion
// correctly releases its snapshot guard. Polling Pending proves admission
// to Tokio's write-preferring queue before the cleanup attempts another read.
let read_gate = set_disks.disks.read().await;
let writer = set_disks.disks.write();
tokio::pin!(writer);
assert!(matches!(
futures::poll!(tokio::task::unconstrained(writer.as_mut())),
std::task::Poll::Pending
));
assert!(
set_disks.disks.try_read().is_err(),
"the pending writer must already block new readers before the cleanup resumes"
);
drop(read_gate);
barrier.release();
let writer_guard = tokio::time::timeout(Duration::from_secs(5), writer)
.await
.expect("a queued topology writer must not deadlock with multipart cleanup's disk snapshot");
// A reconnect can publish the same handles; this test isolates admission
// order without changing the disks that contain the committed object.
drop(writer_guard);
tokio::time::timeout(Duration::from_secs(10), complete)
.await
.expect("multipart cleanup must finish after the topology writer releases")
.expect("completion task should not panic")
.expect("completion should preserve the successful object commit");
let mut reader = tokio::time::timeout(
Duration::from_secs(10),
set_disks.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()),
)
.await
.expect("GET should finish after completion")
.expect("the completed object should remain readable");
let mut observed_body = Vec::new();
tokio::time::timeout(Duration::from_secs(10), reader.stream.read_to_end(&mut observed_body))
.await
.expect("the completed object body should finish streaming")
.expect("the completed object body should be readable");
assert_eq!(observed_body, body);
assert!(matches!(
set_disks.check_upload_id_exists(bucket, object, &upload_id, false).await,
Err(StorageError::InvalidUploadID(..))
));
for dir in &temp_dirs {
assert!(
!dir.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&upload_id_path).exists(),
"successful completion must remove its upload staging from every disk"
);
}
}
#[tokio::test(flavor = "multi_thread")] #[tokio::test(flavor = "multi_thread")]
#[serial] #[serial]
async fn complete_releases_object_lock_before_cleanup_and_keeps_upload_lock() { async fn complete_releases_object_lock_before_cleanup_and_keeps_upload_lock() {
+447 -31
View File
@@ -299,11 +299,11 @@ use crate::error::is_err_invalid_upload_id;
use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppressed}; use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppressed};
use crate::object_api::{ use crate::object_api::{
NamespaceLockFence, ReplicationStatusWritebackCondition, ReplicationStatusWritebackMode, NamespaceLockFence, ReplicationStatusWritebackCondition, ReplicationStatusWritebackMode,
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, WriteCompletion,
}; };
use crate::services::notification_sys::RemoteVersionStateFleetProofToken; use crate::services::notification_sys::RemoteVersionStateFleetProofToken;
use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease, tier_destination_id_from_metadata}; use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease, tier_destination_id_from_metadata};
use crate::set_disk::core::io_primitives::{RenameTailCleanup, finish_rename_tail_heal}; use crate::set_disk::core::io_primitives::{RenameRollbackReceipt, RenameTailCleanup, finish_rename_tail_heal};
#[cfg(test)] #[cfg(test)]
use crate::storage_api_contracts::namespace::NamespaceLocking; use crate::storage_api_contracts::namespace::NamespaceLocking;
#[cfg(test)] #[cfg(test)]
@@ -3548,6 +3548,7 @@ impl SetDisks {
(None, None, None) (None, None, None)
}; };
let mut tmp_cleanup_owned = false; let mut tmp_cleanup_owned = false;
let rollback_receipt = RenameRollbackReceipt::default();
let operation = async { let operation = async {
let erasure = Arc::new(erasure_from_file_info(&fi, false)?); let erasure = Arc::new(erasure_from_file_info(&fi, false)?);
@@ -4256,6 +4257,7 @@ impl SetDisks {
let commit_bucket = bucket.to_owned(); let commit_bucket = bucket.to_owned();
let commit_object = object.to_owned(); let commit_object = object.to_owned();
let commit_tmp_dir = tmp_dir.clone(); let commit_tmp_dir = tmp_dir.clone();
let commit_rollback_receipt = rollback_receipt.clone();
let commit_object_lock_guard = object_lock_guard.take(); let commit_object_lock_guard = object_lock_guard.take();
let commit_decommission_object_lock_guard = decommission_object_lock_guard.take(); let commit_decommission_object_lock_guard = decommission_object_lock_guard.take();
let commit_publication_guard = publication_commit_guard.take(); let commit_publication_guard = publication_commit_guard.take();
@@ -4266,13 +4268,17 @@ impl SetDisks {
// complete rename fan-out drains. Keep this path synchronous so // complete rename fan-out drains. Keep this path synchronous so
// its terminal state is known before the coordinator releases // its terminal state is known before the coordinator releases
// remote leases. // remote leases.
let commit_allows_early_ack = !(opts.data_movement && opts.has_decommission_capacity_reservation()) let commit_owns_namespace_guard = commit_object_lock_guard.is_some()
&& (commit_object_lock_guard.is_some() || commit_decommission_object_lock_guard.is_some()
|| commit_decommission_object_lock_guard.is_some() || commit_publication_guard.is_some();
|| commit_publication_guard.is_some()) let commit_allows_early_ack = opts.write_completion == WriteCompletion::Quorum
&& !(opts.data_movement && opts.has_decommission_capacity_reservation())
&& commit_owns_namespace_guard
&& commit_scanner_publication_scope.is_none(); && commit_scanner_publication_scope.is_none();
// Full-tail callers also transfer owned guards to the coordinator:
// cancelling their ACK waiter must not cancel an in-flight rename.
let detach_commit_owner = commit_scanner_publication_scope.is_some() let detach_commit_owner = commit_scanner_publication_scope.is_some()
|| commit_allows_early_ack || commit_owns_namespace_guard
|| commit_bucket_lifecycle_guard.is_some() || commit_bucket_lifecycle_guard.is_some()
|| quota_mutation_fence; || quota_mutation_fence;
let commit_write_path_label = write_path.metric_label(); let commit_write_path_label = write_path.metric_label();
@@ -4452,7 +4458,11 @@ impl SetDisks {
write_quorum, write_quorum,
commit_scanner_publication_lease_tokens.as_ref(), commit_scanner_publication_lease_tokens.as_ref(),
) )
.with_publication_scope(commit_scanner_publication_scope.clone()), .with_publication_scope(commit_scanner_publication_scope.clone())
.with_rollback_receipt(commit_rollback_receipt.clone())
.with_namespace_commit_guard(
(!is_meta_bucketname(&commit_bucket)).then(|| commit_set.ctx.begin_namespace_commit()),
),
) )
.await; .await;
if let Some(scope) = commit_scanner_publication_scope.as_ref() { if let Some(scope) = commit_scanner_publication_scope.as_ref() {
@@ -4585,6 +4595,11 @@ impl SetDisks {
let rename_commit = match rename_result { let rename_commit = match rename_result {
Ok(commit) => commit, Ok(commit) => commit,
Err(err) => { Err(err) => {
if commit_rollback_receipt.is_incomplete() {
// Incomplete undo retains the staging source and
// rollback backup for recovery; cleanup is unsafe.
return Err(err.into());
}
if let Err(cleanup_err) = commit_set.delete_all(RUSTFS_META_TMP_BUCKET, &commit_tmp_dir).await { if let Err(cleanup_err) = commit_set.delete_all(RUSTFS_META_TMP_BUCKET, &commit_tmp_dir).await {
warn!(tmp_dir = %commit_tmp_dir, error = ?cleanup_err, "failed to cleanup put_object temporary data"); warn!(tmp_dir = %commit_tmp_dir, error = ?cleanup_err, "failed to cleanup put_object temporary data");
} else if issue3031_diag_enabled() { } else if issue3031_diag_enabled() {
@@ -4617,9 +4632,8 @@ impl SetDisks {
request.object_version_id = committed_version_id request.object_version_id = committed_version_id
.or_else(|| commit_version_suspended.then(Uuid::nil)) .or_else(|| commit_version_suspended.then(Uuid::nil))
.map(|version_id| version_id.to_string()); .map(|version_id| version_id.to_string());
tokio::spawn(async move { let heal_set = commit_set.clone();
let _ = rustfs_heal_contracts::heal_channel::send_heal_request(request).await; tokio::spawn(async move { heal_set.submit_rename_tail_heal(request).await });
});
} }
let rename_stage_elapsed = rename_stage_start.elapsed(); let rename_stage_elapsed = rename_stage_start.elapsed();
@@ -4885,7 +4899,7 @@ impl SetDisks {
); );
} }
}); });
} else { } else if !rollback_receipt.is_incomplete() {
// Failure path (quorum loss / rollback): keep the cleanup inline so // Failure path (quorum loss / rollback): keep the cleanup inline so
// a failed PUT never returns while its tmp shards are still on disk // a failed PUT never returns while its tmp shards are still on disk
// (state-residue hardening tracked by backlog#864 / backlog#898). // (state-residue hardening tracked by backlog#864 / backlog#898).
@@ -17494,27 +17508,69 @@ mod put_object_tmp_cleanup_tests {
} }
#[tokio::test] #[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn put_object_failure_cleans_tmp_workspace_inline() { async fn put_object_failure_cleans_tmp_workspace_inline() {
let (temp_dirs, _disk_stores, set_disks) = hermetic_set_disks(4).await; temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
for write_completion in [WriteCompletion::Quorum, WriteCompletion::TailDrained] {
let (temp_dirs, _disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "tmp-clean-missing-bucket";
let object = "orphan-object";
let barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::BeforeNamespace);
let writer = Arc::clone(&set_disks);
let put = tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(vec![9u8; TEST_OBJECT_SIZE]);
writer
.put_object(
bucket,
object,
&mut reader,
&ObjectOptions {
write_completion,
..Default::default()
},
)
.await
});
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
.await
.expect("missing-bucket PUT must stage before rename");
let staged = non_trash_tmp_entries(&temp_dirs).await;
assert_eq!(staged.len(), 4, "every disk must have a staged workspace before rejection");
for workspace in staged {
let mut entries = tokio::fs::read_dir(&workspace)
.await
.expect("staged workspace should be readable");
let mut shards = 0;
while let Some(entry) = entries.next_entry().await.expect("staged data directory should be readable") {
if entry.file_type().await.expect("staged entry type").is_dir() {
let part = tokio::fs::metadata(entry.path().join("part.1"))
.await
.expect("staging must contain an actual erasure shard");
assert!(part.len() > 0, "the shard must be written before the missing-bucket failure");
shards += 1;
}
}
assert_eq!(shards, 1);
}
assert!(temp_dirs.iter().all(|dir| !dir.path().join(bucket).exists()));
barrier.release();
let err = tokio::time::timeout(Duration::from_secs(30), put)
.await
.expect("missing-bucket PUT must finish")
.expect("PUT task should join")
.expect_err("put_object into a missing bucket volume must fail");
assert!(matches!(err, StorageError::VolumeNotFound), "original disk error expected: {err}");
// The bucket volume is never created, so the shards are written into // No polling: known pre-publication rejection must clean staging
// the tmp workspace and the commit fails at rename_data with a quorum // inline, before PUT returns (backlog#864 / backlog#898).
// error — exercising the failure-path cleanup. let leftovers = non_trash_tmp_entries(&temp_dirs).await;
let mut reader = PutObjReader::from_vec(vec![9u8; TEST_OBJECT_SIZE]); assert!(
let err = set_disks leftovers.is_empty(),
.put_object("tmp-clean-missing-bucket", "orphan-object", &mut reader, &ObjectOptions::default()) "failed PUT must not leave tmp shards behind, leftovers: {leftovers:?}, err: {err}"
.await );
.expect_err("put_object into a missing bucket volume must fail"); }
})
// No polling: the failure path must clean the tmp workspace inline, .await;
// before put_object returns (backlog#864 / backlog#898 hardening).
let leftovers = non_trash_tmp_entries(&temp_dirs).await;
assert!(
leftovers.is_empty(),
"failed PUT must not leave tmp shards behind, leftovers: {leftovers:?}, err: {err}"
);
drop(temp_dirs);
} }
#[tokio::test] #[tokio::test]
@@ -18157,6 +18213,354 @@ mod put_object_tmp_cleanup_tests {
.await; .await;
} }
async fn make_completion_test_bucket(disks: &[DiskStore], bucket: &str) {
for disk in disks {
disk.make_volume(bucket)
.await
.expect("completion test bucket should be created");
}
}
/// Observe the actual metadata quorum while the remaining rename is parked.
/// A completed task count alone can race tasks that have not started yet.
async fn wait_for_paused_tail_metadata_quorum(disks: &[DiskStore], bucket: &str, object: &str) {
tokio::time::timeout(Duration::from_secs(30), async {
loop {
let mut committed = 0;
for disk in disks {
match disk.read_version("", bucket, object, "", &ReadOptions::default()).await {
Ok(_) => committed += 1,
Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) => {}
Err(err) => panic!("unexpected metadata error while observing {bucket}/{object}: {err}"),
}
}
if committed == 3 {
break;
}
tokio::task::yield_now().await;
}
})
.await
.expect("three disks must publish metadata while the fourth rename remains paused");
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn tail_drained_put_waits_for_tail_and_allows_immediate_cas() {
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
for size in [4096, 1024 * 1024] {
let (_dirs, disks, set) = hermetic_set_disks(4).await;
let bucket = "put-full-tail-cas";
let object = "full-tail-cas-object";
make_completion_test_bucket(&disks, bucket).await;
let tasks = rename_fanout_barrier::observe_tasks(object);
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let writer = Arc::clone(&set);
let put = tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(vec![b'1'; size]);
writer
.put_object(
bucket,
object,
&mut reader,
&ObjectOptions {
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
.await
});
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
.await
.expect("full-tail PUT must reach the rename barrier");
wait_for_paused_tail_metadata_quorum(&disks, bucket, object).await;
assert!(!put.is_finished(), "full-tail PUT must remain pending after metadata quorum");
let mut lock_probe = Box::pin(set.acquire_write_lock_diag("full_tail_probe", bucket, object));
assert!(
futures::poll!(lock_probe.as_mut()).is_pending(),
"the owned namespace guard must remain held"
);
barrier.release();
let written = tokio::time::timeout(Duration::from_secs(30), put)
.await
.expect("full-tail PUT should finish after release")
.expect("full-tail PUT task should join")
.expect("full-tail PUT must commit");
assert_eq!(tasks.running(), 0, "full-tail response must follow every rename task");
drop(
tokio::time::timeout(Duration::from_secs(5), lock_probe)
.await
.expect("same-key lock should be available on return")
.expect("same-key lock probe should succeed"),
);
for disk in &disks {
disk.read_version("", bucket, object, "", &ReadOptions::default())
.await
.expect("successful full-tail PUT must publish on every healthy disk");
}
drop(barrier);
let mut replacement = PutObjReader::from_vec(b"cas successor".to_vec());
set.put_object(
bucket,
object,
&mut replacement,
&ObjectOptions {
write_completion: WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_match: written.etag,
..Default::default()
}),
..Default::default()
},
)
.await
.expect("immediate same-key CAS must acquire the namespace guard");
let mut read = set
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("CAS successor must be immediately readable");
let mut body = Vec::new();
read.stream.read_to_end(&mut body).await.expect("successor body must drain");
assert_eq!(body, b"cas successor");
}
})
.await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn tail_drained_put_preserves_quorum_success_and_heals_failed_tail() {
let (_dirs, disks, set) = hermetic_set_disks(4).await;
let bucket = "put-full-tail-heal";
let object = "full-tail-heal-object";
make_completion_test_bucket(&disks, bucket).await;
let mut heals = set.capture_test_rename_tail_heals();
let tasks = rename_fanout_barrier::observe_tasks(object);
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let _fault = rename_fault_injection::fail_rename_on(object, &[0]);
let writer = Arc::clone(&set);
let put = tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
writer
.put_object(
bucket,
object,
&mut reader,
&ObjectOptions {
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
.await
});
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
.await
.expect("failed tail must first reach the rename barrier");
wait_for_paused_tail_metadata_quorum(&disks, bucket, object).await;
assert!(!put.is_finished(), "committed quorum must still wait for the failing tail");
barrier.release();
tokio::time::timeout(Duration::from_secs(30), put)
.await
.expect("failed tail should drain")
.expect("PUT task should join")
.expect("a minority tail error must not negate committed quorum");
assert_eq!(tasks.running(), 0);
let heal = tokio::time::timeout(Duration::from_secs(30), heals.recv())
.await
.expect("failed tail must schedule heal")
.expect("heal capture must remain connected");
assert_eq!(heal.bucket, bucket);
assert_eq!(heal.object_prefix.as_deref(), Some(object));
let info = set
.get_object_info(bucket, object, &ObjectOptions::default())
.await
.expect("committed object must remain readable despite the failed tail");
assert_eq!(info.size, TEST_OBJECT_SIZE as i64);
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn tail_drained_put_rejects_quorum_minus_one() {
let (_dirs, disks, set) = hermetic_set_disks(4).await;
let bucket = "put-full-tail-no-quorum";
let object = "full-tail-no-quorum-object";
make_completion_test_bucket(&disks, bucket).await;
let _fault = rename_fault_injection::fail_rename_on(object, &[0, 1]);
let tasks = rename_fanout_barrier::observe_tasks(object);
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
let err = set
.put_object(
bucket,
object,
&mut reader,
&ObjectOptions {
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
.await
.expect_err("draining two successful disks cannot satisfy write quorum three");
assert!(
matches!(err, Error::ErasureWriteQuorum | Error::InsufficientWriteQuorum(_, _)),
"original quorum error expected: {err}"
);
assert_eq!(tasks.running(), 0, "failed fan-out and rollback must complete before return");
assert!(
set.get_object_info(bucket, object, &ObjectOptions::default()).await.is_err(),
"failed fresh write must not become visible"
);
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn put_incomplete_rollback_preserves_staging_and_old_version_backup() {
use crate::set_disk::core::io_primitives::rollback_fault_injection;
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
for write_completion in [WriteCompletion::Quorum, WriteCompletion::TailDrained] {
for fault in [
rollback_fault_injection::Fault::Io,
rollback_fault_injection::Fault::VolumeNotFoundAfterRename,
] {
let (dirs, disks, set) = hermetic_set_disks(4).await;
let bucket = "put-incomplete-undo";
let object = "incomplete-undo-object";
make_completion_test_bucket(&disks, bucket).await;
let mut old_reader = PutObjReader::from_vec(vec![b'0'; TEST_OBJECT_SIZE]);
set.put_object(
bucket,
object,
&mut old_reader,
&ObjectOptions {
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
.await
.expect("old generation should be completely committed");
wait_for_tmp_workspace_to_drain(&dirs, "old PUT must leave no unrelated staging").await;
let old = disks[0]
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.expect("old metadata must be readable");
let old_data_dir = old.data_dir.expect("non-inline old version needs a data directory");
let tasks = rename_fanout_barrier::observe_tasks(object);
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let _rename_fault = rename_fault_injection::fail_rename_on(object, &[2, 3]);
let _undo_fault = rollback_fault_injection::arm(object, 0, fault);
let writer = Arc::clone(&set);
let put = tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
writer
.put_object(
bucket,
object,
&mut reader,
&ObjectOptions {
write_completion,
..Default::default()
},
)
.await
});
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
.await
.expect("overwrite must enter the actual rename fan-out before failure injection");
barrier.release();
let err = tokio::time::timeout(Duration::from_secs(30), put)
.await
.expect("incomplete undo must return without hanging")
.expect("PUT task should join")
.expect_err("two renamed disks cannot satisfy write quorum three");
assert!(
matches!(err, Error::ErasureWriteQuorum | Error::InsufficientWriteQuorum(_, _)),
"original quorum error expected: {err}"
);
assert_eq!(tasks.running(), 0, "every rename and undo task must be reaped before return");
let leftovers = non_trash_tmp_entries(&dirs).await;
assert!(!leftovers.is_empty(), "incomplete undo must retain the new staging source for recovery");
let backups = dirs
.iter()
.filter(|dir| {
dir.path()
.join(bucket)
.join(object)
.join(old_data_dir.to_string())
.join(crate::disk::STORAGE_FORMAT_FILE_BACKUP)
.exists()
})
.count();
assert_eq!(backups, 1, "exactly the failed undo disk must retain its old-version backup");
// The remaining three disks still serve the old generation;
// the failed minority must never become an acknowledged write.
let mut read = set
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("old generation must remain readable after incomplete rollback");
let mut body = Vec::new();
read.stream
.read_to_end(&mut body)
.await
.expect("old generation should stream");
assert_eq!(body, vec![b'0'; TEST_OBJECT_SIZE]);
}
}
})
.await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn tail_drained_put_owned_commit_survives_waiter_cancellation() {
let (dirs, disks, set) = hermetic_set_disks(4).await;
let bucket = RUSTFS_META_BUCKET;
let object = "full-tail-cancelled-receipt";
// Internal config writes do not own a bucket lifecycle guard. The object
// guard alone must keep the full-tail coordinator alive after cancellation.
let tasks = rename_fanout_barrier::observe_tasks(object);
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let writer = Arc::clone(&set);
let put = tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
writer
.put_object(
bucket,
object,
&mut reader,
&ObjectOptions {
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
.await
});
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
.await
.expect("cancelled receipt must first reach the rename barrier");
wait_for_paused_tail_metadata_quorum(&disks, bucket, object).await;
put.abort();
assert!(put.await.expect_err("ACK waiter should cancel").is_cancelled());
let mut lock_probe = Box::pin(set.acquire_write_lock_diag("cancelled_full_tail_probe", bucket, object));
assert!(
futures::poll!(lock_probe.as_mut()).is_pending(),
"owned coordinator must retain the namespace guard after waiter cancellation"
);
barrier.release();
drop(
tokio::time::timeout(Duration::from_secs(30), lock_probe)
.await
.expect("cancelled coordinator must eventually release its guard")
.expect("post-commit lock probe should succeed"),
);
assert_eq!(tasks.running(), 0, "cancelled coordinator must reap every rename task");
for disk in &disks {
disk.read_version("", bucket, object, "", &ReadOptions::default())
.await
.expect("caller cancellation must not interrupt committed receipt materialization");
}
wait_for_tmp_workspace_to_drain(&dirs, "cancelled full-tail commit should release staging ownership").await;
}
#[tokio::test] #[tokio::test]
#[serial_test::serial(capacity_dirty_scope)] #[serial_test::serial(capacity_dirty_scope)]
async fn no_lock_put_waits_for_rename_tail_under_outer_guard() { async fn no_lock_put_waits_for_rename_tail_under_outer_guard() {
@@ -18184,6 +18588,7 @@ mod put_object_tmp_cleanup_tests {
&mut reader, &mut reader,
&ObjectOptions { &ObjectOptions {
no_lock: true, no_lock: true,
write_completion: WriteCompletion::TailDrained,
..Default::default() ..Default::default()
}, },
) )
@@ -18209,7 +18614,18 @@ mod put_object_tmp_cleanup_tests {
put.await put.await
.expect("no-lock PUT task should join") .expect("no-lock PUT task should join")
.expect("no-lock PUT should commit after the rename tail releases"); .expect("no-lock PUT should commit after the rename tail releases");
let mut lock_probe = Box::pin(set_disks.acquire_write_lock_diag("borrowed_full_tail_probe", bucket, object));
assert!(
futures::poll!(lock_probe.as_mut()).is_pending(),
"full-tail PUT must not release the caller's outer guard"
);
drop(outer_guard); drop(outer_guard);
drop(
tokio::time::timeout(Duration::from_secs(5), lock_probe)
.await
.expect("outer owner releasing its guard should unblock the probe")
.expect("post-outer-guard probe should succeed"),
);
}) })
.await; .await;
} }
@@ -18,6 +18,7 @@ use super::{
}; };
use crate::bucket::lifecycle::lifecycle::{TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time}; use crate::bucket::lifecycle::lifecycle::{TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time};
use crate::ecstore_validation_blackbox::make_local_set_disks; use crate::ecstore_validation_blackbox::make_local_set_disks;
use crate::object_api::WriteCompletion;
use crate::services::tier::test_util::register_mock_tier; use crate::services::tier::test_util::register_mock_tier;
use crate::storage_api_contracts::bucket::BucketOperations; use crate::storage_api_contracts::bucket::BucketOperations;
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
@@ -72,7 +73,7 @@ async fn transition_and_restore_reclaim_prior_metadata_generations() {
object, object,
&mut reader, &mut reader,
&ObjectOptions { &ObjectOptions {
no_lock: true, write_completion: WriteCompletion::TailDrained,
..Default::default() ..Default::default()
}, },
) )
@@ -185,7 +186,7 @@ async fn prepared_snapshot_transition_duplicate_and_late_get_use_committed_remot
object, object,
&mut reader, &mut reader,
&ObjectOptions { &ObjectOptions {
no_lock: true, write_completion: WriteCompletion::TailDrained,
..Default::default() ..Default::default()
}, },
) )
+12 -2
View File
@@ -1059,6 +1059,7 @@ mod tests {
use crate::storage_api_contracts::{ use crate::storage_api_contracts::{
bucket::{BucketOperations as _, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp}, bucket::{BucketOperations as _, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp},
list::ListOperations as _, list::ListOperations as _,
namespace::NamespaceLocking as _,
object::{ObjectIO as _, ObjectOperations as _}, object::{ObjectIO as _, ObjectOperations as _},
}; };
use crate::store::{ECStore, init_local_disks_with_instance_ctx}; use crate::store::{ECStore, init_local_disks_with_instance_ctx};
@@ -1486,10 +1487,19 @@ mod tests {
.put_object(bucket, object, &mut reader, &ObjectOptions::default()) .put_object(bucket, object, &mut reader, &ObjectOptions::default())
.await .await
.expect("object should be written"); .expect("object should be written");
let lock = ecstore.pools[0].disk_set[0]
.new_ns_lock(bucket, object)
.await
.expect("fixture namespace lock should be created");
drop(
lock.get_write_lock(Duration::from_secs(30))
.await
.expect("fixture rename tail should finish before checking its generation"),
);
assert_eq!( assert_eq!(
ecstore.scanner_namespace_mutation_generation(), ecstore.scanner_namespace_mutation_generation(),
generation_before_put.saturating_add(1), generation_before_put.saturating_add(3),
"successful object creation should advance scanner namespace activity" "successful object creation must observe the logical mutation and both fanout boundaries"
); );
ecstore ecstore
.get_object_info(bucket, object, &ObjectOptions::default()) .get_object_info(bucket, object, &ObjectOptions::default())
+731 -47
View File
@@ -787,6 +787,12 @@ impl ECStore {
pub fn single_pool(&self) -> bool { pub fn single_pool(&self) -> bool {
self.pools.len() == 1 self.pools.len() == 1
} }
/// The set-local create-only check is atomic only when every object
/// mutation uses that same, enabled namespace lock domain.
pub fn supports_atomic_create_only_write_back(&self) -> bool {
!self.ctx.lock_manager().is_disabled() && self.pools.len() == 1 && self.pools[0].disk_set.len() == 1
}
} }
#[cfg(test)] #[cfg(test)]
@@ -2127,7 +2133,7 @@ mod tests {
.iter() .iter()
.map(|&drives_per_set| (1, drives_per_set)) .map(|&drives_per_set| (1, drives_per_set))
.collect::<Vec<_>>(); .collect::<Vec<_>>();
build_isolated_test_store_with_layout(temp_dir, cmd_line, &pool_layouts, shutdown).await build_isolated_test_store_with_layout(temp_dir, cmd_line, &pool_layouts, shutdown, None).await
} }
async fn build_isolated_test_store_with_layout( async fn build_isolated_test_store_with_layout(
@@ -2135,6 +2141,7 @@ mod tests {
cmd_line: &str, cmd_line: &str,
pool_layouts: &[(usize, usize)], pool_layouts: &[(usize, usize)],
shutdown: CancellationToken, shutdown: CancellationToken,
instance_ctx: Option<Arc<crate::runtime::instance::InstanceContext>>,
) -> ( ) -> (
Arc<crate::runtime::instance::InstanceContext>, Arc<crate::runtime::instance::InstanceContext>,
Arc<crate::store::ECStore>, Arc<crate::store::ECStore>,
@@ -2167,7 +2174,7 @@ mod tests {
let endpoint_pools = EndpointServerPools(pools); let endpoint_pools = EndpointServerPools(pools);
crate::services::notification_sys::install_cross_pool_fence_fleet_proof_for_test(); crate::services::notification_sys::install_cross_pool_fence_fleet_proof_for_test();
let instance_ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); let instance_ctx = instance_ctx.unwrap_or_else(|| Arc::new(crate::runtime::instance::InstanceContext::new()));
crate::store::init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone()) crate::store::init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
.await .await
.expect("register local disks into the fresh context"); .expect("register local disks into the fresh context");
@@ -2535,6 +2542,348 @@ mod tests {
shutdown.cancel(); shutdown.cancel();
} }
#[cfg(feature = "test-util")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial_test::serial(storage_class_env)]
async fn early_ack_put_tails_block_scanner_publication_until_all_renames_finish() {
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
let temp_dir = tempfile::tempdir().expect("create scanner PUT tail store dir");
let (ctx, store, shutdown) =
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "scanner-put-tails", &[4])).await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
let bucket = format!("scanner-put-tails-{}", Uuid::new_v4());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create scanner PUT tail bucket");
let set = &store.pools[0].disk_set[0];
let objects = [("scanner-tail-a", vec![0xA1; 273]), ("scanner-tail-b", vec![0xB2; 379])];
temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
let (active, blocked, movement_generation) = store.scanner_data_movement_activity().await;
assert!(!active && !blocked);
assert!(ctx.scanner_publication_state_allowed(), "the set admission cache should start allowed");
let (old_lease, _) = store
.acquire_scanner_publication_lease(movement_generation, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
.await
.expect("publication lease should be admitted before either PUT starts");
let barriers: Vec<_> = objects
.iter()
.map(|(object, _)| {
crate::set_disk::rename_fanout_barrier::arm(object, 0, crate::set_disk::rename_fanout_barrier::PHASE_RENAME)
})
.collect();
let trackers: Vec<_> = objects
.iter()
.map(|(object, _)| crate::set_disk::rename_fanout_barrier::observe_tasks(object))
.collect();
let puts: Vec<_> = objects
.iter()
.map(|(object, body)| {
let put_store = Arc::clone(&store);
let put_bucket = bucket.clone();
let object = *object;
let body = body.clone();
tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(body);
put_store
.put_object(&put_bucket, object, &mut reader, &ObjectOptions::default())
.await
})
})
.collect();
let committed = tokio::time::timeout(Duration::from_secs(30), async {
for barrier in &barriers {
barrier.wait_until_paused().await;
}
let mut committed = Vec::with_capacity(puts.len());
for put in puts {
committed.push(
put.await
.expect("early-ACK PUT task should join while its tail is paused")
.expect("root PUT should return after quorum without waiting for its tail"),
);
}
committed
})
.await
.expect("both root PUTs must quorum-ACK while their tail disks remain paused");
assert!(trackers.iter().all(|tracker| tracker.running() >= 1));
assert!(ctx.namespace_commits_pending());
assert!(
ctx.scanner_publication_state_allowed(),
"pending PUT tails must not disable scanner namespace walks"
);
let (active, blocked, observed_movement_generation) = store.scanner_data_movement_activity().await;
assert!(!active, "ordinary PUT tails are not decommission or rebalance work");
assert!(!blocked, "ordinary PUT tails must not block the movement-only scan baseline");
assert_eq!(observed_movement_generation, movement_generation);
assert!(store.scanner_data_usage_publication_blocked().await);
assert!(store.scanner_data_usage_publication_admission_guard().await.is_some());
assert!(set.scanner_data_usage_publication_admission_guard().await.is_some());
for error in [
store
.acquire_scanner_publication_lease(
movement_generation,
crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL,
)
.await
.expect_err("a new remote publication lease must reject pending PUT tails"),
store
.validate_scanner_publication_lease(old_lease, movement_generation)
.await
.expect_err("an existing remote lease must not bypass pending PUT tails"),
store
.acquire_scanner_publication_lease_guard(old_lease)
.await
.expect_err("target-side publication admission must reject pending PUT tails"),
] {
assert!(
error.to_string().contains("blocked"),
"publication must fail because of active tails: {error}"
);
}
store.release_scanner_publication_lease(old_lease).await;
for (index, barrier) in barriers.iter().enumerate() {
let commit_generation = ctx.namespace_commit_generation();
let namespace_generation = store.scanner_namespace_mutation_generation();
barrier.release();
tokio::time::timeout(Duration::from_secs(30), async {
while trackers[index].running() != 0 || ctx.namespace_commit_generation() <= commit_generation {
tokio::task::yield_now().await;
}
if index + 1 == barriers.len() {
while ctx.namespace_commits_pending() {
tokio::task::yield_now().await;
}
}
})
.await
.expect("released tail must drain and publish its terminal namespace generation");
assert!(store.scanner_namespace_mutation_generation() > namespace_generation);
let pending = index + 1 < barriers.len();
assert_eq!(ctx.namespace_commits_pending(), pending);
assert_eq!(store.scanner_data_usage_publication_blocked().await, pending);
assert!(!store.scanner_data_movement_activity().await.1);
assert!(store.scanner_data_usage_publication_admission_guard().await.is_some());
assert!(set.scanner_data_usage_publication_admission_guard().await.is_some());
}
let (lease, generation) = store
.acquire_scanner_publication_lease(movement_generation, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
.await
.expect("remote publication lease should resume after both tails drain");
store
.validate_scanner_publication_lease(lease, generation)
.await
.expect("a resumed remote publication lease should validate");
drop(
store
.acquire_scanner_publication_lease_guard(lease)
.await
.expect("target-side publication admission should resume after both tails drain"),
);
assert!(store.release_scanner_publication_lease(lease).await);
let disks = set.disk_inventory().await;
assert_eq!(disks.len(), 4);
for ((object, body), committed) in objects.iter().zip(&committed) {
let logical_size = i64::try_from(body.len()).expect("fixture payload size should fit i64");
let etag = committed.etag.as_ref().expect("root PUT should return a committed ETag");
for (disk_index, disk) in disks.iter().enumerate() {
let file_info = disk
.as_ref()
.expect("every fixture disk should remain online")
.read_version(
"",
&bucket,
object,
"",
&crate::disk::ReadOptions {
read_data: true,
..Default::default()
},
)
.await
.unwrap_or_else(|err| panic!("disk {disk_index} should publish {object} after its tail finishes: {err}"));
assert_eq!(file_info.size, logical_size);
assert_eq!(file_info.metadata.get(http::header::ETAG.as_str()), Some(etag));
assert!(
file_info.inline_data(),
"small fixture payloads should have an inline shard on every disk"
);
let inline_data = file_info.data.as_ref().expect("every disk should retain its inline shard");
let erasure = crate::erasure::coding::Erasure::try_new_with_options(
file_info.erasure.data_blocks,
file_info.erasure.parity_blocks,
file_info.erasure.block_size,
file_info.uses_legacy_checksum,
)
.expect("persisted erasure geometry should be valid");
let shard_size =
usize::try_from(erasure.shard_file_size(logical_size)).expect("fixture shard size should fit usize");
crate::erasure::coding::bitrot_verify(
Cursor::new(inline_data.clone()),
inline_data.len(),
shard_size,
rustfs_utils::HashAlgorithm::HighwayHash256S,
erasure.shard_size(),
)
.await
.unwrap_or_else(|err| panic!("disk {disk_index} should retain a complete valid shard for {object}: {err}"));
}
let mut reader = store
.get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("fully drained PUT should be readable");
let mut actual = Vec::new();
reader.stream.read_to_end(&mut actual).await.expect("PUT body should drain");
assert_eq!(&actual, body);
}
let generation_before_internal_put = ctx.namespace_commit_generation();
let internal_object = "scanner-tail-regression/internal-metadata";
let internal_body = b"scanner metadata must not invalidate its own publication";
let mut internal_reader = PutObjReader::from_vec(internal_body.to_vec());
store
.put_object(RUSTFS_META_BUCKET, internal_object, &mut internal_reader, &ObjectOptions::default())
.await
.expect("internal metadata PUT should commit without scanner self-invalidation");
let internal_lock = set
.new_ns_lock(RUSTFS_META_BUCKET, internal_object)
.await
.expect("internal metadata tail lock should be available");
drop(
internal_lock
.get_write_lock(Duration::from_secs(30))
.await
.expect("internal metadata tail should drain"),
);
assert_eq!(ctx.namespace_commit_generation(), generation_before_internal_put);
assert!(!ctx.namespace_commits_pending());
assert!(store.scanner_data_usage_publication_admission_guard().await.is_some());
assert!(set.scanner_data_usage_publication_admission_guard().await.is_some());
let mut internal_reader = store
.get_object_reader(RUSTFS_META_BUCKET, internal_object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("internal metadata should remain readable");
let mut actual = Vec::new();
internal_reader
.stream
.read_to_end(&mut actual)
.await
.expect("internal metadata body should drain");
assert_eq!(actual, internal_body);
})
.await;
shutdown.cancel();
}
#[cfg(feature = "test-util")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial_test::serial(storage_class_env)]
async fn cancelled_early_ack_put_keeps_scanner_publication_blocked_until_tail_finishes() {
let temp_dir = tempfile::tempdir().expect("create cancelled scanner PUT tail store dir");
let (ctx, store, shutdown) =
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "scanner-cancelled-put-tail", &[4])).await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
let bucket = format!("scanner-cancelled-put-tail-{}", Uuid::new_v4());
let object = "scanner-cancelled-tail";
let body = vec![0xC3; 273];
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create cancelled scanner PUT tail bucket");
temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
let tracker = crate::set_disk::rename_fanout_barrier::observe_tasks(object);
let tail =
crate::set_disk::rename_fanout_barrier::arm(object, 0, crate::set_disk::rename_fanout_barrier::PHASE_RENAME);
let quorum = crate::set_disk::PutObjectCommitBarrier::install(
&bucket,
object,
crate::set_disk::PutObjectCommitPause::AfterRenameQuorum,
);
let handoff = crate::set_disk::PutObjectCommitBarrier::install(
&bucket,
object,
crate::set_disk::PutObjectCommitPause::AfterRenameHandoff,
);
let put_store = Arc::clone(&store);
let put_bucket = bucket.clone();
let put_body = body.clone();
let put = tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(put_body);
put_store
.put_object(&put_bucket, object, &mut reader, &ObjectOptions::default())
.await
});
tokio::time::timeout(Duration::from_secs(30), tail.wait_until_paused())
.await
.expect("cancelled PUT should pause one disk before rename");
quorum.wait_until_paused().await;
put.abort();
assert!(
put.await
.expect_err("caller should be cancelled after rename quorum")
.is_cancelled()
);
quorum.release();
handoff.wait_until_paused().await;
assert!(tracker.running() >= 1);
assert!(ctx.namespace_commits_pending());
assert!(!store.scanner_data_movement_activity().await.1);
assert!(store.scanner_data_usage_publication_blocked().await);
assert!(store.scanner_data_usage_publication_admission_guard().await.is_some());
assert!(
store.pools[0].disk_set[0]
.scanner_data_usage_publication_admission_guard()
.await
.is_some()
);
let generation = store.scanner_namespace_mutation_generation();
handoff.release();
tail.release();
tokio::time::timeout(Duration::from_secs(30), async {
while tracker.running() != 0 || ctx.namespace_commits_pending() {
tokio::task::yield_now().await;
}
})
.await
.expect("cancelled request's detached fanout must release scanner admission after finishing");
assert!(store.scanner_namespace_mutation_generation() > generation);
assert!(!store.scanner_data_usage_publication_blocked().await);
assert!(store.scanner_data_usage_publication_admission_guard().await.is_some());
for (disk_index, disk) in store.pools[0].disk_set[0].disk_inventory().await.iter().enumerate() {
let file_info = disk
.as_ref()
.expect("cancelled PUT fixture disk should remain online")
.read_version("", &bucket, object, "", &crate::disk::ReadOptions::default())
.await
.unwrap_or_else(|err| panic!("cancelled PUT must still publish on disk {disk_index}: {err}"));
assert_eq!(file_info.size, i64::try_from(body.len()).expect("fixture body size should fit i64"));
}
let mut reader = store
.get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("a cancelled caller must not discard its quorum-committed object");
let mut actual = Vec::new();
reader
.stream
.read_to_end(&mut actual)
.await
.expect("cancelled PUT body should drain");
assert_eq!(actual, body);
})
.await;
shutdown.cancel();
}
#[cfg(feature = "test-util")] #[cfg(feature = "test-util")]
#[test] #[test]
#[serial_test::serial(storage_class_env)] #[serial_test::serial(storage_class_env)]
@@ -2979,6 +3328,35 @@ mod tests {
#[cfg(feature = "test-util")] #[cfg(feature = "test-util")]
const DECOMMISSION_TEST_FAULT_STAGE_TIERED: &str = "decommission_tiered_object"; const DECOMMISSION_TEST_FAULT_STAGE_TIERED: &str = "decommission_tiered_object";
fn decommission_retry_fault_hook(
bucket: &str,
object: &str,
faults: Arc<AtomicUsize>,
) -> crate::core::pools::DecommissionTestFaultDecision {
let target_bucket = bucket.to_string();
let target_object = object.to_string();
Arc::new(move |stage, bucket, object, attempt, succeeded| {
if !succeeded
|| attempt >= crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS
|| stage != DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT
|| bucket != target_bucket
|| object != target_object
{
return false;
}
// Entry retries reset the local attempt; real copy errors can skip
// successful attempts. Only injected faults spend this global budget.
// A real failure may consume an attempt, so preserve the final chance.
faults
.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |faults| {
(faults < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS.saturating_sub(1))
.then_some(faults.saturating_add(1))
})
.is_ok()
})
}
async fn seed_decommission_source( async fn seed_decommission_source(
store: &Arc<crate::store::ECStore>, store: &Arc<crate::store::ECStore>,
bucket: &str, bucket: &str,
@@ -4991,6 +5369,7 @@ mod tests {
"decommission-delete-fence", "decommission-delete-fence",
&[(2, 4), (1, 4)], &[(2, 4), (1, 4)],
CancellationToken::new(), CancellationToken::new(),
None,
)) ))
.await; .await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
@@ -5120,6 +5499,43 @@ mod tests {
shutdown.cancel(); shutdown.cancel();
} }
#[test]
fn decommission_retry_fault_budget_counts_successes_across_attempt_changes() {
let cases: &[&[(usize, bool, bool)]] = &[
&[(1, true, true), (2, true, true), (3, true, false)],
&[(1, true, true), (1, true, true), (2, true, false)],
&[(1, true, true), (3, true, false), (3, true, false)],
&[(1, true, true), (2, false, false), (1, true, true), (2, true, false)],
&[(1, true, true), (2, false, false), (3, true, false)],
&[(3, true, false), (4, true, false)],
];
for case in cases {
let faults = Arc::new(AtomicUsize::new(0));
let hook = decommission_retry_fault_hook("bucket", "object", Arc::clone(&faults));
for (stage, bucket, object, succeeded) in [
("other-stage", "bucket", "object", true),
(DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "other-bucket", "object", true),
(DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "other-object", true),
(DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "object", false),
] {
assert!(!hook(stage, bucket, object, 1, succeeded));
}
assert_eq!(faults.load(Ordering::SeqCst), 0, "unrelated or failed copies must not consume faults");
let mut expected_faults = 0;
for &(attempt, succeeded, expected) in *case {
assert_eq!(
hook(DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "object", attempt, succeeded),
expected,
"fault plan {case:?} at attempt {attempt}"
);
expected_faults += usize::from(expected);
assert_eq!(faults.load(Ordering::SeqCst), expected_faults);
}
}
}
#[test] #[test]
#[serial_test::serial(storage_class_env)] #[serial_test::serial(storage_class_env)]
fn decommission_entry_retries_source_changed_without_canceling_other_bucket() { fn decommission_entry_retries_source_changed_without_canceling_other_bucket() {
@@ -5214,31 +5630,8 @@ mod tests {
)); ));
let ordinary_faults = Arc::new(AtomicUsize::new(0)); let ordinary_faults = Arc::new(AtomicUsize::new(0));
let ordinary_faults_for_hook = Arc::clone(&ordinary_faults); let fault_hook = decommission_retry_fault_hook(&other_bucket, other_object, Arc::clone(&ordinary_faults));
let fault_bucket = other_bucket.clone(); let _fault_guard = crate::core::pools::DecommissionTestFaultGuard::install(fault_hook);
let _fault_guard = crate::core::pools::DecommissionTestFaultGuard::install(Arc::new(
move |stage, bucket, object, attempt, succeeded| {
let candidate = succeeded
&& stage == DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT
&& bucket == fault_bucket.as_str()
&& object == other_object;
if !candidate {
return false;
}
// Keep the fault budget global across any
// entry-level re-list; its inner attempt counter
// restarts after SourceChanged.
ordinary_faults_for_hook
.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |faults| {
let next_fault = faults.saturating_add(1);
(faults < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS.saturating_sub(1)
&& attempt == next_fault)
.then_some(next_fault)
})
.is_ok()
},
));
let rx = CancellationToken::new(); let rx = CancellationToken::new();
let source_changed_exhaustions = Arc::new(AtomicUsize::new(0)); let source_changed_exhaustions = Arc::new(AtomicUsize::new(0));
@@ -5275,6 +5668,15 @@ mod tests {
changed_result.expect("SourceChanged entry retry must converge"); changed_result.expect("SourceChanged entry retry must converge");
other_result.expect("other bucket entry must continue through ordinary copy retries"); other_result.expect("other bucket entry must continue through ordinary copy retries");
assert_eq!(
store.pool_meta.read().await.pools[0]
.decommission
.as_ref()
.expect("decommission progress should remain available")
.items_decommission_failed,
0,
"entry completion must not hide an exhausted copy failure"
);
assert!(!rx.is_cancelled(), "entry-level SourceChanged must not cancel the shared worker token"); assert!(!rx.is_cancelled(), "entry-level SourceChanged must not cancel the shared worker token");
assert_eq!(mutation_calls.load(Ordering::SeqCst), 2, "entry must be re-listed after SourceChanged"); assert_eq!(mutation_calls.load(Ordering::SeqCst), 2, "entry must be re-listed after SourceChanged");
assert_eq!(ordinary_faults.load(Ordering::SeqCst), 2, "ordinary copy must consume the retry budget"); assert_eq!(ordinary_faults.load(Ordering::SeqCst), 2, "ordinary copy must consume the retry budget");
@@ -5903,6 +6305,7 @@ mod tests {
"reverse-decommission-fixed-target", "reverse-decommission-fixed-target",
&[(1, 4), (1, 4)], &[(1, 4), (1, 4)],
CancellationToken::new(), CancellationToken::new(),
None,
)) ))
.await; .await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
@@ -6324,6 +6727,7 @@ mod tests {
"multi-set-decommission-source-cleanup", "multi-set-decommission-source-cleanup",
&[(2, 4)], &[(2, 4)],
CancellationToken::new(), CancellationToken::new(),
None,
)) ))
.await; .await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
@@ -8045,10 +8449,15 @@ mod tests {
); );
assert!(com::read_config(store.pools[0].clone(), &second_page_path).await.is_ok()); assert!(com::read_config(store.pools[0].clone(), &second_page_path).await.is_ok());
com::save_config(store.pools[target_pool_idx].clone(), &second_page_path, receipt_bytes.clone()) let full_tail = ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
..Default::default()
};
com::save_config_with_opts(store.pools[target_pool_idx].clone(), &second_page_path, receipt_bytes.clone(), &full_tail)
.await .await
.expect("second page receipt should restore"); .expect("second page receipt should restore");
com::save_config(store.pools[target_pool_idx].clone(), &second_page_path, b"{corrupt".to_vec()) com::save_config_with_opts(store.pools[target_pool_idx].clone(), &second_page_path, b"{corrupt".to_vec(), &full_tail)
.await .await
.expect("second page receipt should corrupt deterministically"); .expect("second page receipt should corrupt deterministically");
let corrupt = store let corrupt = store
@@ -8834,18 +9243,17 @@ mod tests {
const MANIFEST_COUNT: usize = 10; const MANIFEST_COUNT: usize = 10;
let temp_dir = tempfile::tempdir().expect("create fast manifest pass recovery store dir"); let temp_dir = tempfile::tempdir().expect("create fast manifest pass recovery store dir");
let (ctx, store, _shutdown) = let mut instance_ctx = crate::runtime::instance::InstanceContext::new();
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-delete-fast-manifest-pass", &[4])).await; instance_ctx.suppress_tier_delete_journal_recovery_for_test();
let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
temp_dir.path(),
"tier-delete-fast-manifest-pass",
&[(1, 4)],
CancellationToken::new(),
Some(Arc::new(instance_ctx)),
))
.await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = "tier-delete-fast-manifest-pass-bucket";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("fast manifest pass bucket should be created");
let incarnation = store
.bucket_incarnation_id(bucket)
.await
.expect("fast manifest pass bucket incarnation should resolve");
let tier_name = "FAST-MANIFEST-PASS"; let tier_name = "FAST-MANIFEST-PASS";
let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name) let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
@@ -8853,9 +9261,19 @@ mod tests {
.expect("fast manifest pass tier lease should resolve") .expect("fast manifest pass tier lease should resolve")
.backend_identity(); .backend_identity();
for index in 0..MANIFEST_COUNT { for index in 0..MANIFEST_COUNT {
// Pagination must not depend on same-bucket lock wait deadlines.
let bucket = format!("tier-delete-fast-manifest-pass-{index}");
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("fast manifest pass bucket should be created");
let incarnation = store
.bucket_incarnation_id(&bucket)
.await
.expect("fast manifest pass bucket incarnation should resolve");
install_aborting_dispatch_fixture( install_aborting_dispatch_fixture(
store.clone(), store.clone(),
bucket, &bucket,
incarnation, incarnation,
&format!("manifest-page-{index:06}/"), &format!("manifest-page-{index:06}/"),
tier_name, tier_name,
@@ -8886,12 +9304,78 @@ mod tests {
"one production pass must cross the default eight-manifest page limit" "one production pass must cross the default eight-manifest page limit"
); );
assert_eq!(stats.manifests.scanned, MANIFEST_COUNT); assert_eq!(stats.manifests.scanned, MANIFEST_COUNT);
assert_eq!(stats.manifests.deleted, MANIFEST_COUNT); assert_eq!(stats.manifests.deleted, MANIFEST_COUNT, "full recovery result: {stats:?}");
assert_eq!(stats.manifests.failed, 0); assert_eq!(stats.manifests.failed, 0, "full recovery result: {stats:?}");
assert_eq!(manifest_marker, None); assert_eq!(manifest_marker, None);
assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0); assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
assert_eq!(tier_delete_journal_count(store).await, 0); assert_eq!(tier_delete_journal_count(store).await, 0);
assert_eq!(backend.remove_count().await, 0, "rollback recovery must not call the remote tier"); assert_eq!(backend.remove_count().await, 0, "rollback recovery must not call the remote tier");
shutdown.cancel();
}
#[cfg(feature = "test-util")]
#[tokio::test]
#[serial_test::serial(storage_class_env)]
async fn tier_delete_manual_pass_retains_manifest_owned_by_startup_recovery() {
let temp_dir = tempfile::tempdir().expect("create automatic recovery ownership store dir");
let (ctx, store, shutdown) =
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-delete-auto-owner", &[4])).await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = "tier-delete-auto-owner-bucket";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("automatic recovery bucket should be created");
let incarnation = store.bucket_incarnation_id(bucket).await.expect("bucket incarnation");
let tier_name = "AUTO-OWNER";
let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
.await
.expect("automatic recovery tier lease")
.backend_identity();
// The automatic worker must not observe a partially installed fixture.
let lifecycle_guard = store
.acquire_bucket_lifecycle_write_lock(bucket)
.await
.expect("fixture lifecycle lock");
let (manifest_name, entries) =
install_aborting_dispatch_fixture(store.clone(), bucket, incarnation, "auto-owner/", tier_name, identity, 1).await;
let journal_name = tier_delete_journal_object_name(&entries[0]);
let hook = TierDeleteDispatchRollbackTestHook::install_slow_delete(&journal_name, &journal_name);
drop(lifecycle_guard);
ctx.wake_tier_delete_journal_recovery();
tokio::time::timeout(Duration::from_secs(30), hook.wait_until_delete_paused())
.await
.expect("startup recovery should own the manifest before a manual pass");
assert!(tier_delete_dispatch_manifest_recovery_inflight_for_test(&store, &manifest_name));
let stats = recover_tier_delete_dispatch_manifests(store.clone(), 8, None)
.await
.expect("manual recovery scan");
assert_eq!(stats.scanned, 1, "{stats:?}");
assert_eq!(stats.retained, 1, "{stats:?}");
assert_eq!(stats.deleted, 0, "{stats:?}");
assert_eq!(stats.failed, 0, "{stats:?}");
assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 1);
assert_eq!(tier_delete_journal_count(store.clone()).await, 1);
hook.release_delete();
tokio::time::timeout(Duration::from_secs(30), async {
loop {
let manifest_gone = matches!(com::read_config(store.clone(), &manifest_name).await, Err(Error::ConfigNotFound));
if manifest_gone && !tier_delete_dispatch_manifest_recovery_inflight_for_test(&store, &manifest_name) {
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("automatic recovery should converge without a manual retry");
assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
assert_eq!(tier_delete_journal_count(store).await, 0);
assert_eq!(backend.remove_count().await, 0, "rollback must not delete from the remote tier");
shutdown.cancel();
} }
#[cfg(feature = "test-util")] #[cfg(feature = "test-util")]
@@ -10266,8 +10750,17 @@ mod tests {
const JOURNAL_COUNT: usize = 40; const JOURNAL_COUNT: usize = 40;
let temp_dir = tempfile::tempdir().expect("create rollback retry store dir"); let temp_dir = tempfile::tempdir().expect("create rollback retry store dir");
let (ctx, store, _shutdown) = // Manual retries must own progress between fault removal and the next attempt.
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-rollback-retry", &[4])).await; let mut instance_ctx = crate::runtime::instance::InstanceContext::new();
instance_ctx.suppress_tier_delete_journal_recovery_for_test();
let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
temp_dir.path(),
"dispatch-rollback-retry",
&[(1, 4)],
CancellationToken::new(),
Some(Arc::new(instance_ctx)),
))
.await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = "dispatch-rollback-retry-bucket"; let bucket = "dispatch-rollback-retry-bucket";
store store
@@ -10341,6 +10834,7 @@ mod tests {
assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0); assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
assert_eq!(backend.remove_count().await, 0, "rollback retries must never call the remote tier"); assert_eq!(backend.remove_count().await, 0, "rollback retries must never call the remote tier");
shutdown.cancel();
} }
#[cfg(feature = "test-util")] #[cfg(feature = "test-util")]
@@ -11575,6 +12069,7 @@ mod tests {
pool_index: usize, pool_index: usize,
bucket: &str, bucket: &str,
object: &str, object: &str,
minio_unversioned: bool,
) { ) {
for disk_index in 0..4 { for disk_index in 0..4 {
let metadata_path = let metadata_path =
@@ -11608,6 +12103,11 @@ mod tests {
] { ] {
rustfs_utils::http::metadata_compat::remove_bytes(&mut object_meta.meta_sys, suffix); rustfs_utils::http::metadata_compat::remove_bytes(&mut object_meta.meta_sys, suffix);
} }
if minio_unversioned {
object_meta
.meta_sys
.insert("x-minio-internal-transitioned-versionID".to_string(), Vec::new());
}
*shallow = rustfs_filemeta::FileMetaShallowVersion::try_from(version) *shallow = rustfs_filemeta::FileMetaShallowVersion::try_from(version)
.expect("legacy transitioned version should re-encode"); .expect("legacy transitioned version should re-encode");
} }
@@ -11618,6 +12118,152 @@ mod tests {
} }
} }
#[cfg(feature = "test-util")]
async fn read_store_body(
store: &Arc<crate::store::ECStore>,
bucket: &str,
object: &str,
range: Option<HTTPRangeSpec>,
opts: &ObjectOptions,
) -> Vec<u8> {
let mut reader = store
.get_object_reader(bucket, object, range, HeaderMap::new(), opts)
.await
.expect("object reader should open");
let mut body = Vec::new();
reader.stream.read_to_end(&mut body).await.expect("object body should drain");
body
}
#[cfg(feature = "test-util")]
#[tokio::test]
#[serial_test::serial(storage_class_env)]
async fn legacy_unknown_unversioned_transition_supports_head_get_and_range_without_backfill() {
let temp_dir = tempfile::tempdir().expect("create legacy unknown unversioned store dir");
let (ctx, store, _shutdown) =
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-unknown-unversioned-read", &[4])).await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let tier_name = "LEGACY-UNKNOWN-UNVERSIONED-READ";
let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
backend.set_put_remote_version(Some(String::new())).await;
let bucket = "legacy-unknown-unversioned-read-bucket";
let object = "object.bin";
let payload = b"legacy unversioned remote tier object remains readable".repeat(1024);
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("legacy source bucket should be created");
let mut reader = PutObjReader::from_vec(payload.clone());
let source = store
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
.await
.expect("legacy source should be written");
store
.transition_object(
bucket,
object,
&ObjectOptions {
transition: TransitionOptions {
status: TRANSITION_PENDING.to_string(),
tier: tier_name.to_string(),
etag: source.etag.clone().expect("legacy source should have an etag"),
..Default::default()
},
mod_time: source.mod_time,
..Default::default()
},
)
.await
.expect("legacy source should transition");
rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 0, bucket, object, true).await;
backend.clear_op_log().await;
let opts = ObjectOptions {
metadata_cache_safe: false,
..Default::default()
};
let head = store
.get_object_info(bucket, object, &opts)
.await
.expect("legacy transitioned HEAD should use local metadata");
assert_eq!(head.transition_version_state, rustfs_filemeta::TransitionVersionState::Unknown);
assert!(head.transitioned_object.version_id.is_empty());
assert_eq!(
head.user_defined
.get("x-minio-internal-transitioned-versionID")
.map(String::as_str),
Some(""),
"the MinIO empty version-key provenance must survive xl.meta decoding"
);
assert!(
!rustfs_utils::http::metadata_compat::contains_key_str(
&head.user_defined,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
),
"the compatibility read must not synthesize version-state metadata"
);
let full_body = read_store_body(&store, bucket, object, None, &opts).await;
assert_eq!(full_body, payload);
let range = HTTPRangeSpec {
is_suffix_length: false,
start: 7,
end: 38,
};
let ranged_body = read_store_body(&store, bucket, object, Some(range), &opts).await;
assert_eq!(ranged_body, &payload[7..=38]);
let after_read = store.pools[0]
.get_disks_by_key(object)
.load_file_info_versions_exact(bucket, object)
.await
.expect("legacy metadata should remain readable after GET")
.expect("legacy object metadata should remain on disk")
.versions
.into_iter()
.find(|version| version.transition_status == rustfs_filemeta::TRANSITION_COMPLETE)
.expect("legacy transitioned source should remain visible after GET");
assert_eq!(after_read.transition_version_state, rustfs_filemeta::TransitionVersionState::Unknown);
assert!(after_read.transition_version.is_none());
assert!(after_read.transition_version_id.is_none());
assert_eq!(
after_read
.metadata
.get("x-minio-internal-transitioned-versionID")
.map(String::as_str),
Some(""),
"the MinIO empty version-key provenance must remain after GET and Range GET"
);
assert!(
!rustfs_utils::http::metadata_compat::contains_key_str(
&after_read.metadata,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
),
"the compatibility read must remain side-effect free"
);
assert_eq!(
backend.op_log().await,
vec![
MockWarmOp::Probe {
object: after_read.transitioned_objname.clone(),
},
MockWarmOp::Get {
object: after_read.transitioned_objname.clone(),
},
MockWarmOp::Probe {
object: after_read.transitioned_objname.clone(),
},
MockWarmOp::Get {
object: after_read.transitioned_objname,
},
],
"legacy reads should probe before each unversioned GET and never mutate local metadata"
);
assert_eq!(backend.remove_count().await, 0);
}
#[cfg(feature = "test-util")] #[cfg(feature = "test-util")]
#[tokio::test] #[tokio::test]
#[serial_test::serial(storage_class_env)] #[serial_test::serial(storage_class_env)]
@@ -11658,7 +12304,7 @@ mod tests {
) )
.await .await
.expect("legacy source should transition"); .expect("legacy source should transition");
rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 0, bucket, object).await; rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 0, bucket, object, false).await;
let legacy = store.pools[0] let legacy = store.pools[0]
.get_disks_by_key(object) .get_disks_by_key(object)
.load_file_info_versions_exact(bucket, object) .load_file_info_versions_exact(bucket, object)
@@ -12799,7 +13445,7 @@ mod tests {
.expect("merge-loser source should transition"); .expect("merge-loser source should transition");
copy_test_xlmeta_between_pools(temp_dir.path(), 0, 1, bucket, object).await; copy_test_xlmeta_between_pools(temp_dir.path(), 0, 1, bucket, object).await;
} }
rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 1, bucket, "legacy/item.bin").await; rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 1, bucket, "legacy/item.bin", false).await;
backend.set_remove_failure(true); backend.set_remove_failure(true);
store.pools[1] store.pools[1]
.delete_object(bucket, "hidden/item.bin", ObjectOptions::default()) .delete_object(bucket, "hidden/item.bin", ObjectOptions::default())
@@ -13016,6 +13662,7 @@ mod tests {
"partial-set-prefix-delete", "partial-set-prefix-delete",
&[(2, 4)], &[(2, 4)],
CancellationToken::new(), CancellationToken::new(),
None,
)) ))
.await; .await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
@@ -16388,6 +17035,7 @@ mod tests {
"prepared-directory-recovery", "prepared-directory-recovery",
&[(2, 4)], &[(2, 4)],
shutdown, shutdown,
None,
)) ))
.await; .await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
@@ -16866,6 +17514,10 @@ mod tests {
.find(|version| version.version_id == history.version_id) .find(|version| version.version_id == history.version_id)
.expect("transitioned history should exist"); .expect("transitioned history should exist");
transitioned.transition_version_state = rustfs_filemeta::TransitionVersionState::Unknown; transitioned.transition_version_state = rustfs_filemeta::TransitionVersionState::Unknown;
rustfs_utils::http::metadata_compat::remove_str(
&mut transitioned.metadata,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
);
metadata metadata
.add_version(transitioned) .add_version(transitioned)
.expect("unknown state should replace the transitioned version"); .expect("unknown state should replace the transitioned version");
@@ -17036,6 +17688,38 @@ mod tests {
.expect("test thread should complete"); .expect("test thread should complete");
} }
#[cfg(feature = "test-util")]
#[tokio::test]
#[serial_test::serial(storage_class_env)]
async fn odm_write_back_requires_one_set_and_enabled_namespace_locking() {
for (layout, locking, supported) in [
(&[(1, 4)][..], true, true),
(&[(1, 4), (1, 4)][..], true, false),
(&[(2, 4)][..], true, false),
(&[(1, 4)][..], false, false),
] {
temp_env::async_with_vars([("RUSTFS_LOCK_ENABLED", Some(if locking { "true" } else { "false" }))], async {
let dir = tempfile::tempdir().expect("isolated topology");
let shutdown = CancellationToken::new();
let (_ctx, store, _) = without_storage_class_env(build_isolated_test_store_with_layout(
dir.path(),
"odm-topology",
layout,
shutdown.clone(),
None,
))
.await;
assert_eq!(
store.supports_atomic_create_only_write_back(),
supported,
"layout={layout:?}, locking={locking}"
);
shutdown.cancel();
})
.await;
}
}
#[cfg(feature = "test-util")] #[cfg(feature = "test-util")]
#[tokio::test] #[tokio::test]
#[serial_test::serial(storage_class_env)] #[serial_test::serial(storage_class_env)]
+9 -8
View File
@@ -425,7 +425,7 @@ pub(crate) mod init_format;
pub(crate) mod list_objects; pub(crate) mod list_objects;
mod multipart; mod multipart;
mod object; mod object;
#[cfg(any(test, feature = "test-util"))] #[cfg(feature = "test-util")]
pub use object::DeleteAfterObjectLockSnapshotBarrier; pub use object::DeleteAfterObjectLockSnapshotBarrier;
pub(crate) use object::{ pub(crate) use object::{
DecommissionFixedReadAnchor, ObjectLockDiagGuard, RemoteTuplePublicationCommitGuard, RemoteTuplePublicationFence, DecommissionFixedReadAnchor, ObjectLockDiagGuard, RemoteTuplePublicationCommitGuard, RemoteTuplePublicationFence,
@@ -848,7 +848,7 @@ impl ECStore {
} }
pub fn scanner_namespace_mutation_generation(&self) -> u64 { pub fn scanner_namespace_mutation_generation(&self) -> u64 {
list_objects::scanner_namespace_mutation_generation() list_objects::scanner_namespace_mutation_generation().saturating_add(self.ctx.namespace_commit_generation())
} }
pub async fn scanner_data_movement_active(&self) -> bool { pub async fn scanner_data_movement_active(&self) -> bool {
@@ -857,7 +857,7 @@ impl ECStore {
} }
/// Return the storage-owned movement state and generation as one /// Return the storage-owned movement state and generation as one
/// authenticated activity snapshot. The read lock is acquired before /// authenticated activity snapshot. The read lock is acquired before
/// the state locks (cancelers, pool metadata, then rebalance metadata), /// the state locks (cancelers, pool metadata, then rebalance metadata),
/// matching the transition writer order and preventing a terminal state /// matching the transition writer order and preventing a terminal state
/// from being reported with the preceding generation. /// from being reported with the preceding generation.
@@ -886,11 +886,12 @@ impl ECStore {
/// Returns whether scanner metadata may still be hidden by a local /// Returns whether scanner metadata may still be hidden by a local
/// data-movement state. Terminal failed/canceled decommission entries /// data-movement state. Terminal failed/canceled decommission entries
/// remain suspended until an operator clears or retries them, so they are /// remain suspended until an operator clears or retries them, so they are
/// a publication barrier even after the worker has stopped. /// a publication barrier even after the worker has stopped. Active PUT
/// rename fanouts also defer publication, including post-ACK tails.
pub async fn scanner_data_usage_publication_blocked(&self) -> bool { pub async fn scanner_data_usage_publication_blocked(&self) -> bool {
let operation_gate = self.ctx.data_movement_operation_gate(); let operation_gate = self.ctx.data_movement_operation_gate();
let _operation_guard = operation_gate.read_owned().await; let _operation_guard = operation_gate.read_owned().await;
self.scanner_data_usage_publication_snapshot_blocked().await self.scanner_data_usage_publication_snapshot_blocked().await || self.ctx.namespace_commits_pending()
} }
pub async fn scanner_data_movement_pause_status(&self) -> ScannerDataMovementPauseStatus { pub async fn scanner_data_movement_pause_status(&self) -> ScannerDataMovementPauseStatus {
@@ -1070,7 +1071,7 @@ impl ECStore {
{ {
return Err(Error::other("scanner publication lease generation is stale")); return Err(Error::other("scanner publication lease generation is stale"));
} }
if self.scanner_data_movement_snapshot_locked().await.1 { if self.scanner_data_movement_snapshot_locked().await.1 || self.ctx.namespace_commits_pending() {
return Err(Error::other("scanner publication lease is blocked by data movement")); return Err(Error::other("scanner publication lease is blocked by data movement"));
} }
@@ -1109,7 +1110,7 @@ impl ECStore {
{ {
return Err(Error::other("scanner publication lease generation is stale")); return Err(Error::other("scanner publication lease generation is stale"));
} }
if self.scanner_data_movement_snapshot_locked().await.1 { if self.scanner_data_movement_snapshot_locked().await.1 || self.ctx.namespace_commits_pending() {
return Err(Error::other("scanner publication lease is blocked by data movement")); return Err(Error::other("scanner publication lease is blocked by data movement"));
} }
if !self.ctx.scanner_publication_lease_is_active(token).await { if !self.ctx.scanner_publication_lease_is_active(token).await {
@@ -1129,7 +1130,7 @@ impl ECStore {
if self.ctx.data_movement_generation_exhausted() || self.ctx.data_movement_operation_epoch_exhausted() { if self.ctx.data_movement_generation_exhausted() || self.ctx.data_movement_operation_epoch_exhausted() {
return Err(Error::other("scanner publication lease generation is exhausted")); return Err(Error::other("scanner publication lease generation is exhausted"));
} }
if self.scanner_data_movement_snapshot_locked().await.1 { if self.scanner_data_movement_snapshot_locked().await.1 || self.ctx.namespace_commits_pending() {
return Err(Error::other("scanner publication lease is blocked by data movement")); return Err(Error::other("scanner publication lease is blocked by data movement"));
} }
let Some(lease_generation) = self.ctx.scanner_publication_lease_generation(token).await else { let Some(lease_generation) = self.ctx.scanner_publication_lease_generation(token).await else {
+115 -10
View File
@@ -297,6 +297,20 @@ fn transitioned_version_from_bytes(value: Option<&[u8]>, state: TransitionVersio
} }
} }
fn transition_version_metadata_value(raw: &[u8], decoded: Option<&str>) -> String {
decoded.map(str::to_owned).unwrap_or_else(|| {
if raw.is_empty() {
String::new()
} else {
String::from_utf8_lossy(raw).into_owned()
}
})
}
fn is_transition_version_metadata_key(key: &str) -> bool {
strip_internal_prefix_preserving_case(key).is_some_and(|suffix| suffix.eq_ignore_ascii_case(SUFFIX_TRANSITIONED_VERSION_ID))
}
fn validate_transition_version_state(state: TransitionVersionState, version: Option<&str>) -> Result<()> { fn validate_transition_version_state(state: TransitionVersionState, version: Option<&str>) -> Result<()> {
let valid = match state { let valid = match state {
TransitionVersionState::Unknown | TransitionVersionState::KnownDisabled => version.is_none(), TransitionVersionState::Unknown | TransitionVersionState::KnownDisabled => version.is_none(),
@@ -366,14 +380,26 @@ impl<'a> DerivedInternalMetadata<'a> {
} }
*slot = Some(value.as_slice()); *slot = Some(value.as_slice());
} }
fn merge_consistent<'a>(canonical: Option<&'a [u8]>, legacy: Option<&'a [u8]>) -> Result<Option<&'a [u8]>> {
if let (Some(canonical), Some(legacy)) = (canonical, legacy)
&& canonical != legacy
{
return Err(Error::FileCorrupt);
}
Ok(canonical.or(legacy))
}
Ok(Self { Ok(Self {
checksum: canonical.checksum.or(legacy.checksum), checksum: canonical.checksum.or(legacy.checksum),
part_checksums: canonical.part_checksums.or(legacy.part_checksums), part_checksums: canonical.part_checksums.or(legacy.part_checksums),
transition_status: canonical.transition_status.or(legacy.transition_status), transition_status: merge_consistent(canonical.transition_status, legacy.transition_status)?,
transitioned_object: canonical.transitioned_object.or(legacy.transitioned_object), transitioned_object: merge_consistent(canonical.transitioned_object, legacy.transitioned_object)?,
transitioned_version: canonical.transitioned_version.or(legacy.transitioned_version), transitioned_version: merge_consistent(canonical.transitioned_version, legacy.transitioned_version)?,
transitioned_version_state: canonical.transitioned_version_state.or(legacy.transitioned_version_state), transitioned_version_state: merge_consistent(
transition_tier: canonical.transition_tier.or(legacy.transition_tier), canonical.transitioned_version_state,
legacy.transitioned_version_state,
)?,
transition_tier: merge_consistent(canonical.transition_tier, legacy.transition_tier)?,
}) })
} }
} }
@@ -438,8 +464,14 @@ impl FileInfo {
} }
} }
fn set_transition_version_state(meta_sys: &mut HashMap<String, Vec<u8>>, state: TransitionVersionState) { fn set_transition_version_state(
if state == TransitionVersionState::Unknown { meta_sys: &mut HashMap<String, Vec<u8>>,
state: TransitionVersionState,
source_metadata: &HashMap<String, String>,
) {
if state == TransitionVersionState::Unknown
&& !rustfs_utils::http::metadata_compat::contains_key_str(source_metadata, SUFFIX_TRANSITIONED_VERSION_STATE)
{
remove_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE); remove_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE);
} else { } else {
insert_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE, state.as_str().as_bytes().to_vec()); insert_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE, state.as_str().as_bytes().to_vec());
@@ -2643,6 +2675,11 @@ impl MetaObject {
if derived_metadata.transitioned_version_state.is_some() { if derived_metadata.transitioned_version_state.is_some() {
validate_transition_version_state(transition_version_state, transition_version.as_deref())?; validate_transition_version_state(transition_version_state, transition_version.as_deref())?;
} }
for (key, value) in &self.meta_sys {
if is_transition_version_metadata_key(key) {
metadata.insert(key.to_owned(), transition_version_metadata_value(value, transition_version.as_deref()));
}
}
let transition_version_id = transition_version.as_deref().and_then(|value| Uuid::parse_str(value).ok()); let transition_version_id = transition_version.as_deref().and_then(|value| Uuid::parse_str(value).ok());
let transition_tier = derived_metadata let transition_tier = derived_metadata
.transition_tier .transition_tier
@@ -2689,7 +2726,7 @@ impl MetaObject {
} else { } else {
remove_bytes(&mut self.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID); remove_bytes(&mut self.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID);
} }
set_transition_version_state(&mut self.meta_sys, fi.transition_version_state); set_transition_version_state(&mut self.meta_sys, fi.transition_version_state, &fi.metadata);
insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITION_TIER, fi.transition_tier.as_bytes().to_vec()); insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITION_TIER, fi.transition_tier.as_bytes().to_vec());
if let Some(destination_id) = get_str(&fi.metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID) { if let Some(destination_id) = get_str(&fi.metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID) {
insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID, destination_id.into_bytes()); insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID, destination_id.into_bytes());
@@ -2830,7 +2867,7 @@ impl From<FileInfo> for MetaObject {
insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, transition_version); insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, transition_version);
} }
if !value.transition_status.is_empty() { if !value.transition_status.is_empty() {
set_transition_version_state(&mut meta_sys, value.transition_version_state); set_transition_version_state(&mut meta_sys, value.transition_version_state, &value.metadata);
} }
if !value.transition_tier.is_empty() { if !value.transition_tier.is_empty() {
@@ -2985,6 +3022,12 @@ impl MetaDeleteMarker {
fi.transition_version_state = transition_version_state_from_bytes(derived_metadata.transitioned_version_state)?; fi.transition_version_state = transition_version_state_from_bytes(derived_metadata.transitioned_version_state)?;
fi.transition_version = fi.transition_version =
transitioned_version_from_bytes(derived_metadata.transitioned_version, fi.transition_version_state); transitioned_version_from_bytes(derived_metadata.transitioned_version, fi.transition_version_state);
for (key, value) in &self.meta_sys {
if is_transition_version_metadata_key(key) {
fi.metadata
.insert(key.to_owned(), transition_version_metadata_value(value, fi.transition_version.as_deref()));
}
}
fi.transition_version_id = fi.transition_version.as_deref().and_then(|value| Uuid::parse_str(value).ok()); fi.transition_version_id = fi.transition_version.as_deref().and_then(|value| Uuid::parse_str(value).ok());
if derived_metadata.transitioned_version_state.is_some() { if derived_metadata.transitioned_version_state.is_some() {
validate_transition_version_state(fi.transition_version_state, fi.transition_version.as_deref())?; validate_transition_version_state(fi.transition_version_state, fi.transition_version.as_deref())?;
@@ -3152,7 +3195,7 @@ impl From<FileInfo> for MetaDeleteMarker {
insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, transition_version); insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, transition_version);
} }
if !value.transition_status.is_empty() || value.tier_free_version() { if !value.transition_status.is_empty() || value.tier_free_version() {
set_transition_version_state(&mut meta_sys, value.transition_version_state); set_transition_version_state(&mut meta_sys, value.transition_version_state, &value.metadata);
} }
if !value.transition_tier.is_empty() { if !value.transition_tier.is_empty() {
insert_bytes(&mut meta_sys, SUFFIX_TRANSITION_TIER, value.transition_tier.as_bytes().to_vec()); insert_bytes(&mut meta_sys, SUFFIX_TRANSITION_TIER, value.transition_tier.as_bytes().to_vec());
@@ -4574,6 +4617,7 @@ mod tests {
.into_fileinfo("b", "k", false) .into_fileinfo("b", "k", false)
.expect("into_fileinfo"); .expect("into_fileinfo");
assert_eq!(fi.transition_version_id, None); assert_eq!(fi.transition_version_id, None);
assert_eq!(get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID), Some(String::new()));
} }
#[test] #[test]
@@ -4585,6 +4629,10 @@ mod tests {
.into_fileinfo("b", "k", false) .into_fileinfo("b", "k", false)
.expect("into_fileinfo"); .expect("into_fileinfo");
assert_eq!(fi.transition_version_id, None); assert_eq!(fi.transition_version_id, None);
assert!(
get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID).is_some_and(|value| !value.is_empty()),
"nil UUID bytes must remain distinguishable from an empty MinIO version"
);
} }
#[test] #[test]
@@ -4598,6 +4646,7 @@ mod tests {
assert_eq!(fi.transition_version_id, Some(id)); assert_eq!(fi.transition_version_id, Some(id));
assert_eq!(fi.transition_version, Some(id.to_string())); assert_eq!(fi.transition_version, Some(id.to_string()));
assert_eq!(fi.transition_version_state, TransitionVersionState::Unknown); assert_eq!(fi.transition_version_state, TransitionVersionState::Unknown);
assert_eq!(get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID), Some(id.to_string()));
} }
#[test] #[test]
@@ -4637,6 +4686,36 @@ mod tests {
assert_eq!(fi.transition_version_state, TransitionVersionState::Unknown); assert_eq!(fi.transition_version_state, TransitionVersionState::Unknown);
} }
#[test]
fn meta_object_transition_version_state_explicit_unknown_is_not_legacy_missing() {
let mut metadata = HashMap::new();
rustfs_utils::http::metadata_compat::insert_str(
&mut metadata,
SUFFIX_TRANSITIONED_VERSION_STATE,
TransitionVersionState::Unknown.as_str().to_string(),
);
let fi = FileInfo {
transition_status: "complete".to_string(),
transition_version_state: TransitionVersionState::Unknown,
metadata,
..Default::default()
};
let object = MetaObject::from(fi);
assert_eq!(
get_consistent_bytes(&object.meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE),
Some(b"unknown".as_slice())
);
let decoded = object
.into_fileinfo("b", "k", false)
.expect("explicit unknown state should decode");
assert_eq!(decoded.transition_version_state, TransitionVersionState::Unknown);
assert_eq!(
rustfs_utils::http::metadata_compat::get_consistent_str(&decoded.metadata, SUFFIX_TRANSITIONED_VERSION_STATE,),
Some("unknown")
);
}
#[test] #[test]
fn meta_object_transition_version_state_exact_round_trips_dual_keys() { fn meta_object_transition_version_state_exact_round_trips_dual_keys() {
let id = sample_version_id(); let id = sample_version_id();
@@ -4753,6 +4832,10 @@ mod tests {
.expect("invalid transition version bytes must not fail the object read"); .expect("invalid transition version bytes must not fail the object read");
assert_eq!(fi.transition_version_id, None); assert_eq!(fi.transition_version_id, None);
assert_eq!(fi.transition_version, None); assert_eq!(fi.transition_version, None);
assert!(
get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID).is_some_and(|value| !value.is_empty()),
"invalid raw bytes must remain distinguishable from an empty MinIO version"
);
} }
#[test] #[test]
@@ -4795,6 +4878,10 @@ mod tests {
.into_fileinfo("b", "k", false) .into_fileinfo("b", "k", false)
.expect("nil tier version should remain an absent remote version"); .expect("nil tier version should remain an absent remote version");
assert_eq!(fi.transition_version_id, None); assert_eq!(fi.transition_version_id, None);
assert!(
get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID).is_some_and(|value| !value.is_empty()),
"nil UUID bytes must remain distinguishable from an empty MinIO version"
);
} }
#[test] #[test]
@@ -4812,6 +4899,7 @@ mod tests {
.expect("legacy binary UUID tier version should decode"); .expect("legacy binary UUID tier version should decode");
assert_eq!(fi.transition_version_id, Some(id)); assert_eq!(fi.transition_version_id, Some(id));
assert_eq!(fi.transition_version, Some(id.to_string())); assert_eq!(fi.transition_version, Some(id.to_string()));
assert_eq!(get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID), Some(id.to_string()));
} }
#[test] #[test]
@@ -4910,6 +4998,23 @@ mod tests {
assert_eq!(err, Error::FileCorrupt); assert_eq!(err, Error::FileCorrupt);
} }
#[test]
fn meta_object_transition_version_state_mixed_case_alias_conflict_fails_closed() {
let sys = HashMap::from([
(
format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_TRANSITIONED_VERSION_STATE}"),
b"unknown".to_vec(),
),
("X-Minio-Internal-transitioned-version-state".to_string(), b"exact".to_vec()),
]);
let err = make_meta_object_with_sys(sys)
.into_fileinfo("b", "k", false)
.expect_err("mixed-case transition state aliases must agree");
assert_eq!(err, Error::FileCorrupt);
}
#[test] #[test]
fn version_header_sorts_before_prefers_object_over_delete_marker_on_equal_mod_time() { fn version_header_sorts_before_prefers_object_over_delete_marker_on_equal_mod_time() {
let object = FileMetaVersionHeader { let object = FileMetaVersionHeader {
+60 -12
View File
@@ -45,6 +45,11 @@ use tracing::{debug, error, info, warn};
use super::{DiskError, Endpoint, HealDiskExt as _, local_disk_map_read}; use super::{DiskError, Endpoint, HealDiskExt as _, local_disk_map_read};
const KEEP_HEAL_TASK_STATUS_DURATION: Duration = Duration::from_secs(10 * 60); const KEEP_HEAL_TASK_STATUS_DURATION: Duration = Duration::from_secs(10 * 60);
// Each cache includes alias tokens in its count and byte budget. Eviction
// removes every token sharing a snapshot; neither cache retains repair state.
const MAX_COMPLETED_HEAL_TOKENS: usize = 1024;
const MAX_COMPLETED_HEAL_BYTES: usize = 64 * 1024 * 1024;
const MAX_COMPLETED_HEAL_RESULT_BYTES: usize = 1024 * 1024;
const DISPLACED_HEAL_REASON: &str = "reason=displaced; retry_hint=submit_again"; const DISPLACED_HEAL_REASON: &str = "reason=displaced; retry_hint=submit_again";
const LOG_COMPONENT_HEAL: &str = "heal"; const LOG_COMPONENT_HEAL: &str = "heal";
const LOG_SUBSYSTEM_DISK_SCANNER: &str = "disk_scanner"; const LOG_SUBSYSTEM_DISK_SCANNER: &str = "disk_scanner";
@@ -180,6 +185,8 @@ fn record_displaced_terminal(
request: &HealRequest, request: &HealRequest,
) -> Arc<CompletedHealStatus> { ) -> Arc<CompletedHealStatus> {
let terminal = Arc::new(CompletedHealStatus { let terminal = Arc::new(CompletedHealStatus {
progress: None,
retained_bytes: std::sync::OnceLock::new(),
heal_type: request.heal_type.clone(), heal_type: request.heal_type.clone(),
status: HealTaskStatus::Failed { status: HealTaskStatus::Failed {
error: format!("heal task displaced by a higher-priority request ({DISPLACED_HEAL_REASON})"), error: format!("heal task displaced by a higher-priority request ({DISPLACED_HEAL_REASON})"),
@@ -193,6 +200,7 @@ fn record_displaced_terminal(
let mut terminals = lock_displaced_terminals(registry); let mut terminals = lock_displaced_terminals(registry);
prune_completed_heal_statuses(&mut terminals); prune_completed_heal_statuses(&mut terminals);
terminals.insert(request.id.clone(), Arc::clone(&terminal)); terminals.insert(request.id.clone(), Arc::clone(&terminal));
prune_completed_heal_statuses(&mut terminals);
terminal terminal
} }
@@ -209,9 +217,15 @@ async fn remove_displaced_task_aliases(
.collect::<Vec<_>>(); .collect::<Vec<_>>();
let mut displaced_terminals = lock_displaced_terminals(terminals); let mut displaced_terminals = lock_displaced_terminals(terminals);
prune_completed_heal_statuses(&mut displaced_terminals); prune_completed_heal_statuses(&mut displaced_terminals);
for alias_id in alias_ids { if displaced_terminals
displaced_terminals.insert(alias_id, Arc::clone(terminal)); .get(task_id)
.is_some_and(|current| Arc::ptr_eq(current, terminal))
{
for alias_id in alias_ids {
displaced_terminals.insert(alias_id, Arc::clone(terminal));
}
} }
prune_completed_heal_statuses(&mut displaced_terminals);
aliases.retain(|alias_id, alias| alias_id != task_id && alias.task_id != task_id); aliases.retain(|alias_id, alias| alias_id != task_id && alias.task_id != task_id);
} }
@@ -222,6 +236,36 @@ async fn remove_task_aliases_for_task(registry: &Arc<Mutex<HashMap<String, HealT
.retain(|alias_id, alias| alias_id != task_id && alias.task_id != task_id); .retain(|alias_id, alias| alias_id != task_id && alias.task_id != task_id);
} }
// Callers hold active ownership until publication. Lock order is active ->
// retrying (when needed) -> aliases -> completed; queries release aliases
// before looking up active state. Publishing aliases before removing their
// mapping keeps both an already-resolved token and a new lookup valid.
async fn publish_completed_heal(
completed_heals: &Mutex<HashMap<String, Arc<CompletedHealStatus>>>,
task_aliases: &Mutex<HashMap<String, HealTaskAlias>>,
task_id: &str,
completed: CompletedHealStatus,
terminal: bool,
) {
let completed = Arc::new(completed);
completed.retained_bytes();
let mut aliases = task_aliases.lock().await;
let mut retained = completed_heals.lock().await;
if let Some(previous) = retained.get(task_id).cloned() {
for entry in retained.values_mut().filter(|entry| Arc::ptr_eq(entry, &previous)) {
*entry = Arc::clone(&completed);
}
}
retained.insert(task_id.to_owned(), Arc::clone(&completed));
if terminal {
for (alias_id, _) in aliases.iter().filter(|(_, alias)| alias.task_id == task_id) {
retained.insert(alias_id.clone(), Arc::clone(&completed));
}
aliases.retain(|alias_id, alias| alias_id != task_id && alias.task_id != task_id);
}
prune_completed_heal_statuses(&mut retained);
}
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct HealTaskReport { pub struct HealTaskReport {
pub status: HealTaskStatus, pub status: HealTaskStatus,
@@ -268,7 +312,7 @@ fn completed_task_report(completed: &CompletedHealStatus, since: Option<u64>) ->
let result_items = match since { let result_items = match since {
None => completed.seqed_items.iter().map(|(_, item)| item.clone()).collect(), None => completed.seqed_items.iter().map(|(_, item)| item.clone()).collect(),
Some(cursor) => { Some(cursor) => {
if cursor + 1 < completed.min_seq { if cursor.saturating_add(1) < completed.min_seq {
lagged = true; lagged = true;
} }
completed completed
@@ -283,7 +327,7 @@ fn completed_task_report(completed: &CompletedHealStatus, since: Option<u64>) ->
status: completed.status.clone(), status: completed.status.clone(),
result_items, result_items,
result_items_truncated: completed.result_items_truncated || lagged, result_items_truncated: completed.result_items_truncated || lagged,
progress: None, progress: completed.progress.clone(),
next_seq: completed.next_seq, next_seq: completed.next_seq,
min_seq: completed.min_seq, min_seq: completed.min_seq,
} }
@@ -1847,14 +1891,14 @@ impl HealManager {
pub async fn get_task_progress(&self, task_id: &str) -> Result<HealProgress> { pub async fn get_task_progress(&self, task_id: &str) -> Result<HealProgress> {
let canonical_task_id = self.canonical_task_id(task_id).await; let canonical_task_id = self.canonical_task_id(task_id).await;
let active_heals = self.active_heals.lock().await; let progress = match self.lookup_task_state(&canonical_task_id, None).await {
if let Some(task) = active_heals.get(&canonical_task_id) { TaskStateLookup::Active(task) => Some(task.get_progress().await),
Ok(task.get_progress().await) TaskStateLookup::Completed(completed) => completed.progress.clone(),
} else { _ => None,
Err(Error::TaskNotFound { };
task_id: task_id.to_string(), progress.ok_or_else(|| Error::TaskNotFound {
}) task_id: task_id.to_string(),
} })
} }
/// Cancel task /// Cancel task
@@ -1864,6 +1908,8 @@ impl HealManager {
let mut active_heals = self.active_heals.lock().await; let mut active_heals = self.active_heals.lock().await;
if let Some(task) = active_heals.get(&canonical_task_id) { if let Some(task) = active_heals.get(&canonical_task_id) {
task.cancel().await?; task.cancel().await?;
let completed = CompletedHealStatus::snapshot(task, HealTaskStatus::Cancelled).await;
publish_completed_heal(&self.completed_heals, &self.task_aliases, &canonical_task_id, completed, true).await;
active_heals.remove(&canonical_task_id); active_heals.remove(&canonical_task_id);
publish_active_heal_count(&active_heals); publish_active_heal_count(&active_heals);
info!( info!(
@@ -1940,6 +1986,8 @@ impl HealManager {
for task_id in &task_ids { for task_id in &task_ids {
if let Some(task) = active_heals.get(task_id) { if let Some(task) = active_heals.get(task_id) {
task.cancel().await?; task.cancel().await?;
let completed = CompletedHealStatus::snapshot(task, HealTaskStatus::Cancelled).await;
publish_completed_heal(&self.completed_heals, &self.task_aliases, task_id, completed, true).await;
} }
active_heals.remove(task_id); active_heals.remove(task_id);
cancelled += 1; cancelled += 1;
+129
View File
@@ -82,6 +82,8 @@ pub(super) enum QueuePushOutcome {
pub(super) struct CompletedHealStatus { pub(super) struct CompletedHealStatus {
pub(super) heal_type: HealType, pub(super) heal_type: HealType,
pub(super) status: HealTaskStatus, pub(super) status: HealTaskStatus,
pub(super) progress: Option<HealProgress>,
pub(super) retained_bytes: std::sync::OnceLock<usize>,
pub(super) result_items_truncated: bool, pub(super) result_items_truncated: bool,
pub(super) completed_at: SystemTime, pub(super) completed_at: SystemTime,
/// Sequence-stamped retained window, archived with the completion so /// Sequence-stamped retained window, archived with the completion so
@@ -92,6 +94,133 @@ pub(super) struct CompletedHealStatus {
pub(super) min_seq: u64, pub(super) min_seq: u64,
} }
impl CompletedHealStatus {
// Account for owned capacities, including nested drive arrays. Aliases
// conservatively charge the shared allocation again, keeping both token
// count and retained payload bounded without a second ownership index.
pub(super) fn retained_bytes(&self) -> usize {
*self.retained_bytes.get_or_init(|| self.measure_retained_bytes())
}
fn measure_retained_bytes(&self) -> usize {
let mut bytes = size_of::<Self>();
let mut add = |amount: usize| bytes = bytes.saturating_add(amount);
match &self.heal_type {
HealType::Cluster => {}
HealType::Bucket { bucket } => add(bucket.capacity()),
HealType::Object {
bucket,
object,
version_id,
}
| HealType::ECDecode {
bucket,
object,
version_id,
} => {
add(bucket.capacity());
add(object.capacity());
add(version_id.as_ref().map_or(0, String::capacity));
}
HealType::Prefix { bucket, prefix } => {
add(bucket.capacity());
add(prefix.capacity());
}
HealType::Metadata { bucket, object } => {
add(bucket.capacity());
add(object.capacity());
}
HealType::ErasureSet { buckets, set_disk_id } => {
add(buckets.capacity().saturating_mul(size_of::<String>()));
for bucket in buckets {
add(bucket.capacity());
}
add(set_disk_id.capacity());
}
}
if let HealTaskStatus::Failed { error } | HealTaskStatus::Retrying { error, .. } = &self.status {
add(error.capacity());
}
add(self
.progress
.as_ref()
.and_then(|progress| progress.current_object.as_ref())
.map_or(0, String::capacity));
add(self.seqed_items.capacity().saturating_mul(size_of::<(u64, HealResultItem)>()));
for (_, item) in &self.seqed_items {
add(Self::result_item_heap_bytes(item));
}
bytes
}
fn result_item_heap_bytes(item: &HealResultItem) -> usize {
let mut bytes = 0usize;
let mut add = |amount: usize| bytes = bytes.saturating_add(amount);
for value in [
&item.heal_item_type,
&item.bucket,
&item.object,
&item.version_id,
&item.detail,
] {
add(value.capacity());
}
for infos in [&item.before, &item.after] {
add(infos
.drives
.capacity()
.saturating_mul(size_of::<rustfs_madmin::heal_commands::HealDriveInfo>()));
for drive in &infos.drives {
add(drive.uuid.capacity());
add(drive.endpoint.capacity());
add(drive.state.capacity());
}
}
bytes
}
pub(super) fn bound_result_window(&mut self) {
let mut bytes = 0usize;
let retained = self
.seqed_items
.iter()
.rev()
.take_while(|(_, item)| {
bytes = bytes
.saturating_add(size_of::<(u64, HealResultItem)>())
.saturating_add(Self::result_item_heap_bytes(item));
bytes <= MAX_COMPLETED_HEAL_RESULT_BYTES
})
.count();
let truncated = retained < self.seqed_items.len();
if truncated {
self.seqed_items.drain(..self.seqed_items.len() - retained);
self.seqed_items.shrink_to_fit();
self.min_seq = self.seqed_items.first().map_or(self.next_seq, |(seq, _)| *seq);
self.result_items_truncated = true;
self.retained_bytes.take();
}
}
pub(super) async fn snapshot(task: &HealTask, status: HealTaskStatus) -> Self {
let seqed_items = task.get_seqed_result_items().await;
let (next_seq, min_seq) = task.result_seq_cursors();
let mut snapshot = Self {
heal_type: task.heal_type.clone(),
status,
progress: Some(task.get_progress().await),
retained_bytes: std::sync::OnceLock::new(),
result_items_truncated: task.result_items_truncated(),
completed_at: SystemTime::now(),
seqed_items,
next_seq,
min_seq,
};
snapshot.bound_result_window();
snapshot
}
}
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub(super) struct HealTaskAlias { pub(super) struct HealTaskAlias {
pub(super) task_id: String, pub(super) task_id: String,
+75 -39
View File
@@ -264,7 +264,7 @@ impl HealManager {
error: error.clone(), error: error.clone(),
retry_attempt: request.retry_attempts, retry_attempt: request.retry_attempts,
}); });
let retry_request_for_queue = retry_request; let mut retry_request_for_queue = retry_request;
let retry_cancel_token = retry_request_for_queue.as_ref().map(|_| CancellationToken::new()); let retry_cancel_token = retry_request_for_queue.as_ref().map(|_| CancellationToken::new());
if retry_request_for_queue.is_none() { if retry_request_for_queue.is_none() {
replacement_recovery_anchors_clone replacement_recovery_anchors_clone
@@ -272,7 +272,35 @@ impl HealManager {
.unwrap_or_else(|poisoned| poisoned.into_inner()) .unwrap_or_else(|poisoned| poisoned.into_inner())
.remove(&task_id); .remove(&task_id);
} }
let mut completed_status = match retry_request_for_status {
Some(status) => status,
None => task.get_status().await,
};
let mut completed_status_entry = CompletedHealStatus::snapshot(&task, completed_status.clone()).await;
let completed_progress = task.get_progress().await;
#[cfg(test)]
tests::pause_completed_retention_before_publish(&task_id, &completed_status).await;
let mut active_heals_guard = active_heals_clone.lock().await; let mut active_heals_guard = active_heals_clone.lock().await;
let owns_completion = active_heals_guard.contains_key(&task_id);
let cancelled_completion = if owns_completion {
false
} else {
// Cancellation can win while a finished worker waits
// for active ownership. It must not resurrect a retry
// or replace an acknowledged cancellation with success.
retry_request_for_queue = None;
completed_heals_clone
.lock()
.await
.get(&task_id)
.is_some_and(|completed| completed.status == HealTaskStatus::Cancelled)
};
if cancelled_completion {
completed_status = HealTaskStatus::Cancelled;
completed_status_entry.status = HealTaskStatus::Cancelled;
}
let terminal_completion = !matches!(completed_status, HealTaskStatus::Retrying { .. });
let successful_completion = matches!(completed_status, HealTaskStatus::Completed);
// Keep retry ownership continuous: status snapshots acquire // Keep retry ownership continuous: status snapshots acquire
// these locks in the same active -> retrying order. // these locks in the same active -> retrying order.
let mut retrying_heals_guard = if let (Some((request, _, error)), Some(cancel_token)) = let mut retrying_heals_guard = if let (Some((request, _, error)), Some(cancel_token)) =
@@ -295,6 +323,16 @@ impl HealManager {
} else { } else {
None None
}; };
if owns_completion || cancelled_completion {
publish_completed_heal(
&completed_heals_clone,
&task_aliases_clone,
&task_id,
completed_status_entry,
terminal_completion,
)
.await;
}
let completed_task = active_heals_guard.remove(&task_id); let completed_task = active_heals_guard.remove(&task_id);
if let Some(completed_task) = completed_task.as_ref() { if let Some(completed_task) = completed_task.as_ref() {
publish_active_heal_count(&active_heals_guard); publish_active_heal_count(&active_heals_guard);
@@ -304,33 +342,10 @@ impl HealManager {
drop(retrying_heals_guard.take()); drop(retrying_heals_guard.take());
drop(active_heals_guard); drop(active_heals_guard);
if let Some(completed_task) = completed_task { #[cfg(test)]
let completed_status = if let Some(status) = retry_request_for_status { tests::pause_completed_retention_handoff(&task_id).await;
status
} else { if completed_task.is_some() {
completed_task.get_status().await
};
let terminal_completion = !matches!(completed_status, HealTaskStatus::Retrying { .. });
let successful_completion = matches!(completed_status, HealTaskStatus::Completed);
let completed_progress = completed_task.get_progress().await;
// Single snapshot of the retained window: the task is
// finished and already off the active map, so there is
// no concurrent writer to race with.
let seqed_items = completed_task.get_seqed_result_items().await;
let (next_seq, min_seq) = completed_task.result_seq_cursors();
let completed_status_entry = CompletedHealStatus {
heal_type: completed_task.heal_type.clone(),
status: completed_status.clone(),
result_items_truncated: completed_task.result_items_truncated(),
completed_at: SystemTime::now(),
seqed_items,
next_seq,
min_seq,
};
let mut completed_heals_guard = completed_heals_clone.lock().await;
prune_completed_heal_statuses(&mut completed_heals_guard);
completed_heals_guard.insert(task_id.clone(), Arc::new(completed_status_entry));
drop(completed_heals_guard);
// update statistics // update statistics
let mut stats = statistics_clone.write().await; let mut stats = statistics_clone.write().await;
match completed_status { match completed_status {
@@ -352,10 +367,6 @@ impl HealManager {
} else { } else {
release_mrf_repair_notice_targets(notice_targets); release_mrf_repair_notice_targets(notice_targets);
} }
task_aliases_clone
.lock()
.await
.retain(|alias_id, alias| alias_id != &task_id && alias.task_id != task_id);
} }
} }
@@ -718,17 +729,42 @@ pub(super) fn heal_request_set_key_for_task(task: &HealTask) -> Option<String> {
} }
pub(super) fn prune_completed_heal_statuses(completed_heals: &mut HashMap<String, Arc<CompletedHealStatus>>) { pub(super) fn prune_completed_heal_statuses(completed_heals: &mut HashMap<String, Arc<CompletedHealStatus>>) {
let Ok(now) = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH) else { prune_completed_heal_statuses_at(completed_heals, SystemTime::now());
return; }
};
pub(super) fn prune_completed_heal_statuses_at(completed_heals: &mut HashMap<String, Arc<CompletedHealStatus>>, now: SystemTime) {
completed_heals.retain(|_, completed| { completed_heals.retain(|_, completed| {
completed now.duration_since(completed.completed_at)
.completed_at .map(|age| age <= KEEP_HEAL_TASK_STATUS_DURATION)
.duration_since(SystemTime::UNIX_EPOCH)
.map(|completed_at| now.saturating_sub(completed_at) <= KEEP_HEAL_TASK_STATUS_DURATION)
.unwrap_or(false) .unwrap_or(false)
}); });
let entry_bytes = |key: &String, value: &Arc<CompletedHealStatus>| {
key.capacity()
.saturating_add(size_of::<(String, Arc<CompletedHealStatus>)>())
.saturating_add(value.retained_bytes())
};
let mut bytes = completed_heals
.iter()
.fold(0usize, |total, (key, value)| total.saturating_add(entry_bytes(key, value)));
while completed_heals.len() > MAX_COMPLETED_HEAL_TOKENS || bytes > MAX_COMPLETED_HEAL_BYTES {
let Some(oldest) = completed_heals
.iter()
.min_by(|(left_id, left), (right_id, right)| {
left.completed_at.cmp(&right.completed_at).then_with(|| left_id.cmp(right_id))
})
.map(|(_, value)| Arc::clone(value))
else {
break;
};
completed_heals.retain(|key, value| {
if Arc::ptr_eq(value, &oldest) {
bytes = bytes.saturating_sub(entry_bytes(key, value));
false
} else {
true
}
});
}
} }
pub(super) fn can_schedule_request( pub(super) fn can_schedule_request(
+348 -3
View File
@@ -101,6 +101,326 @@ async fn process_manager_queue_once(manager: &HealManager) {
struct MockStorage; struct MockStorage;
fn completed_retention_fixture(completed_at: SystemTime) -> CompletedHealStatus {
CompletedHealStatus {
heal_type: HealType::Cluster,
status: HealTaskStatus::Completed,
progress: Some(HealProgress {
objects_scanned: 9,
objects_healed: 8,
objects_failed: 1,
..Default::default()
}),
retained_bytes: std::sync::OnceLock::new(),
result_items_truncated: false,
completed_at,
seqed_items: vec![(3, HealResultItem::default()), (4, HealResultItem::default())],
next_seq: 5,
min_seq: 3,
}
}
#[test]
fn completed_retention_cursor_boundaries_preserve_progress() {
let completed = completed_retention_fixture(SystemTime::now());
for (cursor, count, lagged) in [
(0, 2, true),
(1, 2, true),
(2, 2, false),
(3, 1, false),
(4, 0, false),
(5, 0, false),
(u64::MAX, 0, false),
] {
let report = completed_task_report(&completed, Some(cursor));
assert_eq!(report.result_items.len(), count, "cursor={cursor}");
assert_eq!(report.result_items_truncated, lagged, "cursor={cursor}");
assert_eq!(report.progress, completed.progress);
assert_eq!((report.next_seq, report.min_seq), (5, 3));
}
assert_eq!(completed_task_report(&completed, None).result_items.len(), 2);
}
#[tokio::test]
async fn completed_retention_displaced_alias_does_not_resurrect_evicted_snapshot() {
let manager = HealManager::new(Arc::new(MockStorage), None);
let request = HealRequest::bucket("bucket".to_string());
manager.insert_task_alias("alias", &request.id).await;
let terminal = record_displaced_terminal(&manager.displaced_terminals, &request);
lock_displaced_terminals(&manager.displaced_terminals).remove(&request.id);
remove_displaced_task_aliases(&manager.task_aliases, &manager.displaced_terminals, &request.id, &terminal).await;
for token in [&request.id, &"alias".to_string()] {
assert!(matches!(manager.get_task_report(token).await, Err(Error::TaskNotFound { .. })));
}
assert!(manager.task_aliases.lock().await.is_empty());
assert!(lock_displaced_terminals(&manager.displaced_terminals).is_empty());
}
#[test]
fn completed_retention_count_ttl_and_alias_eviction_are_bounded() {
let now = SystemTime::now();
let mut entries = HashMap::new();
let oldest = Arc::new(completed_retention_fixture(now - KEEP_HEAL_TASK_STATUS_DURATION));
entries.insert("oldest".to_string(), Arc::clone(&oldest));
entries.insert("oldest-alias".to_string(), Arc::clone(&oldest));
for index in 2..MAX_COMPLETED_HEAL_TOKENS {
entries.insert(format!("task-{index}"), Arc::new(completed_retention_fixture(now)));
}
prune_completed_heal_statuses_at(&mut entries, now);
assert_eq!(entries.len(), MAX_COMPLETED_HEAL_TOKENS);
entries.insert("cap-plus-one".to_string(), Arc::new(completed_retention_fixture(now)));
prune_completed_heal_statuses_at(&mut entries, now);
assert_eq!(entries.len(), MAX_COMPLETED_HEAL_TOKENS - 1);
assert!(!entries.contains_key("oldest"));
assert!(!entries.contains_key("oldest-alias"));
entries.clear();
entries.insert("ttl-boundary".to_string(), oldest);
entries.insert(
"expired".to_string(),
Arc::new(completed_retention_fixture(
now - KEEP_HEAL_TASK_STATUS_DURATION - Duration::from_nanos(1),
)),
);
entries.insert("future".to_string(), Arc::new(completed_retention_fixture(now + Duration::from_nanos(1))));
prune_completed_heal_statuses_at(&mut entries, now);
assert_eq!(entries.len(), 1);
assert!(entries.contains_key("ttl-boundary"));
prune_completed_heal_statuses_at(&mut entries, now + Duration::from_nanos(1));
assert!(entries.is_empty());
}
#[test]
fn completed_retention_total_byte_cap_and_cap_plus_one() {
let now = SystemTime::now();
let key = "large".to_string();
let mut entry = completed_retention_fixture(now);
let base_bytes = entry.retained_bytes() + key.capacity() + size_of::<(String, Arc<CompletedHealStatus>)>();
entry.retained_bytes.take();
entry.status = HealTaskStatus::Failed {
error: "x".repeat(MAX_COMPLETED_HEAL_BYTES - base_bytes),
};
assert_eq!(
entry.retained_bytes() + key.capacity() + size_of::<(String, Arc<CompletedHealStatus>)>(),
MAX_COMPLETED_HEAL_BYTES
);
let mut entries = HashMap::from([(key, Arc::new(entry))]);
prune_completed_heal_statuses_at(&mut entries, now);
assert_eq!(entries.len(), 1, "exact byte cap remains retained");
let mut over = Arc::try_unwrap(entries.remove("large").expect("entry retained")).expect("entry not shared");
over.retained_bytes.take();
if let HealTaskStatus::Failed { error } = &mut over.status {
*error = "x".repeat(error.len() + 1);
}
entries.insert("large".to_string(), Arc::new(over));
prune_completed_heal_statuses_at(&mut entries, now);
assert!(entries.is_empty(), "oversized metadata cannot escape total byte bound");
}
#[tokio::test]
async fn completed_retention_large_window_keeps_cursors_and_progress() {
let task = HealTask::from_request(HealRequest::bucket("bucket".to_string()), Arc::new(MockStorage));
let mut snapshot = completed_retention_fixture(SystemTime::now());
snapshot.seqed_items[0].1.detail = "x".repeat(MAX_COMPLETED_HEAL_RESULT_BYTES);
snapshot.bound_result_window();
assert_eq!(snapshot.seqed_items.len(), 1);
assert_eq!((snapshot.min_seq, snapshot.next_seq), (4, 5));
assert!(snapshot.result_items_truncated);
assert!(snapshot.retained_bytes() < MAX_COMPLETED_HEAL_RESULT_BYTES);
let report = completed_task_report(&snapshot, Some(0));
assert_eq!(report.progress.expect("progress retained").objects_scanned, 9);
assert!(report.result_items_truncated);
let active_max = task.get_result_items_since(Some(u64::MAX)).await;
assert!(active_max.items.is_empty());
assert!(!active_max.lagged);
}
#[test]
fn completed_retention_result_byte_cap_and_cap_plus_one() {
for extra in [0, 1] {
let mut snapshot = completed_retention_fixture(SystemTime::now());
snapshot.seqed_items = vec![(
4,
HealResultItem {
detail: "x".repeat(MAX_COMPLETED_HEAL_RESULT_BYTES - size_of::<(u64, HealResultItem)>() + extra),
..Default::default()
},
)];
snapshot.min_seq = 4;
snapshot.bound_result_window();
assert_eq!(snapshot.seqed_items.len(), 1 - extra);
assert_eq!(snapshot.result_items_truncated, extra == 1);
assert_eq!(snapshot.min_seq, if extra == 0 { 4 } else { 5 });
assert_eq!(snapshot.next_seq, 5);
assert_eq!(snapshot.progress.as_ref().expect("progress retained").objects_scanned, 9);
}
}
#[derive(Default)]
struct CompletedRetentionHook {
started: Notify,
execute: Notify,
handoff: Notify,
finish: Notify,
pause_before_publish: bool,
before_publish: Notify,
publish: Notify,
prepared_status: Mutex<Option<HealTaskStatus>>,
}
static COMPLETED_RETENTION_HOOKS: LazyLock<Mutex<HashMap<String, Arc<CompletedRetentionHook>>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
pub(super) async fn pause_completed_retention_handoff(task_id: &str) {
let hook = COMPLETED_RETENTION_HOOKS.lock().await.get(task_id).cloned();
if let Some(hook) = hook {
hook.handoff.notify_one();
hook.finish.notified().await;
}
}
pub(super) async fn pause_completed_retention_before_publish(task_id: &str, status: &HealTaskStatus) {
let hook = COMPLETED_RETENTION_HOOKS.lock().await.get(task_id).cloned();
if let Some(hook) = hook.filter(|hook| hook.pause_before_publish) {
*hook.prepared_status.lock().await = Some(status.clone());
hook.before_publish.notify_one();
hook.publish.notified().await;
}
}
#[tokio::test]
async fn completed_retention_cancel_wins_over_a_prepared_retry_snapshot() {
let bucket = "completed-retention-retry-cancel";
let manager = HealManager::new(Arc::new(MockStorage), None);
let request = HealRequest::object(bucket.to_string(), "object".to_string(), None);
let task_id = request.id.clone();
let duplicate = HealRequest::object(bucket.to_string(), "object".to_string(), None);
let alias = duplicate.id.clone();
let hook = Arc::new(CompletedRetentionHook {
pause_before_publish: true,
..Default::default()
});
{
let mut hooks = COMPLETED_RETENTION_HOOKS.lock().await;
hooks.insert(bucket.to_string(), Arc::clone(&hook));
hooks.insert(task_id.clone(), Arc::clone(&hook));
}
manager.submit_heal_request(request).await.expect("admit original");
manager.submit_heal_request(duplicate).await.expect("admit alias");
process_manager_queue_once(&manager).await;
tokio::time::timeout(Duration::from_secs(5), hook.started.notified())
.await
.expect("scheduler starts");
let task = manager.active_heals.lock().await.get(&task_id).cloned().expect("active task");
task.progress.write().await.update_object_progress(1, 1, 0, 0, 4096);
hook.execute.notify_one();
tokio::time::timeout(Duration::from_secs(5), hook.before_publish.notified())
.await
.expect("retry snapshot prepared");
manager.cancel_task(&alias).await.expect("cancel wins active ownership");
assert!(matches!(*hook.prepared_status.lock().await, Some(HealTaskStatus::Retrying { .. })));
hook.publish.notify_one();
tokio::time::timeout(Duration::from_secs(5), hook.handoff.notified())
.await
.expect("scheduler finishes handoff");
for token in [&task_id, &alias] {
let report = manager.get_task_report(token).await.expect("cancelled token retained");
assert_eq!(report.status, HealTaskStatus::Cancelled);
assert_eq!(report.progress.expect("frozen progress").objects_scanned, 1);
}
assert!(!manager.retrying_heals.lock().await.contains_key(&task_id));
assert!(!manager.heal_queue.lock().await.contains_request_id(&task_id));
hook.finish.notify_one();
COMPLETED_RETENTION_HOOKS
.lock()
.await
.retain(|key, _| key != bucket && key != &task_id);
}
#[tokio::test]
async fn completed_retention_scheduler_preserves_progress_aliases_and_atomic_handoff() {
for outcome in ["success", "failed", "cancelled"] {
let bucket = format!("completed-retention-{outcome}");
let hook = Arc::new(CompletedRetentionHook::default());
let manager = Arc::new(HealManager::new(Arc::new(MockStorage), None));
let request = HealRequest::object(bucket.clone(), "object".to_string(), None);
let task_id = request.id.clone();
let duplicate = HealRequest::object(bucket.clone(), "object".to_string(), None);
let alias = duplicate.id.clone();
{
let mut hooks = COMPLETED_RETENTION_HOOKS.lock().await;
hooks.insert(bucket.clone(), Arc::clone(&hook));
hooks.insert(task_id.clone(), Arc::clone(&hook));
}
manager.submit_heal_request(request).await.expect("admit original");
manager.submit_heal_request(duplicate).await.expect("admit alias");
process_manager_queue_once(&manager).await;
tokio::time::timeout(Duration::from_secs(5), hook.started.notified())
.await
.expect("scheduler reaches storage");
let task = manager
.active_heals
.lock()
.await
.get(&task_id)
.cloned()
.expect("task is active");
task.progress.write().await.update_object_progress(1, 1, 0, 0, 4096);
let before = manager.get_task_report(&alias).await.expect("alias resolves active progress");
assert_eq!(before.progress.as_ref().expect("active progress").objects_scanned, 1);
let poll_manager = Arc::clone(&manager);
let poll_alias = alias.clone();
let stop = CancellationToken::new();
let poll_stop = stop.clone();
let polling = tokio::spawn(async move {
while !poll_stop.is_cancelled() {
let report = poll_manager
.get_task_report(&poll_alias)
.await
.expect("handoff must never return NotFound");
assert!(report.progress.expect("progress never disappears").objects_scanned >= 1);
tokio::task::yield_now().await;
}
});
if outcome == "cancelled" {
manager.cancel_task(&alias).await.expect("cancel active task by alias");
} else {
hook.execute.notify_one();
}
tokio::time::timeout(Duration::from_secs(5), hook.handoff.notified())
.await
.expect("scheduler archives terminal");
assert!(!manager.active_heals.lock().await.contains_key(&task_id));
let expected = task.get_progress().await;
for token in [&task_id, &alias] {
assert_eq!(manager.get_task_progress(token).await.expect("terminal progress query"), expected);
let report = manager
.get_task_report_for_path_since(&format!("{bucket}/object"), token, Some(u64::MAX))
.await
.expect("terminal token remains queryable at handoff");
assert_eq!(report.progress.as_ref(), Some(&expected));
assert!(report.result_items.is_empty());
match outcome {
"success" => assert_eq!(report.status, HealTaskStatus::Completed),
"failed" => assert!(matches!(report.status, HealTaskStatus::Failed { .. })),
_ => assert_eq!(report.status, HealTaskStatus::Cancelled),
}
}
let retained = manager.completed_heals.lock().await;
assert!(Arc::ptr_eq(&retained[&task_id], &retained[&alias]));
drop(retained);
stop.cancel();
polling.await.expect("concurrent polling succeeds");
// Archived progress must not alias a mutable live progress object.
task.progress.write().await.objects_scanned = 999;
assert_eq!(manager.get_task_report(&alias).await.expect("frozen report").progress, Some(expected));
hook.finish.notify_one();
COMPLETED_RETENTION_HOOKS
.lock()
.await
.retain(|key, _| key != &bucket && key != &task_id);
}
}
#[async_trait::async_trait] #[async_trait::async_trait]
impl HealStorageAPI for MockStorage { impl HealStorageAPI for MockStorage {
async fn get_object_meta(&self, _bucket: &str, _object: &str) -> Result<Option<HealObjectInfo>> { async fn get_object_meta(&self, _bucket: &str, _object: &str) -> Result<Option<HealObjectInfo>> {
@@ -123,6 +443,12 @@ impl HealStorageAPI for MockStorage {
} }
async fn object_exists(&self, bucket: &str, _object: &str) -> Result<bool> { async fn object_exists(&self, bucket: &str, _object: &str) -> Result<bool> {
let hook = COMPLETED_RETENTION_HOOKS.lock().await.get(bucket).cloned();
if let Some(hook) = hook {
hook.started.notify_one();
hook.execute.notified().await;
return Ok(true);
}
Ok(bucket == "retry-transition") Ok(bucket == "retry-transition")
} }
@@ -133,13 +459,18 @@ impl HealStorageAPI for MockStorage {
_version_id: Option<&str>, _version_id: Option<&str>,
_opts: &HealOpts, _opts: &HealOpts,
) -> Result<(HealResultItem, Option<Error>)> { ) -> Result<(HealResultItem, Option<Error>)> {
if bucket == "completed-retention-failed" {
return Err(Error::TaskExecutionFailed {
message: "retention fixture failure".to_string(),
});
}
if let Some(hook) = manager_recovery_test_hook() { if let Some(hook) = manager_recovery_test_hook() {
*hook *hook
.heal_object_calls .heal_object_calls
.lock() .lock()
.expect("manager recovery object call lock should not poison") += 1; .expect("manager recovery object call lock should not poison") += 1;
} }
if bucket == "retry-transition" { if matches!(bucket, "retry-transition" | "completed-retention-retry-cancel") {
return Ok(( return Ok((
HealResultItem::default(), HealResultItem::default(),
Some(Error::Storage(EcstoreError::InsufficientReadQuorum( Some(Error::Storage(EcstoreError::InsufficientReadQuorum(
@@ -1145,7 +1476,13 @@ async fn test_active_duplicate_token_can_query_and_cancel_original_task() {
.expect("duplicate token should cancel merged active task"); .expect("duplicate token should cancel merged active task");
assert!(manager.active_heals.lock().await.get(&active_task_id).is_none()); assert!(manager.active_heals.lock().await.get(&active_task_id).is_none());
assert!(matches!(manager.get_task_status(&active_task_id).await, Err(Error::TaskNotFound { .. }))); assert_eq!(
manager
.get_task_status(&active_task_id)
.await
.expect("cancelled task remains queryable"),
HealTaskStatus::Cancelled
);
} }
#[tokio::test] #[tokio::test]
@@ -1638,6 +1975,8 @@ async fn insert_retrying_request(manager: &HealManager, request: HealRequest) ->
manager.completed_heals.lock().await.insert( manager.completed_heals.lock().await.insert(
task_id, task_id,
Arc::new(CompletedHealStatus { Arc::new(CompletedHealStatus {
progress: None,
retained_bytes: std::sync::OnceLock::new(),
heal_type: request.heal_type, heal_type: request.heal_type,
status: HealTaskStatus::Retrying { status: HealTaskStatus::Retrying {
error: "Lock acquisition timeout".to_string(), error: "Lock acquisition timeout".to_string(),
@@ -2053,7 +2392,7 @@ async fn admin_force_start_cancels_overlapping_active_task_first() {
"the overlapping admin task must be cancelled (removed from the active table) before the new one starts" "the overlapping admin task must be cancelled (removed from the active table) before the new one starts"
); );
assert!( assert!(
matches!(manager.get_task_status(&old_id).await, Err(Error::TaskNotFound { .. })), matches!(manager.get_task_status(&old_id).await, Ok(HealTaskStatus::Cancelled)),
"a cancelled task must no longer resolve as an active heal" "a cancelled task must no longer resolve as an active heal"
); );
} }
@@ -2360,6 +2699,8 @@ async fn test_retrying_completion_outranks_the_queue_for_the_same_id() {
manager.completed_heals.lock().await.insert( manager.completed_heals.lock().await.insert(
task_id.clone(), task_id.clone(),
Arc::new(CompletedHealStatus { Arc::new(CompletedHealStatus {
progress: None,
retained_bytes: std::sync::OnceLock::new(),
heal_type: request.heal_type.clone(), heal_type: request.heal_type.clone(),
status: HealTaskStatus::Retrying { status: HealTaskStatus::Retrying {
error: "transient disk failure".to_string(), error: "transient disk failure".to_string(),
@@ -2395,6 +2736,8 @@ async fn test_get_task_status_reads_recent_completed_status() {
manager.completed_heals.lock().await.insert( manager.completed_heals.lock().await.insert(
"completed-token".to_string(), "completed-token".to_string(),
Arc::new(CompletedHealStatus { Arc::new(CompletedHealStatus {
progress: None,
retained_bytes: std::sync::OnceLock::new(),
heal_type: HealType::Bucket { heal_type: HealType::Bucket {
bucket: "bucket".to_string(), bucket: "bucket".to_string(),
}, },
@@ -2424,6 +2767,8 @@ async fn test_get_task_report_for_path_reads_completed_items() {
manager.completed_heals.lock().await.insert( manager.completed_heals.lock().await.insert(
"completed-token".to_string(), "completed-token".to_string(),
Arc::new(CompletedHealStatus { Arc::new(CompletedHealStatus {
progress: None,
retained_bytes: std::sync::OnceLock::new(),
heal_type: HealType::Object { heal_type: HealType::Object {
bucket: "bucket".to_string(), bucket: "bucket".to_string(),
object: "object".to_string(), object: "object".to_string(),
+4
View File
@@ -45,6 +45,10 @@ use uuid::Uuid;
use crate::heal::task::{HealOptions, HealPriority, HealRequest, HealType}; use crate::heal::task::{HealOptions, HealPriority, HealRequest, HealType};
/// Read-only inspection of committed MRF checkpoints. The legacy consumer
/// remains unchanged until ownership-aware replay is deployed.
pub mod snapshot;
/// Journal location inside the metadata bucket, following the resume-state /// Journal location inside the metadata bucket, following the resume-state
/// layout. /// layout.
pub(crate) const MRF_JOURNAL_PATH: &str = "buckets/.heal/mrf/journal.bin"; pub(crate) const MRF_JOURNAL_PATH: &str = "buckets/.heal/mrf/journal.bin";
+681
View File
@@ -0,0 +1,681 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Reader-first support for owner-local MRF checkpoints.
//!
//! Each of two slots has a payload and a commit manifest. The manifest binds
//! the writer identity, persistent sequence, length and whole-payload digest.
//! Replacing the inactive slot must leave the previous committed slot intact.
//! Production publication and reclamation are deliberately not enabled here.
//! An unreadable commit path cannot prove that only legacy data exists. This
//! explicit inspection API fails closed and never mutates recovery anchors.
//! It is not wired into the legacy consumer: that transition requires the
//! ownership-aware replay and producer handoff before writer activation.
//! One surviving committed replica supports process restart recovery only;
//! this reader does not establish a replication quorum or a power-loss policy.
use super::{MRF_JOURNAL_PATH, MRF_SCOPED_JOURNAL_PATH, decode_journal};
use crate::heal::RUSTFS_META_BUCKET;
use crate::heal::storage_api::owner::{EcstoreDiskAPI, EcstoreDiskError, EcstoreDiskStore};
use sha2::{Digest, Sha256};
use std::collections::HashMap;
use tokio::io::AsyncReadExt;
use uuid::Uuid;
// Root-level control files avoid requiring a new directory before the first
// atomic commit. They remain inside the storage owner's metadata volume.
const PAYLOAD_PATHS: [&str; 2] = [".heal-mrf-snapshot.0.bin", ".heal-mrf-snapshot.1.bin"];
const MANIFEST_PATHS: [&str; 2] = [".heal-mrf-commit.0.bin", ".heal-mrf-commit.1.bin"];
const MAGIC: &[u8; 8] = b"RFMRFC01";
const MANIFEST_LEN: usize = 8 + 1 + 16 + 8 + 8 + 32 + 32;
const VERSION: u8 = 1;
#[derive(Debug, thiserror::Error)]
pub enum SnapshotError {
#[error("MRF checkpoint has an invalid or incomplete commit record")]
Corrupt,
#[error("MRF checkpoint format is unsupported")]
Unsupported,
#[error("MRF checkpoint exceeds the configured byte limit")]
TooLarge,
#[error("MRF checkpoint replicas disagree at the same sequence")]
Conflict,
#[error("MRF checkpoint storage is unavailable")]
Disk(#[source] EcstoreDiskError),
#[error("MRF checkpoint body could not be read")]
Read(#[source] std::io::Error),
}
#[derive(Debug, PartialEq, Eq)]
struct Manifest {
owner: Uuid,
sequence: u64,
payload_len: usize,
payload_digest: [u8; 32],
}
impl Manifest {
fn decode(bytes: &[u8], limit: usize) -> Result<Self, SnapshotError> {
if bytes.len() != MANIFEST_LEN || &bytes[..8] != MAGIC {
return Err(SnapshotError::Corrupt);
}
if bytes[8] != VERSION {
return Err(SnapshotError::Unsupported);
}
let signed = MANIFEST_LEN - 32;
let checksum: [u8; 32] = Sha256::digest(&bytes[..signed]).into();
if checksum != bytes[signed..] {
return Err(SnapshotError::Corrupt);
}
let owner = Uuid::from_slice(&bytes[9..25]).map_err(|_| SnapshotError::Corrupt)?;
let sequence = u64::from_le_bytes(bytes[25..33].try_into().map_err(|_| SnapshotError::Corrupt)?);
let payload_len = u64::from_le_bytes(bytes[33..41].try_into().map_err(|_| SnapshotError::Corrupt)?);
let payload_len = usize::try_from(payload_len).map_err(|_| SnapshotError::TooLarge)?;
if owner.is_nil() || sequence == 0 || sequence == u64::MAX {
return Err(SnapshotError::Corrupt);
}
if payload_len > limit {
return Err(SnapshotError::TooLarge);
}
Ok(Self {
owner,
sequence,
payload_len,
payload_digest: bytes[41..73].try_into().map_err(|_| SnapshotError::Corrupt)?,
})
}
}
#[derive(Debug)]
pub struct CommittedSnapshot {
manifest: Manifest,
payload: Vec<u8>,
}
impl CommittedSnapshot {
/// Persistent single-writer sequence, not a process UUID ordering.
pub fn sequence(&self) -> u64 {
self.manifest.sequence
}
/// Identity recorded by the committed checkpoint's writer.
pub fn owner(&self) -> Uuid {
self.manifest.owner
}
/// Complete, checksum-validated record bytes. Inspection does not consume
/// these records or acknowledge completion to any producer.
pub fn payload(&self) -> &[u8] {
&self.payload
}
fn decode(manifest: &[u8], payload: Vec<u8>, limit: usize) -> Result<Self, SnapshotError> {
let manifest = Manifest::decode(manifest, limit)?;
let checksum: [u8; 32] = Sha256::digest(&payload).into();
if payload.len() != manifest.payload_len || checksum != manifest.payload_digest {
return Err(SnapshotError::Corrupt);
}
if decode_journal(&payload).1 != 0 {
return Err(SnapshotError::Corrupt);
}
Ok(Self { manifest, payload })
}
}
#[derive(Debug)]
pub enum RecoverySnapshot {
/// An intact legacy snapshot, without a comparable commit sequence.
Legacy(Vec<u8>),
/// A committed checkpoint requiring ownership-aware replay before use.
Committed(CommittedSnapshot),
}
async fn read_bounded(disk: &EcstoreDiskStore, path: &str, limit: usize) -> Result<Option<Vec<u8>>, SnapshotError> {
let reader = match EcstoreDiskAPI::read_file(disk.as_ref(), RUSTFS_META_BUCKET, path).await {
Ok(reader) => reader,
Err(EcstoreDiskError::FileNotFound | EcstoreDiskError::VolumeNotFound) => return Ok(None),
Err(error) => return Err(SnapshotError::Disk(error)),
};
let maximum = limit.checked_add(1).ok_or(SnapshotError::TooLarge)?;
let maximum = u64::try_from(maximum).map_err(|_| SnapshotError::TooLarge)?;
let mut bytes = Vec::new();
reader
.take(maximum)
.read_to_end(&mut bytes)
.await
.map_err(SnapshotError::Read)?;
if bytes.len() > limit {
return Err(SnapshotError::TooLarge);
}
Ok(Some(bytes))
}
fn select_snapshot(selected: &mut Option<CommittedSnapshot>, candidate: CommittedSnapshot) -> Result<(), SnapshotError> {
if let Some(current) = selected {
if current.manifest.sequence == candidate.manifest.sequence
&& (current.manifest != candidate.manifest || current.payload != candidate.payload)
{
return Err(SnapshotError::Conflict);
}
if current.manifest.sequence >= candidate.manifest.sequence {
return Ok(());
}
}
*selected = Some(candidate);
Ok(())
}
async fn read_committed(disks: &[EcstoreDiskStore], limit: usize) -> Result<Option<CommittedSnapshot>, SnapshotError> {
let mut selected = None;
let mut damaged = None;
let mut identities = HashMap::new();
for disk in disks {
for (manifest_path, payload_path) in MANIFEST_PATHS.into_iter().zip(PAYLOAD_PATHS) {
let candidate = async {
let Some(manifest) = read_bounded(disk, manifest_path, MANIFEST_LEN).await? else {
return Ok(None);
};
let header = Manifest::decode(&manifest, limit)?;
let payload = read_bounded(disk, payload_path, header.payload_len)
.await?
.ok_or(SnapshotError::Corrupt)?;
CommittedSnapshot::decode(&manifest, payload, limit).map(Some)
}
.await;
match candidate {
Ok(Some(candidate)) => {
let identity = (
candidate.manifest.owner,
candidate.manifest.payload_len,
candidate.manifest.payload_digest,
);
if identities
.insert(candidate.manifest.sequence, identity)
.is_some_and(|previous| previous != identity)
{
return Err(SnapshotError::Conflict);
}
select_snapshot(&mut selected, candidate)?;
}
Ok(None) => {}
// A future committed format may supersede all readable slots.
Err(SnapshotError::Unsupported) => return Err(SnapshotError::Unsupported),
Err(error) => damaged = Some(error),
}
}
}
match (selected, damaged) {
(Some(snapshot), _) => Ok(Some(snapshot)),
(None, Some(error)) => Err(error),
(None, None) => Ok(None),
}
}
async fn read_legacy(disks: &[EcstoreDiskStore], path: &str, limit: usize) -> Result<Option<Vec<u8>>, SnapshotError> {
let mut selected = None;
let mut incomplete: Option<Vec<u8>> = None;
for disk in disks {
match read_bounded(disk, path, limit).await {
Ok(Some(payload)) if decode_journal(&payload).1 == 0 => {
if selected.as_ref().is_some_and(|current| *current != payload) {
// Legacy snapshots have no sequence. There is no evidence
// that the first, longest or nonempty replica is newest.
return Err(SnapshotError::Conflict);
}
selected = Some(payload);
}
Ok(Some(payload)) => {
if let Some(previous) = &incomplete {
if previous.starts_with(&payload) {
continue;
}
if !payload.starts_with(previous) {
return Err(SnapshotError::Corrupt);
}
}
incomplete = Some(payload);
}
Ok(None) => {}
Err(error) => return Err(error),
}
}
if let Some(prefix) = incomplete
&& !selected.as_ref().is_some_and(|payload| payload.starts_with(&prefix))
{
// In particular, an empty O_TRUNC replica cannot supersede another
// replica containing intact records followed by a torn tail.
return Err(SnapshotError::Corrupt);
}
Ok(selected)
}
/// Inspect local MRF checkpoints without replaying, acknowledging or deleting.
///
/// `max_bytes` bounds each payload read. Every local replica is examined and
/// ambiguous identities, unavailable proof or unsupported formats return a
/// typed error. This API must not authorize a writer without the separate
/// ownership and mixed-version activation checks.
pub async fn inspect_local_recovery_snapshot(max_bytes: usize) -> Result<Option<RecoverySnapshot>, SnapshotError> {
read_recovery_snapshot(&super::journal_disks().await, max_bytes).await
}
async fn read_recovery_snapshot(disks: &[EcstoreDiskStore], limit: usize) -> Result<Option<RecoverySnapshot>, SnapshotError> {
if let Some(snapshot) = read_committed(disks, limit).await? {
return Ok(Some(RecoverySnapshot::Committed(snapshot)));
}
// RUSTFS_COMPAT_TODO(backlog-2263): inspect retained legacy MRF journals. Remove after all supported upgrade and rollback readers understand committed snapshots and retained journals have migrated.
if let Some(payload) = read_legacy(disks, MRF_SCOPED_JOURNAL_PATH, limit).await? {
return Ok(Some(RecoverySnapshot::Legacy(payload)));
}
Ok(read_legacy(disks, MRF_JOURNAL_PATH, limit)
.await?
.map(RecoverySnapshot::Legacy))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::heal::mrf_queue::encode_intent;
use crate::heal::storage_api::owner::{EcstoreConditionalFileUpdate, EcstoreDiskBytes};
use crate::heal::{DiskOption, Endpoint, new_disk};
use rustfs_common::mrf_channel::{MrfIntent, MrfKind, MrfScope};
use std::sync::Arc;
use tempfile::TempDir;
fn payload(object: &str) -> Vec<u8> {
let intent = MrfIntent {
bucket: Arc::from("bucket"),
object: Arc::from(object),
version_id: None,
kind: MrfKind::PartialWrite,
scope: None,
lease: None,
enqueued_at_ms: 1234,
attempts: 0,
};
let mut bytes = Vec::new();
assert!(encode_intent(&intent, &mut bytes), "fixture must encode a full record");
bytes
}
fn manifest(owner: Uuid, sequence: u64, payload: &[u8]) -> Vec<u8> {
let mut bytes = Vec::with_capacity(MANIFEST_LEN);
bytes.extend_from_slice(MAGIC);
bytes.push(VERSION);
bytes.extend_from_slice(owner.as_bytes());
bytes.extend_from_slice(&sequence.to_le_bytes());
bytes.extend_from_slice(&u64::try_from(payload.len()).expect("fixture length fits").to_le_bytes());
bytes.extend_from_slice(&Sha256::digest(payload));
bytes.extend_from_slice(&Sha256::digest(&bytes));
bytes
}
async fn disk(root: &TempDir, name: &str) -> EcstoreDiskStore {
let path = root.path().join(name);
std::fs::create_dir_all(&path).expect("create disk directory");
let endpoint = Endpoint::try_from(path.to_string_lossy().as_ref()).expect("valid disk endpoint");
let disk = new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("open disk");
let result = EcstoreDiskAPI::make_volume(disk.as_ref(), RUSTFS_META_BUCKET).await;
assert!(
matches!(result, Ok(()) | Err(EcstoreDiskError::VolumeExists)),
"metadata volume: {result:?}"
);
disk
}
// Exercise the existing storage owner's atomic CAS primitive. No production
// caller publishes this format until ownership-aware replay is available.
async fn install(disk: &EcstoreDiskStore, path: &str, bytes: &[u8]) {
let expected = EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, path).await.ok();
let result = EcstoreDiskAPI::compare_and_update_file(
disk.as_ref(),
RUSTFS_META_BUCKET,
path,
expected,
Some(EcstoreDiskBytes::copy_from_slice(bytes)),
)
.await
.expect("atomic snapshot slot write");
assert_eq!(result, EcstoreConditionalFileUpdate::Updated);
}
async fn commit(disk: &EcstoreDiskStore, slot: usize, owner: Uuid, sequence: u64, bytes: &[u8]) {
install(disk, PAYLOAD_PATHS[slot], bytes).await;
install(disk, MANIFEST_PATHS[slot], &manifest(owner, sequence, bytes)).await;
}
#[test]
fn manifest_validates_identity_sequence_length_and_digest() {
let bytes = payload("object");
let owner = Uuid::new_v4();
assert!(CommittedSnapshot::decode(&manifest(owner, 1, &bytes), bytes.clone(), bytes.len()).is_ok());
for (owner, sequence) in [(Uuid::nil(), 1), (owner, 0), (owner, u64::MAX)] {
assert!(matches!(
Manifest::decode(&manifest(owner, sequence, &bytes), bytes.len()),
Err(SnapshotError::Corrupt)
));
}
assert!(matches!(
Manifest::decode(&manifest(owner, 1, &bytes), bytes.len() - 1),
Err(SnapshotError::TooLarge)
));
let mut corrupt = manifest(owner, 1, &bytes);
corrupt[25] ^= 1;
assert!(matches!(Manifest::decode(&corrupt, bytes.len()), Err(SnapshotError::Corrupt)));
let mut unsupported = manifest(owner, 1, &bytes);
unsupported[8] = 2;
assert!(matches!(Manifest::decode(&unsupported, bytes.len()), Err(SnapshotError::Unsupported)));
}
#[test]
fn whole_payload_integrity_is_required_even_with_a_valid_manifest() {
let bytes = payload("object");
let owner = Uuid::new_v4();
let header = manifest(owner, 1, &bytes);
assert!(matches!(
CommittedSnapshot::decode(&header, bytes[..bytes.len() - 1].to_vec(), bytes.len()),
Err(SnapshotError::Corrupt)
));
let invalid = b"not an MRF record".to_vec();
assert!(matches!(
CommittedSnapshot::decode(&manifest(owner, 2, &invalid), invalid, bytes.len()),
Err(SnapshotError::Corrupt)
));
}
#[tokio::test]
async fn newest_complete_replica_wins_in_both_disk_orders() {
let root = TempDir::new().expect("test directory");
let first = disk(&root, "first").await;
let second = disk(&root, "second").await;
let owner = Uuid::new_v4();
commit(&first, 0, owner, 1, &payload("old")).await;
commit(&second, 1, owner, 2, &payload("new")).await;
for disks in [vec![first.clone(), second.clone()], vec![second.clone(), first.clone()]] {
let recovered = read_committed(&disks, 4096)
.await
.expect("read replicas")
.expect("committed snapshot");
assert_eq!(recovered.manifest.sequence, 2);
assert_eq!(recovered.payload, payload("new"));
}
}
#[tokio::test]
async fn divergent_commits_at_same_sequence_fail_closed() {
let root = TempDir::new().expect("test directory");
let first = disk(&root, "first").await;
let second = disk(&root, "second").await;
let owner = Uuid::new_v4();
commit(&first, 0, owner, 7, &payload("a")).await;
commit(&second, 1, owner, 7, &payload("b")).await;
assert!(matches!(read_committed(&[first, second], 4096).await, Err(SnapshotError::Conflict)));
}
#[tokio::test]
async fn newer_slot_does_not_hide_a_conflicting_commit_history() {
let root = TempDir::new().expect("test directory");
let first = disk(&root, "first").await;
let second = disk(&root, "second").await;
let owner = Uuid::new_v4();
commit(&first, 0, owner, 8, &payload("newest")).await;
commit(&first, 1, owner, 7, &payload("a")).await;
commit(&second, 1, owner, 7, &payload("b")).await;
assert!(matches!(read_committed(&[first, second], 4096).await, Err(SnapshotError::Conflict)));
}
#[tokio::test]
async fn uncommitted_or_torn_successor_preserves_previous_slot() {
let root = TempDir::new().expect("test directory");
let disk = disk(&root, "disk").await;
let owner = Uuid::new_v4();
let old = payload("old");
let next = payload("next");
commit(&disk, 0, owner, 1, &old).await;
install(&disk, PAYLOAD_PATHS[1], &next).await;
let recovered = read_committed(std::slice::from_ref(&disk), 4096)
.await
.expect("staged payload is not a commit")
.expect("old snapshot");
assert_eq!(recovered.payload, old);
install(&disk, MANIFEST_PATHS[1], &manifest(owner, 2, &next)[..20]).await;
let recovered = read_committed(std::slice::from_ref(&disk), 4096)
.await
.expect("torn manifest preserves old slot")
.expect("old snapshot");
assert_eq!(recovered.manifest.sequence, 1);
install(&disk, MANIFEST_PATHS[1], &manifest(owner, 2, &next)).await;
install(&disk, PAYLOAD_PATHS[1], b"torn").await;
let recovered = read_committed(&[disk], 4096)
.await
.expect("torn payload preserves old slot")
.expect("old snapshot");
assert_eq!(recovered.manifest.sequence, 1);
}
#[tokio::test]
async fn stale_manifest_cas_cannot_replace_committed_anchor() {
let root = TempDir::new().expect("test directory");
let disk = disk(&root, "disk").await;
let owner = Uuid::new_v4();
let bytes = payload("object");
commit(&disk, 0, owner, 1, &bytes).await;
let result = EcstoreDiskAPI::compare_and_update_file(
disk.as_ref(),
RUSTFS_META_BUCKET,
MANIFEST_PATHS[0],
None,
Some(manifest(owner, 2, &bytes).into()),
)
.await
.expect("CAS call");
assert_eq!(result, EcstoreConditionalFileUpdate::Mismatch);
let recovered = read_committed(&[disk], 4096)
.await
.expect("read old anchor")
.expect("snapshot");
assert_eq!(recovered.manifest.sequence, 1);
}
#[tokio::test]
async fn legacy_import_requires_complete_consistent_replicas() {
let root = TempDir::new().expect("test directory");
let first = disk(&root, "first").await;
let second = disk(&root, "second").await;
let bytes = payload("object");
for (disk, data) in [(&first, &bytes[..bytes.len() - 1]), (&second, bytes.as_slice())] {
EcstoreDiskAPI::write_all(
disk.as_ref(),
RUSTFS_META_BUCKET,
MRF_SCOPED_JOURNAL_PATH,
EcstoreDiskBytes::copy_from_slice(data),
)
.await
.expect("legacy fixture");
}
let disks = [first.clone(), second];
assert!(
matches!(read_recovery_snapshot(&disks, 4096).await.expect("intact legacy replica"), Some(RecoverySnapshot::Legacy(data)) if data == bytes)
);
EcstoreDiskAPI::write_all(first.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH, payload("different").into())
.await
.expect("divergent fixture");
assert!(matches!(read_recovery_snapshot(&disks, 4096).await, Err(SnapshotError::Conflict)));
}
#[tokio::test]
async fn committed_inspection_leaves_payload_and_manifest_unchanged() {
let root = TempDir::new().expect("test directory");
let disk = disk(&root, "disk").await;
let owner = Uuid::new_v4();
let bytes = payload("object");
commit(&disk, 0, owner, 3, &bytes).await;
assert!(matches!(
read_recovery_snapshot(std::slice::from_ref(&disk), 4096)
.await
.expect("new snapshot"),
Some(RecoverySnapshot::Committed(_))
));
assert_eq!(
EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, MANIFEST_PATHS[0])
.await
.expect("manifest retained")
.as_ref(),
manifest(owner, 3, &bytes)
);
assert_eq!(
EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, PAYLOAD_PATHS[0])
.await
.expect("payload retained")
.as_ref(),
bytes
);
}
#[tokio::test]
async fn legacy_inspection_rejects_complete_subsets_and_scope_ambiguity() {
let scoped = |set_index| {
let intent = MrfIntent {
bucket: Arc::from("bucket"),
object: Arc::from("a"),
version_id: None,
kind: MrfKind::PartialWrite,
scope: Some(MrfScope {
pool_index: 0,
set_index,
}),
lease: None,
enqueued_at_ms: 1234,
attempts: 0,
};
let mut bytes = Vec::new();
assert!(encode_intent(&intent, &mut bytes), "scoped fixture must encode");
bytes
};
let mut superset = payload("a");
superset.extend_from_slice(&payload("b"));
for (case, first_bytes, second_bytes) in [
("complete-subset", payload("a"), superset),
("different-set", scoped(1), scoped(2)),
("unknown-scope", payload("a"), scoped(1)),
] {
let root = TempDir::new().expect("test directory");
let first = disk(&root, "first").await;
let second = disk(&root, "second").await;
for (disk, bytes) in [(&first, &first_bytes), (&second, &second_bytes)] {
assert_eq!(decode_journal(bytes).1, 0, "{case}: complete fixture");
EcstoreDiskAPI::write_all(
disk.as_ref(),
RUSTFS_META_BUCKET,
MRF_SCOPED_JOURNAL_PATH,
EcstoreDiskBytes::copy_from_slice(bytes),
)
.await
.expect("write legacy replica");
}
for disks in [vec![first.clone(), second.clone()], vec![second.clone(), first.clone()]] {
assert!(
matches!(read_recovery_snapshot(&disks, 4096).await, Err(SnapshotError::Conflict)),
"{case}: neither replica order proves a latest snapshot"
);
}
for (disk, bytes) in [(&first, &first_bytes), (&second, &second_bytes)] {
assert_eq!(
EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH)
.await
.expect("legacy evidence retained")
.as_ref(),
bytes.as_slice(),
"{case}: inspection must preserve both source replicas"
);
}
}
}
#[tokio::test]
async fn oversized_or_corrupt_scoped_snapshot_never_falls_back_to_legacy() {
let root = TempDir::new().expect("test directory");
let disk = disk(&root, "disk").await;
EcstoreDiskAPI::write_all(disk.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH, vec![0; 1025].into())
.await
.expect("oversized fixture");
EcstoreDiskAPI::write_all(disk.as_ref(), RUSTFS_META_BUCKET, MRF_JOURNAL_PATH, payload("old").into())
.await
.expect("legacy fixture");
assert!(matches!(
read_recovery_snapshot(std::slice::from_ref(&disk), 1024).await,
Err(SnapshotError::TooLarge)
));
assert!(matches!(read_recovery_snapshot(&[disk], 2048).await, Err(SnapshotError::Corrupt)));
}
#[tokio::test]
async fn empty_legacy_replica_cannot_erase_records_in_a_torn_replica() {
let root = TempDir::new().expect("test directory");
let first = disk(&root, "first").await;
let second = disk(&root, "second").await;
let mut incomplete = payload("durable-object");
incomplete.extend_from_slice(b"torn");
EcstoreDiskAPI::write_all(first.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH, Vec::new().into())
.await
.expect("empty truncated replica");
EcstoreDiskAPI::write_all(second.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH, incomplete.clone().into())
.await
.expect("records and torn tail");
for disks in [vec![first.clone(), second.clone()], vec![second.clone(), first.clone()]] {
assert!(matches!(read_recovery_snapshot(&disks, 4096).await, Err(SnapshotError::Corrupt)));
}
assert_eq!(
EcstoreDiskAPI::read_all(second.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH)
.await
.expect("recovery anchor preserved")
.as_ref(),
incomplete
);
}
#[tokio::test]
async fn unreadable_commit_record_never_implies_legacy_only() {
let root = TempDir::new().expect("test directory");
let disk = disk(&root, "disk").await;
let legacy = payload("old");
EcstoreDiskAPI::write_all(disk.as_ref(), RUSTFS_META_BUCKET, MRF_JOURNAL_PATH, legacy.clone().into())
.await
.expect("legacy fixture");
// Opening a directory as a record either fails at open or at read,
// depending on the platform. Neither outcome proves absence.
std::fs::create_dir(root.path().join("disk").join(RUSTFS_META_BUCKET).join(MANIFEST_PATHS[0]))
.expect("unreadable manifest fixture");
let recovered = read_recovery_snapshot(std::slice::from_ref(&disk), 4096).await;
assert!(
matches!(recovered, Err(SnapshotError::Disk(_) | SnapshotError::Read(_))),
"must preserve unavailable proof: {recovered:?}"
);
assert_eq!(
EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, MRF_JOURNAL_PATH)
.await
.expect("legacy remains")
.as_ref(),
legacy
);
}
}
+1 -1
View File
@@ -999,7 +999,7 @@ impl HealTask {
let items = match since { let items = match since {
None => result_items.iter().map(|(_, item)| item.clone()).collect::<Vec<_>>(), None => result_items.iter().map(|(_, item)| item.clone()).collect::<Vec<_>>(),
Some(cursor) => { Some(cursor) => {
if cursor + 1 < min_seq { if cursor.saturating_add(1) < min_seq {
lagged = true; lagged = true;
} }
result_items result_items
@@ -44,7 +44,9 @@ use walkdir::WalkDir;
mod storage_api; mod storage_api;
use storage_api::integration::{BucketOperations, ECStore, MakeBucketOptions, ObjectIO as _, ObjectOperations as _}; use storage_api::integration::{
BucketOperations, ECStore, MakeBucketOptions, NamespaceLocking as _, ObjectIO as _, ObjectOperations as _,
};
/// 256 KiB + change: large enough to be stored as non-inline erasure shards /// 256 KiB + change: large enough to be stored as non-inline erasure shards
/// (so each data version materializes as an on-disk `part.*` file we can assert /// (so each data version materializes as an on-disk `part.*` file we can assert
@@ -106,6 +108,7 @@ async fn put_versioned(ecstore: &Arc<ECStore>, bucket: &str, object: &str, data:
.put_object(bucket, object, &mut reader, &opts) .put_object(bucket, object, &mut reader, &opts)
.await .await
.expect("versioned put_object failed"); .expect("versioned put_object failed");
wait_for_put_tail(ecstore, bucket, object).await;
info.version_id info.version_id
.map(|u| u.to_string()) .map(|u| u.to_string())
.expect("versioned put must return a version id") .expect("versioned put must return a version id")
@@ -117,6 +120,7 @@ async fn put_unversioned(ecstore: &Arc<ECStore>, bucket: &str, object: &str, dat
.put_object(bucket, object, &mut reader, &ObjectOptions::default()) .put_object(bucket, object, &mut reader, &ObjectOptions::default())
.await .await
.expect("unversioned put_object failed"); .expect("unversioned put_object failed");
wait_for_put_tail(ecstore, bucket, object).await;
} }
/// Create a delete-marker as the latest version (versioned:true, no version_id) /// Create a delete-marker as the latest version (versioned:true, no version_id)
@@ -160,20 +164,16 @@ fn xl_meta_path(obj_dir: &Path) -> PathBuf {
obj_dir.join("xl.meta") obj_dir.join("xl.meta")
} }
async fn wait_for_two_version_copies(disks: &[PathBuf], bucket: &str, object: &str) { async fn wait_for_put_tail(ecstore: &Arc<ECStore>, bucket: &str, object: &str) {
tokio::time::timeout(Duration::from_secs(5), async { // Shards and xl.meta can exist before the detached PUT owner finishes.
loop { let lock = ecstore
if disks.iter().all(|disk| { .new_ns_lock(bucket, object)
let object_dir = object_dir(disk, bucket, object); .await
xl_meta_path(&object_dir).exists() && count_part_files(&object_dir) >= 2 .expect("fixture namespace lock should be created");
}) { let _settled = lock
break; .get_write_lock(Duration::from_secs(30))
} .await
tokio::time::sleep(Duration::from_millis(10)).await; .expect("PUT rename tail must finish before inspecting or wiping the fixture");
}
})
.await
.expect("PUT rename tails must converge before wiping the versioned fixture");
} }
fn recreate_heal_opts() -> HealOpts { fn recreate_heal_opts() -> HealOpts {
@@ -305,7 +305,13 @@ mod serial_tests {
let data_v2 = versioned_test_data(20); let data_v2 = versioned_test_data(20);
let v1 = put_versioned(&ecstore, bucket, object, &data_v1).await; // OLD, non-latest let v1 = put_versioned(&ecstore, bucket, object, &data_v1).await; // OLD, non-latest
let v2 = put_versioned(&ecstore, bucket, object, &data_v2).await; // latest let v2 = put_versioned(&ecstore, bucket, object, &data_v2).await; // latest
wait_for_two_version_copies(&disk_paths, bucket, object).await; assert!(
disk_paths.iter().all(|disk| {
let dir = object_dir(disk, bucket, object);
xl_meta_path(&dir).exists() && count_part_files(&dir) >= 2
}),
"both versions must exist on every disk before wiping the fixture"
);
// ── Pre-wipe: prove the fixture actually has 2 versions on disk[0] ── // ── Pre-wipe: prove the fixture actually has 2 versions on disk[0] ──
let obj_dir0 = object_dir(&disk_paths[0], bucket, object); let obj_dir0 = object_dir(&disk_paths[0], bucket, object);
+1
View File
@@ -23,6 +23,7 @@ pub(crate) mod integration {
pub(crate) use rustfs_ecstore::api::storage::ECStore; pub(crate) use rustfs_ecstore::api::storage::ECStore;
pub(crate) use rustfs_storage_api::BucketOperations; pub(crate) use rustfs_storage_api::BucketOperations;
pub(crate) use rustfs_storage_api::MakeBucketOptions; pub(crate) use rustfs_storage_api::MakeBucketOptions;
pub(crate) use rustfs_storage_api::NamespaceLocking;
pub(crate) use rustfs_storage_api::ObjectIO; pub(crate) use rustfs_storage_api::ObjectIO;
pub(crate) use rustfs_storage_api::ObjectOperations; pub(crate) use rustfs_storage_api::ObjectOperations;
} }
+460 -44
View File
@@ -43,6 +43,10 @@ const ERR_LIFECYCLE_BUCKET_LOCKED: &str =
"ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an object locked bucket"; "ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an object locked bucket";
const ERR_LIFECYCLE_TOO_MANY_RULES: &str = "Lifecycle configuration should have at most 1000 rules"; const ERR_LIFECYCLE_TOO_MANY_RULES: &str = "Lifecycle configuration should have at most 1000 rules";
const ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS: &str = "'Days' for Expiration action must be a positive integer"; const ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS: &str = "'Days' for Expiration action must be a positive integer";
const ERR_LIFECYCLE_EXPIRATION_DAYS_DATE_CONFLICT: &str = "Expiration cannot specify both Days and Date";
const ERR_LIFECYCLE_MULTIPLE_TRANSITIONS: &str = "Only one Transition action per lifecycle rule is supported";
const ERR_LIFECYCLE_MULTIPLE_NONCURRENT_TRANSITIONS: &str =
"Only one NoncurrentVersionTransition action per lifecycle rule is supported";
const ERR_LIFECYCLE_INVALID_NONCURRENT_EXPIRATION_DAYS: &str = const ERR_LIFECYCLE_INVALID_NONCURRENT_EXPIRATION_DAYS: &str =
"'NoncurrentDays' for NoncurrentVersionExpiration action must be a positive integer"; "'NoncurrentDays' for NoncurrentVersionExpiration action must be a positive integer";
const ERR_LIFECYCLE_INVALID_ABORT_INCOMPLETE_MPU_DAYS: &str = const ERR_LIFECYCLE_INVALID_ABORT_INCOMPLETE_MPU_DAYS: &str =
@@ -361,6 +365,12 @@ impl Lifecycle for BucketLifecycleConfiguration {
{ {
return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_EXPIRED_OBJECT_ALL_VERSIONS)); return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_EXPIRED_OBJECT_ALL_VERSIONS));
} }
if expiration.days.is_some() && expiration.date.is_some() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
ERR_LIFECYCLE_EXPIRATION_DAYS_DATE_CONFLICT,
));
}
if let Some(expiration_date) = &expiration.date { if let Some(expiration_date) = &expiration.date {
let date = OffsetDateTime::from(expiration_date.clone()); let date = OffsetDateTime::from(expiration_date.clone());
if date.hour() != 0 || date.minute() != 0 || date.second() != 0 || date.nanosecond() != 0 { if date.hour() != 0 || date.minute() != 0 || date.second() != 0 || date.nanosecond() != 0 {
@@ -394,11 +404,20 @@ impl Lifecycle for BucketLifecycleConfiguration {
} }
} }
if let Some(transitions) = &r.transitions { if let Some(transitions) = &r.transitions {
if transitions.len() > 1 {
return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, ERR_LIFECYCLE_MULTIPLE_TRANSITIONS));
}
for transition in transitions { for transition in transitions {
TransitionOps::validate(transition)?; TransitionOps::validate(transition)?;
} }
} }
if let Some(noncurrent_transitions) = &r.noncurrent_version_transitions { if let Some(noncurrent_transitions) = &r.noncurrent_version_transitions {
if noncurrent_transitions.len() > 1 {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
ERR_LIFECYCLE_MULTIPLE_NONCURRENT_TRANSITIONS,
));
}
for transition in noncurrent_transitions { for transition in noncurrent_transitions {
NoncurrentVersionTransitionOps::validate(transition)?; NoncurrentVersionTransitionOps::validate(transition)?;
} }
@@ -473,6 +492,8 @@ impl Lifecycle for BucketLifecycleConfiguration {
} }
async fn eval(&self, obj: &ObjectOpts) -> Event { async fn eval(&self, obj: &ObjectOpts) -> Event {
// A single-object lookup cannot prove how many newer historical versions
// survive. Count-dependent actions wait for the complete-group evaluator.
self.eval_inner(obj, OffsetDateTime::now_utc(), 0).await self.eval_inner(obj, OffsetDateTime::now_utc(), 0).await
} }
@@ -536,23 +557,8 @@ impl Lifecycle for BucketLifecycleConfiguration {
return Event::default(); return Event::default();
}; };
if let Some(restore_expires) = obj.restore_expires if let Some(event) = obj.restored_copy_expiry(now) {
&& restore_expires.unix_timestamp() != 0 events.push(event);
&& now.unix_timestamp() > restore_expires.unix_timestamp()
{
let mut action = IlmAction::DeleteRestoredAction;
if !obj.is_latest {
action = IlmAction::DeleteRestoredVersionAction;
}
events.push(Event {
action,
due: Some(now),
rule_id: "".into(),
noncurrent_days: 0,
newer_noncurrent_versions: 0,
storage_class: "".into(),
});
} }
if let Some(ref lc_rules) = self.filter_rules(obj).await { if let Some(ref lc_rules) = self.filter_rules(obj).await {
@@ -611,17 +617,12 @@ impl Lifecycle for BucketLifecycleConfiguration {
continue; continue;
} }
if !obj.is_latest
&& let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration
&& let Some(retain_newer_noncurrent_versions) = noncurrent_version_expiration.newer_noncurrent_versions
&& newer_noncurrent_versions < usize::try_from(retain_newer_noncurrent_versions).unwrap_or(usize::MAX)
{
continue;
}
if !obj.is_latest if !obj.is_latest
&& let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration && let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration
&& let Some(noncurrent_days) = noncurrent_version_expiration.noncurrent_days && let Some(noncurrent_days) = noncurrent_version_expiration.noncurrent_days
&& noncurrent_version_expiration
.newer_noncurrent_versions
.is_none_or(|retain| usize::try_from(retain).is_ok_and(|retain| newer_noncurrent_versions >= retain))
{ {
if let Some(successor_mod_time) = obj.successor_mod_time { if let Some(successor_mod_time) = obj.successor_mod_time {
let expected_expiry = expected_expiry_time(successor_mod_time, noncurrent_days); let expected_expiry = expected_expiry_time(successor_mod_time, noncurrent_days);
@@ -651,7 +652,11 @@ impl Lifecycle for BucketLifecycleConfiguration {
&& let Some(noncurrent_version_transition) = rule && let Some(noncurrent_version_transition) = rule
.noncurrent_version_transitions .noncurrent_version_transitions
.as_ref() .as_ref()
.filter(|transitions| transitions.len() == 1)
.and_then(|transitions| transitions.first()) .and_then(|transitions| transitions.first())
&& noncurrent_version_transition
.newer_noncurrent_versions
.is_none_or(|retain| usize::try_from(retain).is_ok_and(|retain| newer_noncurrent_versions >= retain))
&& let Some(storage_class) = noncurrent_version_transition.storage_class.as_ref() && let Some(storage_class) = noncurrent_version_transition.storage_class.as_ref()
&& !storage_class.as_str().is_empty() && !storage_class.as_str().is_empty()
&& !obj.delete_marker && !obj.delete_marker
@@ -735,7 +740,11 @@ impl Lifecycle for BucketLifecycleConfiguration {
} }
if obj.transition_status != TRANSITION_COMPLETE if obj.transition_status != TRANSITION_COMPLETE
&& let Some(transition) = rule.transitions.as_ref().and_then(|transitions| transitions.first()) && let Some(transition) = rule
.transitions
.as_ref()
.filter(|transitions| transitions.len() == 1)
.and_then(|transitions| transitions.first())
&& let Some(storage_class) = transition.storage_class.as_ref() && let Some(storage_class) = transition.storage_class.as_ref()
&& !storage_class.as_str().is_empty() && !storage_class.as_str().is_empty()
{ {
@@ -758,18 +767,15 @@ impl Lifecycle for BucketLifecycleConfiguration {
} }
if !events.is_empty() { if !events.is_empty() {
// Select the winning event using a strict total order (MinIO semantics): // Eligible expiration takes precedence over transition, even when a
// the earliest `due` wins, and ties break toward delete-type actions. A // failed transition has an earlier deadline. Within each action class,
// missing `due` is treated as UNIX_EPOCH. This replaces a hand-written // prefer the earliest deadline using a deterministic total order.
// `sort_by` comparator that was not a strict weak ordering (it could return
// `Ordering::Less` for both `(a, b)` and `(b, a)`), which panics on the
// repository toolchain and did not deterministically pick the earliest event.
let event = events let event = events
.iter() .iter()
.min_by_key(|event| { .min_by_key(|event| {
( (
event.due.unwrap_or(OffsetDateTime::UNIX_EPOCH).unix_timestamp(),
ilm_action_priority_rank(&event.action), ilm_action_priority_rank(&event.action),
event.due.unwrap_or(OffsetDateTime::UNIX_EPOCH).unix_timestamp(),
) )
}) })
.cloned() .cloned()
@@ -1042,6 +1048,27 @@ impl ObjectOpts {
pub fn expired_object_deletemarker(&self) -> bool { pub fn expired_object_deletemarker(&self) -> bool {
self.delete_marker && self.is_latest && self.num_versions == 1 self.delete_marker && self.is_latest && self.num_versions == 1
} }
pub(crate) fn restored_copy_expiry(&self, now: OffsetDateTime) -> Option<Event> {
let restore_expires = self.restore_expires?;
// Restore metadata alone does not prove that a durable remote copy exists.
if self.transition_status != TRANSITION_COMPLETE
|| restore_expires.unix_timestamp() == 0
|| now.unix_timestamp() <= restore_expires.unix_timestamp()
{
return None;
}
let action = if self.is_latest {
IlmAction::DeleteRestoredAction
} else {
IlmAction::DeleteRestoredVersionAction
};
expiration_action_has_valid_target(action, self.version_id, self.is_latest, self.delete_marker).then(|| Event {
action,
due: Some(now),
..Default::default()
})
}
} }
/// Returns whether an expiry action has enough identity to target the object /// Returns whether an expiry action has enough identity to target the object
@@ -1064,11 +1091,8 @@ pub fn expiration_action_has_valid_target(
} }
} }
/// Total-order rank for lifecycle actions used to break `due` ties. /// Eligible logical expiration takes precedence over transition and restore-copy
/// /// cleanup. Deadlines break ties within an action class.
/// Delete-type actions rank before every other action so that, when two events
/// share the same `due`, a delete wins (MinIO semantics). The concrete numeric
/// values only matter relative to each other.
fn ilm_action_priority_rank(action: &IlmAction) -> u8 { fn ilm_action_priority_rank(action: &IlmAction) -> u8 {
match action { match action {
IlmAction::DeleteAllVersionsAction IlmAction::DeleteAllVersionsAction
@@ -4159,6 +4183,392 @@ mod tests {
assert_eq!(event.action, IlmAction::NoneAction); assert_eq!(event.action, IlmAction::NoneAction);
} }
mod adversarial_regressions {
use super::*;
use s3s::dto::NoncurrentVersionExpiration;
fn run(test: impl std::future::Future<Output = ()>) {
with_default_ilm_process_time(|| {
tokio::runtime::Builder::new_current_thread()
.build()
.expect("lifecycle regression runtime should build")
.block_on(test);
});
}
fn noncurrent_object() -> ObjectOpts {
ObjectOpts {
name: "logs/object".to_string(),
mod_time: Some(datetime!(2020-01-01 00:00:00 UTC)),
successor_mod_time: Some(datetime!(2020-01-02 00:00:00 UTC)),
version_id: Some(Uuid::from_u128(1)),
size: 1024 * 1024,
..Default::default()
}
}
#[test]
#[serial]
fn noncurrent_transition_retains_the_requested_newer_versions() {
run(async {
let mut rule = enabled_rule(None, None, Some("retain-two-hot-versions"));
rule.filter = Some(LifecycleRuleFilter::default());
rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition {
noncurrent_days: Some(1),
newer_noncurrent_versions: Some(2),
storage_class: Some(TransitionStorageClass::from_static("WARM")),
}]);
let lc = Arc::new(BucketLifecycleConfiguration {
rules: vec![rule],
expiry_updated_at: None,
});
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("valid noncurrent transition policy");
let objects = (0..4)
.map(|index| ObjectOpts {
mod_time: Some(datetime!(2020-01-05 00:00:00 UTC) - Duration::days(index)),
successor_mod_time: (index > 0).then_some(datetime!(2020-01-06 00:00:00 UTC) - Duration::days(index)),
version_id: Some(Uuid::from_u128(u128::try_from(index + 1).expect("small version index"))),
is_latest: index == 0,
num_versions: 4,
..noncurrent_object()
})
.collect::<Vec<_>>();
let actions = crate::Evaluator::new(lc)
.eval(&objects)
.await
.expect("complete version chain should evaluate")
.into_iter()
.map(|event| event.action)
.collect::<Vec<_>>();
assert_eq!(
actions,
[
IlmAction::NoneAction,
IlmAction::NoneAction,
IlmAction::NoneAction,
IlmAction::TransitionVersionAction
],
"the two newest noncurrent versions must remain in their current storage class"
);
});
}
#[test]
#[serial]
fn noncurrent_transition_checks_count_age_and_single_object_context() {
run(async {
let mut rule = enabled_rule(None, None, Some("retain-two"));
rule.filter = Some(LifecycleRuleFilter::default());
rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition {
noncurrent_days: Some(3),
newer_noncurrent_versions: Some(2),
storage_class: Some(TransitionStorageClass::from_static("WARM")),
}]);
let mut lc = BucketLifecycleConfiguration {
rules: vec![rule],
expiry_updated_at: None,
};
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("valid counted transition");
let object = noncurrent_object();
let now = datetime!(2020-01-10 00:00:00 UTC);
for (newer, expected) in [
(0, IlmAction::NoneAction),
(1, IlmAction::NoneAction),
(2, IlmAction::TransitionVersionAction),
(3, IlmAction::TransitionVersionAction),
] {
assert_eq!(lc.eval_inner(&object, now, newer).await.action, expected, "newer count: {newer}");
}
assert_eq!(
lc.eval_inner(&object, datetime!(2020-01-04 00:00:00 UTC), 2).await.action,
IlmAction::NoneAction,
"the retention count does not replace the age condition"
);
assert_eq!(
lc.eval(&object).await.action,
IlmAction::NoneAction,
"a single-object lookup must not assume a complete version history"
);
for retain in [None, Some(0), Some(-1), Some(i32::MAX)] {
lc.rules[0]
.noncurrent_version_transitions
.as_mut()
.expect("transition exists")[0]
.newer_noncurrent_versions = retain;
let expected = if matches!(retain, None | Some(0)) {
IlmAction::TransitionVersionAction
} else {
IlmAction::NoneAction
};
assert_eq!(lc.eval_inner(&object, now, 2).await.action, expected, "retention: {retain:?}");
}
});
}
#[test]
#[serial]
fn noncurrent_expiration_and_transition_have_independent_retention_counts() {
run(async {
let mut rule = enabled_rule(None, None, Some("independent-counts"));
rule.filter = Some(LifecycleRuleFilter::default());
rule.noncurrent_version_expiration = Some(NoncurrentVersionExpiration {
noncurrent_days: Some(90),
newer_noncurrent_versions: Some(4),
});
rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition {
noncurrent_days: Some(30),
newer_noncurrent_versions: Some(2),
storage_class: Some(TransitionStorageClass::from_static("WARM")),
}]);
let lc = BucketLifecycleConfiguration {
rules: vec![rule],
expiry_updated_at: None,
};
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("valid independent retention limits");
let object = noncurrent_object();
let now = datetime!(2020-05-01 00:00:00 UTC);
for (newer, expected) in [
(1, IlmAction::NoneAction),
(2, IlmAction::TransitionVersionAction),
(3, IlmAction::TransitionVersionAction),
(4, IlmAction::DeleteVersionAction),
] {
assert_eq!(lc.eval_inner(&object, now, newer).await.action, expected, "newer count: {newer}");
}
});
}
#[test]
#[serial]
fn expiration_retention_does_not_skip_an_independent_transition() {
run(async {
let mut rule = enabled_rule(None, None, Some("transition-then-expire"));
rule.filter = Some(LifecycleRuleFilter::default());
rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition {
noncurrent_days: Some(1),
newer_noncurrent_versions: None,
storage_class: Some(TransitionStorageClass::from_static("WARM")),
}]);
let mut lc = BucketLifecycleConfiguration {
rules: vec![rule],
expiry_updated_at: None,
};
let object = noncurrent_object();
let now = datetime!(2020-01-10 00:00:00 UTC);
let transition_only = lc.eval_inner(&object, now, 0).await;
assert_eq!(transition_only.action, IlmAction::TransitionVersionAction);
lc.rules[0].noncurrent_version_expiration = Some(NoncurrentVersionExpiration {
noncurrent_days: Some(90),
newer_noncurrent_versions: Some(2),
});
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("valid combined policy");
let combined = lc.eval_inner(&object, now, 0).await;
assert_eq!(combined.action, transition_only.action, "retention limits expiration, not transition");
assert_eq!(combined.storage_class, transition_only.storage_class);
});
}
#[test]
#[serial]
fn current_transition_rejects_multiple_stages_in_any_order() {
run(async {
let mut rule = enabled_rule(None, None, Some("two-current-transitions"));
rule.transitions = Some(vec![
Transition {
date: Some(datetime!(2020-03-01 00:00:00 UTC).into()),
days: None,
storage_class: Some(TransitionStorageClass::from_static("COLD")),
},
Transition {
date: Some(datetime!(2020-01-03 00:00:00 UTC).into()),
days: None,
storage_class: Some(TransitionStorageClass::from_static("WARM")),
},
]);
let mut lc = BucketLifecycleConfiguration {
rules: vec![rule],
expiry_updated_at: None,
};
let object = ObjectOpts {
is_latest: true,
..noncurrent_object()
};
let now = datetime!(2020-01-10 00:00:00 UTC);
for status in [ExpirationStatus::ENABLED, ExpirationStatus::DISABLED] {
lc.rules[0].status = ExpirationStatus::from_static(status);
for _ in 0..2 {
let err = lc
.validate(&ObjectLockConfiguration::default())
.await
.expect_err("multiple transition stages must be rejected");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
assert_eq!(err.to_string(), ERR_LIFECYCLE_MULTIPLE_TRANSITIONS);
assert_eq!(
lc.eval_inner(&object, now, 0).await.action,
IlmAction::NoneAction,
"legacy multi-stage configurations must not silently execute their first stage"
);
lc.rules[0]
.transitions
.as_mut()
.expect("transition array is present")
.reverse();
}
}
lc.rules[0]
.transitions
.as_mut()
.expect("transition array is present")
.remove(0);
lc.rules[0].status = ExpirationStatus::from_static(ExpirationStatus::ENABLED);
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("one stage is supported");
let event = lc.eval_inner(&object, now, 0).await;
assert_eq!(event.action, IlmAction::TransitionAction);
assert_eq!(event.storage_class, "WARM");
});
}
#[test]
#[serial]
fn noncurrent_transition_rejects_multiple_stages_in_any_order() {
run(async {
let mut rule = enabled_rule(None, None, Some("two-noncurrent-transitions"));
rule.noncurrent_version_transitions = Some(vec![
NoncurrentVersionTransition {
noncurrent_days: Some(30),
newer_noncurrent_versions: None,
storage_class: Some(TransitionStorageClass::from_static("COLD")),
},
NoncurrentVersionTransition {
noncurrent_days: Some(1),
newer_noncurrent_versions: None,
storage_class: Some(TransitionStorageClass::from_static("WARM")),
},
]);
let mut lc = BucketLifecycleConfiguration {
rules: vec![rule],
expiry_updated_at: None,
};
let object = noncurrent_object();
let now = datetime!(2020-01-10 00:00:00 UTC);
for status in [ExpirationStatus::ENABLED, ExpirationStatus::DISABLED] {
lc.rules[0].status = ExpirationStatus::from_static(status);
for _ in 0..2 {
let err = lc
.validate(&ObjectLockConfiguration::default())
.await
.expect_err("multiple noncurrent transition stages must be rejected");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
assert_eq!(err.to_string(), ERR_LIFECYCLE_MULTIPLE_NONCURRENT_TRANSITIONS);
assert_eq!(
lc.eval_inner(&object, now, 0).await.action,
IlmAction::NoneAction,
"legacy multi-stage configurations must not silently execute their first stage"
);
lc.rules[0]
.noncurrent_version_transitions
.as_mut()
.expect("transition array is present")
.reverse();
}
}
lc.rules[0]
.noncurrent_version_transitions
.as_mut()
.expect("transition array is present")
.remove(0);
lc.rules[0].status = ExpirationStatus::from_static(ExpirationStatus::ENABLED);
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("one stage is supported");
let event = lc.eval_inner(&object, now, 0).await;
assert_eq!(event.action, IlmAction::TransitionVersionAction);
assert_eq!(event.storage_class, "WARM");
});
}
#[test]
#[serial]
fn expiration_rejects_simultaneous_days_and_date() {
run(async {
let mut lc = BucketLifecycleConfiguration {
rules: vec![enabled_rule(
Some(LifecycleExpiration {
days: Some(1),
..Default::default()
}),
None,
Some("ambiguous-expiry"),
)],
expiry_updated_at: None,
};
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("a single Days expiration is valid");
lc.rules[0].expiration.as_mut().expect("expiration is present").date =
Some(datetime!(2099-01-01 00:00:00 UTC).into());
let err = lc
.validate(&ObjectLockConfiguration::default())
.await
.expect_err("Days and Date are mutually exclusive; accepting both silently overrides Days");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
assert_eq!(err.to_string(), ERR_LIFECYCLE_EXPIRATION_DAYS_DATE_CONFLICT);
});
}
#[test]
#[serial]
fn overdue_transition_does_not_starve_permanent_expiration() {
run(async {
let mut rule = enabled_rule(
Some(LifecycleExpiration {
days: Some(90),
..Default::default()
}),
None,
Some("archive-then-delete"),
);
rule.transitions = Some(vec![Transition {
days: Some(30),
date: None,
storage_class: Some(TransitionStorageClass::from_static("WARM")),
}]);
let lc = BucketLifecycleConfiguration {
rules: vec![rule],
expiry_updated_at: None,
};
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("valid transition and expiration policy");
let object = ObjectOpts {
is_latest: true,
version_id: None,
transition_status: TRANSITION_PENDING.to_string(),
..noncurrent_object()
};
let before_expiration = lc.eval_inner(&object, datetime!(2020-02-15 00:00:00 UTC), 0).await;
assert_eq!(before_expiration.action, IlmAction::TransitionAction);
let overdue = lc.eval_inner(&object, datetime!(2020-05-01 00:00:00 UTC), 0).await;
assert_eq!(
overdue.action,
IlmAction::DeleteAction,
"an unavailable tier must not prevent permanent expiration indefinitely"
);
});
}
}
/// Property-based tests for the rule evaluator (backlog#1148 ilm-14, /// Property-based tests for the rule evaluator (backlog#1148 ilm-14,
/// follow-up to backlog#1030 / rustfs#4455). /// follow-up to backlog#1030 / rustfs#4455).
/// ///
@@ -4169,7 +4579,7 @@ mod tests {
/// ///
/// * `eval_inner` never panics and is deterministic for a fixed input; /// * `eval_inner` never panics and is deterministic for a fixed input;
/// * the winning event matches an independently recomputed candidate set: /// * the winning event matches an independently recomputed candidate set:
/// earliest `due` wins, ties break toward delete-class actions (the /// eligible expiration wins over transition, then earliest `due` wins (the
/// `min_by_key` selection that replaced the rustfs#4455 comparator); /// `min_by_key` selection that replaced the rustfs#4455 comparator);
/// * `expected_expiry_time` is monotonically non-decreasing in `days` and /// * `expected_expiry_time` is monotonically non-decreasing in `days` and
/// always lands on the processing boundary, both at production defaults /// always lands on the processing boundary, both at production defaults
@@ -4458,8 +4868,8 @@ mod tests {
/// consider for a live current version under `selection`-shaped rules /// consider for a live current version under `selection`-shaped rules
/// (expiration and first-transition only, no filters): expiration /// (expiration and first-transition only, no filters): expiration
/// fires when `now >= due`, transition when `now > due` and the object /// fires when `now >= due`, transition when `now > due` and the object
/// has not already transitioned. Selection semantics under test: /// has not already transitioned. Eligible expiration wins over transition;
/// earliest due wins, ties prefer delete-class. /// the earliest deadline wins within the selected action class.
fn oracle_candidates(lc: &BucketLifecycleConfiguration, obj: &ObjectOpts, now: OffsetDateTime) -> Vec<Candidate> { fn oracle_candidates(lc: &BucketLifecycleConfiguration, obj: &ObjectOpts, now: OffsetDateTime) -> Vec<Candidate> {
let mod_time = obj.mod_time.expect("selection strategy always sets mod_time"); let mod_time = obj.mod_time.expect("selection strategy always sets mod_time");
let mut candidates = Vec::new(); let mut candidates = Vec::new();
@@ -4548,8 +4958,8 @@ mod tests {
/// Differential test of winner selection (the rustfs#4455 fix): /// Differential test of winner selection (the rustfs#4455 fix):
/// for a live current version under randomized expiration and /// for a live current version under randomized expiration and
/// transition rules, `eval_inner`'s winner must carry the /// transition rules, `eval_inner`'s winner must carry the
/// minimum `(due, rank)` of the independently recomputed /// earliest expiration from the independently recomputed candidate
/// candidate set — earliest due wins, ties prefer delete-class — /// set, or the earliest transition when no expiration is eligible,
/// and must be `NoneAction` exactly when that set is empty. /// and must be `NoneAction` exactly when that set is empty.
#[test] #[test]
#[serial] #[serial]
@@ -4578,7 +4988,13 @@ mod tests {
// Oracle and evaluator must observe the same (pinned) time env. // Oracle and evaluator must observe the same (pinned) time env.
let (event, expected) = with_production_time_env(|| { let (event, expected) = with_production_time_env(|| {
let expected = oracle_candidates(&lc, &obj, now).into_iter().min(); let candidates = oracle_candidates(&lc, &obj, now);
let expected = candidates
.iter()
.filter(|(_, rank)| *rank == 0)
.min()
.copied()
.or_else(|| candidates.into_iter().min());
let rt = tokio::runtime::Builder::new_current_thread() let rt = tokio::runtime::Builder::new_current_thread()
.enable_all() .enable_all()
.build() .build()

Some files were not shown because too many files have changed in this diff Show More