Compare commits

..

1 Commits

Author SHA1 Message Date
overtrue 693800db8e fix(ecstore): retry decommission entries safely on source changes
A single object's SourceChanged during decommission cleanup no longer
cancels the shared worker token and fails the whole pool operation.
Cleanup preflight and source-cleanup outcomes now retry per entry with
bounded attempts and cancellation-aware backoff, applied uniformly to
ordinary versions, delete markers, and tiered copies (removing the
try-once-only branches); every retry re-lists the entry and redoes
version multiset validation before touching the source. Only quorum
loss, unrecoverable system errors, or exceeding a pool-level
SourceChanged exhaustion threshold still fails the decommission, and
exhausted entries never delete their source versions.

Retry attempts, backoff, and deferred/exhausted reasons are logged per
entry for observability. Heavy regression tests spawn on dedicated
32MiB stacks following the existing store-test pattern.

Fixes rustfs/backlog#1913
2026-08-23 10:37:55 +08:00
283 changed files with 5199 additions and 49064 deletions
-20
View File
@@ -1,20 +0,0 @@
# Report-only calibration baseline from https://github.com/rustfs/rustfs/actions/runs/29394996173.
# Update counts only with a linked coverage run and a reviewed explanation.
phase = "report-only"
allowed_drop_percentage_points = 1.0
[crates."crates/iam"]
covered = 5149
count = 8131
[crates."crates/kms"]
covered = 2950
count = 4200
[crates."crates/policy"]
covered = 4636
count = 5464
[crates."crates/crypto"]
covered = 469
count = 494
+2 -2
View File
@@ -1,2 +1,2 @@
sha256-darwin=88ee9684ece0e27294f2b3f0c9c8fe62890feff76aa47279d42dab0af3196fe2
sha256-linux=d13337936af6778b1d2b2b255ae7fd350fdec94034be46daf738bd577653f799
sha256-darwin=9f767b37ed8b1c82da62ea441462d75487785c8086e56f08fb6f6cd89c6e2e52
sha256-linux=fbdaf42b220958d4b1e8880e0f8b5a7992d38e21051bb60596dd4538424757d6
-1
View File
@@ -36,7 +36,6 @@ script-tests: ## Run shell script tests
./scripts/test_manual_transition_runbooks.sh
./scripts/check_embedded_secrets.sh --self-test
python3 ./scripts/check_test_wiring.py --self-test
python3 ./scripts/check_security_coverage.py --self-test
python3 ./scripts/check_scheduled_validation_freshness.py --self-test
python3 ./scripts/s3-tests/test_report_compat.py
bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
+8 -19
View File
@@ -78,12 +78,6 @@ test-group = 'embedded-test-ports'
filter = 'package(rustfs-ecstore) & test(manual_transition_page_checkpoint_persists_durable_job_progress)'
test-group = 'ecstore-serial-flaky'
# The durable ILM decommission regressions build isolated multi-pool stores and
# deliberately take source or target disks offline while checking fencing.
[[profile.default.overrides]]
filter = 'package(rustfs-ecstore) & (test(decommission_migrates_and_verifies_registered_durable_ilm_records) | test(decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success) | test(decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup) | test(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page) | test(decommission_durable_ilm_recovery_keeps_multiple_active_sources))'
test-group = 'ecstore-serial-flaky'
# Serialize the bucket-incarnation / lifecycle-fence tests. They drive
# init_bucket_metadata_sys and bucket_metadata_sys_of, i.e. process-global
# OnceLock state that serial_test's #[serial] cannot protect across nextest's
@@ -113,10 +107,9 @@ filter = 'package(e2e_test) & test(/^inline_fast_path_cluster_test::/)'
test-group = 'e2e-inline-boundaries'
# Vault KMS tests share the fixed dev-server port 8200. serial_test's #[serial]
# does not cross nextest process boundaries, so keep every Vault-backed test in
# one group.
# does not cross nextest process boundaries, so keep these tests in one group.
[[profile.default.overrides]]
filter = 'package(e2e_test) & (test(/^kms::kms_vault_test::/) | test(/^kms::configured_roundtrip_test::test_configured_vault_kms_admin_and_versioned_cleanup$/))'
filter = 'package(e2e_test) & test(/^kms::kms_vault_test::/)'
test-group = 'e2e-vault'
# ---------------------------------------------------------------------------
@@ -197,10 +190,6 @@ test-group = 'embedded-test-ports'
filter = 'package(rustfs-ecstore) & test(manual_transition_page_checkpoint_persists_durable_job_progress)'
test-group = 'ecstore-serial-flaky'
[[profile.ci.overrides]]
filter = 'package(rustfs-ecstore) & (test(decommission_migrates_and_verifies_registered_durable_ilm_records) | test(decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success) | test(decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup) | test(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page) | test(decommission_durable_ilm_recovery_keeps_multiple_active_sources))'
test-group = 'ecstore-serial-flaky'
# Serialize the bucket-incarnation / lifecycle-fence tests under the ci profile
# too (see the matching default-profile override near the top). No retries.
[[profile.ci.overrides]]
@@ -316,7 +305,7 @@ slow-timeout = { period = "60s", terminate-after = 2, grace-period = "10s" }
# the target incl. multipart and the resync path, SSE-C and
# target-without-KMS stay fail-closed), and one guards event/history
# observers.
# * 13 `_real_dual_node` site-replication tests — each spawns TWO full rustfs
# * 12 `_real_dual_node` site-replication tests — each spawns TWO full rustfs
# servers and drives the cross-process site-replication control plane.
# * 1 `_real_three_node` site-replication test.
# * 1 `_real_single_node` service-account round-trip test.
@@ -336,8 +325,8 @@ slow-timeout = { period = "60s", terminate-after = 2, grace-period = "10s" }
#
# Wired by .github/workflows/e2e-replication-nightly.yml (schedule +
# workflow_dispatch), which builds the rustfs binary once, installs awscurl so
# the STS dual-node test actually exercises its path (the test fails when
# awscurl is absent), and routes scheduled failures
# the STS dual-node test actually exercises its path (it skips gracefully with
# a visible log line when awscurl is absent), and routes scheduled failures
# through .github/actions/schedule-failure-issue (ci-8). Explicit division of
# labor with e2e-full: these tests run only in the consolidated nightly
# workflow, not in the merge/main lane.
@@ -407,10 +396,10 @@ path = "junit.xml"
# object_lambda) — too heavy for the merge budget; they run in the
# e2e-nightly serial cluster-fault lane.
# * replication_extension_test — repl-1 already splits it into the PR
# `e2e-smoke` (20 fast) and `e2e-repl-nightly` (56 slow) lanes and reserves
# `e2e-smoke` (20 fast) and `e2e-repl-nightly` (55 slow) lanes and reserves
# it for those, so e2e-full does not double-run it.
# * #[ignore]d tests — nextest skips them by default (no --run-ignored); the
# manual-localhost:9000 reliant tests are ci-13's migration.
# manual-localhost:9000 reliant/policy tests are ci-13's migration.
#
# Each e2e test spawns its own single-node rustfs server on a random port with
# an isolated temp dir (crates/e2e_test/src/common.rs), so the set is
@@ -454,5 +443,5 @@ filter = 'package(e2e_test) & test(/^inline_fast_path_cluster_test::/)'
test-group = 'e2e-inline-boundaries'
[[profile.e2e-full.overrides]]
filter = 'package(e2e_test) & (test(/^kms::kms_vault_test::/) | test(/^kms::configured_roundtrip_test::test_configured_vault_kms_admin_and_versioned_cleanup$/))'
filter = 'package(e2e_test) & test(/^kms::kms_vault_test::/)'
test-group = 'e2e-vault'
-27
View File
@@ -681,19 +681,6 @@ jobs:
cache-save-if: 'false'
install-build-packaging-tools: 'false'
- name: Set up Python
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: "3.12"
- name: Install awscurl
run: |
python3 -m pip install --user --upgrade pip "awscurl==0.44"
echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"
- name: Verify awscurl
run: test -x "$AWSCURL_PATH"
# Download after the cache restore so the freshly built binary from the
# build job always wins over anything restored into target/debug.
- name: Download debug binary
@@ -816,20 +803,6 @@ jobs:
- name: Verify awscurl
run: test -x "$AWSCURL_PATH"
- name: Install mc
env:
MC_VERSION: RELEASE.2025-08-13T08-35-41Z
MC_SHA256: 01f866e9c5f9b87c2b09116fa5d7c06695b106242d829a8bb32990c00312e891
run: |
MC_BINARY="mc.linux-amd64.${MC_VERSION}"
curl -fsSLo "$RUNNER_TEMP/mc" "https://github.com/minio/mc/releases/download/${MC_VERSION}/${MC_BINARY}"
echo "${MC_SHA256} $RUNNER_TEMP/mc" | sha256sum --check --status
chmod +x "$RUNNER_TEMP/mc"
echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
- name: Verify mc
run: mc --version
- name: Install Vault
run: |
VAULT_VERSION="1.17.6"
+12 -29
View File
@@ -12,12 +12,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Workspace line-coverage baseline and security-crate calibration
# (backlog#1153 infra-5/infra-6).
# Weekly workspace line-coverage baseline (backlog#1153 infra-5).
#
# NON-BLOCKING by design: the weekly job gives coverage a visible baseline and
# trend, while relevant pull requests run a report-only security-crate
# comparison. Neither job is a required check during calibration.
# NON-BLOCKING by design: this workflow only runs on schedule and manual
# dispatch, so it never attaches a status to a PR and must never be made a
# required check. It exists to give coverage a visible baseline and trend
# (per-crate table in the job summary, lcov artifact kept 90 days) — the
# per-crate ratchet for the security-critical crates builds on it later
# (backlog#1153 infra-6, report-only first per the ci-11 ladder).
#
# Measurement scope matches the PR test gate (ci.yml "Run tests"):
# `--workspace --exclude e2e_test` with the `ci` nextest profile. Doctests are
@@ -29,17 +31,6 @@
name: coverage
on:
pull_request:
branches: [main]
paths:
- "crates/iam/**"
- "crates/kms/**"
- "crates/policy/**"
- "crates/crypto/**"
- ".config/coverage-baselines.toml"
- "scripts/coverage_per_crate.py"
- "scripts/check_security_coverage.py"
- ".github/workflows/coverage.yml"
workflow_dispatch:
schedule:
# 07:00 UTC Sunday — staggered clear of the other Sunday crons: ci (00:00),
@@ -48,10 +39,6 @@ on:
# e2e-replication-nightly (04:00) and performance-ab (06:00) lanes.
- cron: "43 7 * * 0"
concurrency:
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name != 'schedule' }}
# Only alert-on-failure needs more than read access; it declares its own
# job-level `issues: write`.
permissions:
@@ -59,14 +46,12 @@ permissions:
jobs:
coverage:
name: Workspace line coverage
name: Workspace coverage (weekly)
runs-on: sm-standard-4
# The instrumented build cannot reuse the regular CI cache (different
# RUSTFLAGS), so a cold run rebuilds the workspace before running the
# full suite. Two later exact-head runs exhausted 150 minutes before the
# report steps, so allow one additional 90-minute cold-run margin while
# keeping the calibration job bounded.
timeout-minutes: 240
# RUSTFLAGS), so a cold week rebuilds the workspace before running the
# full suite; give it double the test job's 60-minute budget.
timeout-minutes: 120
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
# Match the PR gate's nextest semantics (ci.yml runs `--profile ci`):
@@ -106,9 +91,7 @@ jobs:
cargo llvm-cov report --json --output-path target/llvm-cov/coverage.json
- name: Write per-crate summary
run: |
python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"
python3 scripts/check_security_coverage.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"
run: python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"
- name: Upload coverage artifact
if: always()
@@ -75,7 +75,11 @@ jobs:
cache-save-if: ${{ github.ref == 'refs/heads/main' }}
install-build-packaging-tools: 'false'
# The STS dual-node test requires awscurl and fails if it is unavailable.
# awscurl lets the STS dual-node test actually exercise its path. Without
# it the test skips gracefully with a visible log line
# (`awscurl_available()` in crates/e2e_test/src/common.rs), so the lane
# still passes — installing it just upgrades that one test from skip to
# real coverage.
- name: Set up Python
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
@@ -83,7 +87,7 @@ jobs:
- name: Install awscurl
run: |
python3 -m pip install --user --upgrade pip "awscurl==0.44"
python3 -m pip install --user --upgrade pip awscurl
echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"
- name: Verify awscurl
@@ -192,11 +196,8 @@ jobs:
cache-save-if: 'false'
install-build-packaging-tools: 'false'
- name: Install and verify protocol socket oracle
run: |
sudo apt-get update -qq
sudo apt-get install -y -qq iproute2
ss -tn state CLOSE-WAIT >/dev/null
- name: Verify protocol socket oracle
run: ss -tn state CLOSE-WAIT >/dev/null
# The suite owns fixed protocol ports and serializes its internal cases.
- name: Verify protocol e2e membership
+2 -89
View File
@@ -21,9 +21,6 @@
# suite and reports promotion candidates. Regressions, unclassified tests,
# incomplete execution, and infrastructure errors fail the job; classified
# failures for not-yet-implemented features remain informational.
# - Non-blocking upstream HEAD canary: collects current upstream node IDs and
# reports new, removed, duplicate, or overlapping classifications without
# making upstream drift a release gate.
# - Manual runs (workflow_dispatch): same, with configurable mode/scope.
#
# All test execution is delegated to scripts/s3-tests/run.sh (single source of
@@ -181,14 +178,9 @@ jobs:
- name: Install Python tools
run: |
python3 -m pip install --user --upgrade pip "awscurl==0.44" "tox==4.60.0"
python3 -m pip install --user --upgrade pip awscurl tox
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
- name: Verify Python tools
run: |
test "$(python3 -c 'import importlib.metadata as m; print(m.version("awscurl"))')" = "0.44"
test "$(python3 -c 'import importlib.metadata as m; print(m.version("tox"))')" = "4.60.0"
- name: Enable buildx
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
@@ -311,7 +303,7 @@ jobs:
- name: Wait for RustFS ready
run: |
for _ in {1..120}; do
if curl -sf "http://${S3_HOST}:${S3_PORT}/health/ready" >/dev/null 2>&1; then
if curl -sf "http://${S3_HOST}:${S3_PORT}/health" >/dev/null 2>&1; then
echo "RustFS is ready"
exit 0
fi
@@ -362,85 +354,6 @@ jobs:
name: s3tests-${{ env.TEST_MODE }}-shard-${{ matrix.shard-index }}
path: artifacts/**
upstream-head-canary:
name: Upstream HEAD classification canary
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
continue-on-error: true
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: "3.12"
- name: Install collection tool
run: |
python3 -m pip install --user "tox==4.60.0"
python3 - <<'PY'
from importlib.metadata import version
assert version("tox") == "4.60.0"
PY
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
- name: Compare upstream HEAD classifications
id: upstream-compare
run: |
ARTIFACT_DIR="artifacts/s3tests-upstream-head"
UPSTREAM_DIR="${RUNNER_TEMP}/s3-tests-upstream"
mkdir -p "${ARTIFACT_DIR}"
git clone --depth 1 https://github.com/ceph/s3-tests.git "${UPSTREAM_DIR}"
git -C "${UPSTREAM_DIR}" rev-parse HEAD > "${ARTIFACT_DIR}/upstream-sha.txt"
cp "${UPSTREAM_DIR}/s3tests.conf.SAMPLE" "${UPSTREAM_DIR}/s3tests.conf"
(
cd "${UPSTREAM_DIR}"
S3TEST_CONF="${UPSTREAM_DIR}/s3tests.conf" tox -- \
-q --collect-only s3tests/functional/test_s3.py \
-m "not rustfs_never_marker"
) 2>&1 | tee "${ARTIFACT_DIR}/collect.log"
grep -E '^s3tests/functional/test_s3\.py::' \
"${ARTIFACT_DIR}/collect.log" > "${ARTIFACT_DIR}/collected-nodeids.txt"
python3 scripts/s3-tests/report_compat.py \
--lists-dir scripts/s3-tests \
--collected-nodeids "${ARTIFACT_DIR}/collected-nodeids.txt" \
--check-classifications-only 2>&1 | tee "${ARTIFACT_DIR}/classification-drift.txt"
- name: Publish canary report
if: always()
env:
CANARY_OUTCOME: ${{ steps.upstream-compare.outcome }}
run: |
{
echo "## ceph/s3-tests upstream HEAD canary"
echo
if [ -f artifacts/s3tests-upstream-head/upstream-sha.txt ]; then
echo "Upstream HEAD: $(cat artifacts/s3tests-upstream-head/upstream-sha.txt)"
fi
echo
echo '```text'
if [ -s artifacts/s3tests-upstream-head/classification-drift.txt ]; then
cat artifacts/s3tests-upstream-head/classification-drift.txt
elif [ "${CANARY_OUTCOME}" != "success" ]; then
echo "Canary did not complete; inspect the collection log artifact."
else
echo "No classification drift detected."
fi
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
- name: Upload canary artifacts
if: always() && env.ACT != 'true'
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: s3tests-upstream-head
path: artifacts/s3tests-upstream-head/**
retention-days: 14
alert-on-failure:
name: Alert on scheduled failure
needs: [s3tests]
+1 -1
View File
@@ -155,7 +155,7 @@ jobs:
- name: Wait for RustFS ready
run: |
for _ in {1..60}; do
if curl -sf http://127.0.0.1:9000/health/ready >/dev/null 2>&1; then
if curl -sf http://127.0.0.1:9000/health >/dev/null 2>&1; then
echo "RustFS is ready"
exit 0
fi
+6 -3
View File
@@ -39,10 +39,11 @@ jobs:
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
steps:
- name: Checkout repository
- name: Checkout main branch
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
ref: main
- name: Setup Rust environment
uses: ./.github/actions/setup
@@ -88,10 +89,11 @@ jobs:
# either casing.
NO_PROXY: 127.0.0.1,localhost
steps:
- name: Checkout repository
- name: Checkout main branch
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
ref: main
- name: Setup Rust environment
uses: ./.github/actions/setup
@@ -176,10 +178,11 @@ jobs:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
NO_PROXY: 127.0.0.1,localhost
steps:
- name: Checkout repository
- name: Checkout main branch
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
ref: main
- name: Setup Rust environment
uses: ./.github/actions/setup
+4 -8
View File
@@ -121,14 +121,14 @@ jobs:
candidate_sha="$(git rev-parse HEAD)"
if [[ "${{ github.event_name }}" == "schedule" ]]; then
baseline_sha="${SCHEDULED_BASELINE_SHA:-$candidate_sha}"
if ! git merge-base --is-ancestor "$baseline_sha" "$candidate_sha"; then
echo "::error::scheduled baseline $baseline_sha is not an ancestor of candidate $candidate_sha" >&2
exit 1
fi
else
baseline_sha="$(git rev-parse origin/main)"
fi
git cat-file -e "${baseline_sha}^{commit}"
if ! git merge-base --is-ancestor "$baseline_sha" "$candidate_sha"; then
echo "::error::baseline $baseline_sha is not an ancestor of candidate $candidate_sha; update the selected ref before comparing" >&2
exit 1
fi
echo "baseline_sha=$baseline_sha" >> "$GITHUB_OUTPUT"
echo "candidate_sha=$candidate_sha" >> "$GITHUB_OUTPUT"
echo "baseline commit: $baseline_sha"
@@ -342,10 +342,6 @@ jobs:
if: always()
run: |
status="${{ steps.ab.outputs.status }}"
if [[ -z "$status" ]]; then
echo "::error::warp A/B setup failed before the rig ran. Check the first failed workflow step." >&2
exit 1
fi
if [[ "$status" != "0" ]]; then
echo "::error::warp A/B budget gate failed (exit $status). See the step summary / gate.md artifact." >&2
exit "$status"
+1 -2
View File
@@ -30,8 +30,7 @@ make build-docker BUILD_OS=ubuntu22.04
- Crate membership: `Cargo.toml` `[workspace].members`
- Architecture, layering, crate map: [ARCHITECTURE.md](ARCHITECTURE.md)
- Migration guardrails & readiness contracts: [docs/architecture/](docs/architecture/README.md)
- CI workflow steps: `.github/workflows/`; event, timeout, and required-status
matrix: [docs/testing/ci-gates.md](docs/testing/ci-gates.md)
- CI gates: `.github/workflows/ci.yml` (source of truth; never copy its steps into docs)
- Test-layer taxonomy, per-layer entry commands, serial/nextest rules, flake
policy: [docs/testing/README.md](docs/testing/README.md)
- Tier/ILM transition debugging (xl.meta inspection, versionId tracing):
-2
View File
@@ -70,8 +70,6 @@ make pre-pr
> For the full test-layer taxonomy (unit / ecstore black-box / e2e / s3s-e2e / S3 compatibility / chaos / fuzz / bench), each layer's entry command, the naming conventions the migration gate depends on, and the serial/nextest rules, see [docs/testing/README.md](docs/testing/README.md).
> For the event, timeout, required-status, and local reproduction matrix, see [docs/testing/ci-gates.md](docs/testing/ci-gates.md).
### 🔒 Automated Pre-commit Hooks
#### What `make pre-commit` and `make pre-pr` actually run
Generated
+81 -132
View File
@@ -68,16 +68,16 @@ dependencies = [
[[package]]
name = "aes-gcm"
version = "0.11.1"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f2b8006a0c83f52b62ba44a97b58bf76fe2f70a329e588f67f89691d93d498f"
checksum = "fdf011db2e21ce0d575593d749db5554b47fed37aff429e4dc50bc91ac93a028"
dependencies = [
"aead",
"aes 0.9.2",
"cipher 0.5.2",
"ctr",
"ctutils",
"ghash",
"subtle",
"zeroize",
]
@@ -333,12 +333,6 @@ dependencies = [
"password-hash",
]
[[package]]
name = "array-init"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc"
[[package]]
name = "arrayvec"
version = "0.7.8"
@@ -819,12 +813,11 @@ dependencies = [
[[package]]
name = "async_zip"
version = "0.0.19"
version = "0.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb7f5f40e1eb30949a266fc900d37fd3267c7baf50c3705ac09c5d8ced5def63"
checksum = "0d8c50d65ce1b0e0cb65a785ff615f78860d7754290647d3b983208daa4f85e6"
dependencies = [
"async-compression",
"binrw",
"crc32fast",
"futures-lite",
"pin-project",
@@ -876,9 +869,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
[[package]]
name = "aws-config"
version = "1.11.0"
version = "1.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a767267da9e2c2e189b2f9df8b5657e850ecf5352644734ba130d4a57095cf1b"
checksum = "1b180a3c8b55960db3426d8964b8745e652466a1a49fe1a2eda828046d30b5e4"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -971,9 +964,9 @@ dependencies = [
[[package]]
name = "aws-sdk-kms"
version = "1.116.0"
version = "1.115.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "484ecdbea2a1cfc0e6eea69ce0a665f93913671b303ba40b2361b1d826544e7e"
checksum = "d5b034f8b7ceadb873d0bc607c30bb4b0be68e09a84c837174e7c2c6878ff882"
dependencies = [
"arc-swap",
"aws-credential-types",
@@ -997,9 +990,9 @@ dependencies = [
[[package]]
name = "aws-sdk-s3"
version = "1.143.0"
version = "1.142.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a0ade5433c9561daac7c0c6bc910f1240b4f8ec0d6148b0b463aac0691d747c9"
checksum = "f9e15a5c55e05f4b0b7e483160b3c85cccdf77cff02c95504f3e71d460855cd2"
dependencies = [
"arc-swap",
"aws-credential-types",
@@ -1034,9 +1027,9 @@ dependencies = [
[[package]]
name = "aws-sdk-sso"
version = "1.107.0"
version = "1.106.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "769b0abd0f89cfe11da5099986dd493e4f94347ce9a4562cb86ddecfe926b6c0"
checksum = "2d0efcee834347b6705eca3eea2defd88242f43774f55d7326604222e3c86260"
dependencies = [
"arc-swap",
"aws-credential-types",
@@ -1060,9 +1053,9 @@ dependencies = [
[[package]]
name = "aws-sdk-ssooidc"
version = "1.109.0"
version = "1.108.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f4075b8a2c8cda4076a3dcc43b9d6dabd93e0c2502abeaaf7e14aaead9bb312b"
checksum = "a59312a04cf19c962cfee32b64ecfee758f8786407ff6da5b30fff46ae96f201"
dependencies = [
"arc-swap",
"aws-credential-types",
@@ -1086,9 +1079,9 @@ dependencies = [
[[package]]
name = "aws-sdk-sts"
version = "1.112.0"
version = "1.111.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f582002918346a3e685be1b391c7bea155073088cea6bd4e4b7663df9e43b6c"
checksum = "120e7eb63457a9e547f9986fe3b273f77c43679da4d04f46359fa881c5e19b6e"
dependencies = [
"arc-swap",
"aws-credential-types",
@@ -1551,30 +1544,6 @@ dependencies = [
"serde",
]
[[package]]
name = "binrw"
version = "0.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ad120d555272286c1017d25165ab8bd74806f13fc85b258484ec7e4ce75458f"
dependencies = [
"array-init",
"binrw_derive",
"bytemuck",
]
[[package]]
name = "binrw_derive"
version = "0.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6df92e0e9baae4dc82c7bad7715ca40c0a5c71539057bf2ea04a5c29c980410b"
dependencies = [
"either",
"owo-colors",
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]]
name = "bit-set"
version = "0.8.0"
@@ -1662,9 +1631,9 @@ dependencies = [
[[package]]
name = "blocking"
version = "1.7.0"
version = "1.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a70e4329df6cb94385eed412ec92375c3cdd8a6e502493d1229b6414e4036dfa"
checksum = "e83f8d02be6967315521be875afa792a316e28d57b5a2d401897e2a7921b7f21"
dependencies = [
"async-channel",
"async-task",
@@ -1753,12 +1722,6 @@ version = "3.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
[[package]]
name = "bytemuck"
version = "1.25.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797"
[[package]]
name = "byteorder"
version = "1.5.0"
@@ -1895,9 +1858,9 @@ dependencies = [
[[package]]
name = "cc"
version = "1.4.4"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273"
checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d"
dependencies = [
"find-msvc-tools",
"jobserver",
@@ -2362,9 +2325,9 @@ dependencies = [
[[package]]
name = "crc32fast"
version = "1.5.1"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550"
checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
dependencies = [
"cfg-if",
]
@@ -2559,6 +2522,12 @@ dependencies = [
"subtle",
]
[[package]]
name = "cty"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35"
[[package]]
name = "curve25519-dalek"
version = "4.1.3"
@@ -3971,9 +3940,9 @@ dependencies = [
[[package]]
name = "either"
version = "1.18.0"
version = "1.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34"
checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d"
[[package]]
name = "elliptic-curve"
@@ -4507,7 +4476,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5"
dependencies = [
"polyval",
"zeroize",
]
[[package]]
@@ -5090,9 +5058,9 @@ dependencies = [
[[package]]
name = "hotpath"
version = "0.24.0"
version = "0.23.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2645642a23d4061ec15a4a6e74f851a3145c3125356846cfc7772ff9c6f2737"
checksum = "dce755d457a63bdd0c95e4c91511daad1b58b33209543b7f38027b676f387e5e"
dependencies = [
"arc-swap",
"async-channel",
@@ -5124,9 +5092,9 @@ dependencies = [
[[package]]
name = "hotpath-macros"
version = "0.24.0"
version = "0.23.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89a3d3cdf9b0d4d3d4f6d4a29798f3b9170401ba500eaa58dd8f890f926af0f1"
checksum = "a903af89a8429cb07790c3818bc15270b394f80af1bc254e5ccf9c7de2961770"
dependencies = [
"proc-macro2",
"quote",
@@ -5135,15 +5103,15 @@ dependencies = [
[[package]]
name = "hotpath-macros-meta"
version = "0.24.0"
version = "0.23.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e84cd2417fa60938241cf1cd6c03e09953f5c821122dc5da9b8f27975d136c5b"
checksum = "bcc0ab94ffbb2ee77f4a897df02b5a137a10cf24d69bda936e59aff4dd456e61"
[[package]]
name = "hotpath-meta"
version = "0.24.0"
version = "0.23.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d2c145b67b1a4e7bcefa918995e212c30a49a85f05cc5962fe1f717878d560b"
checksum = "053481f6cec8f775a3276c7f6e2f21123111d28261e4edc15ea7421c445964bb"
dependencies = [
"hotpath-macros-meta",
]
@@ -5410,9 +5378,9 @@ checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa"
[[package]]
name = "icu_provider"
version = "2.3.1"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73"
checksum = "92a7ed671a6aad807a8651a2e1782a6598fda9ce5185dd8158549e95a91c6428"
dependencies = [
"displaydoc",
"icu_locale_core",
@@ -5799,9 +5767,9 @@ dependencies = [
[[package]]
name = "keccak"
version = "0.2.2"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8f198d1db720e4940b5a493201d199d9f24f568f8f746bd13706243a2f71598"
checksum = "ffd9697dc4a9a62e2da93389f34400b77a28f0287711263cabb203b3ccb9c0e4"
dependencies = [
"cfg-if",
"cpufeatures 0.3.0",
@@ -6020,6 +5988,15 @@ version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
[[package]]
name = "libmimalloc-sys"
version = "0.1.49"
source = "git+https://github.com/xonatius/mimalloc_rust.git?rev=6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11#6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11"
dependencies = [
"cc",
"cty",
]
[[package]]
name = "libredox"
version = "0.1.20"
@@ -6113,9 +6090,9 @@ dependencies = [
[[package]]
name = "log"
version = "0.4.34"
version = "0.4.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
[[package]]
name = "lru"
@@ -6305,7 +6282,7 @@ dependencies = [
"hashbrown 0.16.1",
"indexmap 2.14.0",
"metrics",
"ordered-float 5.5.0",
"ordered-float 5.3.0",
"quanta",
"radix_trie",
"rand 0.9.5",
@@ -6420,6 +6397,14 @@ dependencies = [
"synstructure 0.13.2",
]
[[package]]
name = "mimalloc"
version = "0.1.52"
source = "git+https://github.com/xonatius/mimalloc_rust.git?rev=6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11#6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11"
dependencies = [
"libmimalloc-sys",
]
[[package]]
name = "mime"
version = "0.3.17"
@@ -7253,9 +7238,9 @@ dependencies = [
[[package]]
name = "ordered-float"
version = "5.5.0"
version = "5.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c7c9e0d9b23589f26070720bac724174bfec1083e82f7854cdd0267518343c0"
checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e"
dependencies = [
"num-traits",
]
@@ -7290,12 +7275,6 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e"
[[package]]
name = "owo-colors"
version = "4.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d"
[[package]]
name = "p12-keystore"
version = "0.2.1"
@@ -7838,7 +7817,6 @@ dependencies = [
"cpubits",
"cpufeatures 0.3.0",
"universal-hash",
"zeroize",
]
[[package]]
@@ -8309,16 +8287,6 @@ name = "quick-xml"
version = "0.41.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1"
dependencies = [
"memchr",
"serde",
]
[[package]]
name = "quick-xml"
version = "0.42.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b"
dependencies = [
"encoding_rs",
"memchr",
@@ -8983,9 +8951,9 @@ dependencies = [
[[package]]
name = "russh"
version = "0.63.0"
version = "0.62.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00cf00190c315093734a8d405225bd8773a219bc86538a9b73bfc51145b33995"
checksum = "9decb68e4e44e1079700e54f17c8f23806ec53d7e0db73ab1c71d9dabc666812"
dependencies = [
"aes 0.9.2",
"aws-lc-rs",
@@ -9194,20 +9162,22 @@ dependencies = [
"insta",
"jiff",
"libc",
"libmimalloc-sys",
"libsystemd",
"matchit 0.9.2",
"md-5 0.11.0",
"metrics",
"metrics-util",
"mimalloc",
"mime_guess",
"opentelemetry",
"opentelemetry_sdk",
"p256 0.14.0",
"p256 0.13.2",
"parking_lot",
"percent-encoding",
"pin-project-lite",
"proptest",
"quick-xml 0.42.0",
"quick-xml",
"rand 0.10.2",
"rcgen",
"regex",
@@ -9234,8 +9204,6 @@ dependencies = [
"rustfs-lock",
"rustfs-log-analyzer",
"rustfs-madmin",
"rustfs-mimalloc",
"rustfs-mimalloc-sys",
"rustfs-notify",
"rustfs-object-capacity",
"rustfs-object-data-cache",
@@ -9473,7 +9441,7 @@ dependencies = [
"path-absolutize",
"pin-project-lite",
"proptest",
"quick-xml 0.42.0",
"quick-xml",
"rand 0.10.2",
"ratelimit",
"rcgen",
@@ -9619,7 +9587,6 @@ dependencies = [
"serde",
"serde_json",
"serial_test",
"sha2 0.11.0",
"temp-env",
"tempfile",
"thiserror 2.0.20",
@@ -9908,24 +9875,6 @@ dependencies = [
"tokio",
]
[[package]]
name = "rustfs-mimalloc"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a406f4aa07084301d485beec873af6dccc8e3f8762da244743df92038b1db1a6"
dependencies = [
"rustfs-mimalloc-sys",
]
[[package]]
name = "rustfs-mimalloc-sys"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3051b819175f58445d4c369a72f0ab88149f3885ba8bea2aff3be01f53fe7cd"
dependencies = [
"cc",
]
[[package]]
name = "rustfs-notify"
version = "1.0.0-rc.3"
@@ -9940,7 +9889,7 @@ dependencies = [
"jiff",
"metrics",
"percent-encoding",
"quick-xml 0.42.0",
"quick-xml",
"rayon",
"rustc-hash",
"rustfs-config",
@@ -10713,9 +10662,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"
[[package]]
name = "rustls-webpki"
version = "0.103.15"
version = "0.103.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2"
checksum = "0527518605e68109d875e248ea259b6758801cf165e4b2c2733ae3b51f12535a"
dependencies = [
"aws-lc-rs",
"ring",
@@ -10759,7 +10708,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
[[package]]
name = "s3s"
version = "0.15.0-alpha.1"
source = "git+https://github.com/rustfs/s3s.git?rev=e080e38c56a3b43acbacce55710d765a5ce9003d#e080e38c56a3b43acbacce55710d765a5ce9003d"
source = "git+https://github.com/rustfs/s3s.git?rev=ed70cb048cc4be168419d461cb9ac3c2c7fa6d5a#ed70cb048cc4be168419d461cb9ac3c2c7fa6d5a"
dependencies = [
"arc-swap",
"arrayvec",
@@ -10786,7 +10735,7 @@ dependencies = [
"nom 8.0.0",
"numeric_cast",
"pin-project-lite",
"quick-xml 0.41.0",
"quick-xml",
"regex",
"serde",
"serde_json",
@@ -12731,9 +12680,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]]
name = "uuid"
version = "1.25.0"
version = "1.24.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f053576934f05a761a402421fbbe3d425d9366f75f978806a037b3ca481abecc"
checksum = "2cefc03fd367c0c6d4305de1b312cf00248c4114f4a0418ce6a6af769e3b0bd9"
dependencies = [
"getrandom 0.4.3",
"js-sys",
@@ -13453,9 +13402,9 @@ dependencies = [
[[package]]
name = "zerovec-derive"
version = "0.11.6"
version = "0.11.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da"
checksum = "9f212a141d820099d57ffafb9569be9617a6f27d3dc881fbee8fb56642f917a9"
dependencies = [
"proc-macro2",
"quote",
+13 -15
View File
@@ -135,7 +135,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.3" }
# Async Runtime and Networking
async-channel = "2.5.0"
async_zip = { default-features = false, version = "0.0.19" }
async_zip = { default-features = false, version = "0.0.18" }
mysql_async = { default-features = false, version = "0.37" }
async-compression = { version = "0.4.43" }
async-recursion = "1.1.1"
@@ -178,7 +178,7 @@ byteorder = "1.5.0"
flatbuffers = "25.12.19"
form_urlencoded = "1.2.2"
prost = "0.14.4"
quick-xml = "0.42.0"
quick-xml = "0.41.0"
rmp = { version = "0.8.15" }
rmp-serde = { version = "1.3.1" }
serde = { version = "1.0.229" }
@@ -191,7 +191,7 @@ serde_urlencoded = "0.7.1"
# matching stable releases are not available yet, while previous stable lines
# have incompatible APIs. Keep them exact-pinned and monitor upstream for stable
# releases.
aes-gcm = { version = "=0.11.1" }
aes-gcm = { version = "=0.11.0" }
argon2 = { version = "=0.6.0-rc.8" }
blake2 = "=0.11.0-rc.6"
chacha20poly1305 = { version = "=0.11.0" }
@@ -200,7 +200,6 @@ hmac = { version = "0.13.0" }
jsonwebtoken = { version = "11.0.0" }
openidconnect = { default-features = false, version = "4.0" }
pbkdf2 = "0.13.0"
p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
rsa = { version = "=0.10.0-rc.18" }
rustls = { default-features = false, version = "0.23.43" }
rustls-native-certs = "0.8"
@@ -210,7 +209,6 @@ sha1 = "0.11.0"
sha2 = "0.11.0"
subtle = "2.6"
zeroize = { version = "1.9.0" }
proptest = "1"
# Time and Date
chrono = { version = "0.4.45" }
@@ -229,11 +227,11 @@ arc-swap = "1.9.2"
astral-tokio-tar = "0.6.4"
atoi = "3.1.0"
atomic_enum = "0.3.0"
aws-config = { version = "1.11.0" }
aws-config = { version = "1.10.1" }
aws-credential-types = { version = "1.3.0" }
aws-sdk-kms = { default-features = false, version = "1.116.0" }
aws-sdk-s3 = { default-features = false, version = "1.143.0" }
aws-sdk-sts = { default-features = false, version = "1.112.0" }
aws-sdk-kms = { default-features = false, version = "1.115.0" }
aws-sdk-s3 = { default-features = false, version = "1.142.0" }
aws-sdk-sts = { default-features = false, version = "1.111.0" }
aws-smithy-http-client = { default-features = false, version = "1.4.0" }
aws-smithy-runtime-api = { version = "1.15.0" }
aws-smithy-types = { version = "1.6.2" }
@@ -293,7 +291,7 @@ rustify = { version = "0.7", default-features = false }
rustix = { version = "1.1.4" }
rust-embed = { version = "8.12.0" }
rustc-hash = { version = "2.1.3" }
s3s = { git = "https://github.com/rustfs/s3s.git", rev = "e080e38c56a3b43acbacce55710d765a5ce9003d" }
s3s = { git = "https://github.com/rustfs/s3s.git", rev = "ed70cb048cc4be168419d461cb9ac3c2c7fa6d5a" }
serial_test = "4.0.1"
shadow-rs = { default-features = false, version = "2.0.0" }
siphasher = "1.0.3"
@@ -316,7 +314,7 @@ tracing-subscriber = { version = "0.3.23" }
transform-stream = "0.3.1"
url = "2.5.8"
urlencoding = "2.1.3"
uuid = { version = "1.25.0" }
uuid = { version = "1.24.1" }
vaultrs = { version = "0.8.0" }
tar = "0.4.46"
walkdir = "2.5.0"
@@ -345,16 +343,16 @@ libunftp = { version = "0.23.0" }
unftp-core = "0.1.0"
suppaftp = { version = "10.0.2" }
rcgen = { version = "0.14.9", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
russh = { version = "0.63.0" }
russh = { version = "0.62.7" }
russh-sftp = "2.4.0"
# WebDAV
dav-server = "0.11.0"
# Performance Analysis and Memory Profiling
rustfs-mimalloc = { version = "0.5.0" }
rustfs-mimalloc-sys = { version = "0.5.0" }
hotpath = { version = "0.24.0", default-features = false }
mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11" }
libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11", features = ["extended"] }
hotpath = { version = "0.23.3", default-features = false }
# Snapshot testing for output format regression detection
insta = { version = "1.48" }
-104
View File
@@ -918,15 +918,7 @@ pub struct Metrics {
scanner_dirty_usage_last_cycle_dirty_buckets: AtomicU64,
scanner_dirty_usage_last_cycle_cleared_buckets: AtomicU64,
scanner_usage_last_save_unix_secs: AtomicU64,
scanner_usage_last_durable_success_unix_secs: AtomicU64,
scanner_usage_last_publication_unix_secs: AtomicU64,
scanner_usage_last_publication_state: Mutex<String>,
scanner_usage_last_publication_reason: Mutex<String>,
scanner_usage_last_save_result: AtomicU8,
scanner_usage_deferred_pending: AtomicBool,
scanner_usage_deferred_total: AtomicU64,
scanner_usage_last_deferred_unix_secs: AtomicU64,
scanner_usage_last_deferred_reason: Mutex<String>,
scanner_source_work: Vec<ScannerSourceWorkCounters>,
current_scan_cycle_source_work_start: Vec<ScannerSourceWorkCounters>,
last_scan_cycle_source_work: Vec<ScannerSourceWorkCounters>,
@@ -1224,22 +1216,6 @@ pub struct ScannerUsageFreshnessSnapshot {
pub last_usage_save_unix_secs: u64,
pub last_usage_save_result: String,
pub last_usage_save_result_code: u64,
#[serde(default)]
pub last_durable_success_unix_secs: u64,
#[serde(default)]
pub last_publication_unix_secs: u64,
#[serde(default)]
pub last_publication_state: String,
#[serde(default)]
pub last_publication_reason: String,
#[serde(default)]
pub deferred_pending: bool,
#[serde(default)]
pub deferred_total: u64,
#[serde(default)]
pub last_deferred_unix_secs: u64,
#[serde(default)]
pub last_deferred_reason: String,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
@@ -1969,15 +1945,7 @@ impl Metrics {
scanner_dirty_usage_last_cycle_dirty_buckets: AtomicU64::new(0),
scanner_dirty_usage_last_cycle_cleared_buckets: AtomicU64::new(0),
scanner_usage_last_save_unix_secs: AtomicU64::new(0),
scanner_usage_last_durable_success_unix_secs: AtomicU64::new(0),
scanner_usage_last_publication_unix_secs: AtomicU64::new(0),
scanner_usage_last_publication_state: Mutex::new(String::new()),
scanner_usage_last_publication_reason: Mutex::new(String::new()),
scanner_usage_last_save_result: AtomicU8::new(ScannerUsageSaveResult::Unknown as u8),
scanner_usage_deferred_pending: AtomicBool::new(false),
scanner_usage_deferred_total: AtomicU64::new(0),
scanner_usage_last_deferred_unix_secs: AtomicU64::new(0),
scanner_usage_last_deferred_reason: Mutex::new(String::new()),
scanner_source_work: ScannerWorkSource::all()
.iter()
.map(|_| ScannerSourceWorkCounters::default())
@@ -2302,44 +2270,6 @@ impl Metrics {
.store(unix_now_secs(), Ordering::Relaxed);
}
/// Record an intentional retryable usage publication deferral separately
/// from the last durable save result.
pub fn record_scanner_usage_deferred(&self, reason: impl Into<String>) {
let reason = reason.into();
self.record_scanner_usage_publication("deferred", reason.clone());
self.scanner_usage_deferred_pending.store(true, Ordering::Release);
self.scanner_usage_deferred_total.fetch_add(1, Ordering::Relaxed);
self.scanner_usage_last_deferred_unix_secs
.store(unix_now_secs(), Ordering::Relaxed);
let mut last_reason = match self.scanner_usage_last_deferred_reason.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
*last_reason = reason;
}
pub fn record_scanner_usage_durable_success(&self) {
self.record_scanner_usage_publication("success", "");
self.scanner_usage_last_durable_success_unix_secs
.store(unix_now_secs(), Ordering::Relaxed);
self.scanner_usage_deferred_pending.store(false, Ordering::Release);
}
pub fn record_scanner_usage_publication(&self, state: &str, reason: impl Into<String>) {
self.scanner_usage_last_publication_unix_secs
.store(unix_now_secs(), Ordering::Relaxed);
let mut publication_state = match self.scanner_usage_last_publication_state.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
*publication_state = state.to_string();
let mut publication_reason = match self.scanner_usage_last_publication_reason.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
*publication_reason = reason.into();
}
pub fn record_scanner_source_work(&self, source: ScannerWorkSource, work: ScannerSourceWorkUpdate) {
if let Some(counters) = self.scanner_source_work.get(source.index()) {
counters.add(work);
@@ -3362,23 +3292,6 @@ impl Metrics {
last_usage_save_unix_secs: self.scanner_usage_last_save_unix_secs.load(Ordering::Relaxed),
last_usage_save_result: usage_save_result.as_str().to_string(),
last_usage_save_result_code: usage_save_result as u8 as u64,
last_durable_success_unix_secs: self.scanner_usage_last_durable_success_unix_secs.load(Ordering::Relaxed),
last_publication_unix_secs: self.scanner_usage_last_publication_unix_secs.load(Ordering::Relaxed),
last_publication_state: match self.scanner_usage_last_publication_state.lock() {
Ok(state) => state.clone(),
Err(poisoned) => poisoned.into_inner().clone(),
},
last_publication_reason: match self.scanner_usage_last_publication_reason.lock() {
Ok(reason) => reason.clone(),
Err(poisoned) => poisoned.into_inner().clone(),
},
deferred_pending: self.scanner_usage_deferred_pending.load(Ordering::Acquire),
deferred_total: self.scanner_usage_deferred_total.load(Ordering::Relaxed),
last_deferred_unix_secs: self.scanner_usage_last_deferred_unix_secs.load(Ordering::Relaxed),
last_deferred_reason: match self.scanner_usage_last_deferred_reason.lock() {
Ok(reason) => reason.clone(),
Err(poisoned) => poisoned.into_inner().clone(),
},
};
m.throttle_idle_mode_enabled = self.scanner_throttle_idle_mode_enabled.load(Ordering::Relaxed);
m.throttle_sleep_factor = self.scanner_throttle_sleep_factor_micros.load(Ordering::Relaxed) as f64 / 1_000_000.0;
@@ -4750,7 +4663,6 @@ mod tests {
metrics.record_scanner_dirty_usage_cycle_snapshot(1);
metrics.record_scanner_dirty_usage_cycle_clear(1, 1);
metrics.record_scanner_usage_save_result(ScannerUsageSaveResult::Success);
metrics.record_scanner_usage_deferred("data_movement");
let report = metrics.report().await;
@@ -4762,22 +4674,6 @@ mod tests {
assert!(report.usage_freshness.last_usage_save_unix_secs > 0);
assert_eq!(report.usage_freshness.last_usage_save_result, "success");
assert_eq!(report.usage_freshness.last_usage_save_result_code, 1);
assert!(report.usage_freshness.deferred_pending);
assert_eq!(report.usage_freshness.deferred_total, 1);
assert!(report.usage_freshness.last_deferred_unix_secs > 0);
assert_eq!(report.usage_freshness.last_deferred_reason, "data_movement");
metrics.record_scanner_usage_durable_success();
let report = metrics.report().await;
assert!(!report.usage_freshness.deferred_pending);
assert_eq!(report.usage_freshness.deferred_total, 1);
assert!(report.usage_freshness.last_durable_success_unix_secs > 0);
assert_eq!(report.usage_freshness.last_publication_state, "success");
metrics.record_scanner_usage_publication("no_update", "no_update");
let report = metrics.report().await;
assert_eq!(report.usage_freshness.last_publication_state, "no_update");
assert_eq!(report.usage_freshness.last_publication_reason, "no_update");
}
#[tokio::test]
+11 -334
View File
@@ -23,32 +23,21 @@
//! unconsumed intents is the consumer's job (see `rustfs-heal`
//! `heal::mrf_queue`), mirroring MinIO's `.heal/mrf/list.bin`.
use std::collections::HashMap;
use std::collections::hash_map::RandomState;
use std::hash::{BuildHasher, Hash};
use std::sync::atomic::AtomicU64;
use std::sync::atomic::AtomicUsize;
use std::sync::{
Arc, Mutex, OnceLock,
Arc, OnceLock,
atomic::{AtomicBool, Ordering},
};
use std::time::{Duration, Instant};
use tokio::sync::mpsc;
use uuid::Uuid;
/// Bounded capacity of the global MRF channel. Backpressure is resolved by
/// dropping (and counting) intents, never by blocking the producer.
const MRF_CHANNEL_CAPACITY: usize = 8192;
const MRF_COALESCER_SHARDS: usize = 16;
const MRF_COALESCER_MAX_KEYS: usize = 8192;
const MRF_COALESCER_MAX_BYTES: usize = 16 * 1024 * 1024;
const MRF_COALESCER_TTL: Duration = Duration::from_secs(60);
const MRF_MAX_IDENTITY_COMPONENT: usize = 1024;
/// Why an intent was produced. Drives the heal priority mapping on the
/// consumer side (DecodeFailure -> Urgent, MetadataCorruption -> High,
/// PartialWrite -> Normal).
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum MrfKind {
/// Erasure decode failed while serving a read (read path).
DecodeFailure,
@@ -78,52 +67,12 @@ pub struct MrfIntent {
/// Version the intent targets, as raw UUID bytes.
pub version_id: Option<[u8; 16]>,
pub kind: MrfKind,
/// Stable erasure-set scope when the producer has it. Kept optional so
/// metadata corruption and legacy producers do not invent a scope.
pub scope: Option<MrfScope>,
/// Generation of the node-local ingress lease. It is not persisted in
/// the journal; replayed records acquire a fresh lease when re-enqueued.
pub lease: Option<MrfIngressLease>,
pub enqueued_at_ms: u64,
/// Times this intent has already been offered to the heal manager.
/// Dropped by the consumer once it reaches `MRF_MAX_ATTEMPTS`.
pub attempts: u8,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct MrfScope {
pub pool_index: u32,
pub set_index: u32,
}
/// Opaque generation used to release exactly the admission that created an
/// ingress entry. A generation prevents a late terminal callback from
/// deleting a newer retry for the same identity (ABA).
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct MrfIngressLease(u64);
impl MrfIngressLease {
const fn new(value: u64) -> Self {
Self(value)
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum MrfDropReason {
Disabled,
Uninitialized,
Full,
OversizedIdentity,
CoalescerFull,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum MrfIngressResult {
Enqueued,
Coalesced,
Dropped(MrfDropReason),
}
/// Consumer-side retry ceiling before an intent is given up on.
pub const MRF_MAX_ATTEMPTS: u8 = 3;
@@ -138,159 +87,6 @@ impl MrfIntent {
static GLOBAL_MRF_SENDER: OnceLock<mpsc::Sender<MrfIntent>> = OnceLock::new();
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
struct MrfIdentityKey {
kind: MrfKind,
bucket: Arc<str>,
object: Arc<str>,
version_id: Option<[u8; 16]>,
scope: Option<MrfScope>,
}
#[derive(Debug)]
struct IngressEntry {
lease: MrfIngressLease,
expires_at: Instant,
bytes: usize,
}
type MrfCoalescerShard = Mutex<HashMap<MrfIdentityKey, IngressEntry>>;
type MrfCoalescer = Box<[MrfCoalescerShard]>;
static MRF_COALESCER: OnceLock<MrfCoalescer> = OnceLock::new();
static NEXT_MRF_LEASE: AtomicU64 = AtomicU64::new(1);
static MRF_COALESCER_COUNT: AtomicUsize = AtomicUsize::new(0);
static MRF_COALESCER_BYTES: AtomicUsize = AtomicUsize::new(0);
static MRF_HASH_STATE: OnceLock<RandomState> = OnceLock::new();
fn coalescer() -> &'static [MrfCoalescerShard] {
MRF_COALESCER.get_or_init(|| {
(0..MRF_COALESCER_SHARDS)
.map(|_| Mutex::new(HashMap::new()))
.collect::<Vec<_>>()
.into_boxed_slice()
})
}
fn key_shard(key: &MrfIdentityKey) -> usize {
let hash = MRF_HASH_STATE.get_or_init(RandomState::new).hash_one(key);
usize::try_from(hash).unwrap_or(0) % MRF_COALESCER_SHARDS
}
fn canonical_version(version_id: Option<Uuid>) -> Option<[u8; 16]> {
version_id
.filter(|version| !version.is_nil())
.map(|version| *version.as_bytes())
}
fn canonical_identity(
kind: MrfKind,
version_id: Option<[u8; 16]>,
scope: Option<MrfScope>,
) -> (Option<[u8; 16]>, Option<MrfScope>) {
let version_id = version_id.filter(|bytes| *bytes != [0; 16]);
match kind {
MrfKind::MetadataCorruption => (None, None),
MrfKind::DecodeFailure | MrfKind::PartialWrite => (version_id, scope),
}
}
fn identity_estimated_bytes(key: &MrfIdentityKey) -> usize {
64usize
.saturating_add(key.bucket.len())
.saturating_add(key.object.len())
.saturating_add(key.version_id.map_or(0, |_| 16))
.saturating_add(key.scope.map_or(0, |_| 8))
}
fn reserve(counter: &AtomicUsize, limit: usize, amount: usize) -> bool {
let mut current = counter.load(Ordering::Relaxed);
loop {
let Some(next) = current.checked_add(amount) else {
return false;
};
if next > limit {
return false;
}
match counter.compare_exchange_weak(current, next, Ordering::Relaxed, Ordering::Relaxed) {
Ok(_) => return true,
Err(observed) => current = observed,
}
}
}
fn coalescer_admit(key: MrfIdentityKey) -> Result<MrfIngressLease, MrfIngressResult> {
let shard = key_shard(&key);
let mut entries = coalescer()[shard]
.lock()
.map_err(|_| MrfIngressResult::Dropped(MrfDropReason::CoalescerFull))?;
let now = Instant::now();
let before = entries.len();
let mut expired_bytes = 0usize;
entries.retain(|_, entry| {
if entry.expires_at > now {
true
} else {
expired_bytes = expired_bytes.saturating_add(entry.bytes);
false
}
});
let evicted = before.saturating_sub(entries.len());
if evicted > 0 {
MRF_COALESCER_COUNT.fetch_sub(evicted, Ordering::Relaxed);
MRF_COALESCER_BYTES.fetch_sub(expired_bytes, Ordering::Relaxed);
let evicted = u64::try_from(evicted).unwrap_or(u64::MAX);
metrics::counter!("rustfs_heal_mrf_coalescer_expired_total").increment(evicted);
metrics::counter!("rustfs_heal_mrf_coalescer_evictions_total").increment(evicted);
}
if entries.contains_key(&key) {
metrics::counter!("rustfs_heal_mrf_coalesced_total").increment(1);
return Err(MrfIngressResult::Coalesced);
}
let bytes = identity_estimated_bytes(&key);
let count_reserved = reserve(&MRF_COALESCER_COUNT, MRF_COALESCER_MAX_KEYS, 1);
let bytes_reserved = count_reserved && reserve(&MRF_COALESCER_BYTES, MRF_COALESCER_MAX_BYTES, bytes);
if !count_reserved || !bytes_reserved {
if count_reserved {
MRF_COALESCER_COUNT.fetch_sub(1, Ordering::Relaxed);
}
metrics::counter!("rustfs_heal_mrf_dropped_total", "reason" => "coalescer_full").increment(1);
return Err(MrfIngressResult::Dropped(MrfDropReason::CoalescerFull));
}
let lease = MrfIngressLease::new(NEXT_MRF_LEASE.fetch_add(1, Ordering::Relaxed));
if entries
.insert(
key,
IngressEntry {
lease,
expires_at: now + MRF_COALESCER_TTL,
bytes,
},
)
.is_some()
{
MRF_COALESCER_COUNT.fetch_sub(1, Ordering::Relaxed);
MRF_COALESCER_BYTES.fetch_sub(bytes, Ordering::Relaxed);
metrics::counter!("rustfs_heal_mrf_coalesced_total").increment(1);
return Err(MrfIngressResult::Coalesced);
}
Ok(lease)
}
fn coalescer_release(key: &MrfIdentityKey, lease: Option<MrfIngressLease>) {
let Some(lease) = lease else {
return;
};
if let Ok(mut entries) = coalescer()[key_shard(key)].lock() {
let should_remove = entries.get(key).is_some_and(|entry| entry.lease == lease);
if should_remove {
let bytes = entries.remove(key).map(|entry| entry.bytes).unwrap_or(0);
MRF_COALESCER_COUNT.fetch_sub(1, Ordering::Relaxed);
MRF_COALESCER_BYTES.fetch_sub(bytes, Ordering::Relaxed);
}
}
}
/// Delivery kill-switch, set from `RUSTFS_HEAL_MRF_ENABLE`. Producers check
/// this before touching the channel so the disabled path stays allocation- and
/// sync-free.
@@ -326,90 +122,21 @@ pub fn init_mrf_channel() -> Result<mpsc::Receiver<MrfIntent>, &'static str> {
/// This runs on IO error paths, so it stays synchronous and cheap: one
/// bounded allocation for the two `Arc<str>` handles plus the channel slot.
pub fn try_send_mrf_intent(kind: MrfKind, bucket: &str, object: &str, version_id: Option<Uuid>) -> bool {
matches!(
try_send_mrf_intent_typed(kind, bucket, object, version_id, None),
MrfIngressResult::Enqueued
)
}
/// Typed ingress result. `Coalesced` means an equivalent in-flight channel
/// intent already exists; it is not a second executable or durable admission.
pub fn try_send_mrf_intent_typed(
kind: MrfKind,
bucket: &str,
object: &str,
version_id: Option<Uuid>,
scope: Option<MrfScope>,
) -> MrfIngressResult {
if !mrf_delivery_enabled() {
return MrfIngressResult::Dropped(MrfDropReason::Disabled);
return false;
}
let Some(sender) = GLOBAL_MRF_SENDER.get() else {
return MrfIngressResult::Dropped(MrfDropReason::Uninitialized);
};
if bucket.len() > MRF_MAX_IDENTITY_COMPONENT || object.len() > MRF_MAX_IDENTITY_COMPONENT {
return MrfIngressResult::Dropped(MrfDropReason::OversizedIdentity);
}
let (version_id, scope) = canonical_identity(kind, canonical_version(version_id), scope);
let key = MrfIdentityKey {
kind,
bucket: Arc::from(bucket),
object: Arc::from(object),
version_id,
scope,
};
let lease = match coalescer_admit(key.clone()) {
Ok(lease) => lease,
Err(result) => return result,
return false;
};
let intent = MrfIntent {
bucket: key.bucket.clone(),
object: key.object.clone(),
version_id: key.version_id,
bucket: Arc::from(bucket),
object: Arc::from(object),
version_id: version_id.map(|vid| *vid.as_bytes()),
kind,
scope,
lease: Some(lease),
enqueued_at_ms: unix_now_ms(),
attempts: 0,
};
match sender.try_send(intent) {
Ok(()) => MrfIngressResult::Enqueued,
Err(mpsc::error::TrySendError::Full(_)) => {
coalescer_release(&key, Some(lease));
metrics::counter!("rustfs_heal_mrf_dropped_total", "reason" => "channel_full").increment(1);
MrfIngressResult::Dropped(MrfDropReason::Full)
}
Err(mpsc::error::TrySendError::Closed(_)) => {
coalescer_release(&key, Some(lease));
MrfIngressResult::Dropped(MrfDropReason::Uninitialized)
}
}
}
/// Release the ingress key once the consumer owns the intent.
pub fn release_mrf_intent(intent: &MrfIntent) {
release_mrf_identity(intent.kind, &intent.bucket, &intent.object, intent.version_id, intent.scope, intent.lease);
}
pub fn release_mrf_identity(
kind: MrfKind,
bucket: &str,
object: &str,
version_id: Option<[u8; 16]>,
scope: Option<MrfScope>,
lease: Option<MrfIngressLease>,
) {
let (version_id, scope) = canonical_identity(kind, version_id, scope);
coalescer_release(
&MrfIdentityKey {
kind,
bucket: Arc::from(bucket),
object: Arc::from(object),
version_id,
scope,
},
lease,
);
sender.try_send(intent).is_ok()
}
fn unix_now_ms() -> u64 {
@@ -417,8 +144,7 @@ fn unix_now_ms() -> u64 {
// failure would be a bug rather than something to handle here.
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.ok()
.and_then(|d| u64::try_from(d.as_millis()).ok())
.map(|d| d.as_millis() as u64)
.unwrap_or(0)
}
@@ -489,60 +215,12 @@ mod tests {
object: Arc::from("object"),
version_id: Some([0u8; 16]),
kind: MrfKind::DecodeFailure,
scope: None,
lease: None,
enqueued_at_ms: 0,
attempts: 0,
};
assert!(intent.estimated_bytes() >= intent.bucket.len() + intent.object.len());
}
#[test]
fn ingress_duplicate_identity_coalesces_and_releases_for_retry() {
let key = MrfIdentityKey {
kind: MrfKind::DecodeFailure,
bucket: Arc::from("ingress-test-bucket"),
object: Arc::from("ingress-test-object"),
version_id: Some([9; 16]),
scope: Some(MrfScope {
pool_index: 3,
set_index: 4,
}),
};
let lease = coalescer_admit(key.clone()).expect("first identity should be admitted");
for _ in 0..999 {
assert_eq!(coalescer_admit(key.clone()), Err(MrfIngressResult::Coalesced));
}
coalescer_release(&key, Some(lease));
let retry_lease = coalescer_admit(key.clone()).expect("released identity must admit a retry");
coalescer_release(&key, Some(retry_lease));
}
#[test]
fn ingress_identity_preserves_kind_scope_and_version_boundaries() {
let (nil_version, nil_scope) = canonical_identity(
MrfKind::DecodeFailure,
Some([0; 16]),
Some(MrfScope {
pool_index: 1,
set_index: 2,
}),
);
assert_eq!(nil_version, None, "nil UUID is the unversioned identity");
assert!(nil_scope.is_some());
let (metadata_version, metadata_scope) = canonical_identity(
MrfKind::MetadataCorruption,
Some([7; 16]),
Some(MrfScope {
pool_index: 1,
set_index: 2,
}),
);
assert_eq!(metadata_version, None);
assert_eq!(metadata_scope, None);
}
#[tokio::test]
async fn try_send_delivers_and_respects_capacity() {
let mut receiver = init_mrf_channel().expect("first initialization should succeed");
@@ -552,7 +230,6 @@ mod tests {
let intent = receiver.recv().await.expect("intent should arrive");
assert_eq!(intent.kind, MrfKind::DecodeFailure);
assert_eq!(intent.bucket.as_ref(), "b");
release_mrf_intent(&intent);
// Disable delivery: producers become no-ops.
set_mrf_delivery_enabled(false);
@@ -562,8 +239,8 @@ mod tests {
// Fill the bounded channel past capacity: excess intents are dropped,
// never blocking.
let mut accepted = 0;
for index in 0..(MRF_CHANNEL_CAPACITY + 64) {
if try_send_mrf_intent(MrfKind::PartialWrite, "b", &format!("o-{index}"), None) {
for _ in 0..(MRF_CHANNEL_CAPACITY + 64) {
if try_send_mrf_intent(MrfKind::PartialWrite, "b", "o", None) {
accepted += 1;
}
}
-19
View File
@@ -168,19 +168,6 @@ pub const DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED: bool = false;
const _: () = assert!(!DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_WRITE);
const _: () = assert!(!DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED);
/// Request writing pool metadata version 2.
///
/// This remains ineffective until [`ENV_POOL_META_V2_FLEET_CONFIRMED`] is also enabled.
pub const ENV_POOL_META_V2_WRITE: &str = "RUSTFS_POOL_META_V2_WRITE";
pub const DEFAULT_POOL_META_V2_WRITE: bool = false;
/// Operator-attested confirmation that every pool metadata reader and writer understands version 2.
pub const ENV_POOL_META_V2_FLEET_CONFIRMED: &str = "RUSTFS_POOL_META_V2_FLEET_CONFIRMED";
pub const DEFAULT_POOL_META_V2_FLEET_CONFIRMED: bool = false;
const _: () = assert!(!DEFAULT_POOL_META_V2_WRITE);
const _: () = assert!(!DEFAULT_POOL_META_V2_FLEET_CONFIRMED);
// =============================================================================
// Concurrent Request Fix - Timeout and Backpressure Configuration
// =============================================================================
@@ -749,10 +736,4 @@ mod remote_version_state_tests {
"RUSTFS_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED"
);
}
#[test]
fn pool_meta_v2_gate_uses_stable_environment_names() {
assert_eq!(super::ENV_POOL_META_V2_WRITE, "RUSTFS_POOL_META_V2_WRITE");
assert_eq!(super::ENV_POOL_META_V2_FLEET_CONFIRMED, "RUSTFS_POOL_META_V2_FLEET_CONFIRMED");
}
}
+2 -2
View File
@@ -60,9 +60,9 @@ pub const DEFAULT_RNG_SEED: Option<u64> = None; // None means random
/// Dedicated blocking thread pool for fsync/fdatasync operations.
/// When > 1, fsync operations are isolated from the main blocking pool to
/// prevent device-bound fsync from starving read operations (pread/stat/open).
/// Default 64 isolates fsync from the main blocking pool to prevent device-bound fsync from starving read I/O.
/// Default 0 means auto (no isolation, use main runtime).
pub const ENV_FSYNC_BLOCKING_THREADS: &str = "RUSTFS_RUNTIME_FSYNC_BLOCKING_THREADS";
pub const DEFAULT_FSYNC_BLOCKING_THREADS: usize = 64;
pub const DEFAULT_FSYNC_BLOCKING_THREADS: usize = 0;
// Dial9 Tokio Telemetry Default values
pub const DEFAULT_RUNTIME_DIAL9_ENABLED: bool = false; // Disabled by default
File diff suppressed because it is too large Load Diff
+11 -14
View File
@@ -72,7 +72,7 @@ The reason string on each attribute is the classifier. Current classes:
- **Needs a pre-started server** — `"requires running RustFS server at
localhost:9000"` / `"Connects to existing rustfs server"`. These are the
`reliant/*` tests; start a server first (e.g.
`reliant/*` and `policy/test_runner` tests; start a server first (e.g.
[`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh)) or use
`--run-ignored`.
- **Heavy / external tool** — `"Starts a rustfs server; enable when running
@@ -123,7 +123,7 @@ via `create_s3_client(idx)` / `create_all_clients()`. See
| `find_available_port` | Random free port (isolation primitive) |
| `rustfs_binary_path` / `_with_features` | Locate/build the binary; honors `RUSTFS_BUILD_FEATURES` |
| `requested_rustfs_build_features` / `rustfs_build_feature_enabled` | Feature-gate a test to what the binary was built with |
| `execute_awscurl` / `awscurl_post` / `_get` / `_put` / `_delete` / `awscurl_post_sts_form_urlencoded` | Admin/STS API calls via `awscurl`; missing binaries are test failures |
| `awscurl_available` + `execute_awscurl` / `awscurl_post` / `_get` / `_put` / `_delete` / `awscurl_post_sts_form_urlencoded` | Admin/STS API calls via `awscurl` (skip gracefully when absent) |
| `replication_fast_env` | Env vars that shrink replication timers (from repl-4); pass to `start_rustfs_server_with_env` |
| `local_http_client` / `init_logging` | Loopback HTTP client; idempotent tracing init |
| `RustFSTestClusterEnvironment` (`new`/`start`/`start_node`/`stop_node`/`create_all_clients`) | Multi-node harness |
@@ -189,7 +189,7 @@ cargo nextest run --profile e2e-smoke -p e2e_test
cargo nextest run --profile e2e-full -p e2e_test
# Cluster fault nightly lane
cargo nextest run --profile e2e-nightly -p e2e_test
# Replication nightly lane; awscurl is required for STS paths
# Replication nightly lane; install awscurl so STS paths do not skip
cargo nextest run --profile e2e-repl-nightly -p e2e_test
# Fixed-port protocol nightly lane
RUSTFS_BUILD_FEATURES=ftps,webdav,sftp \
@@ -221,8 +221,9 @@ The `s3s-e2e` CI job selects a random `RUSTFS_TEST_PORT` (see the `e2e-tests`
job) to dodge this; local single-node tests already use random ports, so a
lingering orphan is usually the cause of a spurious bind failure.
**`awscurl` not found.** `awscurl`-dependent tests fail closed with a process
spawn error. Install the pinned CI version before running their profiles.
**`awscurl` not found.** `awscurl`-dependent tests skip gracefully with a
visible log line (`awscurl_available()`); install `awscurl` to actually run
them.
## Related
@@ -257,9 +258,10 @@ A test module may join the smoke filter only if every test in it is:
2. **Single-node** — spawns its own server via
`RustFSTestEnvironment`/`start_rustfs_server` on a random port with an
isolated temp dir. No `RustFSTestClusterEnvironment`, no fixed ports.
3. **Hermetic dependencies** — no pre-started server at `localhost:9000`, no
Vault, and no fixed protocol ports. Any required CLI must be pinned and
installed by the workflow; a missing CLI must fail the test.
3. **Dependency-free** — no pre-started server at `localhost:9000`, no Vault,
no fixed protocol ports. Tools that may be absent on the runner (e.g.
`awscurl`) are acceptable only when the test skips gracefully with a
visible log line (see `bucket_policy_check_test.rs`).
4. **Not `#[ignore]`** — ignored tests are activation work (backlog#1149
ci-13 / backlog#1148 ilm-3), not smoke candidates.
@@ -276,9 +278,4 @@ listed by `cargo nextest list -p e2e_test`. Regenerate it when adding or
moving e2e tests so acceptance numbers in the test-strategy issues
(backlog#1147#1155) stay auditable. When a profile membership change is
intentional, review its JSON listing before updating the matching
`.config/e2e-*-selection.txt` test-ID digest. Update only the platform that
produced the listing:
```bash
python3 scripts/check_test_wiring.py --update-profile e2e-full /path/to/listing.json linux
```
`.config/e2e-*-selection.txt` test-ID digest.
+4 -10
View File
@@ -33,7 +33,6 @@
mod tests {
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client, rustfs_binary_path};
use aws_sdk_s3::config::{Credentials, Region};
use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::{Client, Config};
use http::header::HOST;
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
@@ -369,15 +368,10 @@ mod tests {
reqwest::StatusCode::FORBIDDEN,
"stale root must be rejected on the admin API after rotation, body: {body}"
);
let s3_old = s3_client_with(&env, &old_ak, &old_sk)
.list_buckets()
.send()
.await
.expect_err("stale root must be rejected on the S3 plane after rotation");
assert_eq!(
s3_old.as_service_error().and_then(ProvideErrorMetadata::code),
Some("InvalidAccessKeyId"),
"stale root must receive InvalidAccessKeyId after rotation: {s3_old:?}"
let s3_old = s3_client_with(&env, &old_ak, &old_sk).list_buckets().send().await;
assert!(
s3_old.is_err(),
"stale root must be rejected on the S3 plane after rotation, got: {s3_old:?}"
);
env.stop_server();
+14 -32
View File
@@ -30,7 +30,6 @@ use crate::common::{
RustFSTestEnvironment, admin_ok, admin_request, admin_request_with_session_token, build_test_sts_client, init_logging,
};
use aws_sdk_s3::config::{Credentials, Region};
use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::{Client, Config};
use reqwest::StatusCode;
@@ -412,13 +411,8 @@ async fn test_admin_user_policy_service_account_crud_lifecycle() -> TestResult {
.key("before-attach")
.body(ByteStream::from_static(b"x"))
.send()
.await
.expect_err("user without a policy must not be able to write to the bucket");
assert_eq!(
denied.as_service_error().and_then(ProvideErrorMetadata::code),
Some("AccessDenied"),
"user without a policy must receive AccessDenied: {denied:?}"
);
.await;
assert!(denied.is_err(), "user without a policy must not be able to write to {bucket}");
// --- attach policy: the credential actually gains S3 access -----------------
admin_ok(
@@ -505,19 +499,13 @@ async fn test_admin_user_policy_service_account_crud_lifecycle() -> TestResult {
.body(ByteStream::from_static(b"x"))
.send()
.await;
match revoked {
Ok(_) if tokio::time::Instant::now() >= deadline => {
return Err("deleted service account credential still works".into());
}
Ok(_) => sleep(Duration::from_millis(500)).await,
Err(error) => {
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
if matches!(code, Some("AccessDenied" | "InvalidAccessKeyId")) {
break;
}
return Err(format!("deleted service account must fail with an authorization error, got {error:?}").into());
}
if revoked.is_err() {
break;
}
if tokio::time::Instant::now() >= deadline {
return Err("deleted service account credential still works".into());
}
sleep(Duration::from_millis(500)).await;
}
// Disable then remove the user; the credential must stop working.
@@ -537,19 +525,13 @@ async fn test_admin_user_policy_service_account_crud_lifecycle() -> TestResult {
.body(ByteStream::from_static(b"x"))
.send()
.await;
match disabled {
Ok(_) if tokio::time::Instant::now() >= deadline => {
return Err("disabled user credential still works".into());
}
Ok(_) => sleep(Duration::from_millis(500)).await,
Err(error) => {
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
if matches!(code, Some("AccessDenied" | "InvalidAccessKeyId")) {
break;
}
return Err(format!("disabled user must fail with an authorization error, got {error:?}").into());
}
if disabled.is_err() {
break;
}
if tokio::time::Instant::now() >= deadline {
return Err("disabled user credential still works".into());
}
sleep(Duration::from_millis(500)).await;
}
admin_ok(
@@ -52,6 +52,10 @@ fn create_user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key:
#[tokio::test]
async fn test_bucket_policy_authenticated_user() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if !crate::common::awscurl_available() {
info!("Skipping test_bucket_policy_authenticated_user because awscurl is not available");
return Ok(());
}
info!("Starting test_bucket_policy_authenticated_user...");
let mut env = RustFSTestEnvironment::new().await?;
+4 -24
View File
@@ -199,18 +199,8 @@ mod tests {
);
// And the object must not have been stored.
let error = client
.head_object()
.bucket(bucket)
.key(key)
.send()
.await
.expect_err("Object must not exist after a rejected mismatched-checksum PutObject");
assert_eq!(
error.raw_response().map(|response| response.status().as_u16()),
Some(404),
"Rejected mismatched-checksum PutObject absence probe must return HTTP 404, got {error:?}"
);
let head = client.head_object().bucket(bucket).key(key).send().await;
assert!(head.is_err(), "Object must not exist after a rejected mismatched-checksum PutObject");
info!("PASSED: PutObject rejects mismatched SHA256 and stores nothing");
}
@@ -562,18 +552,8 @@ mod tests {
msg.contains("BadDigest") || msg.to_lowercase().contains("digest") || msg.to_lowercase().contains("checksum"),
"{header}: expected a BadDigest/checksum error, got: {msg}"
);
let error = client
.head_object()
.bucket(bucket)
.key(&bad_key)
.send()
.await
.expect_err("nothing must be stored after a rejected PutObject");
assert_eq!(
error.raw_response().map(|response| response.status().as_u16()),
Some(404),
"{header}: rejected PutObject absence probe must return HTTP 404, got {error:?}"
);
let head = client.head_object().bucket(bucket).key(&bad_key).send().await;
assert!(head.is_err(), "{header}: nothing must be stored after a rejected PutObject");
info!("PASSED additional-checksum verify-on-write: {header}");
}
+16 -77
View File
@@ -15,27 +15,16 @@
use crate::common::RustFSTestClusterEnvironment;
use aws_sdk_s3::Client;
use aws_sdk_s3::error::SdkError;
use aws_sdk_s3::types::{CorsConfiguration, CorsRule};
use bytes::Bytes;
use std::sync::Arc;
use tokio::sync::Barrier;
use tracing::{info, warn};
const BUCKET: &str = "conditional-put-race-bucket";
const BUCKET_METADATA_RELOAD_BUCKET: &str = "bucket-metadata-reload-barrier";
async fn cleanup_object(client: &Client, key: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
client.delete_object().bucket(BUCKET).key(key).send().await?;
Ok(())
}
async fn assert_bucket_cors_missing(client: &Client) {
let result = client.get_bucket_cors().bucket(BUCKET_METADATA_RELOAD_BUCKET).send().await;
match result {
Err(SdkError::ServiceError(error)) => {
assert_eq!(error.err().meta().code(), Some("NoSuchCORSConfiguration"));
}
result => panic!("expected the peer to report a missing CORS configuration: {result:?}"),
async fn cleanup_object(client: &Client, key: &str) {
if let Err(e) = client.delete_object().bucket(BUCKET).key(key).send().await {
warn!("Failed to delete object '{}' from bucket '{}' during cleanup: {:?}", key, BUCKET, e);
}
}
@@ -82,13 +71,14 @@ async fn run_race_iteration(
test_key: &str,
iteration: usize,
) -> Result<usize, Box<dyn std::error::Error + Send + Sync>> {
cleanup_object(&clients[0], test_key).await?;
cleanup_object(&clients[0], test_key).await;
tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
match clients[0].head_object().bucket(BUCKET).key(test_key).send().await {
Ok(_) => return Err(format!("object still exists after cleanup in iteration {iteration}").into()),
Err(error) if error.as_service_error().is_some_and(|error| error.is_not_found()) => {}
Err(error) => return Err(format!("failed to verify cleanup in iteration {iteration}: {error:?}").into()),
let head_result = clients[0].head_object().bucket(BUCKET).key(test_key).send().await;
if head_result.is_ok() {
warn!("Warning: Object still exists after cleanup, skipping iteration {}", iteration);
return Ok(0);
}
info!("\n=== Iteration {} ===", iteration);
@@ -130,16 +120,14 @@ async fn run_race_iteration(
info!("Result: {} out of {} succeeded", success_count, clients.len());
if had_error {
return Err("one or more conditional PUTs failed unexpectedly".into());
}
if success_count > 1 {
info!(">>> RACE CONDITION DETECTED!");
} else if success_count == 1 {
info!(">>> Correct behavior: exactly 1 writer succeeded.");
} else if had_error {
return Err("all conditional PUTs failed (e.g. cluster/bucket not ready)".into());
} else {
return Err("no conditional PUT succeeded".into());
info!(">>> Unexpected: no writers succeeded.");
}
Ok(success_count)
@@ -179,7 +167,7 @@ async fn test_conditional_put_race_cluster() -> Result<(), Box<dyn std::error::E
}
}
cleanup_object(&clients[0], &test_key).await?;
cleanup_object(&clients[0], &test_key).await;
tokio::time::sleep(tokio::time::Duration::from_millis(50)).await;
}
@@ -189,7 +177,7 @@ async fn test_conditional_put_race_cluster() -> Result<(), Box<dyn std::error::E
info!("Total iterations: {}", iterations);
info!("Correct (1 winner): {}", correct_count);
info!("Race conditions: {}", races_detected);
info!("Failed iterations: {}", error_count);
info!("Errors (skipped): {}", error_count);
assert_eq!(races_detected, 0, "Race conditions detected: {}/{}", races_detected, iterations);
assert_eq!(
@@ -197,10 +185,6 @@ async fn test_conditional_put_race_cluster() -> Result<(), Box<dyn std::error::E
"{} iteration(s) failed due to errors (e.g. cluster not ready)",
error_count
);
assert_eq!(
correct_count, iterations,
"only {correct_count}/{iterations} iterations observed exactly one winner"
);
Ok(())
}
@@ -217,7 +201,7 @@ async fn test_conditional_put_basic_cluster() -> Result<(), Box<dyn std::error::
let client = cluster.create_s3_client(0)?;
let test_key = "basic-conditional-put";
cleanup_object(&client, test_key).await?;
cleanup_object(&client, test_key).await;
let result = client
.put_object()
@@ -249,51 +233,6 @@ async fn test_conditional_put_basic_cluster() -> Result<(), Box<dyn std::error::
assert_eq!(code, "PreconditionFailed");
}
cleanup_object(&client, test_key).await?;
Ok(())
}
#[tokio::test]
async fn test_bucket_cors_write_is_visible_on_peer_before_response() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
crate::common::init_logging();
let mut cluster = RustFSTestClusterEnvironment::new(2).await?;
cluster.start().await?;
cluster.create_test_bucket(BUCKET_METADATA_RELOAD_BUCKET).await?;
let writer = cluster.create_s3_client(0)?;
let reader = cluster.create_s3_client(1)?;
assert_bucket_cors_missing(&reader).await;
let rule = CorsRule::builder()
.allowed_methods("GET")
.allowed_origins("https://example.com")
.build()?;
let configuration = CorsConfiguration::builder().cors_rules(rule).build()?;
writer
.put_bucket_cors()
.bucket(BUCKET_METADATA_RELOAD_BUCKET)
.cors_configuration(configuration)
.send()
.await?;
let response = reader.get_bucket_cors().bucket(BUCKET_METADATA_RELOAD_BUCKET).send().await?;
let rules = response.cors_rules();
assert_eq!(
rules.len(),
1,
"peer should observe the committed CORS rule before the write response returns"
);
assert_eq!(rules[0].allowed_methods(), ["GET"]);
assert_eq!(rules[0].allowed_origins(), ["https://example.com"]);
writer
.delete_bucket_cors()
.bucket(BUCKET_METADATA_RELOAD_BUCKET)
.send()
.await?;
assert_bucket_cors_missing(&reader).await;
writer.delete_bucket().bucket(BUCKET_METADATA_RELOAD_BUCKET).send().await?;
cleanup_object(&client, test_key).await;
Ok(())
}
+19 -66
View File
@@ -310,17 +310,6 @@ pub fn rustfs_binary_path() -> PathBuf {
rustfs_binary_path_with_features(requested_rustfs_build_features().as_deref())
}
fn resolve_rustfs_binary_path(workspace: &Path, configured_target_dir: Option<&Path>) -> PathBuf {
let mut path = match configured_target_dir {
Some(path) if path.is_absolute() => path.to_path_buf(),
Some(path) => workspace.join(path),
None => workspace.join("target"),
};
path.push(if cfg!(debug_assertions) { "debug" } else { "release" });
path.push(format!("rustfs{}", std::env::consts::EXE_SUFFIX));
path
}
/// Resolve the RustFS binary relative to the workspace, optionally requesting build features.
pub fn rustfs_binary_path_with_features(requested_features: Option<&str>) -> PathBuf {
if let Some(path) = std::env::var_os("CARGO_BIN_EXE_rustfs") {
@@ -328,9 +317,11 @@ pub fn rustfs_binary_path_with_features(requested_features: Option<&str>) -> Pat
}
let requested_features = requested_features.and_then(normalize_rustfs_build_features);
let workspace = workspace_root();
let configured_target_dir = std::env::var_os("CARGO_TARGET_DIR").map(PathBuf::from);
let binary_path = resolve_rustfs_binary_path(&workspace, configured_target_dir.as_deref());
let mut binary_path = workspace_root();
binary_path.push("target");
let profile_dir = if cfg!(debug_assertions) { "debug" } else { "release" };
binary_path.push(profile_dir);
binary_path.push(format!("rustfs{}", std::env::consts::EXE_SUFFIX));
let features_match = binary_features_match(&binary_path, requested_features.as_deref());
let source_is_newer = workspace_sources_newer_than_binary(&binary_path);
@@ -347,7 +338,7 @@ pub fn rustfs_binary_path_with_features(requested_features: Option<&str>) -> Pat
}
info!("Building RustFS binary to ensure it's up to date...");
build_rustfs_binary(requested_features.as_deref(), &binary_path);
build_rustfs_binary(requested_features.as_deref());
info!("Using RustFS binary at {:?}", binary_path);
binary_path
@@ -449,7 +440,7 @@ fn path_is_newer_than(binary_modified: std::time::SystemTime, path: &Path) -> bo
}
/// Build the RustFS binary using cargo
fn build_rustfs_binary(requested_features: Option<&str>, binary_path: &Path) {
fn build_rustfs_binary(requested_features: Option<&str>) {
let workspace = workspace_root();
info!("Building RustFS binary from workspace: {:?}", workspace);
@@ -485,7 +476,11 @@ fn build_rustfs_binary(requested_features: Option<&str>, binary_path: &Path) {
panic!("Failed to build RustFS binary. Error: {stderr}");
}
let stamp_path = rustfs_binary_features_stamp_path(binary_path);
let mut binary_path = workspace;
binary_path.push("target");
binary_path.push(if cfg!(debug_assertions) { "debug" } else { "release" });
binary_path.push(format!("rustfs{}", std::env::consts::EXE_SUFFIX));
let stamp_path = rustfs_binary_features_stamp_path(&binary_path);
if let Err(err) = stdfs::write(&stamp_path, requested_features.unwrap_or_default()) {
warn!("Failed to write RustFS feature stamp {:?}: {}", stamp_path, err);
}
@@ -499,20 +494,15 @@ fn awscurl_binary_path() -> PathBuf {
.unwrap_or_else(|| PathBuf::from("awscurl"))
}
fn verify_awscurl_path(path: &Path) -> std::io::Result<()> {
let output = Command::new(path).arg("--help").output()?;
if output.status.success() {
return Ok(());
pub fn awscurl_available() -> bool {
let path = awscurl_binary_path();
if path.components().count() > 1 || path.is_absolute() {
return path.is_file();
}
Err(std::io::Error::other(format!(
"awscurl prerequisite check failed: {}",
String::from_utf8_lossy(&output.stderr).trim()
)))
}
pub fn require_awscurl() -> std::io::Result<()> {
verify_awscurl_path(&awscurl_binary_path())
std::env::var_os("PATH")
.map(|paths| std::env::split_paths(&paths).any(|dir| dir.join(&path).is_file()))
.unwrap_or(false)
}
// Global initialization
@@ -1757,22 +1747,6 @@ mod tests {
assert_eq!(normalize_rustfs_build_features(" , "), None);
}
#[test]
fn missing_awscurl_is_a_prerequisite_failure() {
let missing = std::env::temp_dir().join(format!("missing-awscurl-{}", Uuid::new_v4()));
let error = verify_awscurl_path(&missing).expect_err("a missing awscurl binary must fail the test prerequisite");
assert_eq!(error.kind(), ErrorKind::NotFound);
}
#[test]
fn available_awscurl_client_passes_prerequisite_check() {
let executable = std::env::current_exe().expect("the test executable should have a path");
verify_awscurl_path(&executable).expect("an available client with a working help command should pass");
}
#[test]
fn capture_log_path_uses_temp_directory_basename() {
assert_eq!(
@@ -1781,27 +1755,6 @@ mod tests {
);
}
#[test]
fn resolves_rustfs_binary_in_configured_cargo_target_directory() {
let workspace = Path::new("workspace");
let profile = if cfg!(debug_assertions) { "debug" } else { "release" };
let binary = format!("rustfs{}", std::env::consts::EXE_SUFFIX);
assert_eq!(
resolve_rustfs_binary_path(workspace, None),
workspace.join("target").join(profile).join(&binary)
);
assert_eq!(
resolve_rustfs_binary_path(workspace, Some(Path::new("custom-target"))),
workspace.join("custom-target").join(profile).join(&binary)
);
let absolute = std::env::temp_dir().join("rustfs-e2e-custom-target");
assert_eq!(
resolve_rustfs_binary_path(workspace, Some(&absolute)),
absolute.join(profile).join(binary)
);
}
#[test]
fn full_feature_enables_any_required_feature() {
assert!(rustfs_build_feature_enabled(Some("full"), "sftp"));
+29 -59
View File
@@ -4,8 +4,7 @@ use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path};
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
use std::fs;
use std::io;
use std::path::{Path, PathBuf};
use std::path::PathBuf;
use std::process::Command;
use tracing::info;
@@ -32,58 +31,30 @@ fn generate_high_ratio_binary_data(size: usize, seed: u8) -> Vec<u8> {
.collect()
}
fn find_part_files(temp_dir: &str, bucket: &str, object_key: &str) -> io::Result<Vec<PathBuf>> {
fn find_part_files(temp_dir: &str, bucket: &str, object_key: &str) -> Vec<PathBuf> {
let bucket_path = PathBuf::from(temp_dir).join(bucket);
let mut part_files = Vec::new();
fn scan_dir(dir: &Path, target: &str, results: &mut Vec<PathBuf>) -> io::Result<()> {
let entries = fs::read_dir(dir)
.map_err(|error| io::Error::new(error.kind(), format!("failed to read {}: {error}", dir.display())))?;
for entry in entries {
let entry = entry
.map_err(|error| io::Error::new(error.kind(), format!("failed to read entry in {}: {error}", dir.display())))?;
let path = entry.path();
let file_type = entry
.file_type()
.map_err(|error| io::Error::new(error.kind(), format!("failed to inspect {}: {error}", path.display())))?;
if file_type.is_dir() {
scan_dir(&path, target, results)?;
} else if path
.file_name()
.map(|n| n.to_string_lossy().starts_with("part."))
.unwrap_or(false)
&& path.to_string_lossy().contains(target)
{
if !file_type.is_file() {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("expected regular part file at {}", path.display()),
));
fn scan_dir(dir: &PathBuf, target: &str, results: &mut Vec<PathBuf>) {
if let Ok(entries) = fs::read_dir(dir) {
for entry in entries.flatten() {
let path = entry.path();
if path.is_dir() {
scan_dir(&path, target, results);
} else if path
.file_name()
.map(|n| n.to_string_lossy().starts_with("part."))
.unwrap_or(false)
&& path.to_string_lossy().contains(target)
{
results.push(path);
}
results.push(path);
}
}
Ok(())
}
scan_dir(&bucket_path, object_key, &mut part_files)?;
Ok(part_files)
}
fn part_files_total_size(part_files: &[PathBuf]) -> io::Result<u64> {
part_files.iter().try_fold(0_u64, |total, path| {
let metadata = fs::symlink_metadata(path)
.map_err(|error| io::Error::new(error.kind(), format!("failed to stat {}: {error}", path.display())))?;
if !metadata.file_type().is_file() {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("expected regular part file at {}", path.display()),
));
}
total
.checked_add(metadata.len())
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "on-disk part size overflow"))
})
scan_dir(&bucket_path, object_key, &mut part_files);
part_files
}
async fn start_rustfs_with_compression(env: &mut RustFSTestEnvironment) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
@@ -152,9 +123,8 @@ async fn test_compression_roundtrip() -> Result<(), Box<dyn std::error::Error +
let content_length = head_response.content_length().unwrap_or(0);
assert_eq!(content_length as usize, original_size, "Content-Length should be original size");
let part_files = find_part_files(&env.temp_dir, COMPRESSION_TEST_BUCKET, object_key)?;
assert!(!part_files.is_empty(), "expected on-disk part files for the compressed object");
let total_physical_size = part_files_total_size(&part_files)?;
let part_files = find_part_files(&env.temp_dir, COMPRESSION_TEST_BUCKET, object_key);
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
assert!(
total_physical_size < original_size as u64,
@@ -276,9 +246,9 @@ async fn test_compression_multipart_roundtrip() -> Result<(), Box<dyn std::error
"Content-Length should be the logical object size"
);
let part_files = find_part_files(&env.temp_dir, MULTIPART_COMPRESSION_BUCKET, object_key)?;
let part_files = find_part_files(&env.temp_dir, MULTIPART_COMPRESSION_BUCKET, object_key);
assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object");
let total_physical_size = part_files_total_size(&part_files)?;
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
assert!(
total_physical_size < (total_size / 2) as u64,
"Physical size {total_physical_size} should be well below original size {total_size} (multipart compression applied)"
@@ -396,9 +366,9 @@ async fn test_compression_multipart_high_ratio_binary_roundtrip() -> Result<(),
// This pattern compresses to roughly 1/50 of its logical size, so a comfortably loose 2x
// margin still proves the parts were stored compressed rather than raw or double-encoded.
let part_files = find_part_files(&env.temp_dir, MPU_HIGH_RATIO_BUCKET, object_key)?;
let part_files = find_part_files(&env.temp_dir, MPU_HIGH_RATIO_BUCKET, object_key);
assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object");
let total_physical_size = part_files_total_size(&part_files)?;
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
assert!(
total_physical_size < (total_size as u64) / 2,
"Physical size {total_physical_size} should be far below the logical size {total_size} for high-ratio data"
@@ -552,9 +522,9 @@ async fn test_compression_multipart_upload_part_copy_roundtrip() -> Result<(), B
"Content-Length should be the logical object size"
);
let part_files = find_part_files(&env.temp_dir, MPU_COPY_COMPRESSION_BUCKET, target_key)?;
let part_files = find_part_files(&env.temp_dir, MPU_COPY_COMPRESSION_BUCKET, target_key);
assert!(!part_files.is_empty(), "expected on-disk part files for the copied object");
let total_physical_size = part_files_total_size(&part_files)?;
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
assert!(
total_physical_size < (total_size / 2) as u64,
"Physical size {total_physical_size} should be well below original size {total_size} (copied part compression applied)"
@@ -615,9 +585,9 @@ async fn test_compression_multipart_three_parts_part_number_gets() -> Result<(),
"Content-Length should be the logical object size"
);
let part_files = find_part_files(&env.temp_dir, MPU_THREE_PARTS_BUCKET, object_key)?;
let part_files = find_part_files(&env.temp_dir, MPU_THREE_PARTS_BUCKET, object_key);
assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object");
let total_physical_size = part_files_total_size(&part_files)?;
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
assert!(
total_physical_size < (total_size / 2) as u64,
"Physical size {total_physical_size} should be well below original size {total_size} (multipart compression applied)"
@@ -764,9 +734,9 @@ async fn test_compression_multipart_sse_s3_roundtrip() -> Result<(), Box<dyn std
"HEAD must report SSE-S3"
);
let part_files = find_part_files(&env.temp_dir, MPU_SSE_COMPRESSION_BUCKET, object_key)?;
let part_files = find_part_files(&env.temp_dir, MPU_SSE_COMPRESSION_BUCKET, object_key);
assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object");
let total_physical_size = part_files_total_size(&part_files)?;
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
assert!(
total_physical_size < (total_size / 2) as u64,
"Physical size {total_physical_size} should be well below original size {total_size} (compress-then-encrypt applied)"
+14 -4
View File
@@ -59,6 +59,7 @@ where
/// Regression test for data usage accuracy (issue #1012).
/// Launches rustfs, writes 1000 objects, then asserts admin data usage reports the full count.
#[tokio::test(flavor = "multi_thread")]
#[ignore = "Starts a rustfs server and requires awscurl; enable when running full E2E"]
async fn data_usage_reports_all_objects() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
@@ -85,20 +86,28 @@ async fn data_usage_reports_all_objects() -> Result<(), Box<dyn std::error::Erro
usage
.buckets_usage
.get(TEST_BUCKET)
.map(|bucket_usage| usage.objects_total_count == 1000 && bucket_usage.objects_count == 1000)
.map(|bucket_usage| usage.objects_total_count >= 1000 && bucket_usage.objects_count >= 1000)
.unwrap_or(false)
})
.await?;
// Assert total object count and per-bucket count are exact.
// Assert total object count and per-bucket count are not truncated
let bucket_usage = usage
.buckets_usage
.get(TEST_BUCKET)
.cloned()
.expect("bucket usage should exist");
assert_eq!(usage.objects_total_count, 1000, "total object count should be exact");
assert_eq!(bucket_usage.objects_count, 1000, "bucket object count should be exact");
assert!(
usage.objects_total_count >= 1000,
"total object count should be at least 1000, got {}",
usage.objects_total_count
);
assert!(
bucket_usage.objects_count >= 1000,
"bucket object count should be at least 1000, got {}",
bucket_usage.objects_count
);
env.stop_server();
Ok(())
@@ -107,6 +116,7 @@ async fn data_usage_reports_all_objects() -> Result<(), Box<dyn std::error::Erro
/// Regression test for issue #3898.
/// Versioned buckets should expose versions and delete markers through admin data usage.
#[tokio::test(flavor = "multi_thread")]
#[ignore = "Starts a rustfs server and requires awscurl; enable when running full E2E"]
async fn data_usage_reports_versioned_objects_and_delete_markers() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
+6 -24
View File
@@ -117,18 +117,9 @@ mod tests {
);
// Verify HEAD returns 404
let error = client
.head_object()
.bucket(bucket)
.key("to-delete.txt")
.send()
.await
.expect_err("RT-05 FAIL: HEAD on deleted object should return 404, got success");
assert_eq!(
error.raw_response().map(|response| response.status().as_u16()),
Some(404),
"RT-05 FAIL: HEAD on deleted object must return HTTP 404, got {error:?}"
);
let head = client.head_object().bucket(bucket).key("to-delete.txt").send().await;
assert!(head.is_err(), "RT-05 FAIL: HEAD on deleted object should return error, got success");
info!("RT-05 PASS: delete correctly removes object from LIST and HEAD");
Ok(())
@@ -423,18 +414,9 @@ mod tests {
// All HEAD requests should return 404
for key in &keys {
let error = client
.head_object()
.bucket(bucket)
.key(*key)
.send()
.await
.expect_err("RT-05f FAIL: HEAD on deleted key should return 404, got success");
assert_eq!(
error.raw_response().map(|response| response.status().as_u16()),
Some(404),
"RT-05f FAIL: HEAD on deleted key '{key}' must return HTTP 404, got {error:?}"
);
let head = client.head_object().bucket(bucket).key(*key).send().await;
assert!(head.is_err(), "RT-05f FAIL: HEAD on deleted key '{key}' should return error");
}
// LIST should be empty
@@ -16,9 +16,10 @@
//! session policy** (`Policy` parameter) via `awscurl --service sts` with explicit
//! `Content-Type: application/x-www-form-urlencoded` on `POST /`.
use crate::common::{RustFSTestEnvironment, awscurl_delete, awscurl_post_sts_form_urlencoded, awscurl_put, init_logging};
use crate::common::{
RustFSTestEnvironment, awscurl_available, awscurl_delete, awscurl_post_sts_form_urlencoded, awscurl_put, init_logging,
};
use aws_sdk_s3::config::{Credentials, Region};
use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{Delete, ObjectIdentifier, Tag, Tagging};
use aws_sdk_s3::{Client, Config};
@@ -174,6 +175,11 @@ async fn cleanup_bucket_and_object(admin: &Client, bucket: &str, key: &str) {
#[tokio::test]
async fn test_e2e_iam_policy_existing_object_tag_get_object() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if !awscurl_available() {
info!("Skipping test_e2e_iam_policy_existing_object_tag_get_object: awscurl not available");
return Ok(());
}
let suffix = Uuid::new_v4();
let user = format!("e2eiamtag-{suffix}");
let user_secret = "longSecretKeyForTest123!";
@@ -209,17 +215,10 @@ async fn test_e2e_iam_policy_existing_object_tag_get_object() -> Result<(), Box<
let _ = out.body.collect().await?;
put_object_tag_kv(&admin, &bucket, key, "security", "private").await?;
let denied = uclient
.get_object()
.bucket(&bucket)
.key(key)
.send()
.await
.expect_err("GetObject must be denied when ExistingObjectTag no longer matches IAM policy");
assert_eq!(
denied.as_service_error().and_then(ProvideErrorMetadata::code),
Some("AccessDenied"),
"IAM ExistingObjectTag mismatch must return AccessDenied: {denied:?}"
let denied = uclient.get_object().bucket(&bucket).key(key).send().await;
assert!(
denied.is_err(),
"GetObject must be denied when ExistingObjectTag no longer matches IAM policy"
);
cleanup_bucket_and_object(&admin, &bucket, key).await;
@@ -234,6 +233,11 @@ async fn test_e2e_iam_policy_existing_object_tag_get_object() -> Result<(), Box<
#[tokio::test]
async fn test_e2e_bucket_policy_existing_object_tag_get_object() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if !awscurl_available() {
info!("Skipping test_e2e_bucket_policy_existing_object_tag_get_object: awscurl not available");
return Ok(());
}
let suffix = Uuid::new_v4();
let user = format!("e2ebptag-{suffix}");
let user_secret = "longSecretKeyForTest456!";
@@ -253,13 +257,8 @@ async fn test_e2e_bucket_policy_existing_object_tag_get_object() -> Result<(), B
.bucket(&bucket)
.key(key)
.send()
.await
.expect_err("without bucket policy, user must be denied");
assert_eq!(
deny_before.as_service_error().and_then(ProvideErrorMetadata::code),
Some("AccessDenied"),
"missing bucket policy must return AccessDenied: {deny_before:?}"
);
.await;
assert!(deny_before.is_err(), "without bucket policy, user must be denied");
let bp = serde_json::json!({
"Version": "2012-10-17",
@@ -281,18 +280,8 @@ async fn test_e2e_bucket_policy_existing_object_tag_get_object() -> Result<(), B
let _ = ok.body.collect().await?;
put_object_tag_kv(&admin, &bucket, key, "security", "private").await?;
let denied = uclient
.get_object()
.bucket(&bucket)
.key(key)
.send()
.await
.expect_err("GetObject must fail when tag no longer satisfies bucket policy");
assert_eq!(
denied.as_service_error().and_then(ProvideErrorMetadata::code),
Some("AccessDenied"),
"bucket-policy ExistingObjectTag mismatch must return AccessDenied: {denied:?}"
);
let denied = uclient.get_object().bucket(&bucket).key(key).send().await;
assert!(denied.is_err(), "GetObject must fail when tag no longer satisfies bucket policy");
cleanup_bucket_and_object(&admin, &bucket, key).await;
admin_remove_user(&env, &user).await;
@@ -305,6 +294,11 @@ async fn test_e2e_bucket_policy_existing_object_tag_get_object() -> Result<(), B
#[tokio::test]
async fn test_e2e_sts_assume_role_session_policy_existing_object_tag() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if !awscurl_available() {
info!("Skipping test_e2e_sts_assume_role_session_policy_existing_object_tag: awscurl not available");
return Ok(());
}
let suffix = Uuid::new_v4();
let parent = format!("e2e-sts-par-{suffix}");
let parent_secret = "longSecretKeyForParentSts99!";
@@ -358,17 +352,10 @@ async fn test_e2e_sts_assume_role_session_policy_existing_object_tag() -> Result
let _ = ok.body.collect().await?;
put_object_tag_kv(&parent_client, &bucket, key, "security", "private").await?;
let denied = session_client
.get_object()
.bucket(&bucket)
.key(key)
.send()
.await
.expect_err("session policy must deny GetObject when ExistingObjectTag no longer matches");
assert_eq!(
denied.as_service_error().and_then(ProvideErrorMetadata::code),
Some("AccessDenied"),
"STS ExistingObjectTag mismatch must return AccessDenied: {denied:?}"
let denied = session_client.get_object().bucket(&bucket).key(key).send().await;
assert!(
denied.is_err(),
"session policy must deny GetObject when ExistingObjectTag no longer matches"
);
cleanup_bucket_and_object(&admin, &bucket, key).await;
@@ -383,6 +370,11 @@ async fn test_e2e_sts_assume_role_session_policy_existing_object_tag() -> Result
#[tokio::test]
async fn test_e2e_sts_session_policy_delete_objects_object_prefix_only() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if !awscurl_available() {
info!("Skipping test_e2e_sts_session_policy_delete_objects_object_prefix_only: awscurl not available");
return Ok(());
}
let suffix = Uuid::new_v4();
let parent = format!("e2e-sts-del-par-{suffix}");
let parent_secret = "longSecretKeyForParentDelete99!";
+36 -118
View File
@@ -14,7 +14,7 @@
//! E2E tests for group management (fixes #2028).
use crate::common::{RustFSTestEnvironment, admin_ok, admin_request, init_logging};
use crate::common::{RustFSTestEnvironment, admin_request, awscurl_delete, awscurl_get, awscurl_put, init_logging};
use aws_sdk_s3::config::{Credentials, Region};
use aws_sdk_s3::{Client, Config};
use tracing::info;
@@ -83,6 +83,7 @@ async fn update_group_members_rejects_invalid_new_group_names() -> Result<(), Bo
/// Test that deleting a group with members fails, and deleting an empty group succeeds.
#[tokio::test(flavor = "multi_thread")]
#[ignore = "requires awscurl and spawns a real RustFS server"]
async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
@@ -90,58 +91,29 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std
env.start_rustfs_server(vec![]).await?;
// 1. Create a user
let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey=testuser1", env.url);
let user_body = serde_json::json!({
"secretKey": "testuser1secret",
"status": "enabled"
});
admin_ok(
&env,
http::Method::PUT,
"/rustfs/admin/v3/add-user?accessKey=testuser1",
Some(user_body.to_string()),
)
.await?;
awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
info!("Created testuser1");
// 2. Create a group with testuser1 as a member
let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
let add_member_body = serde_json::json!({
"group": "testgroup",
"members": ["testuser1"],
"isRemove": false,
"groupStatus": "enabled"
});
admin_ok(
&env,
http::Method::PUT,
"/rustfs/admin/v3/update-group-members",
Some(add_member_body.to_string()),
)
.await?;
awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
info!("Added testuser1 to testgroup");
// 3. Attempt to delete the group while it still has members — should fail
let (delete_status, delete_body) = admin_request(
&env.url,
http::Method::DELETE,
"/rustfs/admin/v3/group/testgroup",
None,
&env.access_key,
&env.secret_key,
)
.await?;
assert_eq!(
delete_status,
reqwest::StatusCode::BAD_REQUEST,
"deleting a non-empty group must return HTTP 400, body: {delete_body}"
);
assert!(
delete_body.contains("<Code>InvalidRequest</Code>"),
"deleting a non-empty group must return InvalidRequest, body: {delete_body}"
);
assert!(
delete_body.contains("<Message>group is not empty</Message>"),
"deleting a non-empty group returned an unexpected message: {delete_body}"
);
let delete_group_url = format!("{}/rustfs/admin/v3/group/testgroup", env.url);
let delete_result = awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await;
assert!(delete_result.is_err(), "deleting a non-empty group should fail");
info!("Delete of non-empty group correctly rejected");
// 4. Remove the member from the group
@@ -151,42 +123,17 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std
"isRemove": true,
"groupStatus": "enabled"
});
admin_ok(
&env,
http::Method::PUT,
"/rustfs/admin/v3/update-group-members",
Some(remove_member_body.to_string()),
)
.await?;
awscurl_put(&update_members_url, &remove_member_body.to_string(), &env.access_key, &env.secret_key).await?;
info!("Removed testuser1 from testgroup");
// 5. Delete the now-empty group — should succeed
admin_ok(&env, http::Method::DELETE, "/rustfs/admin/v3/group/testgroup", None).await?;
awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await?;
info!("Deleted empty testgroup successfully");
// 6. Verify the group no longer exists
let (get_status, get_body) = admin_request(
&env.url,
http::Method::GET,
"/rustfs/admin/v3/group?group=testgroup",
None,
&env.access_key,
&env.secret_key,
)
.await?;
assert_eq!(
get_status,
reqwest::StatusCode::NOT_FOUND,
"a deleted group must return HTTP 404, body: {get_body}"
);
assert!(
get_body.contains("<Code>NoSuchResource</Code>"),
"a deleted group must return NoSuchResource, body: {get_body}"
);
assert!(
get_body.contains("<Message>group &apos;testgroup&apos; does not exist</Message>"),
"a deleted group returned an unexpected message: {get_body}"
);
let get_group_url = format!("{}/rustfs/admin/v3/group?group=testgroup", env.url);
let get_result = awscurl_get(&get_group_url, &env.access_key, &env.secret_key).await;
assert!(get_result.is_err(), "group should no longer exist after deletion");
info!("Confirmed testgroup no longer exists");
Ok(())
@@ -195,6 +142,7 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std
/// Test that a user with only group membership (no explicit user policy) gets group policies
/// and can perform actions allowed by the group (regression test for #2028.1).
#[tokio::test(flavor = "multi_thread")]
#[ignore = "requires awscurl and spawns a real RustFS server"]
async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
@@ -212,56 +160,39 @@ async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn s
"Statement": [{
"Effect": "Allow",
"Action": ["s3:ListAllMyBuckets"],
"Resource": ["arn:aws:s3:::*"]
"Resource": ["*"]
}]
});
admin_ok(
&env,
http::Method::PUT,
&format!("/rustfs/admin/v3/add-canned-policy?name={policy_name}"),
Some(policy_doc.to_string()),
)
.await?;
let add_policy_url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", env.url, policy_name);
awscurl_put(&add_policy_url, &policy_doc.to_string(), &env.access_key, &env.secret_key).await?;
info!("Created canned policy {}", policy_name);
// 2. Create user with no explicit policy
let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, user_name);
let user_body = serde_json::json!({
"secretKey": user_secret,
"status": "enabled"
});
admin_ok(
&env,
http::Method::PUT,
&format!("/rustfs/admin/v3/add-user?accessKey={user_name}"),
Some(user_body.to_string()),
)
.await?;
awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
info!("Created user {} with no explicit policy", user_name);
// 3. Add user to group (creates group with this member; user_group_memberships must be updated)
let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
let add_member_body = serde_json::json!({
"group": group_name,
"members": [user_name],
"isRemove": false,
"groupStatus": "enabled"
});
admin_ok(
&env,
http::Method::PUT,
"/rustfs/admin/v3/update-group-members",
Some(add_member_body.to_string()),
)
.await?;
awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
info!("Added {} to group {}", user_name, group_name);
// 4. Attach policy to group
admin_ok(
&env,
http::Method::PUT,
&format!("/rustfs/admin/v3/set-user-or-group-policy?policyName={policy_name}&userOrGroup={group_name}&isGroup=true"),
Some(String::new()),
)
.await?;
let set_policy_url = format!(
"{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=true",
env.url, policy_name, group_name
);
awscurl_put(&set_policy_url, "", &env.access_key, &env.secret_key).await?;
info!("Attached policy {} to group {}", policy_name, group_name);
// 5. User with only group (no user policy) should be able to list buckets
@@ -278,6 +209,7 @@ async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn s
/// Test that after deleting a user who was the only member of a group, the group can be deleted
/// (regression test for #2028.2: delete group uses backend membership, not stale cache).
#[tokio::test(flavor = "multi_thread")]
#[ignore = "requires awscurl and spawns a real RustFS server"]
async fn test_delete_group_after_deleting_user() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
@@ -289,47 +221,33 @@ async fn test_delete_group_after_deleting_user() -> Result<(), Box<dyn std::erro
let group_name = "soledeletegroup";
// 1. Create user
let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, user_name);
let user_body = serde_json::json!({
"secretKey": user_secret,
"status": "enabled"
});
admin_ok(
&env,
http::Method::PUT,
&format!("/rustfs/admin/v3/add-user?accessKey={user_name}"),
Some(user_body.to_string()),
)
.await?;
awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
info!("Created user {}", user_name);
// 2. Add user to group
let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
let add_member_body = serde_json::json!({
"group": group_name,
"members": [user_name],
"isRemove": false,
"groupStatus": "enabled"
});
admin_ok(
&env,
http::Method::PUT,
"/rustfs/admin/v3/update-group-members",
Some(add_member_body.to_string()),
)
.await?;
awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
info!("Added {} to group {}", user_name, group_name);
// 3. Delete the user (backend and cache update so group membership becomes empty)
admin_ok(
&env,
http::Method::DELETE,
&format!("/rustfs/admin/v3/remove-user?accessKey={user_name}"),
None,
)
.await?;
let remove_user_url = format!("{}/rustfs/admin/v3/remove-user?accessKey={}", env.url, user_name);
awscurl_delete(&remove_user_url, &env.access_key, &env.secret_key).await?;
info!("Deleted user {}", user_name);
// 4. Deleting the group should succeed (backend has empty members; no stale cache)
admin_ok(&env, http::Method::DELETE, &format!("/rustfs/admin/v3/group/{group_name}"), None).await?;
let delete_group_url = format!("{}/rustfs/admin/v3/group/{}", env.url, group_name);
awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await?;
info!("Deleted group {} after user was removed", group_name);
Ok(())
+16 -1
View File
@@ -22,7 +22,9 @@
//! - KMS backend configuration (Local and Vault)
//! - SSE encryption testing utilities
use crate::common::{RustFSTestEnvironment, awscurl_get, awscurl_post, init_logging as common_init_logging, local_http_client};
use crate::common::{
RustFSTestEnvironment, awscurl_available, awscurl_get, awscurl_post, init_logging as common_init_logging, local_http_client,
};
use aws_sdk_s3::Client;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::ServerSideEncryption;
@@ -57,6 +59,15 @@ pub fn init_logging() {
// Additional KMS-specific logging configuration can be added here if needed
}
pub fn skip_if_kms_admin_tool_unavailable(test_name: &str) -> bool {
if awscurl_available() {
return false;
}
info!("Skipping {} because awscurl is not available in PATH", test_name);
true
}
pub fn sse_customer_key_md5_base64(key: &str) -> String {
let mut hasher = Md5::new();
hasher.update(key.as_bytes());
@@ -479,6 +490,10 @@ pub async fn test_kms_key_management(
access_key: &str,
secret_key: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
if skip_if_kms_admin_tool_unavailable("test_kms_key_management") {
return Ok(());
}
info!("Testing KMS key management APIs");
// Test CreateKey
@@ -432,6 +432,7 @@ async fn test_configured_local_kms_admin_and_versioned_cleanup() -> TestResult {
}
#[tokio::test]
#[ignore = "requires a Vault binary"]
async fn test_configured_vault_kms_admin_and_versioned_cleanup() -> TestResult {
let mut env = VaultTestEnvironment::new().await?;
env.start_vault().await?;
@@ -393,10 +393,11 @@ async fn test_kms_multipart_upload_interruption() -> Result<(), Box<dyn std::err
Ok(())
}
/// Test concurrent KMS encryption requests
/// Test KMS resilience to temporary resource constraints
#[tokio::test]
async fn test_kms_concurrent_encryption_requests() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
async fn test_kms_resource_constraints() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
info!("🧪 Testing KMS behavior under resource constraints");
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
@@ -430,27 +431,29 @@ async fn test_kms_concurrent_encryption_requests() -> Result<(), Box<dyn std::er
}
// Wait for all uploads to complete
let mut failures = Vec::new();
let mut successful_uploads = 0;
let mut failed_uploads = 0;
for task in upload_tasks {
let (object_key, result) = task.await?;
let (object_key, result) = task.await.unwrap();
match result {
Ok(_) => {
successful_uploads += 1;
info!("✅ Rapid upload {} succeeded", object_key);
}
Err(e) => {
failed_uploads += 1;
warn!("❌ Rapid upload {} failed: {}", object_key, e);
failures.push(format!("{object_key}: {e}"));
}
}
}
assert!(
failures.is_empty(),
"all 10 concurrent KMS uploads must succeed; failures: {}",
failures.join("; ")
);
info!("📊 Rapid upload results: {} succeeded, {} failed", successful_uploads, failed_uploads);
// We expect most uploads to succeed even under load
assert!(successful_uploads >= 7, "Expected at least 7/10 rapid uploads to succeed");
kms_env.base_env.delete_test_bucket(TEST_BUCKET).await?;
info!("✅ Resource constraints test completed successfully");
Ok(())
}
+5 -1
View File
@@ -20,7 +20,8 @@
//! - Complete encryption/decryption lifecycle
use super::common::{
LocalKMSTestEnvironment, get_kms_status, sse_customer_key_md5_base64, test_kms_key_management, test_sse_c_encryption,
LocalKMSTestEnvironment, get_kms_status, skip_if_kms_admin_tool_unavailable, sse_customer_key_md5_base64,
test_kms_key_management, test_sse_c_encryption,
};
use crate::common::{TEST_BUCKET, init_logging};
use tracing::{error, info};
@@ -28,6 +29,9 @@ use tracing::{error, info};
#[tokio::test]
async fn test_local_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_if_kms_admin_tool_unavailable("test_local_kms_end_to_end") {
return Ok(());
}
info!("Starting Local KMS End-to-End Test");
// Create LocalKMS test environment
+17 -2
View File
@@ -22,8 +22,8 @@ use crate::common::{TEST_BUCKET, init_logging};
use tracing::{error, info};
use super::common::{
VAULT_KEY_NAME, VaultTestEnvironment, get_kms_status, sse_customer_key_md5_base64, start_kms,
test_all_multipart_encryption_types, test_error_scenarios, test_kms_key_management, test_sse_c_encryption,
VAULT_KEY_NAME, VaultTestEnvironment, get_kms_status, skip_if_kms_admin_tool_unavailable, sse_customer_key_md5_base64,
start_kms, test_all_multipart_encryption_types, test_error_scenarios, test_kms_key_management, test_sse_c_encryption,
test_sse_kms_encryption, test_sse_s3_encryption,
};
@@ -62,6 +62,9 @@ impl VaultKmsTestContext {
#[tokio::test]
async fn test_vault_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_if_kms_admin_tool_unavailable("test_vault_kms_end_to_end") {
return Ok(());
}
info!("Starting Vault KMS End-to-End Test with default key {}", VAULT_KEY_NAME);
let context = VaultKmsTestContext::new().await?;
@@ -114,6 +117,9 @@ async fn test_vault_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + S
#[tokio::test]
async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_isolation") {
return Ok(());
}
info!("Starting Vault KMS SSE-C key isolation test");
let context = VaultKmsTestContext::new().await?;
@@ -197,6 +203,9 @@ async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error
#[tokio::test]
async fn test_vault_kms_large_file() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_if_kms_admin_tool_unavailable("test_vault_kms_large_file") {
return Ok(());
}
info!("Starting Vault KMS large file SSE-S3 test");
let context = VaultKmsTestContext::new().await?;
@@ -258,6 +267,9 @@ async fn test_vault_kms_large_file() -> Result<(), Box<dyn std::error::Error + S
#[tokio::test]
async fn test_vault_kms_multipart_upload() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_if_kms_admin_tool_unavailable("test_vault_kms_multipart_upload") {
return Ok(());
}
info!("Starting Vault KMS multipart upload encryption suite");
let context = VaultKmsTestContext::new().await?;
@@ -285,6 +297,9 @@ async fn test_vault_kms_multipart_upload() -> Result<(), Box<dyn std::error::Err
#[tokio::test]
async fn test_vault_kms_key_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_operations") {
return Ok(());
}
info!("Starting Vault KMS key operations test (CRUD)");
let context = VaultKmsTestContext::new().await?;
+2 -12
View File
@@ -131,18 +131,8 @@ mod tests {
// DELETE through the raw key removes the normalized object.
client.delete_object().bucket(bucket).key("//keyname").send().await?;
let error = client
.get_object()
.bucket(bucket)
.key("keyname")
.send()
.await
.expect_err("object must be gone after DELETE with raw key");
assert_eq!(
error.raw_response().map(|response| response.status().as_u16()),
Some(404),
"GET after DELETE with raw key must return HTTP 404, got {error:?}"
);
let result = client.get_object().bucket(bucket).key("keyname").send().await;
assert!(result.is_err(), "object must be gone after DELETE with raw key");
env.stop_server();
info!("Test completed successfully");
-5
View File
@@ -61,11 +61,6 @@ mod get_codec_streaming_compat_test;
#[cfg(test)]
mod version_id_regression_test;
// Receiver-side replication LWW (rustfs/backlog#1953): stale inbound
// replication metadata must not overwrite a newer local category state.
#[cfg(test)]
mod replication_lww_receiver_test;
// Data usage regression tests
#[cfg(test)]
mod data_usage_test;
@@ -41,6 +41,13 @@ async fn create_issue_3107_fixture(root: &Path) -> TestResult {
Ok(())
}
fn mc_available() -> bool {
Command::new("mc")
.arg("--version")
.output()
.is_ok_and(|output| output.status.success())
}
fn run_mc(args: &[&str]) -> TestResult {
let output = Command::new("mc").args(args).output()?;
if !output.status.success() {
@@ -68,7 +75,10 @@ fn count_files(root: &Path) -> usize {
async fn test_mc_mirror_small_bucket_completes_without_list_timeout() -> TestResult {
crate::common::init_logging();
info!("Starting issue #3107 mc mirror regression test");
run_mc(&["--version"])?;
if !mc_available() {
info!("Skipping issue #3107 mc mirror regression test because mc is not installed");
return Ok(());
}
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
@@ -4278,6 +4278,10 @@ async fn test_signed_put_object_extract_preserves_pax_metadata_and_version_id()
async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retention_conditions()
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if !crate::common::awscurl_available() {
return Ok(());
}
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
+14 -5
View File
@@ -11,20 +11,29 @@ The tests cover the following AWS policy variable scenarios:
3. **Variable concatenation** - Combining variables with static text like `prefix-${aws:username}-suffix`
4. **Nested variables** - Complex nested variable patterns like `${${aws:username}-test}`
5. **Deny scenarios** - Testing deny policies with variables
6. **STS credentials** - Variable resolution inherited by temporary credentials
## Prerequisites
- RustFS server binary
- `awscurl` utility for admin API calls
- AWS SDK for Rust (included in the project)
## Running Tests
### Run All Policy Tests Using Unified Test Runner
```bash
# Run all policy tests with comprehensive reporting
# Note: Requires a RustFS server running on localhost:9000
cargo test -p e2e_test policy::test_runner::test_policy_full_suite -- --nocapture --ignored --test-threads=1
# Run only critical policy tests
cargo test -p e2e_test policy::test_runner::test_policy_critical_suite -- --nocapture --ignored --test-threads=1
```
### Run All Policy Tests
```bash
# From the project root directory
cargo test -p e2e_test policy:: -- --nocapture
```
Each test starts an isolated RustFS server on a dynamically allocated local port and cleans it up afterward.
cargo test -p e2e_test policy:: -- --nocapture --ignored --test-threads=1
```
+2
View File
@@ -18,3 +18,5 @@
//! including single-value, multi-value, and nested variable scenarios.
mod policy_variables_test;
mod test_env;
mod test_runner;
@@ -14,17 +14,14 @@
//! Tests for AWS IAM policy variables with single-value, multi-value, and nested scenarios
use crate::common::{
RustFSTestEnvironment, awscurl_delete, awscurl_put, build_test_s3_config, build_test_sts_client, init_logging,
};
use aws_sdk_s3::Client;
use aws_sdk_s3::error::ProvideErrorMetadata;
use crate::common::{awscurl_delete, awscurl_put, init_logging};
use crate::policy::test_env::PolicyTestEnvironment;
use aws_sdk_s3::primitives::ByteStream;
use tracing::info;
/// Helper function to create a regular user with given credentials
async fn create_user(
env: &RustFSTestEnvironment,
env: &PolicyTestEnvironment,
username: &str,
password: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
@@ -39,9 +36,20 @@ async fn create_user(
Ok(())
}
/// Helper function to create an STS user with given credentials
async fn create_sts_user(
env: &PolicyTestEnvironment,
username: &str,
password: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
// For STS, we create a regular user first, then use it to assume roles
create_user(env, username, password).await?;
Ok(())
}
/// Helper function to create and attach a policy
async fn create_and_attach_policy(
env: &RustFSTestEnvironment,
env: &PolicyTestEnvironment,
policy_name: &str,
username: &str,
policy_document: serde_json::Value,
@@ -62,9 +70,9 @@ async fn create_and_attach_policy(
}
/// Helper function to clean up test resources
async fn cleanup_user_and_policy(env: &RustFSTestEnvironment, username: &str, policy_name: &str) {
async fn cleanup_user_and_policy(env: &PolicyTestEnvironment, username: &str, policy_name: &str) {
// Create admin client for cleanup
let admin_client = env.create_s3_client();
let admin_client = env.create_s3_client(&env.access_key, &env.secret_key);
// Delete buckets that might have been created by this user
let bucket_patterns = [
@@ -76,7 +84,7 @@ async fn cleanup_user_and_policy(env: &RustFSTestEnvironment, username: &str, po
format!("{username}-test"),
format!("{username}-sts-bucket"),
format!("{username}-service-bucket"),
format!("{username}-private-bucket"),
"private-test-bucket".to_string(), // For deny test
];
// Try to delete objects and buckets
@@ -113,18 +121,24 @@ async fn cleanup_user_and_policy(env: &RustFSTestEnvironment, username: &str, po
/// Test AWS policy variables with single-value scenarios
#[tokio::test(flavor = "multi_thread")]
#[ignore = "Starts a rustfs server; enable when running full E2E"]
pub async fn test_aws_policy_variables_single_value() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
test_aws_policy_variables_single_value_impl().await
}
/// Implementation function for single-value policy variables test
pub async fn test_aws_policy_variables_single_value_impl() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
info!("Starting AWS policy variables single-value test");
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
let env = PolicyTestEnvironment::with_address("127.0.0.1:9000").await?;
test_aws_policy_variables_single_value_impl_with_env(&env).await
}
async fn test_aws_policy_variables_single_value_impl_with_env(
env: &RustFSTestEnvironment,
/// Implementation function for single-value policy variables test with shared environment
pub async fn test_aws_policy_variables_single_value_impl_with_env(
env: &PolicyTestEnvironment,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
// Create test user
let test_user = "testuser1";
@@ -184,7 +198,9 @@ async fn test_aws_policy_variables_single_value_impl_with_env(
awscurl_put(&attach_policy_url, "", &env.access_key, &env.secret_key).await?;
// Create S3 client for test user
let test_client = env.create_s3_client_with_credentials(test_user, test_password);
let test_client = env.create_s3_client(test_user, test_password);
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
// Test 1: User should be able to list buckets (allowed by policy)
info!("Test 1: User listing buckets");
@@ -241,13 +257,11 @@ async fn test_aws_policy_variables_single_value_impl_with_env(
// Test 6: User should NOT be able to create bucket NOT matching username pattern
info!("Test 6: User attempting to create bucket NOT matching pattern");
let other_bucket_name = "other-user-bucket";
let denied = test_client
.create_bucket()
.bucket(other_bucket_name)
.send()
.await
.expect_err("a bucket outside the username pattern must be denied");
assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
let create_other_result = test_client.create_bucket().bucket(other_bucket_name).send().await;
if create_other_result.is_ok() {
cleanup().await;
return Err("User should NOT be able to create bucket NOT matching username pattern".into());
}
// Cleanup
info!("Cleaning up test resources");
@@ -259,18 +273,24 @@ async fn test_aws_policy_variables_single_value_impl_with_env(
/// Test AWS policy variables with multi-value scenarios
#[tokio::test(flavor = "multi_thread")]
#[ignore = "Starts a rustfs server; enable when running full E2E"]
pub async fn test_aws_policy_variables_multi_value() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
test_aws_policy_variables_multi_value_impl().await
}
/// Implementation function for multi-value policy variables test
pub async fn test_aws_policy_variables_multi_value_impl() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
info!("Starting AWS policy variables multi-value test");
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
let env = PolicyTestEnvironment::with_address("127.0.0.1:9000").await?;
test_aws_policy_variables_multi_value_impl_with_env(&env).await
}
async fn test_aws_policy_variables_multi_value_impl_with_env(
env: &RustFSTestEnvironment,
/// Implementation function for multi-value policy variables test with shared environment
pub async fn test_aws_policy_variables_multi_value_impl_with_env(
env: &PolicyTestEnvironment,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
// Create test user
let test_user = "testuser2";
@@ -318,7 +338,7 @@ async fn test_aws_policy_variables_multi_value_impl_with_env(
create_and_attach_policy(env, policy_name, test_user, policy_document).await?;
// Create S3 client for test user
let test_client = env.create_s3_client_with_credentials(test_user, test_password);
let test_client = env.create_s3_client(test_user, test_password);
// Test 1: User should be able to create buckets matching any of the multi-value patterns
info!("Test 1: User creating first bucket matching multi-value pattern");
@@ -348,13 +368,11 @@ async fn test_aws_policy_variables_multi_value_impl_with_env(
// Test 4: User should NOT be able to create bucket NOT matching any multi-value pattern
info!("Test 4: User attempting to create bucket NOT matching any pattern");
let other_bucket_name = format!("{test_user}-other-bucket");
let denied = test_client
.create_bucket()
.bucket(&other_bucket_name)
.send()
.await
.expect_err("a bucket outside all allowed patterns must be denied");
assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
let create_other_result = test_client.create_bucket().bucket(&other_bucket_name).send().await;
if create_other_result.is_ok() {
cleanup().await;
return Err("User should NOT be able to create bucket NOT matching any multi-value pattern".into());
}
// Test 5: User should be able to list objects in their allowed buckets
info!("Test 5: User listing objects in allowed buckets");
@@ -380,18 +398,24 @@ async fn test_aws_policy_variables_multi_value_impl_with_env(
/// Test AWS policy variables with variable concatenation
#[tokio::test(flavor = "multi_thread")]
#[ignore = "Starts a rustfs server; enable when running full E2E"]
pub async fn test_aws_policy_variables_concatenation() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
test_aws_policy_variables_concatenation_impl().await
}
/// Implementation function for concatenation policy variables test
pub async fn test_aws_policy_variables_concatenation_impl() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
info!("Starting AWS policy variables concatenation test");
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
let env = PolicyTestEnvironment::with_address("127.0.0.1:9000").await?;
test_aws_policy_variables_concatenation_impl_with_env(&env).await
}
async fn test_aws_policy_variables_concatenation_impl_with_env(
env: &RustFSTestEnvironment,
/// Implementation function for concatenation policy variables test with shared environment
pub async fn test_aws_policy_variables_concatenation_impl_with_env(
env: &PolicyTestEnvironment,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
// Create test user
let test_user = "testuser3";
@@ -431,7 +455,10 @@ async fn test_aws_policy_variables_concatenation_impl_with_env(
create_and_attach_policy(env, policy_name, test_user, policy_document).await?;
// Create S3 client for test user
let test_client = env.create_s3_client_with_credentials(test_user, test_password);
let test_client = env.create_s3_client(test_user, test_password);
// Add a small delay to allow policy to propagate
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
// Test: User should be able to create bucket matching concatenated pattern
info!("Test: User creating bucket matching concatenated pattern");
@@ -460,30 +487,41 @@ async fn test_aws_policy_variables_concatenation_impl_with_env(
/// Test AWS policy variables with nested scenarios
#[tokio::test(flavor = "multi_thread")]
#[ignore = "Starts a rustfs server; enable when running full E2E"]
pub async fn test_aws_policy_variables_nested() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
test_aws_policy_variables_nested_impl().await
}
/// Implementation function for nested policy variables test
pub async fn test_aws_policy_variables_nested_impl() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
info!("Starting AWS policy variables nested test");
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
let env = PolicyTestEnvironment::with_address("127.0.0.1:9000").await?;
test_aws_policy_variables_nested_impl_with_env(&env).await
}
/// Test AWS policy variables with STS temporary credentials
#[tokio::test(flavor = "multi_thread")]
#[ignore = "Starts a rustfs server; enable when running full E2E"]
pub async fn test_aws_policy_variables_sts() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
test_aws_policy_variables_sts_impl().await
}
/// Implementation function for STS policy variables test
pub async fn test_aws_policy_variables_sts_impl() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
info!("Starting AWS policy variables STS test");
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
let env = PolicyTestEnvironment::with_address("127.0.0.1:9000").await?;
test_aws_policy_variables_sts_impl_with_env(&env).await
}
async fn test_aws_policy_variables_nested_impl_with_env(
env: &RustFSTestEnvironment,
/// Implementation function for nested policy variables test with shared environment
pub async fn test_aws_policy_variables_nested_impl_with_env(
env: &PolicyTestEnvironment,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
// Create test user
let test_user = "testuser4";
@@ -523,7 +561,10 @@ async fn test_aws_policy_variables_nested_impl_with_env(
create_and_attach_policy(env, policy_name, test_user, policy_document).await?;
// Create S3 client for test user
let test_client = env.create_s3_client_with_credentials(test_user, test_password);
let test_client = env.create_s3_client(test_user, test_password);
// Add a small delay to allow policy to propagate
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
// Test nested variable resolution
info!("Test: Nested variable resolution");
@@ -540,14 +581,14 @@ async fn test_aws_policy_variables_nested_impl_with_env(
return Err(format!("User should be able to create bucket with nested variable: {e}").into());
}
// Verify a valid bucket name outside the resolved resource is denied.
let denied = test_client
.create_bucket()
.bucket("other-user-test")
.send()
.await
.expect_err("a bucket outside the resolved nested variable must be denied");
assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
// Verify bucket creation fails with unresolved variable
let unresolved_bucket = format!("${{}}-test {test_user}");
let create_unresolved = test_client.create_bucket().bucket(&unresolved_bucket).send().await;
if create_unresolved.is_ok() {
cleanup().await;
return Err("User should NOT be able to create bucket with unresolved variable".into());
}
// Cleanup
info!("Cleaning up test resources");
@@ -557,8 +598,9 @@ async fn test_aws_policy_variables_nested_impl_with_env(
Ok(())
}
async fn test_aws_policy_variables_sts_impl_with_env(
env: &RustFSTestEnvironment,
/// Implementation function for STS policy variables test with shared environment
pub async fn test_aws_policy_variables_sts_impl_with_env(
env: &PolicyTestEnvironment,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
// Create test user for STS
let test_user = "testuser-sts";
@@ -570,7 +612,8 @@ async fn test_aws_policy_variables_sts_impl_with_env(
cleanup_user_and_policy(env, test_user, policy_name).await;
};
create_user(env, test_user, test_password).await?;
// Create STS user
create_sts_user(env, test_user, test_password).await?;
// Create policy with STS-compatible variables
let policy_document = serde_json::json!({
@@ -581,11 +624,6 @@ async fn test_aws_policy_variables_sts_impl_with_env(
"Action": ["s3:ListAllMyBuckets"],
"Resource": ["arn:aws:s3:::*"]
},
{
"Effect": "Allow",
"Action": ["sts:AssumeRole"],
"Resource": ["arn:aws:s3:::*"]
},
{
"Effect": "Allow",
"Action": ["s3:CreateBucket"],
@@ -593,12 +631,7 @@ async fn test_aws_policy_variables_sts_impl_with_env(
},
{
"Effect": "Allow",
"Action": ["s3:ListBucket"],
"Resource": [format!("arn:aws:s3:::{}-sts-bucket", "${aws:username}")]
},
{
"Effect": "Allow",
"Action": ["s3:PutObject", "s3:GetObject"],
"Action": ["s3:ListBucket", "s3:PutObject", "s3:GetObject"],
"Resource": [format!("arn:aws:s3:::{}-sts-bucket/*", "${aws:username}")]
}
]
@@ -606,22 +639,11 @@ async fn test_aws_policy_variables_sts_impl_with_env(
create_and_attach_policy(env, policy_name, test_user, policy_document).await?;
let assumed = build_test_sts_client(&env.url, test_user, test_password, None, "policy-variable-sts")
.assume_role()
.role_arn("arn:aws:iam::123456789012:role/policy-variable")
.role_session_name("policy-variable-e2e")
.send()
.await?;
let credentials = assumed
.credentials()
.ok_or("AssumeRole response should contain temporary credentials")?;
let test_client = Client::from_conf(build_test_s3_config(
&env.url,
credentials.access_key_id(),
credentials.secret_access_key(),
Some(credentials.session_token()),
"policy-variable-sts-session",
));
// Create S3 client for test user
let test_client = env.create_s3_client(test_user, test_password);
// Add a small delay to allow policy to propagate
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
// Test: User should be able to create bucket matching STS pattern
info!("Test: User creating bucket matching STS pattern");
@@ -677,18 +699,24 @@ async fn test_aws_policy_variables_sts_impl_with_env(
/// Test AWS policy variables with deny scenarios
#[tokio::test(flavor = "multi_thread")]
#[ignore = "Starts a rustfs server; enable when running full E2E"]
pub async fn test_aws_policy_variables_deny() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
test_aws_policy_variables_deny_impl().await
}
/// Implementation function for deny policy variables test
pub async fn test_aws_policy_variables_deny_impl() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
info!("Starting AWS policy variables deny test");
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
let env = PolicyTestEnvironment::with_address("127.0.0.1:9000").await?;
test_aws_policy_variables_deny_impl_with_env(&env).await
}
async fn test_aws_policy_variables_deny_impl_with_env(
env: &RustFSTestEnvironment,
/// Implementation function for deny policy variables test with shared environment
pub async fn test_aws_policy_variables_deny_impl_with_env(
env: &PolicyTestEnvironment,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
// Create test user
let test_user = "testuser5";
@@ -731,7 +759,10 @@ async fn test_aws_policy_variables_deny_impl_with_env(
create_and_attach_policy(env, policy_name, test_user, policy_document).await?;
// Create S3 client for test user
let test_client = env.create_s3_client_with_credentials(test_user, test_password);
let test_client = env.create_s3_client(test_user, test_password);
// Add a small delay to allow policy to propagate
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
// Test 1: User should be able to create bucket matching username pattern
info!("Test 1: User creating bucket matching username pattern");
@@ -744,14 +775,12 @@ async fn test_aws_policy_variables_deny_impl_with_env(
// Test 2: User should NOT be able to create bucket with "private" in the name (deny rule)
info!("Test 2: User attempting to create bucket with 'private' in name (should be denied)");
let private_bucket_name = format!("{test_user}-private-bucket");
let denied = test_client
.create_bucket()
.bucket(&private_bucket_name)
.send()
.await
.expect_err("the explicit deny must reject a matching bucket name");
assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
let private_bucket_name = "private-test-bucket";
let create_private_result = test_client.create_bucket().bucket(private_bucket_name).send().await;
if create_private_result.is_ok() {
cleanup().await;
return Err("User should NOT be able to create bucket with 'private' in name due to deny rule".into());
}
// Cleanup
info!("Cleaning up test resources");
+100
View File
@@ -0,0 +1,100 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Custom test environment for policy variables tests
//!
//! This module provides a custom test environment that doesn't automatically
//! stop servers when destroyed, addressing the server stopping issue.
use aws_sdk_s3::Client;
use aws_sdk_s3::config::{Config, Credentials, Region};
use std::net::TcpStream;
use std::time::Duration;
use tokio::time::sleep;
use tracing::{info, warn};
// Default credentials
const DEFAULT_ACCESS_KEY: &str = "rustfsadmin";
const DEFAULT_SECRET_KEY: &str = "rustfsadmin";
/// Custom test environment that doesn't automatically stop servers
pub struct PolicyTestEnvironment {
pub temp_dir: String,
pub address: String,
pub url: String,
pub access_key: String,
pub secret_key: String,
}
impl PolicyTestEnvironment {
/// Create a new test environment with specific address
/// This environment won't stop any server when dropped
pub async fn with_address(address: &str) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
let temp_dir = format!("/tmp/rustfs_policy_test_{}", uuid::Uuid::new_v4());
tokio::fs::create_dir_all(&temp_dir).await?;
let url = format!("http://{address}");
Ok(Self {
temp_dir,
address: address.to_string(),
url,
access_key: DEFAULT_ACCESS_KEY.to_string(),
secret_key: DEFAULT_SECRET_KEY.to_string(),
})
}
/// Create an AWS S3 client configured for this RustFS instance
pub fn create_s3_client(&self, access_key: &str, secret_key: &str) -> Client {
let credentials = Credentials::new(access_key, secret_key, None, None, "policy-test");
let config = Config::builder()
.credentials_provider(credentials)
.region(Region::new("us-east-1"))
.endpoint_url(&self.url)
.force_path_style(true)
.behavior_version_latest()
.build();
Client::from_conf(config)
}
/// Wait for RustFS server to be ready by checking TCP connectivity
pub async fn wait_for_server_ready(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
info!("Waiting for RustFS server to be ready on {}", self.address);
for i in 0..30 {
if TcpStream::connect(&self.address).is_ok() {
info!("✅ RustFS server is ready after {} attempts", i + 1);
return Ok(());
}
if i == 29 {
return Err("RustFS server failed to become ready within 30 seconds".into());
}
sleep(Duration::from_secs(1)).await;
}
Ok(())
}
}
// Implement Drop trait that doesn't stop servers
impl Drop for PolicyTestEnvironment {
fn drop(&mut self) {
// Clean up temp directory only, don't stop any server
if let Err(e) = std::fs::remove_dir_all(&self.temp_dir) {
warn!("Failed to clean up temp directory {}: {}", self.temp_dir, e);
}
}
}
+230
View File
@@ -0,0 +1,230 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::common::init_logging;
use crate::policy::test_env::PolicyTestEnvironment;
use std::time::Instant;
use tokio::time::{Duration, sleep};
use tracing::{error, info};
/// Test case definition
#[derive(Debug, Clone)]
pub struct TestDefinition {
pub name: String,
pub is_critical: bool,
}
impl TestDefinition {
pub fn new(name: impl Into<String>, is_critical: bool) -> Self {
Self {
name: name.into(),
is_critical,
}
}
}
/// Test result
#[derive(Debug, Clone)]
pub struct TestResult {
pub test_name: String,
pub success: bool,
pub error_message: Option<String>,
}
impl TestResult {
pub fn success(test_name: String) -> Self {
Self {
test_name,
success: true,
error_message: None,
}
}
pub fn failure(test_name: String, error: String) -> Self {
Self {
test_name,
success: false,
error_message: Some(error),
}
}
}
/// Test suite configuration
#[derive(Debug, Clone, Default)]
pub struct TestSuiteConfig {
pub include_critical_only: bool,
}
/// Policy test suite
pub struct PolicyTestSuite {
tests: Vec<TestDefinition>,
config: TestSuiteConfig,
}
impl PolicyTestSuite {
/// Create default test suite
pub fn new() -> Self {
let tests = vec![
TestDefinition::new("test_aws_policy_variables_single_value", true),
TestDefinition::new("test_aws_policy_variables_multi_value", true),
TestDefinition::new("test_aws_policy_variables_concatenation", true),
TestDefinition::new("test_aws_policy_variables_nested", true),
TestDefinition::new("test_aws_policy_variables_deny", true),
TestDefinition::new("test_aws_policy_variables_sts", true),
];
Self {
tests,
config: TestSuiteConfig::default(),
}
}
/// Configure test suite
pub fn with_config(mut self, config: TestSuiteConfig) -> Self {
self.config = config;
self
}
/// Run test suite
pub async fn run_test_suite(&self) -> Vec<TestResult> {
init_logging();
info!("Starting Policy Variables test suite");
let start_time = Instant::now();
let mut results = Vec::new();
// Create test environment
let env = match PolicyTestEnvironment::with_address("127.0.0.1:9000").await {
Ok(env) => env,
Err(e) => {
error!("Failed to create test environment: {}", e);
return vec![TestResult::failure("env_creation".into(), e.to_string())];
}
};
// Wait for server to be ready
if env.wait_for_server_ready().await.is_err() {
error!("Server is not ready");
return vec![TestResult::failure("server_check".into(), "Server not ready".into())];
}
// Filter tests
let tests_to_run: Vec<&TestDefinition> = self
.tests
.iter()
.filter(|test| !self.config.include_critical_only || test.is_critical)
.collect();
info!("Scheduled {} tests", tests_to_run.len());
// Run tests
for (i, test_def) in tests_to_run.iter().enumerate() {
info!("Running test {}/{}: {}", i + 1, tests_to_run.len(), test_def.name);
let test_start = Instant::now();
let result = self.run_single_test(test_def, &env).await;
let test_duration = test_start.elapsed();
match result {
Ok(_) => {
info!("Test passed: {} ({:.2}s)", test_def.name, test_duration.as_secs_f64());
results.push(TestResult::success(test_def.name.clone()));
}
Err(e) => {
error!("Test failed: {} ({:.2}s): {}", test_def.name, test_duration.as_secs_f64(), e);
results.push(TestResult::failure(test_def.name.clone(), e.to_string()));
}
}
// Delay between tests to avoid resource conflicts
if i < tests_to_run.len() - 1 {
sleep(Duration::from_secs(2)).await;
}
}
// Print summary
self.print_summary(&results, start_time.elapsed());
results
}
/// Run a single test
async fn run_single_test(
&self,
test_def: &TestDefinition,
env: &PolicyTestEnvironment,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
match test_def.name.as_str() {
"test_aws_policy_variables_single_value" => {
super::policy_variables_test::test_aws_policy_variables_single_value_impl_with_env(env).await
}
"test_aws_policy_variables_multi_value" => {
super::policy_variables_test::test_aws_policy_variables_multi_value_impl_with_env(env).await
}
"test_aws_policy_variables_concatenation" => {
super::policy_variables_test::test_aws_policy_variables_concatenation_impl_with_env(env).await
}
"test_aws_policy_variables_nested" => {
super::policy_variables_test::test_aws_policy_variables_nested_impl_with_env(env).await
}
"test_aws_policy_variables_deny" => {
super::policy_variables_test::test_aws_policy_variables_deny_impl_with_env(env).await
}
"test_aws_policy_variables_sts" => {
super::policy_variables_test::test_aws_policy_variables_sts_impl_with_env(env).await
}
_ => Err(format!("Test {} not implemented", test_def.name).into()),
}
}
/// Print test summary
fn print_summary(&self, results: &[TestResult], total_duration: Duration) {
info!("=== Test Suite Summary ===");
info!("Total duration: {:.2}s", total_duration.as_secs_f64());
info!("Total tests: {}", results.len());
let passed = results.iter().filter(|r| r.success).count();
let failed = results.len() - passed;
let success_rate = (passed as f64 / results.len() as f64) * 100.0;
info!("Passed: {} | Failed: {}", passed, failed);
info!("Success rate: {:.1}%", success_rate);
if failed > 0 {
error!("Failed tests:");
for result in results.iter().filter(|r| !r.success) {
error!(" - {}: {}", result.test_name, result.error_message.as_ref().unwrap());
}
}
}
}
/// Test suite
#[tokio::test]
#[ignore = "Connects to existing rustfs server"]
async fn test_policy_critical_suite() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let config = TestSuiteConfig {
include_critical_only: true,
};
let suite = PolicyTestSuite::new().with_config(config);
let results = suite.run_test_suite().await;
let failed = results.iter().filter(|r| !r.success).count();
if failed > 0 {
return Err(format!("Critical tests failed: {failed} failures").into());
}
info!("All critical tests passed");
Ok(())
}
+2 -13
View File
@@ -340,18 +340,7 @@ async fn tampered_presigned_put_returns_signature_does_not_match() -> Result<(),
assert_error_code(&body, "SignatureDoesNotMatch");
// The rejected write must not have created the object.
let error = env
.create_s3_client()
.head_object()
.bucket(BUCKET)
.key(key)
.send()
.await
.expect_err("tampered presigned PUT must not store the object");
assert_eq!(
error.raw_response().map(|response| response.status().as_u16()),
Some(404),
"tampered presigned PUT absence probe must return HTTP 404, got {error:?}"
);
let head = env.create_s3_client().head_object().bucket(BUCKET).key(key).send().await;
assert!(head.is_err(), "tampered presigned PUT must not store the object");
Ok(())
}
@@ -453,28 +453,27 @@ fn watch_session_lifecycle_events(child: &mut Child, counters: Arc<SessionCounte
}
/// Count TCP connections in CLOSE_WAIT against the given local port
/// by shelling out to ss -tn state CLOSE-WAIT. The oracle fails closed
/// if ss is missing or cannot inspect socket state. The contract is zero
/// by shelling out to ss -tn state CLOSE-WAIT. The check is
/// best-effort: if ss is missing on the host the function returns
/// Ok(None) and the caller skips the assertion. The contract is zero
/// CLOSE_WAIT entries attributable to the test.
#[cfg(target_os = "linux")]
async fn count_close_wait_on_port(port: u16, test_id: &str) -> Result<usize> {
let port_filter = format!("sport = :{port}");
let output = Command::new("ss")
.args(["-H", "-t", "-n", "state", "close-wait"])
.arg(&port_filter)
.output()
.await
.map_err(|error| anyhow!("{test_id} failed to run ss CLOSE_WAIT oracle: {error}"))?;
async fn count_close_wait_on_port(port: u16) -> Result<Option<usize>> {
let output = match Command::new("ss").args(["-tn", "state", "CLOSE-WAIT"]).output().await {
Ok(o) => o,
Err(_) => return Ok(None),
};
if !output.status.success() {
return Err(anyhow!(
"{test_id} ss CLOSE_WAIT oracle exited with {}: {}",
output.status,
String::from_utf8_lossy(&output.stderr)
));
return Ok(None);
}
let stdout = String::from_utf8_lossy(&output.stdout);
let count = stdout.lines().filter(|line| !line.trim().is_empty()).count();
Ok(count)
let needle_local = format!(":{port} ");
let needle_local_eol = format!(":{port}\n");
let count = stdout
.lines()
.filter(|l| l.contains(&needle_local) || l.contains(needle_local_eol.trim_end()))
.count();
Ok(Some(count))
}
// CMPTST-01: medium-binary upload then download with SHA256 compare.
@@ -1401,7 +1400,7 @@ pub(crate) mod cmptst_24 {
// the JoinSet flushes finished tasks before the assertion runs.
// 5. Assert the entered/finished session counters balance and that
// no CLOSE_WAIT sockets remain on the bind port (Linux ss(8)
// only; missing or failed socket inspection is an error).
// only; the assertion skips with a warn if ss is unavailable).
pub(crate) async fn run_concurrent_half_close_no_leak() -> Result<()> {
let env = ProtocolTestEnvironment::new().map_err(|e| anyhow!("{}", e))?;
let host_key_dir = PathBuf::from(&env.temp_dir).join("sftp_host_keys");
@@ -1521,13 +1520,15 @@ pub(crate) mod cmptst_24 {
));
}
let close_wait = count_close_wait_on_port(HALF_CLOSE_SFTP_PORT, COMPLIANCE_TEST_OUTPUT_ID).await?;
if close_wait != 0 {
return Err(anyhow!(
"{COMPLIANCE_TEST_OUTPUT_ID} {close_wait} CLOSE_WAIT entries against port {HALF_CLOSE_SFTP_PORT}, expected 0"
));
match count_close_wait_on_port(HALF_CLOSE_SFTP_PORT).await? {
Some(0) => info!("{COMPLIANCE_TEST_OUTPUT_ID}: zero CLOSE_WAIT entries against port {HALF_CLOSE_SFTP_PORT}"),
Some(n) => {
return Err(anyhow!(
"{COMPLIANCE_TEST_OUTPUT_ID} {n} CLOSE_WAIT entries against port {HALF_CLOSE_SFTP_PORT}, expected 0"
));
}
None => info!("{COMPLIANCE_TEST_OUTPUT_ID}: ss(8) unavailable, skipping CLOSE_WAIT assertion"),
}
info!("{COMPLIANCE_TEST_OUTPUT_ID}: zero CLOSE_WAIT entries against port {HALF_CLOSE_SFTP_PORT}");
// Drop the keepalive vector now so the test process does
// not leave the half-closed sockets dangling past the
@@ -1946,7 +1947,7 @@ pub(crate) mod cmptst_25 {
// plus two 15 s ticks worst-case = 60 s) to detect CLOSE_WAIT
// via /proc/net/tcp and cancel the parked session.
// 5. Assert the session task counters balance and CLOSE_WAIT count
// is zero (ss(8) only; missing or failed socket inspection is an error).
// is zero (ss(8) only; skips with a warn when ss is missing).
pub(crate) async fn run_wedge_kill_after_silence_in_close_wait() -> Result<()> {
let env = ProtocolTestEnvironment::new().map_err(|e| anyhow!("{}", e))?;
let host_key_dir = PathBuf::from(&env.temp_dir).join("sftp_host_keys");
@@ -2055,13 +2056,15 @@ pub(crate) mod cmptst_25 {
));
}
let close_wait = count_close_wait_on_port(WEDGE_SFTP_PORT, COMPLIANCE_TEST_OUTPUT_ID).await?;
if close_wait != 0 {
return Err(anyhow!(
"{COMPLIANCE_TEST_OUTPUT_ID} {close_wait} CLOSE_WAIT entries against port {WEDGE_SFTP_PORT}, expected 0"
));
match count_close_wait_on_port(WEDGE_SFTP_PORT).await? {
Some(0) => info!("{COMPLIANCE_TEST_OUTPUT_ID}: zero CLOSE_WAIT entries against port {WEDGE_SFTP_PORT}"),
Some(n) => {
return Err(anyhow!(
"{COMPLIANCE_TEST_OUTPUT_ID} {n} CLOSE_WAIT entries against port {WEDGE_SFTP_PORT}, expected 0"
));
}
None => info!("{COMPLIANCE_TEST_OUTPUT_ID}: ss(8) unavailable, skipping CLOSE_WAIT assertion"),
}
info!("{COMPLIANCE_TEST_OUTPUT_ID}: zero CLOSE_WAIT entries against port {WEDGE_SFTP_PORT}");
drop(keepalive);
info!("PASS {COMPLIANCE_TEST_OUTPUT_ID}: wedged sessions killed by the watchdog");
@@ -26,7 +26,7 @@ use aws_sdk_s3::config::{Credentials, Region};
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
use russh::client::{self, Handle};
use russh::keys::ssh_key::LineEnding;
use russh::keys::{Algorithm, PrivateKey, PublicKeyOrCertificate};
use russh::keys::{Algorithm, PrivateKey, PublicKey};
use russh_sftp::client::SftpSession;
use russh_sftp::protocol::OpenFlags;
use std::path::Path;
@@ -46,7 +46,7 @@ pub struct AcceptAnyServerKey;
impl client::Handler for AcceptAnyServerKey {
type Error = anyhow::Error;
async fn check_server_key(&mut self, _server_public_key: &PublicKeyOrCertificate) -> Result<bool, Self::Error> {
async fn check_server_key(&mut self, _server_public_key: &PublicKey) -> Result<bool, Self::Error> {
Ok(true)
}
}
+56 -13
View File
@@ -18,6 +18,15 @@ use http::{Method, StatusCode};
use tokio::time::{Duration, sleep, timeout};
use tracing::{debug, info};
fn skip_without_awscurl() -> bool {
if crate::common::awscurl_available() {
return false;
}
info!("Skipping quota test because awscurl is not available");
true
}
/// Test environment setup for quota tests
pub struct QuotaTestEnv {
pub env: RustFSTestEnvironment,
@@ -267,6 +276,9 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_basic_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
// Create test bucket
@@ -308,6 +320,9 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_admission_aws_chunked_declared_encoding() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
@@ -356,6 +371,9 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_update_and_clear() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
@@ -388,6 +406,9 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_delete_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
@@ -421,6 +442,9 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_usage_tracking() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
@@ -456,6 +480,9 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_statistics() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
@@ -486,6 +513,9 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_check_api() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
@@ -523,6 +553,9 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_multiple_buckets() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
// Create two buckets in the same environment
@@ -560,6 +593,9 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_error_handling() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
@@ -592,6 +628,9 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_http_endpoints() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
@@ -650,6 +689,9 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_normal_user_permissions() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
@@ -682,26 +724,18 @@ mod integration_tests {
assert!(resp.contains("quota_limit"));
// Normal user sets quota — should be denied
let set_error = awscurl_put(
let set_resp = awscurl_put(
&get_url,
&serde_json::json!({"quota": 2048, "quota_type": "HARD"}).to_string(),
normal_ak,
normal_sk,
)
.await
.expect_err("normal user should not be able to set quota")
.to_string();
assert!(set_error.contains("AccessDenied"), "quota denial must return AccessDenied: {set_error}");
.await;
assert!(set_resp.is_err(), "normal user should not be able to set quota");
// Normal user clears quota — should be denied
let delete_error = awscurl_delete(&get_url, normal_ak, normal_sk)
.await
.expect_err("normal user should not be able to clear quota")
.to_string();
assert!(
delete_error.contains("AccessDenied"),
"quota deletion denial must return AccessDenied: {delete_error}"
);
let del_resp = awscurl_delete(&get_url, normal_ak, normal_sk).await;
assert!(del_resp.is_err(), "normal user should not be able to clear quota");
env.cleanup_bucket().await?;
Ok(())
@@ -710,6 +744,9 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_copy_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
@@ -752,6 +789,9 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_batch_delete() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
@@ -807,6 +847,9 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_multipart_upload() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
+162 -123
View File
@@ -1,26 +1,52 @@
#![cfg(test)]
use crate::common::{RustFSTestEnvironment, TEST_BUCKET, init_logging};
use aws_config::meta::region::RegionProviderChain;
use aws_sdk_s3::Client;
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
use aws_sdk_s3::config::{Credentials, Region};
use aws_sdk_s3::error::SdkError;
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
use bytes::Bytes;
use std::error::Error;
use std::fmt::Debug;
type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
const ENDPOINT: &str = "http://localhost:9000";
const ACCESS_KEY: &str = "rustfsadmin";
const SECRET_KEY: &str = "rustfsadmin";
const BUCKET: &str = "api-test";
fn assert_s3_error_code<T, E>(result: Result<T, SdkError<E>>, expected: &str)
where
T: Debug,
E: ProvideErrorMetadata + Debug,
{
let error = result.expect_err("conditional request must fail");
assert_eq!(
error.as_service_error().and_then(ProvideErrorMetadata::code),
Some(expected),
"unexpected conditional request error: {error:?}"
async fn create_aws_s3_client() -> Result<Client, Box<dyn Error>> {
let region_provider = RegionProviderChain::default_provider().or_else(Region::new("us-east-1"));
let shared_config = aws_config::defaults(aws_config::BehaviorVersion::latest())
.region(region_provider)
.credentials_provider(Credentials::new(ACCESS_KEY, SECRET_KEY, None, None, "static"))
.endpoint_url(ENDPOINT)
.load()
.await;
let client = Client::from_conf(
aws_sdk_s3::Config::from(&shared_config)
.to_builder()
.force_path_style(true)
.build(),
);
Ok(client)
}
/// Setup test bucket, creating it if it doesn't exist
async fn setup_test_bucket(client: &Client) -> Result<(), Box<dyn Error>> {
match client.create_bucket().bucket(BUCKET).send().await {
Ok(_) => {}
Err(SdkError::ServiceError(e)) => {
let e = e.into_err();
let error_code = e.meta().code().unwrap_or("");
if !error_code.eq("BucketAlreadyExists") {
return Err(e.into());
}
}
Err(e) => {
return Err(e.into());
}
}
Ok(())
}
/// Generate test data of specified size
@@ -34,12 +60,7 @@ fn generate_test_data(size: usize) -> Vec<u8> {
}
/// Upload an object and return its ETag
async fn upload_object_with_metadata(
client: &Client,
bucket: &str,
key: &str,
data: &[u8],
) -> Result<String, Box<dyn Error + Send + Sync>> {
async fn upload_object_with_metadata(client: &Client, bucket: &str, key: &str, data: &[u8]) -> Result<String, Box<dyn Error>> {
let response = client
.put_object()
.bucket(bucket)
@@ -48,164 +69,188 @@ async fn upload_object_with_metadata(
.send()
.await?;
response
.e_tag()
.map(str::to_owned)
.ok_or_else(|| std::io::Error::other("put object response did not include an ETag").into())
let etag = response.e_tag().unwrap_or("").to_string();
Ok(etag)
}
async fn object_body(client: &Client, key: &str) -> Result<Bytes, Box<dyn Error + Send + Sync>> {
let response = client.get_object().bucket(TEST_BUCKET).key(key).send().await?;
Ok(response.body.collect().await?.into_bytes())
/// Cleanup test objects from bucket
async fn cleanup_objects(client: &Client, bucket: &str, keys: &[&str]) {
for key in keys {
let _ = client.delete_object().bucket(bucket).key(*key).send().await;
}
}
/// Generate unique test object key
fn generate_test_key(prefix: &str) -> String {
use std::time::{SystemTime, UNIX_EPOCH};
let timestamp = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos();
format!("{prefix}-{timestamp}")
}
#[tokio::test]
async fn test_conditional_put_okay() -> TestResult {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
env.create_test_bucket(TEST_BUCKET).await?;
let client = env.create_s3_client();
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_conditional_put_okay() -> Result<(), Box<dyn std::error::Error>> {
let client = create_aws_s3_client().await?;
setup_test_bucket(&client).await?;
let test_key = "conditional-put-ok";
let test_key = generate_test_key("conditional-put-ok");
let initial_data = generate_test_data(1024); // 1KB test data
let matching_data = generate_test_data(2048); // 2KB updated data
let non_matching_data = generate_test_data(3072); // 3KB updated data
let updated_data = generate_test_data(2048); // 2KB updated data
// Upload initial object and get its ETag
let initial_etag = upload_object_with_metadata(&client, TEST_BUCKET, test_key, &initial_data).await?;
let initial_etag = upload_object_with_metadata(&client, BUCKET, &test_key, &initial_data).await?;
// Test 1: PUT with matching If-Match condition (should succeed)
client
let response1 = client
.put_object()
.bucket(TEST_BUCKET)
.key(test_key)
.body(Bytes::from(matching_data.clone()).into())
.bucket(BUCKET)
.key(&test_key)
.body(Bytes::from(updated_data.clone()).into())
.if_match(&initial_etag)
.send()
.await?;
assert_eq!(object_body(&client, test_key).await?.as_ref(), matching_data);
.await;
assert!(response1.is_ok(), "PUT with matching If-Match should succeed");
// Test 2: PUT with non-matching If-None-Match condition (should succeed)
let fake_etag = "\"fake-etag-12345\"";
client
let response2 = client
.put_object()
.bucket(TEST_BUCKET)
.key(test_key)
.body(Bytes::from(non_matching_data.clone()).into())
.bucket(BUCKET)
.key(&test_key)
.body(Bytes::from(updated_data.clone()).into())
.if_none_match(fake_etag)
.send()
.await?;
assert_eq!(object_body(&client, test_key).await?.as_ref(), non_matching_data);
.await;
assert!(response2.is_ok(), "PUT with non-matching If-None-Match should succeed");
// Cleanup
cleanup_objects(&client, BUCKET, &[&test_key]).await;
Ok(())
}
#[tokio::test]
async fn test_conditional_put_failed() -> TestResult {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
env.create_test_bucket(TEST_BUCKET).await?;
let client = env.create_s3_client();
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_conditional_put_failed() -> Result<(), Box<dyn std::error::Error>> {
let client = create_aws_s3_client().await?;
setup_test_bucket(&client).await?;
let test_key = "conditional-put-failed";
let test_key = generate_test_key("conditional-put-failed");
let initial_data = generate_test_data(1024);
let updated_data = generate_test_data(2048);
// Upload initial object and get its ETag
let initial_etag = upload_object_with_metadata(&client, TEST_BUCKET, test_key, &initial_data).await?;
let initial_etag = upload_object_with_metadata(&client, BUCKET, &test_key, &initial_data).await?;
// Test 1: PUT with non-matching If-Match condition (should fail with 412)
let fake_etag = "\"fake-etag-should-not-match\"";
let response1 = client
.put_object()
.bucket(TEST_BUCKET)
.key(test_key)
.bucket(BUCKET)
.key(&test_key)
.body(Bytes::from(updated_data.clone()).into())
.if_match(fake_etag)
.send()
.await;
assert_s3_error_code(response1, "PreconditionFailed");
assert_eq!(object_body(&client, test_key).await?.as_ref(), initial_data);
assert!(response1.is_err(), "PUT with non-matching If-Match should fail");
if let Err(e) = response1 {
if let SdkError::ServiceError(e) = e {
let e = e.into_err();
let error_code = e.meta().code().unwrap_or("");
assert_eq!("PreconditionFailed", error_code);
} else {
panic!("Unexpected error: {e:?}");
}
}
// Test 2: PUT with matching If-None-Match condition (should fail with 412)
let response2 = client
.put_object()
.bucket(TEST_BUCKET)
.key(test_key)
.bucket(BUCKET)
.key(&test_key)
.body(Bytes::from(updated_data.clone()).into())
.if_none_match(&initial_etag)
.send()
.await;
assert_s3_error_code(response2, "PreconditionFailed");
assert_eq!(object_body(&client, test_key).await?.as_ref(), initial_data);
assert!(response2.is_err(), "PUT with matching If-None-Match should fail");
if let Err(e) = response2 {
if let SdkError::ServiceError(e) = e {
let e = e.into_err();
let error_code = e.meta().code().unwrap_or("");
assert_eq!("PreconditionFailed", error_code);
} else {
panic!("Unexpected error: {e:?}");
}
}
// Cleanup - only need to clean up the initial object since failed PUTs shouldn't create objects
cleanup_objects(&client, BUCKET, &[&test_key]).await;
Ok(())
}
#[tokio::test]
async fn test_conditional_put_when_object_does_not_exist() -> TestResult {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
env.create_test_bucket(TEST_BUCKET).await?;
let client = env.create_s3_client();
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_conditional_put_when_object_does_not_exist() -> Result<(), Box<dyn std::error::Error>> {
let client = create_aws_s3_client().await?;
setup_test_bucket(&client).await?;
let key = "conditional-put-missing";
let key = "some_key";
cleanup_objects(&client, BUCKET, &[key]).await;
// When the object does not exist, the If-Match condition should always fail
let response1 = client
.put_object()
.bucket(TEST_BUCKET)
.bucket(BUCKET)
.key(key)
.body(Bytes::from(generate_test_data(1024)).into())
.if_match("*")
.send()
.await;
assert_s3_error_code(response1, "NoSuchKey");
assert!(response1.is_err());
if let Err(e) = response1 {
if let SdkError::ServiceError(e) = e {
let e = e.into_err();
let error_code = e.meta().code().unwrap_or("");
assert_eq!("NoSuchKey", error_code);
} else {
panic!("Unexpected error: {e:?}");
}
}
// When the object does not exist, the If-None-Match condition should be able to succeed
let created_data = generate_test_data(1024);
client
let response2 = client
.put_object()
.bucket(TEST_BUCKET)
.bucket(BUCKET)
.key(key)
.body(Bytes::from(created_data.clone()).into())
.body(Bytes::from(generate_test_data(1024)).into())
.if_none_match("*")
.send()
.await?;
assert_eq!(object_body(&client, key).await?.as_ref(), created_data);
.await;
assert!(response2.is_ok());
cleanup_objects(&client, BUCKET, &[key]).await;
Ok(())
}
#[tokio::test]
async fn test_conditional_multi_part_upload() -> TestResult {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
env.create_test_bucket(TEST_BUCKET).await?;
let client = env.create_s3_client();
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_conditional_multi_part_upload() -> Result<(), Box<dyn std::error::Error>> {
let client = create_aws_s3_client().await?;
setup_test_bucket(&client).await?;
let test_key = "conditional-multipart-upload";
let test_key = generate_test_key("multipart-upload-ok");
let test_data = generate_test_data(1024);
let initial_etag = upload_object_with_metadata(&client, TEST_BUCKET, test_key, &test_data).await?;
let initial_etag = upload_object_with_metadata(&client, BUCKET, &test_key, &test_data).await?;
let part_size = 5 * 1024 * 1024; // 5MB per part (minimum for multipart)
let num_parts = 3;
let mut parts = Vec::new();
let mut expected_data = Vec::with_capacity(part_size * usize::try_from(num_parts)?);
// Initiate multipart upload
let initiate_response = client
.create_multipart_upload()
.bucket(TEST_BUCKET)
.key(test_key)
.send()
.await?;
let initiate_response = client.create_multipart_upload().bucket(BUCKET).key(&test_key).send().await?;
let upload_id = initiate_response
.upload_id()
@@ -213,13 +258,12 @@ async fn test_conditional_multi_part_upload() -> TestResult {
// Upload parts
for part_number in 1..=num_parts {
let part_data = vec![u8::try_from(part_number)?; part_size];
expected_data.extend_from_slice(&part_data);
let part_data = generate_test_data(part_size);
let upload_part_response = client
.upload_part()
.bucket(TEST_BUCKET)
.key(test_key)
.bucket(BUCKET)
.key(&test_key)
.upload_id(upload_id)
.part_number(part_number)
.body(Bytes::from(part_data).into())
@@ -242,62 +286,57 @@ async fn test_conditional_multi_part_upload() -> TestResult {
// Test 1: Multipart upload with wildcard If-None-Match, should fail
let complete_response = client
.complete_multipart_upload()
.bucket(TEST_BUCKET)
.key(test_key)
.bucket(BUCKET)
.key(&test_key)
.upload_id(upload_id)
.multipart_upload(completed_upload.clone())
.if_none_match("*")
.send()
.await;
assert_s3_error_code(complete_response, "PreconditionFailed");
assert!(complete_response.is_err());
// Test 2: Multipart upload with matching If-None-Match, should fail
let complete_response = client
.complete_multipart_upload()
.bucket(TEST_BUCKET)
.key(test_key)
.bucket(BUCKET)
.key(&test_key)
.upload_id(upload_id)
.multipart_upload(completed_upload.clone())
.if_none_match(initial_etag.clone())
.send()
.await;
assert_s3_error_code(complete_response, "PreconditionFailed");
assert!(complete_response.is_err());
// Test 3: Multipart upload with unmatching If-Match, should fail
let complete_response = client
.complete_multipart_upload()
.bucket(TEST_BUCKET)
.key(test_key)
.bucket(BUCKET)
.key(&test_key)
.upload_id(upload_id)
.multipart_upload(completed_upload.clone())
.if_match("\"abcdef\"")
.send()
.await;
assert_s3_error_code(complete_response, "PreconditionFailed");
let staged_parts = client
.list_parts()
.bucket(TEST_BUCKET)
.key(test_key)
.upload_id(upload_id)
.send()
.await?;
assert_eq!(staged_parts.parts().len(), usize::try_from(num_parts)?);
assert!(complete_response.is_err());
// Test 4: Multipart upload with matching If-Match, should succeed
client
let complete_response = client
.complete_multipart_upload()
.bucket(TEST_BUCKET)
.key(test_key)
.bucket(BUCKET)
.key(&test_key)
.upload_id(upload_id)
.multipart_upload(completed_upload)
.multipart_upload(completed_upload.clone())
.if_match(initial_etag)
.send()
.await?;
assert_eq!(object_body(&client, test_key).await?.as_ref(), expected_data);
.await;
assert!(complete_response.is_ok());
// Cleanup
cleanup_objects(&client, BUCKET, &[&test_key]).await;
Ok(())
}
+153 -174
View File
@@ -13,228 +13,207 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::common::RustFSTestEnvironment;
use crate::common::workspace_root;
use crate::storage_api::node_interact::{
TonicInterceptor, VolumeInfo, WalkDirOptions, gen_tonic_signature_interceptor, node_service_time_out_client,
};
use aws_sdk_s3::primitives::ByteStream;
use futures::future::join_all;
use rmp_serde::{Deserializer, Serializer};
use rustfs_filemeta::MetaCacheEntry;
use rustfs_filemeta::{MetaCacheEntry, MetacacheReader, MetacacheWriter};
use rustfs_protos::proto_gen::node_service::WalkDirRequest;
use rustfs_protos::{
models::{PingBody, PingBodyBuilder},
proto_gen::node_service::{ListVolumesRequest, LocalStorageInfoRequest, MakeVolumeRequest, PingRequest, ReadAllRequest},
proto_gen::node_service::{
ListVolumesRequest, LocalStorageInfoRequest, MakeVolumeRequest, PingRequest, PingResponse, ReadAllRequest,
},
};
use serde::{Deserialize, Serialize};
use std::error::Error;
use std::io::Cursor;
use std::path::PathBuf;
use tokio::spawn;
use tonic::Request;
use tonic::codegen::tokio_stream::StreamExt;
type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
const TEST_RPC_SECRET: &str = "rustfs-internode-signature-e2e-secret";
const CLUSTER_ADDR: &str = "http://localhost:9000";
fn signature_interceptor() -> TonicInterceptor {
TonicInterceptor::Signature(gen_tonic_signature_interceptor())
}
fn rpc_client_error(error: Box<dyn Error>) -> std::io::Error {
std::io::Error::other(error.to_string())
}
async fn start_server() -> Result<RustFSTestEnvironment, Box<dyn Error + Send + Sync>> {
let _ = rustfs_credentials::set_global_rpc_secret(TEST_RPC_SECRET.to_string());
let effective = rustfs_credentials::try_get_rpc_token().expect("RPC secret must resolve in the test process");
assert_eq!(effective, TEST_RPC_SECRET, "the test process uses an unexpected RPC secret");
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server_without_cleanup_with_env(&[
("RUSTFS_RPC_SECRET", TEST_RPC_SECRET),
("RUSTFS_INTERNODE_RPC_SIGNATURE_STRICT", "false"),
("RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT", "false"),
("RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT", "false"),
("RUST_LOG", "error"),
])
.await?;
Ok(env)
}
#[tokio::test]
async fn ping() -> TestResult {
let env = start_server().await?;
#[ignore = "requires running RustFS server at localhost:9000"]
async fn ping() -> Result<(), Box<dyn Error>> {
let mut fbb = flatbuffers::FlatBufferBuilder::new();
let payload = fbb.create_vector(b"hello world");
let mut builder = PingBodyBuilder::new(&mut fbb);
builder.add_payload(payload);
let root = builder.finish();
fbb.finish(root, None);
let mut client = node_service_time_out_client(&env.url, signature_interceptor())
.await
.map_err(rpc_client_error)?;
let response = client
.ping(Request::new(PingRequest {
version: 1,
body: bytes::Bytes::copy_from_slice(fbb.finished_data()),
}))
.await?
.into_inner();
let finished_data = fbb.finished_data();
let decoded_payload = flatbuffers::root::<PingBody>(finished_data);
assert!(decoded_payload.is_ok());
// Create client
let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
// Construct PingRequest
let request = Request::new(PingRequest {
version: 1,
body: bytes::Bytes::copy_from_slice(finished_data),
});
// Send request and get response
let response: PingResponse = client.ping(request).await?.into_inner();
// Print response
let ping_response_body = flatbuffers::root::<PingBody>(&response.body);
if let Err(e) = ping_response_body {
eprintln!("{e}");
} else {
println!("ping_resp:body(flatbuffer): {ping_response_body:?}");
}
assert_eq!(response.version, 1);
let body = flatbuffers::root::<PingBody>(&response.body)?;
assert_eq!(body.payload().expect("ping response must contain a payload").bytes(), b"hello, caller");
Ok(())
}
#[tokio::test]
async fn make_volume() -> TestResult {
let env = start_server().await?;
let mut client = node_service_time_out_client(&env.url, signature_interceptor())
.await
.map_err(rpc_client_error)?;
let response = client
.make_volume(Request::new(MakeVolumeRequest {
disk: env.temp_dir.clone(),
volume: "node-rpc-volume".to_string(),
}))
.await?
.into_inner();
#[ignore = "requires running RustFS server at localhost:9000"]
async fn make_volume() -> Result<(), Box<dyn Error>> {
let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
let request = Request::new(MakeVolumeRequest {
disk: "data".to_string(),
volume: "dandan".to_string(),
});
assert!(response.success, "make_volume failed: {:?}", response.error);
assert!(std::path::Path::new(&env.temp_dir).join("node-rpc-volume").is_dir());
let response = client.make_volume(request).await?.into_inner();
if response.success {
println!("success");
} else {
println!("failed: {:?}", response.error);
}
Ok(())
}
#[tokio::test]
async fn list_volumes() -> TestResult {
let env = start_server().await?;
let mut client = node_service_time_out_client(&env.url, signature_interceptor())
.await
.map_err(rpc_client_error)?;
let created = client
.make_volume(Request::new(MakeVolumeRequest {
disk: env.temp_dir.clone(),
volume: "node-rpc-listed-volume".to_string(),
}))
.await?
.into_inner();
assert!(created.success, "make_volume failed: {:?}", created.error);
#[ignore = "requires running RustFS server at localhost:9000"]
async fn list_volumes() -> Result<(), Box<dyn Error>> {
let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
let request = Request::new(ListVolumesRequest {
disk: "data".to_string(),
});
let response = client
.list_volumes(Request::new(ListVolumesRequest {
disk: env.temp_dir.clone(),
}))
.await?
.into_inner();
assert!(response.success, "list_volumes failed: {:?}", response.error);
let volumes = response
let response = client.list_volumes(request).await?.into_inner();
let volume_infos: Vec<VolumeInfo> = response
.volume_infos
.iter()
.map(|json| serde_json::from_str::<VolumeInfo>(json))
.collect::<Result<Vec<_>, _>>()?;
assert!(volumes.iter().any(|volume| volume.name == "node-rpc-listed-volume"));
.into_iter()
.filter_map(|json_str| serde_json::from_str::<VolumeInfo>(&json_str).ok())
.collect();
println!("{volume_infos:?}");
Ok(())
}
#[tokio::test]
async fn walk_dir() -> TestResult {
let env = start_server().await?;
let s3 = env.create_s3_client();
let bucket = "node-rpc-walk-bucket";
let key = "prefix/object.txt";
env.create_test_bucket(bucket).await?;
s3.put_object()
.bucket(bucket)
.key(key)
.body(ByteStream::from_static(b"walk payload"))
.send()
.await?;
#[ignore = "requires running RustFS server at localhost:9000"]
async fn walk_dir() -> Result<(), Box<dyn Error>> {
println!("walk_dir");
// TODO: use writer
let opts = WalkDirOptions {
bucket: bucket.to_string(),
bucket: "dandan".to_owned(),
base_dir: "".to_owned(),
recursive: true,
..Default::default()
};
let mut encoded = Vec::new();
opts.serialize(&mut Serializer::new(&mut encoded))?;
let mut client = node_service_time_out_client(&env.url, signature_interceptor())
.await
.map_err(rpc_client_error)?;
let mut stream = client
.walk_dir(Request::new(WalkDirRequest {
disk: env.temp_dir.clone(),
walk_dir_options: encoded.into(),
}))
.await?
.into_inner();
let (rd, mut wr) = tokio::io::duplex(1024);
let mut buf = Vec::new();
opts.serialize(&mut Serializer::new(&mut buf))?;
let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
let disk_path = std::env::var_os("RUSTFS_DISK_PATH").map(PathBuf::from).unwrap_or_else(|| {
let mut path = workspace_root();
path.push("target");
path.push(if cfg!(debug_assertions) { "debug" } else { "release" });
path.push("data");
path
});
let request = Request::new(WalkDirRequest {
disk: disk_path.to_string_lossy().into_owned(),
walk_dir_options: buf.into(),
});
let mut response = client.walk_dir(request).await?.into_inner();
let mut entries = Vec::new();
while let Some(response) = stream.next().await {
let response = response?;
assert!(response.success, "walk_dir failed: {:?}", response.error_info);
entries.push(serde_json::from_str::<MetaCacheEntry>(&response.meta_cache_entry)?);
let job1 = spawn(async move {
let mut out = MetacacheWriter::new(&mut wr);
loop {
match response.next().await {
Some(Ok(resp)) => {
if !resp.success {
println!("{}", resp.error_info.unwrap_or_else(|| "".to_string()));
}
let entry = serde_json::from_str::<MetaCacheEntry>(&resp.meta_cache_entry)
.map_err(|_e| std::io::Error::other(format!("Unexpected response: {response:?}")))
.unwrap();
out.write_obj(&entry).await.unwrap();
}
None => {
let _ = out.close().await;
break;
}
_ => {
println!("Unexpected response: {response:?}");
let _ = out.close().await;
break;
}
}
}
});
let job2 = spawn(async move {
let mut reader = MetacacheReader::new(rd);
while let Ok(Some(entry)) = reader.peek().await {
println!("{entry:?}");
}
});
join_all(vec![job1, job2]).await;
Ok(())
}
#[tokio::test]
#[ignore = "requires running RustFS server at localhost:9000"]
async fn read_all() -> Result<(), Box<dyn Error>> {
let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
let request = Request::new(ReadAllRequest {
disk: "data".to_string(),
volume: "ff".to_string(),
path: "format.json".to_string(),
});
let response = client.read_all(request).await?.into_inner();
let volume_infos = response.data;
println!("{}", response.success);
println!("{volume_infos:?}");
Ok(())
}
#[tokio::test]
#[ignore = "requires running RustFS server at localhost:9000"]
async fn storage_info() -> Result<(), Box<dyn Error>> {
let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
let request = Request::new(LocalStorageInfoRequest { metrics: true });
let response = client.local_storage_info(request).await?.into_inner();
if !response.success {
println!("{:?}", response.error_info);
return Ok(());
}
assert!(
entries.iter().any(|entry| entry.name == key),
"walk_dir did not return {key}: {entries:?}"
);
Ok(())
}
#[tokio::test]
async fn read_all() -> TestResult {
let env = start_server().await?;
let mut client = node_service_time_out_client(&env.url, signature_interceptor())
.await
.map_err(rpc_client_error)?;
let volume = "node-rpc-read-volume";
let created = client
.make_volume(Request::new(MakeVolumeRequest {
disk: env.temp_dir.clone(),
volume: volume.to_string(),
}))
.await?
.into_inner();
assert!(created.success, "make_volume failed: {:?}", created.error);
tokio::fs::write(std::path::Path::new(&env.temp_dir).join(volume).join("payload.bin"), b"read payload").await?;
let response = client
.read_all(Request::new(ReadAllRequest {
disk: env.temp_dir.clone(),
volume: volume.to_string(),
path: "payload.bin".to_string(),
}))
.await?
.into_inner();
assert!(response.success, "read_all failed: {:?}", response.error);
assert_eq!(response.data.as_ref(), b"read payload");
Ok(())
}
#[tokio::test]
async fn storage_info() -> TestResult {
let env = start_server().await?;
let mut client = node_service_time_out_client(&env.url, signature_interceptor())
.await
.map_err(rpc_client_error)?;
let response = client
.local_storage_info(Request::new(LocalStorageInfoRequest { metrics: true }))
.await?
.into_inner();
assert!(response.success, "local_storage_info failed: {:?}", response.error_info);
let mut decoder = Deserializer::new(Cursor::new(response.storage_info));
let storage_info: rustfs_madmin::StorageInfo = Deserialize::deserialize(&mut decoder)?;
let expected_disk = std::fs::canonicalize(&env.temp_dir)?;
assert!(!storage_info.disks.is_empty(), "local_storage_info returned no disks");
assert!(
storage_info
.disks
.iter()
.any(|disk| std::path::Path::new(&disk.drive_path) == expected_disk),
"local_storage_info did not include the configured disk: {:?}",
storage_info.disks
);
let info = response.storage_info;
let mut buf = Deserializer::new(Cursor::new(info));
let storage_info: rustfs_madmin::StorageInfo = Deserialize::deserialize(&mut buf).unwrap();
println!("{storage_info:?}");
Ok(())
}
+82 -67
View File
@@ -13,37 +13,55 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::common::{RustFSTestEnvironment, init_logging};
use aws_config::meta::region::RegionProviderChain;
use aws_sdk_s3::Client;
use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::config::{Credentials, Region};
use aws_sdk_s3::types::{
CsvInput, CsvOutput, ExpressionType, FileHeaderInfo, InputSerialization, JsonInput, JsonOutput, JsonType, OutputSerialization,
};
use bytes::Bytes;
use std::error::Error;
use std::time::Duration;
const ENDPOINT: &str = "http://localhost:9000";
const ACCESS_KEY: &str = "rustfsadmin";
const SECRET_KEY: &str = "rustfsadmin";
const BUCKET: &str = "test-sql-bucket";
const CSV_OBJECT: &str = "test-data.csv";
const JSON_OBJECT: &str = "test-data.json";
const SELECT_RESPONSE_TIMEOUT: Duration = Duration::from_secs(30);
type TestResult<T> = Result<T, Box<dyn Error + Send + Sync>>;
async fn create_aws_s3_client() -> Result<Client, Box<dyn Error>> {
let region_provider = RegionProviderChain::default_provider().or_else(Region::new("us-east-1"));
let shared_config = aws_config::defaults(aws_config::BehaviorVersion::latest())
.region(region_provider)
.credentials_provider(Credentials::new(ACCESS_KEY, SECRET_KEY, None, None, "static"))
.endpoint_url(ENDPOINT)
.load()
.await;
async fn create_test_environment() -> TestResult<(RustFSTestEnvironment, Client)> {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
let client = env.create_s3_client();
Ok((env, client))
let client = Client::from_conf(
aws_sdk_s3::Config::from(&shared_config)
.to_builder()
.force_path_style(true) // Important for S3-compatible services
.build(),
);
Ok(client)
}
async fn setup_test_bucket(client: &Client) -> TestResult<()> {
client.create_bucket().bucket(BUCKET).send().await?;
async fn setup_test_bucket(client: &Client) -> Result<(), Box<dyn Error>> {
match client.create_bucket().bucket(BUCKET).send().await {
Ok(_) => {}
Err(e) => {
let error_str = e.to_string();
if !error_str.contains("BucketAlreadyOwnedByYou") && !error_str.contains("BucketAlreadyExists") {
return Err(e.into());
}
}
}
Ok(())
}
async fn upload_test_csv(client: &Client) -> TestResult<()> {
async fn upload_test_csv(client: &Client) -> Result<(), Box<dyn Error>> {
let csv_data = "name,age,city\nAlice,30,New York\nBob,25,Los Angeles\nCharlie,35,Chicago\nDiana,28,Boston";
client
@@ -57,7 +75,7 @@ async fn upload_test_csv(client: &Client) -> TestResult<()> {
Ok(())
}
async fn upload_test_json(client: &Client) -> TestResult<()> {
async fn upload_test_json(client: &Client) -> Result<(), Box<dyn Error>> {
let json_data = r#"{"name":"Alice","age":30,"city":"New York"}
{"name":"Bob","age":25,"city":"Los Angeles"}
{"name":"Charlie","age":35,"city":"Chicago"}
@@ -75,38 +93,33 @@ async fn upload_test_json(client: &Client) -> TestResult<()> {
async fn process_select_response(
mut event_stream: aws_sdk_s3::operation::select_object_content::SelectObjectContentOutput,
) -> TestResult<String> {
tokio::time::timeout(SELECT_RESPONSE_TIMEOUT, async move {
let mut total_data = Vec::new();
let mut saw_end = false;
) -> Result<String, Box<dyn Error>> {
let mut total_data = Vec::new();
while let Some(event) = event_stream.payload.recv().await? {
match event {
aws_sdk_s3::types::SelectObjectContentEventStream::Records(records_event) => {
if let Some(payload) = records_event.payload {
total_data.extend_from_slice(payload.as_ref());
}
while let Ok(Some(event)) = event_stream.payload.recv().await {
match event {
aws_sdk_s3::types::SelectObjectContentEventStream::Records(records_event) => {
if let Some(payload) = records_event.payload {
let data = payload.into_inner();
total_data.extend_from_slice(&data);
}
aws_sdk_s3::types::SelectObjectContentEventStream::End(_) => {
saw_end = true;
break;
}
_ => {}
}
aws_sdk_s3::types::SelectObjectContentEventStream::End(_) => {
break;
}
_ => {
// Handle other event types (Stats, Progress, Cont, etc.)
}
}
}
if !saw_end {
return Err("Select response ended without an End event".into());
}
Ok(String::from_utf8(total_data)?)
})
.await
.map_err(|_| -> Box<dyn Error + Send + Sync> { "Select response timed out".into() })?
Ok(String::from_utf8(total_data)?)
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn test_select_object_content_csv_basic() -> TestResult<()> {
let (_env, client) = create_test_environment().await?;
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_select_object_content_csv_basic() -> Result<(), Box<dyn Error>> {
let client = create_aws_s3_client().await?;
setup_test_bucket(&client).await?;
upload_test_csv(&client).await?;
@@ -145,8 +158,9 @@ async fn test_select_object_content_csv_basic() -> TestResult<()> {
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn test_select_object_content_csv_aggregation() -> TestResult<()> {
let (_env, client) = create_test_environment().await?;
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_select_object_content_csv_aggregation() -> Result<(), Box<dyn Error>> {
let client = create_aws_s3_client().await?;
setup_test_bucket(&client).await?;
upload_test_csv(&client).await?;
@@ -189,15 +203,16 @@ async fn test_select_object_content_csv_aggregation() -> TestResult<()> {
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn test_select_object_content_json_basic() -> TestResult<()> {
let (_env, client) = create_test_environment().await?;
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_select_object_content_json_basic() -> Result<(), Box<dyn Error>> {
let client = create_aws_s3_client().await?;
setup_test_bucket(&client).await?;
upload_test_json(&client).await?;
// Construct JSON query
let sql = "SELECT s.name, s.age FROM S3Object s WHERE s.age > 28";
let json_input = JsonInput::builder().set_type(Some(JsonType::Lines)).build();
let json_input = JsonInput::builder().set_type(Some(JsonType::Document)).build();
let input_serialization = InputSerialization::builder().json(json_input).build();
@@ -229,8 +244,9 @@ async fn test_select_object_content_json_basic() -> TestResult<()> {
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn test_select_object_content_csv_limit() -> TestResult<()> {
let (_env, client) = create_test_environment().await?;
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_select_object_content_csv_limit() -> Result<(), Box<dyn Error>> {
let client = create_aws_s3_client().await?;
setup_test_bucket(&client).await?;
upload_test_csv(&client).await?;
@@ -270,8 +286,9 @@ async fn test_select_object_content_csv_limit() -> TestResult<()> {
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn test_select_object_content_csv_order_by() -> TestResult<()> {
let (_env, client) = create_test_environment().await?;
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_select_object_content_csv_order_by() -> Result<(), Box<dyn Error>> {
let client = create_aws_s3_client().await?;
setup_test_bucket(&client).await?;
upload_test_csv(&client).await?;
@@ -301,10 +318,9 @@ async fn test_select_object_content_csv_order_by() -> TestResult<()> {
println!("CSV Order By result: {result_str}");
// Verify ordered by age descending
assert_eq!(
result_str.lines().filter(|line| !line.trim().is_empty()).count(),
2,
"Should return exactly 2 records"
assert!(
result_str.lines().filter(|line| !line.trim().is_empty()).count() >= 2,
"Should return at least 2 records"
);
// Check if contains highest age records
@@ -315,8 +331,9 @@ async fn test_select_object_content_csv_order_by() -> TestResult<()> {
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn test_select_object_content_error_handling() -> TestResult<()> {
let (_env, client) = create_test_environment().await?;
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_select_object_content_error_handling() -> Result<(), Box<dyn Error>> {
let client = create_aws_s3_client().await?;
setup_test_bucket(&client).await?;
upload_test_csv(&client).await?;
@@ -331,7 +348,7 @@ async fn test_select_object_content_error_handling() -> TestResult<()> {
let output_serialization = OutputSerialization::builder().csv(csv_output).build();
// This query should fail because invalid_column doesn't exist
let error = client
let result = client
.select_object_content()
.bucket(BUCKET)
.key(CSV_OBJECT)
@@ -340,20 +357,18 @@ async fn test_select_object_content_error_handling() -> TestResult<()> {
.input_serialization(input_serialization)
.output_serialization(output_serialization)
.send()
.await
.expect_err("a query referencing an unknown column must fail");
.await;
assert_eq!(
error.as_service_error().and_then(ProvideErrorMetadata::code),
Some("EvaluatorBindingDoesNotExist")
);
// Verify query fails (expected behavior)
assert!(result.is_err(), "Query with invalid column should fail");
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn test_select_object_content_nonexistent_object() -> TestResult<()> {
let (_env, client) = create_test_environment().await?;
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_select_object_content_nonexistent_object() -> Result<(), Box<dyn Error>> {
let client = create_aws_s3_client().await?;
setup_test_bucket(&client).await?;
// Test query on nonexistent object
@@ -366,7 +381,7 @@ async fn test_select_object_content_nonexistent_object() -> TestResult<()> {
let csv_output = CsvOutput::builder().build();
let output_serialization = OutputSerialization::builder().csv(csv_output).build();
let error = client
let result = client
.select_object_content()
.bucket(BUCKET)
.key("nonexistent.csv")
@@ -375,10 +390,10 @@ async fn test_select_object_content_nonexistent_object() -> TestResult<()> {
.input_serialization(input_serialization)
.output_serialization(output_serialization)
.send()
.await
.expect_err("selecting a missing object must fail");
.await;
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("NoSuchKey"));
// Verify query fails (expected behavior)
assert!(result.is_err(), "Query on nonexistent object should fail");
Ok(())
}
+10 -422
View File
@@ -13,8 +13,9 @@
// limitations under the License.
use crate::common::{
RustFSTestEnvironment, admin_create_user, awscurl_post_sts_form_urlencoded, init_logging, local_http_client,
replication_fast_env, rustfs_binary_path, signed_request, signed_request_with_client, signed_request_with_session_token,
RustFSTestEnvironment, admin_create_user, awscurl_available, awscurl_post_sts_form_urlencoded, init_logging,
local_http_client, replication_fast_env, rustfs_binary_path, signed_request, signed_request_with_client,
signed_request_with_session_token,
};
use crate::fake_s3_target::{
FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, FaultAction as FakeTargetFault, Operation as FakeTargetOperation,
@@ -56,7 +57,7 @@ use rustfs_madmin::{
AddServiceAccountReq, ListServiceAccountsResp, PeerInfo, PeerSite, ReplicateAddStatus, ReplicateEditStatus,
ReplicateRemoveStatus, SRRemoveReq, SRResyncOpStatus, SRStatusInfo, SiteReplicationInfo, SyncStatus,
};
use s3s::header::{X_AMZ_REPLICATION_STATUS, X_AMZ_TAGGING};
use s3s::header::X_AMZ_REPLICATION_STATUS;
use sha2::{Digest, Sha256};
use std::collections::BTreeMap;
use std::convert::Infallible;
@@ -2022,7 +2023,6 @@ async fn forward_replication_proxy_request(
client: &reqwest::Client,
request_count: &AtomicU64,
mut replication_enabled: watch::Receiver<bool>,
mut held_tagging: watch::Receiver<Option<String>>,
) -> Response<Full<bytes::Bytes>> {
let (parts, body) = request.into_parts();
let is_replication = parts
@@ -2036,17 +2036,6 @@ async fn forward_replication_proxy_request(
return proxy_error_response("replication gate closed");
}
}
// Content-keyed hold: park only the replication request whose
// `x-amz-tagging` matches the held value, letting every other delivery
// through, so a test can make one specific (stale) delivery the last
// write the backend sees.
if let Some(tagging) = parts.headers.get(X_AMZ_TAGGING).and_then(|value| value.to_str().ok()) {
while held_tagging.borrow().as_deref() == Some(tagging) {
if held_tagging.changed().await.is_err() {
return proxy_error_response("replication tag hold closed");
}
}
}
}
let Some(path_and_query) = parts.uri.path_and_query() else {
@@ -2081,26 +2070,12 @@ async fn start_replication_counting_proxy(
backend_url: &str,
tasks: &mut JoinSet<()>,
) -> Result<(String, Arc<AtomicU64>, watch::Sender<bool>), Box<dyn Error + Send + Sync>> {
let (proxy_url, request_count, replication_enabled, _held_tagging) =
start_replication_counting_proxy_with_tag_hold(backend_url, tasks).await?;
Ok((proxy_url, request_count, replication_enabled))
}
/// [`start_replication_counting_proxy`] plus a content-keyed hold: while the
/// returned `watch::Sender<Option<String>>` holds `Some(tagging)`, replication
/// requests whose `x-amz-tagging` equals `tagging` are parked (and still
/// counted); all other traffic flows. Send `None` to release them.
async fn start_replication_counting_proxy_with_tag_hold(
backend_url: &str,
tasks: &mut JoinSet<()>,
) -> Result<(String, Arc<AtomicU64>, watch::Sender<bool>, watch::Sender<Option<String>>), Box<dyn Error + Send + Sync>> {
let listener = TcpListener::bind("127.0.0.1:0").await?;
let proxy_url = format!("http://{}", listener.local_addr()?);
let backend_url = backend_url.to_string();
let request_count = Arc::new(AtomicU64::new(0));
let task_request_count = request_count.clone();
let (replication_enabled, task_replication_enabled) = watch::channel(true);
let (held_tagging, task_held_tagging) = watch::channel(None);
tasks.spawn(async move {
let client = local_http_client();
let mut connections = JoinSet::new();
@@ -2112,14 +2087,12 @@ async fn start_replication_counting_proxy_with_tag_hold(
let client = client.clone();
let request_count = task_request_count.clone();
let replication_enabled = task_replication_enabled.clone();
let held_tagging = task_held_tagging.clone();
connections.spawn(async move {
let service = service_fn(move |request| {
let backend_url = backend_url.clone();
let client = client.clone();
let request_count = request_count.clone();
let replication_enabled = replication_enabled.clone();
let held_tagging = held_tagging.clone();
async move {
Ok::<_, Infallible>(
forward_replication_proxy_request(
@@ -2128,7 +2101,6 @@ async fn start_replication_counting_proxy_with_tag_hold(
&client,
&request_count,
replication_enabled,
held_tagging,
)
.await,
)
@@ -2141,7 +2113,7 @@ async fn start_replication_counting_proxy_with_tag_hold(
}
}
});
Ok((proxy_url, request_count, replication_enabled, held_tagging))
Ok((proxy_url, request_count, replication_enabled))
}
async fn site_replication_remove(
@@ -6977,395 +6949,6 @@ async fn test_site_replication_active_active_converges_without_loops_real_dual_n
}
}
/// Replication status a site reports for one object version via HEAD
/// (`x-amz-replication-status`), or `None` when the header is absent.
async fn head_replication_status(
client: &Client,
bucket: &str,
key: &str,
version_id: &str,
) -> Result<Option<String>, Box<dyn Error + Send + Sync>> {
let head = client
.head_object()
.bucket(bucket)
.key(key)
.version_id(version_id)
.send()
.await?;
Ok(head.replication_status().map(|status| status.as_str().to_string()))
}
/// Poll one site until the version's replication status is one of `expected`.
async fn wait_for_version_replication_status(
client: &Client,
bucket: &str,
key: &str,
version_id: &str,
expected: &[&str],
site: &str,
) -> Result<String, Box<dyn Error + Send + Sync>> {
let deadline = tokio::time::Instant::now() + Duration::from_secs(60);
loop {
let last = head_replication_status(client, bucket, key, version_id).await?;
if let Some(status) = last.as_deref()
&& expected.contains(&status)
{
return Ok(status.to_string());
}
if tokio::time::Instant::now() >= deadline {
return Err(format!(
"{site}: {bucket}/{key}?versionId={version_id} replication status {last:?} never reached {expected:?}"
)
.into());
}
sleep(Duration::from_millis(200)).await;
}
}
async fn put_single_tag(
client: &Client,
bucket: &str,
key: &str,
version_id: &str,
tag_key: &str,
tag_value: &str,
) -> Result<(), Box<dyn Error + Send + Sync>> {
client
.put_object_tagging()
.bucket(bucket)
.key(key)
.version_id(version_id)
.tagging(
aws_sdk_s3::types::Tagging::builder()
.tag_set(aws_sdk_s3::types::Tag::builder().key(tag_key).value(tag_value).build()?)
.build()?,
)
.send()
.await?;
Ok(())
}
async fn get_single_tag(
client: &Client,
bucket: &str,
key: &str,
version_id: &str,
tag_key: &str,
) -> Result<Option<String>, Box<dyn Error + Send + Sync>> {
let tagging = client
.get_object_tagging()
.bucket(bucket)
.key(key)
.version_id(version_id)
.send()
.await?;
Ok(tagging
.tag_set()
.iter()
.find(|tag| tag.key() == tag_key)
.map(|tag| tag.value().to_string()))
}
/// Poll one site until the version's `tag_key` equals `expected`.
async fn wait_for_single_tag(
client: &Client,
bucket: &str,
key: &str,
version_id: &str,
tag_key: &str,
expected: &str,
site: &str,
) -> Result<(), Box<dyn Error + Send + Sync>> {
let deadline = tokio::time::Instant::now() + Duration::from_secs(60);
loop {
let observed = get_single_tag(client, bucket, key, version_id, tag_key).await?;
if observed.as_deref() == Some(expected) {
return Ok(());
}
if tokio::time::Instant::now() >= deadline {
return Err(format!(
"{site}: {bucket}/{key}?versionId={version_id} tag {tag_key}={observed:?} never became {expected}"
)
.into());
}
sleep(Duration::from_millis(200)).await;
}
}
/// Tag key the dual-node LWW scenario edits on both sites.
const LWW_TAG_KEY: &str = "owner";
/// Assert the version's [`LWW_TAG_KEY`] stays `expected` on both sites for a
/// full quiet window (no late stale delivery flips it back).
async fn assert_tag_stable_on_both_sites(
site_a_client: &Client,
site_b_client: &Client,
bucket: &str,
key: &str,
version_id: &str,
expected: &str,
quiet: Duration,
) -> Result<(), Box<dyn Error + Send + Sync>> {
let deadline = tokio::time::Instant::now() + quiet;
loop {
let on_a = get_single_tag(site_a_client, bucket, key, version_id, LWW_TAG_KEY).await?;
let on_b = get_single_tag(site_b_client, bucket, key, version_id, LWW_TAG_KEY).await?;
assert_eq!(on_a.as_deref(), Some(expected), "site A tag {LWW_TAG_KEY} regressed from the LWW winner");
assert_eq!(on_b.as_deref(), Some(expected), "site B tag {LWW_TAG_KEY} regressed from the LWW winner");
if tokio::time::Instant::now() >= deadline {
return Ok(());
}
sleep(Duration::from_millis(250)).await;
}
}
/// Wait until the counting proxy in front of a site has admitted `expected`
/// replication requests in total (requests held by a closed gate still count).
async fn wait_for_proxy_replication_requests(
counter: &AtomicU64,
expected: u64,
site: &str,
) -> Result<(), Box<dyn Error + Send + Sync>> {
let deadline = tokio::time::Instant::now() + Duration::from_secs(60);
loop {
let observed = counter.load(Ordering::Relaxed);
if observed >= expected {
return Ok(());
}
if tokio::time::Instant::now() >= deadline {
return Err(format!("{site} proxy saw {observed} replication requests, expected at least {expected}").into());
}
sleep(Duration::from_millis(25)).await;
}
}
/// rustfs/backlog#1953 (audit A4/P1-6): receiver-side LWW for replicated
/// metadata categories, exercised end to end over the real dual-node
/// active-active site-replication control plane — sender, worker, status
/// bookkeeping and persisted failure recovery all participate (the single-server
/// `replication_lww_receiver_test` only injects authorized replication PUTs).
///
/// Scenario on one versioned object:
/// 1. reciprocal tag edits in real order (A then B) converge both sites on the
/// newer tag and leave the author COMPLETED / the receiver REPLICA;
/// 2. out-of-order delivery: A's edit is held at B's inbound proxy while B
/// authors a newer edit that reaches A first; releasing the stale delivery
/// must NOT roll B back — both sites settle on B's value and stay there
/// through a quiet window, with no FAILED/PENDING status left behind;
/// 3. persisted retry: B is stopped, A's delivery reaches FAILED, A restarts,
/// then B returns and the scanner-replayed edit converges both sites forward.
/// Durable metadata-MRF serialization/reconstruction is covered separately by
/// `metadata_mrf_roundtrip_preserves_tags_and_admitted_targets`.
#[tokio::test]
async fn test_site_replication_tagging_lww_converges_active_active_real_dual_node() -> TestResult {
init_logging();
match tokio::time::timeout(Duration::from_secs(420), async {
// The scanner is fast for the final persisted-failure recovery phase.
// Step 2 finishes and proves a quiet stable winner before that phase,
// so a later scanner pass cannot mask its stale-delivery assertion.
let mut site_env = replication_fast_env();
site_env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
site_env.extend_from_slice(FAST_SCANNER_ENV);
let mut site_a_env = RustFSTestEnvironment::new().await?;
site_a_env.start_rustfs_server_with_env(vec![], &site_env).await?;
let mut site_b_env = RustFSTestEnvironment::new().await?;
site_b_env.start_rustfs_server_with_env(vec![], &site_env).await?;
let mut proxy_tasks = JoinSet::new();
let (site_a_proxy, site_a_replication_requests, _site_a_replication_enabled, site_a_held_tagging) =
start_replication_counting_proxy_with_tag_hold(&site_a_env.url, &mut proxy_tasks).await?;
let (site_b_proxy, site_b_replication_requests, _site_b_replication_enabled, site_b_held_tagging) =
start_replication_counting_proxy_with_tag_hold(&site_b_env.url, &mut proxy_tasks).await?;
let site_a_client = site_a_env.create_s3_client();
let site_b_client = site_b_env.create_s3_client();
let bucket = "site-repl-tag-lww";
let key = "lww.txt";
let add_status = site_replication_add(
&site_a_env,
&[
PeerSite {
name: "lww-site-a".to_string(),
endpoint: site_a_env.url.clone(),
access_key: site_a_env.access_key.clone(),
secret_key: site_a_env.secret_key.clone(),
..Default::default()
},
PeerSite {
name: "lww-site-b".to_string(),
endpoint: site_b_env.url.clone(),
access_key: site_b_env.access_key.clone(),
secret_key: site_b_env.secret_key.clone(),
..Default::default()
},
],
)
.await?;
assert!(add_status.success, "unexpected site add result: {add_status:?}");
let site_info = wait_for_site_replication_enabled(&site_a_env, 2).await?;
wait_for_site_replication_enabled(&site_b_env, 2).await?;
// Route both directions through the counting proxies so inbound
// replication to B can be held (out-of-order delivery) and observed.
for (env_url, proxy_url, label) in [(&site_a_env.url, &site_a_proxy, "A"), (&site_b_env.url, &site_b_proxy, "B")] {
let mut peer = site_info
.sites
.iter()
.find(|peer| peer.endpoint == *env_url)
.ok_or_else(|| format!("site {label} peer missing from replication info"))?
.clone();
peer.endpoint = proxy_url.clone();
peer.sync_state = SyncStatus::Enable;
let edit = site_replication_edit(&site_a_env, "", &peer).await?;
assert!(edit.success, "unexpected site {label} endpoint edit: {edit:?}");
}
for env in [&site_a_env, &site_b_env] {
wait_for_site_replication_info(env, |info| {
info.sites.iter().any(|peer| peer.endpoint == site_a_proxy)
&& info.sites.iter().any(|peer| peer.endpoint == site_b_proxy)
})
.await?;
}
site_a_client.create_bucket().bucket(bucket).send().await?;
wait_for_bucket_on_target(&site_b_client, bucket).await?;
let version_id = site_a_client
.put_object()
.bucket(bucket)
.key(key)
.body(ByteStream::from_static(b"tag lww payload"))
.send()
.await?
.version_id()
.ok_or("site A PUT omitted version ID")?
.to_string();
wait_for_replicated_object(&site_b_client, bucket, key, "tag lww payload").await?;
wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["COMPLETED"], "site A").await?;
wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["REPLICA"], "site B").await?;
wait_for_proxy_replication_requests(&site_b_replication_requests, 1, "site B").await?;
// --- 1. reciprocal edits in real order: A then B ----------------------
put_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY, "a1").await?;
wait_for_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY, "a1", "site B").await?;
wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["COMPLETED"], "site A").await?;
wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["REPLICA"], "site B").await?;
wait_for_proxy_replication_requests(&site_b_replication_requests, 2, "site B").await?;
put_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY, "b1").await?;
wait_for_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY, "b1", "site A").await?;
wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["COMPLETED"], "site B").await?;
wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["REPLICA"], "site A").await?;
assert_tag_stable_on_both_sites(&site_a_client, &site_b_client, bucket, key, &version_id, "b1", Duration::from_secs(3))
.await?;
// --- 2. concurrent edits, stale delivery last ------------------------
// Both sites edit the same version while each other's delivery is
// parked at the peer's inbound proxy (content-keyed: only the
// `owner=a2` / `owner=b2` replication PUTs wait, everything else
// flows). B's edit is the newer one. Releasing A's stale `a2` first
// makes it the last write B sees while A itself still holds `a2`, so
// nothing A could re-deliver carries the winner: only receiver-side
// LWW on B can keep `b2`. Releasing `b2` afterwards converges A.
site_b_held_tagging.send(Some("owner=a2".to_string()))?;
site_a_held_tagging.send(Some("owner=b2".to_string()))?;
let a2_parked_at = site_b_replication_requests.load(Ordering::Relaxed) + 1;
let b2_parked_at = site_a_replication_requests.load(Ordering::Relaxed) + 1;
put_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY, "a2").await?;
wait_for_proxy_replication_requests(&site_b_replication_requests, a2_parked_at, "site B").await?;
sleep(Duration::from_millis(50)).await;
put_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY, "b2").await?;
wait_for_proxy_replication_requests(&site_a_replication_requests, b2_parked_at, "site A").await?;
assert_eq!(
get_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY)
.await?
.as_deref(),
Some("a2")
);
assert_eq!(
get_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY)
.await?
.as_deref(),
Some("b2")
);
// Release the stale a2 delivery onto B: the newer local b2 must
// survive, and the delivery itself must still succeed (A reaches
// COMPLETED instead of looping through MRF with the stale value).
site_b_held_tagging.send(None)?;
wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["COMPLETED"], "site A").await?;
let stale_deadline = tokio::time::Instant::now() + Duration::from_secs(3);
loop {
assert_eq!(
get_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY)
.await?
.as_deref(),
Some("b2"),
"a stale inbound delivery rolled back site B's newer tag (receiver-side LWW regression)"
);
if tokio::time::Instant::now() >= stale_deadline {
break;
}
sleep(Duration::from_millis(250)).await;
}
// Release b2 onto A: the newer edit wins there and both sites settle.
// B's own version may legitimately read REPLICA here: the stale inbound
// a2 write re-labelled it as a replica write (keeping B's tags); what
// must not remain is PENDING/FAILED.
site_a_held_tagging.send(None)?;
wait_for_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY, "b2", "site A").await?;
wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["COMPLETED", "REPLICA"], "site B")
.await?;
assert_tag_stable_on_both_sites(&site_a_client, &site_b_client, bucket, key, &version_id, "b2", Duration::from_secs(4))
.await?;
for (client, site) in [(&site_a_client, "site A"), (&site_b_client, "site B")] {
let status = head_replication_status(client, bucket, key, &version_id).await?;
assert!(
matches!(status.as_deref(), Some("COMPLETED" | "REPLICA")),
"{site} must not be left PENDING/FAILED after the concurrent edits: {status:?}"
);
}
// --- 3. persisted FAILED state survives a source restart ------------
site_b_env.stop_server();
put_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY, "a3").await?;
wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["FAILED"], "site A").await?;
site_a_env.restart_server_preserving_data(vec![], &site_env).await?;
wait_for_site_replication_enabled(&site_a_env, 2).await?;
site_b_env.restart_server_preserving_data(vec![], &site_env).await?;
wait_for_site_replication_enabled(&site_b_env, 2).await?;
wait_for_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY, "a3", "site B").await?;
wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["COMPLETED"], "site A").await?;
wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["REPLICA"], "site B").await?;
assert_tag_stable_on_both_sites(&site_a_client, &site_b_client, bucket, key, &version_id, "a3", Duration::from_secs(3))
.await?;
// The object itself never forked: one version on each side.
tokio::time::timeout(
Duration::from_secs(70),
assert_replication_converged(&site_a_client, bucket, &site_b_client, bucket),
)
.await??;
let state = list_replication_state(&site_a_client, bucket).await?;
assert_eq!(state.len(), 1, "tag edits must not create new object versions: {state:?}");
assert_eq!(state[0].version_id, version_id);
proxy_tasks.abort_all();
Ok(())
})
.await
{
Ok(result) => result,
Err(_) => Err("site replication tagging LWW test timed out".into()),
}
}
#[tokio::test]
async fn test_site_replication_replicates_policy_backed_user_access_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {
init_logging();
@@ -7698,6 +7281,11 @@ async fn test_site_replication_replicates_multiple_service_accounts_real_dual_no
async fn test_site_replication_replicates_service_accounts_created_from_sts_session_real_dual_node() -> TestResult {
init_logging();
if !awscurl_available() {
eprintln!("Skipping STS site replication service-account test because awscurl is unavailable");
return Ok(());
}
let mut source_env = RustFSTestEnvironment::new().await?;
source_env
.start_rustfs_server_with_env(vec![], LOOPBACK_REPLICATION_TARGET_ENV)
@@ -1,155 +0,0 @@
#![cfg(test)]
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Receiver-side replication LWW over the wire (rustfs/backlog#1953, audit
//! A4/P1-6).
//!
//! In an active-active topology both sites' metadata states arrive at the
//! peer as authorized replication PUTs carrying per-category source
//! timestamps (`x-rustfs-source-replication-tagging-timestamp` header
//! family). Before the fix the receiver applied them unconditionally, so a
//! stale delivery overwrote a newer local state and the two sites diverged
//! permanently while both reported COMPLETED. This test drives one live
//! `rustfs` server with simulated inbound replication PUTs for the same
//! object version and asserts the newer tagging state wins regardless of
//! delivery order, while a stale delivery still succeeds at the object level
//! (a failure would loop through MRF re-delivering the stale value).
//!
//! The real dual-site path (sender, worker, status bookkeeping, MRF replay)
//! is covered by
//! `replication_extension_test::test_site_replication_tagging_lww_converges_active_active_real_dual_node`;
//! this file stays as the fast, single-process receiver check.
use crate::common::{RustFSTestEnvironment, init_logging};
use aws_sdk_s3::Client;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
const HDR_SOURCE_REPLICATION_REQUEST: &str = "x-rustfs-source-replication-request";
const HDR_SOURCE_VERSION_ID: &str = "x-rustfs-source-version-id";
const HDR_SOURCE_MTIME: &str = "x-rustfs-source-mtime";
const HDR_SOURCE_TAGGING_TIMESTAMP: &str = "x-rustfs-source-replication-tagging-timestamp";
const SOURCE_MTIME: &str = "2026-01-01T00:00:00Z";
const T_STALE: &str = "2026-01-01T00:00:01Z";
const T_LOCAL: &str = "2026-02-01T00:00:00Z";
const T_NEWER: &str = "2026-03-01T00:00:00Z";
/// Simulated inbound authorized replication PUT: same object version, tags and
/// the source-authored tagging timestamp carried in transport headers.
async fn inbound_replication_put(
client: &Client,
bucket: &str,
key: &str,
version_id: &str,
tags: &str,
tagging_timestamp: &str,
) -> TestResult {
let version_id = version_id.to_string();
let tagging_timestamp = tagging_timestamp.to_string();
client
.put_object()
.bucket(bucket)
.key(key)
.body(ByteStream::from_static(b"lww-e2e-body"))
.tagging(tags)
.customize()
.mutate_request(move |req| {
req.headers_mut().insert(HDR_SOURCE_REPLICATION_REQUEST, "true");
req.headers_mut().insert(HDR_SOURCE_VERSION_ID, version_id.clone());
req.headers_mut().insert(HDR_SOURCE_MTIME, SOURCE_MTIME);
req.headers_mut()
.insert(HDR_SOURCE_TAGGING_TIMESTAMP, tagging_timestamp.clone());
})
.send()
.await?;
Ok(())
}
async fn tag_value(client: &Client, bucket: &str, key: &str, version_id: &str, tag_key: &str) -> Option<String> {
let tagging = client
.get_object_tagging()
.bucket(bucket)
.key(key)
.version_id(version_id)
.send()
.await
.expect("object tagging should be readable");
tagging
.tag_set()
.iter()
.find(|tag| tag.key() == tag_key)
.map(|tag| tag.value().to_string())
}
#[tokio::test(flavor = "multi_thread")]
async fn receiver_lww_keeps_newer_tags_across_delivery_orders() -> TestResult {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
let client = env.create_s3_client();
let bucket = "replication-lww-receiver";
let key = "object";
client.create_bucket().bucket(bucket).send().await?;
client
.put_bucket_versioning()
.bucket(bucket)
.versioning_configuration(
VersioningConfiguration::builder()
.status(BucketVersioningStatus::Enabled)
.build(),
)
.send()
.await?;
// First delivery establishes version V with tags stamped T_LOCAL.
let version_id = uuid::Uuid::new_v4().to_string();
inbound_replication_put(&client, bucket, key, &version_id, "site=local", T_LOCAL).await?;
assert_eq!(
tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
Some("local"),
"the first delivery must establish the tagged version"
);
// A stale delivery (older source timestamp) must succeed at the object
// level but must NOT overwrite the newer tags.
inbound_replication_put(&client, bucket, key, &version_id, "site=stale", T_STALE).await?;
assert_eq!(
tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
Some("local"),
"a stale inbound delivery must not overwrite newer tags (rustfs/backlog#1953)"
);
// A newer delivery still converges the version onto the newest state.
inbound_replication_put(&client, bucket, key, &version_id, "site=newer", T_NEWER).await?;
assert_eq!(
tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
Some("newer"),
"a newer inbound delivery must overwrite older tags"
);
client
.delete_object()
.bucket(bucket)
.key(key)
.version_id(&version_id)
.send()
.await?;
env.delete_test_bucket(bucket).await.ok();
Ok(())
}
@@ -21,11 +21,12 @@
//! - SSRF prevention (internal/private endpoints rejected for tiering)
//! - Race condition handling (concurrent writes converge without corruption)
use crate::common::{RustFSTestEnvironment, awscurl_put, init_logging, require_awscurl};
use crate::common::{RustFSTestEnvironment, awscurl_available, awscurl_put, init_logging};
use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart, Tag, Tagging};
use std::error::Error;
use tracing::info;
/// Oversized tagging payloads must be rejected by the per-object tag limit.
///
@@ -224,12 +225,16 @@ async fn test_concurrent_object_operations() -> Result<(), Box<dyn Error + Send
/// outcome — the internal endpoint is not accepted — is asserted here.
///
/// The admin API is exercised via signed `awscurl` requests, matching the
/// pattern used by the other admin-API E2E tests in this crate. The full E2E
/// lane installs and verifies the pinned `awscurl` prerequisite.
/// pattern used by the other admin-API E2E tests in this crate; the test is
/// skipped when `awscurl` is not installed.
#[tokio::test]
async fn test_tiering_url_validation() -> Result<(), Box<dyn Error + Send + Sync>> {
init_logging();
require_awscurl()?;
if !awscurl_available() {
info!("Skipping tiering URL validation test because awscurl is not available");
return Ok(());
}
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
+2 -12
View File
@@ -404,18 +404,8 @@ mod tests {
info!("✅ DELETE object succeeded");
// Verify it's deleted
let error = client
.get_object()
.bucket(bucket)
.key(key)
.send()
.await
.expect_err("Object should not exist after DELETE");
assert_eq!(
error.raw_response().map(|response| response.status().as_u16()),
Some(404),
"GET after DELETE must return HTTP 404, got {error:?}"
);
let result = client.get_object().bucket(bucket).key(key).send().await;
assert!(result.is_err(), "Object should not exist after DELETE");
// Cleanup
env.stop_server();
+28 -40
View File
@@ -161,7 +161,7 @@ pub mod bucket {
pub mod objectlock_sys {
pub use crate::bucket::object_lock::objectlock_sys::{
BucketObjectLockSys, ObjectLockBlockReason, add_years, check_object_lock_for_deletion,
check_retention_for_modification, is_retention_active, replication_write_may_pass_worm_gate,
check_retention_for_modification, is_retention_active,
};
}
}
@@ -187,24 +187,22 @@ pub mod bucket {
pub use crate::bucket::replication::{
BucketReplicationResyncStatus, BucketReplicationStat, BucketReplicationStats, BucketStats,
DeleteReplicationConfigSnapshot, DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool, InQueueMetric,
MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts, OperatorRuleContract,
REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
REPLICATE_INCOMING_DELETE, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS,
REPLICATION_WRITABLE_FIELDS, ReplicateDecision, ReplicateObjectInfo, ReplicationBatchAdmission, ReplicationConfig,
MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION,
REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, REPLICATE_INCOMING_DELETE,
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
ReplicateDecision, ReplicateObjectInfo, ReplicationBatchAdmission, ReplicationConfig,
ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationDeleteScheduleInput,
ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge, ReplicationObjectIO,
ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge,
ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage, ReplicationTargetValidationError,
ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, TargetReplicationResyncStatus,
VersionPurgeStatusType, XferStats, assign_site_replication_rule_priorities, commit_force_delete_intent,
complete_force_delete_intent, delete_replication_state_from_config, delete_replication_version_id,
get_global_replication_pool, get_global_replication_stats, get_proxy_targets, init_background_replication,
invalid_replication_config_status_field, is_site_replication_role, is_site_replication_rule,
merge_incoming_replication_config, merge_user_replication_config, persist_force_delete_intent,
read_durable_mrf_backlog, replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map,
replication_target_arn_deployment_id, replication_target_arns, resync_start_conflict_id,
should_remove_replication_target, should_schedule_delete_replication, should_use_existing_delete_replication_info,
should_use_existing_delete_replication_source, site_replication_rule_deployment_id,
VersionPurgeStatusType, XferStats, commit_force_delete_intent, complete_force_delete_intent,
delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool,
get_global_replication_stats, get_proxy_targets, init_background_replication,
invalid_replication_config_status_field, persist_force_delete_intent, read_durable_mrf_backlog,
replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map, replication_target_arns,
resync_start_conflict_id, should_remove_replication_target, should_schedule_delete_replication,
should_use_existing_delete_replication_info, should_use_existing_delete_replication_source,
unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
version_purge_status_to_filemeta,
};
@@ -243,8 +241,8 @@ pub mod cache {
pub mod capacity {
pub use crate::core::pools::{
DecommissionUnresolvedEntry, PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free,
path2_bucket_object, path2_bucket_object_with_base_path,
PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free, path2_bucket_object,
path2_bucket_object_with_base_path,
};
pub use crate::store::utils::is_reserved_or_invalid_bucket;
}
@@ -415,9 +413,9 @@ pub mod notification {
#[cfg(any(test, feature = "test-util"))]
pub use crate::services::notification_sys::rotate_cross_pool_fence_fleet_proof_for_test;
pub use crate::services::notification_sys::{
CrossPoolFenceFleetProofToken, NotificationPeerErr, NotificationSys, ScannerPublicationLeaseGrant,
acquire_cross_pool_fence_fleet_proof, cross_pool_fence_fleet_proof_matches, get_global_notification_sys,
new_global_notification_sys, start_remote_version_state_fleet_probe,
CrossPoolFenceFleetProofToken, NotificationPeerErr, NotificationSys, acquire_cross_pool_fence_fleet_proof,
cross_pool_fence_fleet_proof_matches, get_global_notification_sys, new_global_notification_sys,
start_remote_version_state_fleet_probe,
};
}
@@ -426,10 +424,9 @@ pub mod object {
BLOCK_SIZE_V2, ERASURE_ALGORITHM, EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook,
GetObjectBodyCacheHookLookup, GetObjectBodySource, GetObjectReader, NamespaceLockFence, ObjectEncryptionResolver,
ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, PutObjReader, QuotaAdmission,
RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest,
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, StreamConsumer, get_object_body_cache_plaintext_len,
lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook,
unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest, StreamConsumer,
get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
};
pub use crate::store::{
PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
@@ -443,12 +440,6 @@ pub mod rebalance {
RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord, decode_rebalance_stop_propagation_record,
encode_rebalance_stop_propagation_record,
};
#[cfg(feature = "test-util")]
pub mod test_util {
pub use crate::services::rebalance::PausedRebalanceEntryTestFixture;
pub use crate::services::rebalance::test_store_with_persisted_rebalance_meta;
}
}
pub mod rio {
@@ -462,17 +453,14 @@ pub mod rpc {
pub use crate::cluster::rpc::{
AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
PeerRestClient, PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
ScannerBucketListing, ScannerPeerActivity, ScannerPublicationLease, TONIC_RPC_PREFIX, TonicInterceptor,
build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, decode_heal_bucket_rpc_options,
encode_heal_bucket_rpc_options, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
ScannerBucketListing, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, build_put_file_auth_trailer,
check_and_record_signed_rpc_nonce, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, sign_ns_scanner_capability,
sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer,
verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
verify_tonic_rpc_signature_with_bootstrap,
normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_put_file_capability,
sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
tonic_rpc_auth_failure_reason, verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature,
verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest,
verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
};
}
@@ -499,7 +487,7 @@ pub mod storage {
pub use crate::core::pools::HealLifecycleExpiryContext;
pub use crate::store::HealWalkVersion;
pub use crate::store::{
ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks,
ECStore, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks,
init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
prewarm_local_disk_id_map_with_instance_ctx,
};
+2 -41
View File
@@ -58,8 +58,8 @@ use rustfs_config::{DEFAULT_TRUST_LEAF_CERT_AS_CA, ENV_TRUST_LEAF_CERT_AS_CA, RU
use rustfs_utils::egress::{OutboundUrlError, validate_outbound_url};
use rustfs_utils::http::{
AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_MODE,
AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_OBJECT_TAGGING_LOWER, AMZ_STORAGE_CLASS, AMZ_WEBSITE_REDIRECT_LOCATION, is_amz_header,
is_minio_header, is_rustfs_header, is_standard_header, is_storageclass_header,
AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_STORAGE_CLASS, AMZ_WEBSITE_REDIRECT_LOCATION, is_amz_header, is_minio_header,
is_rustfs_header, is_standard_header, is_storageclass_header,
};
use rustfs_utils::http::{
SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_PROXY_REQUEST,
@@ -1774,22 +1774,6 @@ impl PutObjectOptions {
Self::insert_checked(&mut header, AMZ_BUCKET_REPLICATION_STATUS, self.internal.replication_status.as_str());
}
// MinIO PutObjectOptions.Header parity: object tags travel on the
// `x-amz-tagging` header (form-urlencoded). `replication_put_object_options`
// fills `user_tags` from the source version; without this header the
// whole-object transport delivered a tagless replica, so tag edits
// never reached the peer and the receiver-side LWW comparison
// (rustfs/backlog#1953) had nothing to judge.
if !self.user_tags.is_empty() {
let mut tags: Vec<(&String, &String)> = self.user_tags.iter().collect();
tags.sort();
let mut encoded = url::form_urlencoded::Serializer::new(String::new());
for (key, value) in tags {
encoded.append_pair(key, value);
}
Self::insert_checked(&mut header, AMZ_OBJECT_TAGGING_LOWER, &encoded.finish());
}
for (k, v) in &self.user_metadata {
let Ok(header_value) = HeaderValue::from_str(v) else {
warn!("skipping user metadata header with invalid value: {}", k);
@@ -3211,29 +3195,6 @@ mod tests {
}
}
#[test]
fn put_object_headers_carry_user_tags_on_x_amz_tagging() {
// rustfs/backlog#1953: tag edits replicate through the whole-object
// transport, so the source tags must travel on x-amz-tagging.
let mut opts = PutObjectOptions::default();
opts.user_tags.insert("owner".to_string(), "site a".to_string());
opts.user_tags.insert("env".to_string(), "prod".to_string());
let header = opts.header();
let tagging = header
.get(AMZ_OBJECT_TAGGING_LOWER)
.expect("user tags must be transported on x-amz-tagging")
.to_str()
.expect("tag header must be ASCII");
// Deterministic key order; values are form-urlencoded.
assert_eq!(tagging, "env=prod&owner=site+a");
assert!(
PutObjectOptions::default().header().get(AMZ_OBJECT_TAGGING_LOWER).is_none(),
"a tagless source must not send an empty x-amz-tagging header"
);
}
#[test]
fn put_object_headers_omit_unset_replication_timestamps() {
// UNIX_EPOCH means "never modified on the source"; sending it would
@@ -3739,55 +3739,17 @@ impl ManualTransitionRunReport {
}
pub fn merge_scan_report_preserving_worker(&mut self, scan_report: &ManualTransitionRunReport) {
let previous = self.clone();
let resumed_after_checkpoint = previous.continuation_token.is_some() && scan_report.scanned < previous.scanned;
let mut tier_failure_by_reason = self.tier_failure_by_reason.clone();
for (reason, count) in &scan_report.tier_failure_by_reason {
let current = tier_failure_by_reason.get(reason).copied().unwrap_or_default();
let merged = if resumed_after_checkpoint {
current.saturating_add(*count)
} else {
current.max(*count)
};
tier_failure_by_reason.insert(*reason, merged);
tier_failure_by_reason.insert(*reason, current.max(*count));
}
let transition_completed = self.transition_completed;
let transition_failed = self.transition_failed;
*self = scan_report.clone();
if resumed_after_checkpoint {
self.scanned = previous.scanned.saturating_add(scan_report.scanned);
self.eligible = previous.eligible.saturating_add(scan_report.eligible);
self.enqueued = previous.enqueued.saturating_add(scan_report.enqueued);
self.dry_run_eligible = previous.dry_run_eligible.saturating_add(scan_report.dry_run_eligible);
self.skipped_not_transition = previous
.skipped_not_transition
.saturating_add(scan_report.skipped_not_transition);
self.skipped_tier = previous.skipped_tier.saturating_add(scan_report.skipped_tier);
self.skipped_delete_marker = previous
.skipped_delete_marker
.saturating_add(scan_report.skipped_delete_marker);
self.skipped_directory = previous.skipped_directory.saturating_add(scan_report.skipped_directory);
self.skipped_replication = previous.skipped_replication.saturating_add(scan_report.skipped_replication);
self.skipped_already_transitioned = previous
.skipped_already_transitioned
.saturating_add(scan_report.skipped_already_transitioned);
self.skipped_already_in_flight = previous
.skipped_already_in_flight
.saturating_add(scan_report.skipped_already_in_flight);
self.skipped_queue_full = previous.skipped_queue_full.saturating_add(scan_report.skipped_queue_full);
self.skipped_queue_closed = previous.skipped_queue_closed.saturating_add(scan_report.skipped_queue_closed);
self.skipped_queue_timeout = previous
.skipped_queue_timeout
.saturating_add(scan_report.skipped_queue_timeout);
self.tier_failure = previous.tier_failure.saturating_add(scan_report.tier_failure);
}
self.lifecycle_config_found = previous.lifecycle_config_found || scan_report.lifecycle_config_found;
self.truncated_by_limit = previous.truncated_by_limit || scan_report.truncated_by_limit;
self.truncated_by_duration = previous.truncated_by_duration || scan_report.truncated_by_duration;
self.cancelled = previous.cancelled || scan_report.cancelled;
self.transition_completed = transition_completed;
self.transition_failed = transition_failed;
self.tier_failure = self.tier_failure.saturating_add(transition_failed);
self.tier_failure = scan_report.tier_failure.saturating_add(transition_failed);
self.tier_failure_by_reason = tier_failure_by_reason;
}
@@ -3803,10 +3765,7 @@ struct ManualTransitionContinuationToken {
version_marker: Option<String>,
}
pub(super) fn encode_manual_transition_continuation_token(
marker: Option<String>,
version_marker: Option<String>,
) -> Option<String> {
fn encode_manual_transition_continuation_token(marker: Option<String>, version_marker: Option<String>) -> Option<String> {
if marker.is_none() && version_marker.is_none() {
return None;
}
@@ -4365,16 +4324,6 @@ pub async fn expire_transitioned_object(
lc_event: &lifecycle::Event,
_src: &LcEventSrc,
bucket_incarnation_id: Uuid,
) -> Result<ObjectInfo, std::io::Error> {
expire_transitioned_object_with_lock_lost_signal(api, oi, lc_event, bucket_incarnation_id, None).await
}
async fn expire_transitioned_object_with_lock_lost_signal(
api: Arc<ECStore>,
oi: &ObjectInfo,
lc_event: &lifecycle::Event,
bucket_incarnation_id: Uuid,
lock_lost_signal: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
) -> Result<ObjectInfo, std::io::Error> {
let publication_guard = lifecycle_expiry_publication_guard(&api, oi, bucket_incarnation_id)
.await
@@ -4386,9 +4335,6 @@ async fn expire_transitioned_object_with_lock_lost_signal(
let mut opts = transitioned_object_delete_opts(oi, lc_event.action, versioned, version_suspended, bucket_incarnation_id)
.map_err(std::io::Error::other)?;
opts.add_namespace_lock_guard(&publication_guard);
if let Some(signal) = lock_lost_signal {
opts.add_namespace_lock_lost_signal(signal);
}
opts.delete_replication_config_snapshot = Some(Arc::new(snapshot));
//let tags = LcAuditEvent::new(src, lcEvent).Tags();
if lc_event.action.delete_restored() {
@@ -5047,32 +4993,12 @@ pub async fn apply_expiry_on_transitioned_object(
lc_event: &lifecycle::Event,
src: &LcEventSrc,
bucket_incarnation_id: Uuid,
) -> bool {
apply_expiry_on_transitioned_object_with_lock_lost_signal(api, oi, lc_event, src, bucket_incarnation_id, None).await
}
async fn apply_expiry_on_transitioned_object_with_lock_lost_signal(
api: Arc<ECStore>,
oi: &ObjectInfo,
lc_event: &lifecycle::Event,
_src: &LcEventSrc,
bucket_incarnation_id: Uuid,
lock_lost_signal: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
) -> bool {
if lc_event.action.delete_all() {
return apply_expiry_on_non_transitioned_objects_with_lock_lost_signal(
api,
oi,
lc_event,
bucket_incarnation_id,
lock_lost_signal,
)
.await;
return apply_expiry_on_non_transitioned_objects(api, oi, lc_event, src, bucket_incarnation_id).await;
}
let time_ilm = Metrics::time_ilm(lc_event.action);
if let Err(_err) =
expire_transitioned_object_with_lock_lost_signal(api, oi, lc_event, bucket_incarnation_id, lock_lost_signal).await
{
if let Err(_err) = expire_transitioned_object(api, oi, lc_event, src, bucket_incarnation_id).await {
return false;
}
time_ilm(1)();
@@ -5086,16 +5012,6 @@ pub async fn apply_expiry_on_non_transitioned_objects(
lc_event: &lifecycle::Event,
_src: &LcEventSrc,
bucket_incarnation_id: Uuid,
) -> bool {
apply_expiry_on_non_transitioned_objects_with_lock_lost_signal(api, oi, lc_event, bucket_incarnation_id, None).await
}
async fn apply_expiry_on_non_transitioned_objects_with_lock_lost_signal(
api: Arc<ECStore>,
oi: &ObjectInfo,
lc_event: &lifecycle::Event,
bucket_incarnation_id: Uuid,
lock_lost_signal: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
) -> bool {
let Some(publication_guard) = lifecycle_expiry_publication_guard(&api, oi, bucket_incarnation_id).await else {
return false;
@@ -5126,9 +5042,6 @@ async fn apply_expiry_on_non_transitioned_objects_with_lock_lost_signal(
..Default::default()
};
opts.add_namespace_lock_guard(&publication_guard);
if let Some(signal) = lock_lost_signal {
opts.add_namespace_lock_lost_signal(signal);
}
if lc_event.action.delete_versioned() {
opts.version_id = oi.version_id.map(|v| v.to_string());
@@ -5210,61 +5123,6 @@ async fn enqueue_expiry_rule_with_incarnation(
expiry_state.enqueue_by_days(oi, event, src, bucket_incarnation_id)
}
fn lifecycle_expiry_object_matches(current: &ObjectInfo, expected: &ObjectInfo) -> bool {
current.version_id == expected.version_id
&& current.data_dir == expected.data_dir
&& current.mod_time == expected.mod_time
&& current.etag == expected.etag
&& current.delete_marker == expected.delete_marker
&& current.transitioned_object.name == expected.transitioned_object.name
&& current.transitioned_object.version_id == expected.transitioned_object.version_id
&& current.transitioned_object.tier == expected.transitioned_object.tier
&& current.transitioned_object.status == expected.transitioned_object.status
&& current.restore_expires == expected.restore_expires
}
pub(crate) async fn apply_expiry_rule_for_data_movement(
api: Arc<ECStore>,
event: &lifecycle::Event,
src: &LcEventSrc,
oi: &ObjectInfo,
lock_lost_signal: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
) -> bool {
let Ok(_lifecycle_guard) = api.acquire_bucket_lifecycle_read_lock(&oi.bucket).await else {
return false;
};
let Ok(bucket_incarnation_id) = api.bucket_incarnation_id_from_disk(&oi.bucket).await else {
return false;
};
let current = match api
.get_object_info(
&oi.bucket,
&oi.name,
&ObjectOptions {
version_id: oi.version_id.map(|version_id| version_id.to_string()),
versioned: oi.version_id.is_some(),
expected_bucket_incarnation_id: Some(bucket_incarnation_id),
..Default::default()
},
)
.await
{
Ok(current) => current,
Err(_) => return false,
};
if !lifecycle_expiry_object_matches(&current, oi) {
return false;
}
if oi.transitioned_object.status.is_empty() {
apply_expiry_on_non_transitioned_objects_with_lock_lost_signal(api, oi, event, bucket_incarnation_id, lock_lost_signal)
.await
} else {
apply_expiry_on_transitioned_object_with_lock_lost_signal(api, oi, event, src, bucket_incarnation_id, lock_lost_signal)
.await
}
}
pub(crate) async fn apply_expiry_rule_in(api: Arc<ECStore>, event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool {
let Ok(_lifecycle_guard) = api.acquire_bucket_lifecycle_read_lock(&oi.bucket).await else {
return false;
@@ -5288,7 +5146,17 @@ pub(crate) async fn apply_expiry_rule_in(api: Arc<ECStore>, event: &lifecycle::E
Ok(current) => current,
Err(_) => return false,
};
if !lifecycle_expiry_object_matches(&current, oi) {
if current.version_id != oi.version_id
|| current.data_dir != oi.data_dir
|| current.mod_time != oi.mod_time
|| current.etag != oi.etag
|| current.delete_marker != oi.delete_marker
|| current.transitioned_object.name != oi.transitioned_object.name
|| current.transitioned_object.version_id != oi.transitioned_object.version_id
|| current.transitioned_object.tier != oi.transitioned_object.tier
|| current.transitioned_object.status != oi.transitioned_object.status
|| current.restore_expires != oi.restore_expires
{
return false;
}
enqueue_expiry_rule_with_incarnation(event, src, oi, bucket_incarnation_id).await
@@ -9268,7 +9136,6 @@ mod tests {
assert_eq!(loaded.report.scanned, 37);
assert_eq!(loaded.report.eligible, 11);
assert_eq!(loaded.report.enqueued, 5);
assert_eq!(loaded.cursor_revision, Some(37));
assert!(loaded.lease_expires_at_unix_nanos > 0);
let token = loaded
.report
@@ -9287,30 +9154,6 @@ mod tests {
assert_eq!(admission.lease_id, loaded.lease_id);
assert_eq!(admission.lease_expires_at_unix_nanos, loaded.lease_expires_at_unix_nanos);
let mut same_marker_report = report.clone();
same_marker_report.scanned += 1;
persist_manual_transition_page_checkpoint(
&checkpoint_options,
&same_marker_report,
Some("logs/page-end".to_string()),
Some("opaque-next-version".to_string()),
)
.await
.expect("same-marker version checkpoint should persist through the durable progress sink");
let same_marker_checkpointed = load_manual_transition_job_record(ecstore.clone(), job_id)
.await
.expect("same-marker version checkpoint should reload");
assert_eq!(same_marker_checkpointed.cursor_revision, Some(38));
let (_, version_marker) = decode_manual_transition_continuation_token(
same_marker_checkpointed
.report
.continuation_token
.as_deref()
.expect("same-marker version checkpoint should persist a cursor"),
)
.expect("same-marker version cursor should decode");
assert_eq!(version_marker.as_deref(), Some("opaque-next-version"));
create_test_bucket(&ecstore, &bucket).await;
let lifecycle_xml = format!(
r#"<?xml version="1.0" encoding="UTF-8"?>
@@ -9367,7 +9210,6 @@ mod tests {
assert_eq!(checkpointed.report.scanned, 1000);
assert_eq!(checkpointed.report.eligible, 1000);
assert_eq!(checkpointed.report.dry_run_eligible, 1000);
assert_eq!(checkpointed.cursor_revision, Some(1000));
let token = checkpointed
.report
.continuation_token
File diff suppressed because it is too large Load Diff
@@ -20,16 +20,10 @@ use serde::{Deserialize, Serialize};
use time::OffsetDateTime;
use uuid::Uuid;
#[cfg(test)]
use crate::bucket::lifecycle::bucket_lifecycle_ops::encode_manual_transition_continuation_token;
use crate::bucket::lifecycle::bucket_lifecycle_ops::{
ManualTransitionQueueSnapshot, ManualTransitionRunOptions, ManualTransitionRunReport,
};
use crate::bucket::lifecycle::config_boundary;
use crate::bucket::lifecycle::durable_namespace::{
MANUAL_TRANSITION_JOB_NAMESPACE, MANUAL_TRANSITION_SCOPE_NAMESPACE, MANUAL_TRANSITION_TASK_NAMESPACE,
MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE,
};
use crate::disk::RUSTFS_META_BUCKET;
use crate::error::{Error, Result as EcstoreResult};
use crate::object_api::ObjectOptions;
@@ -40,10 +34,10 @@ use crate::store::ECStore;
pub const MANUAL_TRANSITION_JOB_SCHEMA: &str = "rustfs-manual-transition-job-v1";
pub const MANUAL_TRANSITION_TASK_SCHEMA: &str = "rustfs-manual-transition-task-v1";
pub const MANUAL_TRANSITION_WORKER_RESULT_SCHEMA: &str = "rustfs-manual-transition-worker-result-v1";
pub const MANUAL_TRANSITION_JOB_RECORD_PREFIX: &str = MANUAL_TRANSITION_JOB_NAMESPACE.prefix;
pub const MANUAL_TRANSITION_SCOPE_RECORD_PREFIX: &str = MANUAL_TRANSITION_SCOPE_NAMESPACE.prefix;
pub const MANUAL_TRANSITION_TASK_PREFIX: &str = MANUAL_TRANSITION_TASK_NAMESPACE.prefix;
pub const MANUAL_TRANSITION_WORKER_RESULT_PREFIX: &str = MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE.prefix;
pub const MANUAL_TRANSITION_JOB_RECORD_PREFIX: &str = "ilm/manual-transition/jobs";
pub const MANUAL_TRANSITION_SCOPE_RECORD_PREFIX: &str = "ilm/manual-transition/scopes";
pub const MANUAL_TRANSITION_TASK_PREFIX: &str = "ilm/manual-transition/tasks";
pub const MANUAL_TRANSITION_WORKER_RESULT_PREFIX: &str = "ilm/manual-transition/results";
pub const MAX_MANUAL_TRANSITION_JOB_RECORD_SIZE: usize = 64 * 1024;
pub const MAX_MANUAL_TRANSITION_TASK_RECORD_SIZE: usize = 16 * 1024;
pub const MAX_MANUAL_TRANSITION_WORKER_RESULT_RECORD_SIZE: usize = 8 * 1024;
@@ -201,8 +195,6 @@ pub struct ManualTransitionJobRecord {
pub updated_at_unix_nanos: i128,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub completed_at_unix_nanos: Option<i128>,
#[serde(default, skip_serializing)]
pub cursor_revision: Option<u64>,
pub report: ManualTransitionRunReport,
pub queue_snapshot: ManualTransitionQueueSnapshot,
#[serde(default, skip_serializing_if = "Option::is_none")]
@@ -232,7 +224,6 @@ impl ManualTransitionJobRecord {
created_at_unix_nanos: now,
updated_at_unix_nanos: now,
completed_at_unix_nanos: None,
cursor_revision: None,
report: ManualTransitionRunReport {
bucket: bucket.to_string(),
prefix: options.prefix.clone(),
@@ -247,7 +238,7 @@ impl ManualTransitionJobRecord {
pub fn complete(&mut self, report: ManualTransitionRunReport, queue_snapshot: ManualTransitionQueueSnapshot) {
self.scan_completed = true;
self.merge_scan_report(&report);
self.report.merge_scan_report_preserving_worker(&report);
self.queue_snapshot = queue_snapshot;
self.error = None;
self.mark_terminal_if_worker_drained();
@@ -325,7 +316,7 @@ impl ManualTransitionJobRecord {
}
}
self.queue_snapshot = queue_snapshot;
self.advance_updated_at();
self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
self.mark_terminal_if_worker_drained();
}
@@ -368,14 +359,14 @@ impl ManualTransitionJobRecord {
self.report.tier_failure = scan_tier_failure.saturating_add(transition_failed);
self.report.tier_failure_by_reason = scan_tier_failure_by_reason;
self.queue_snapshot = queue_snapshot;
self.advance_updated_at();
self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
self.mark_terminal_if_worker_drained();
true
}
pub fn mark_cancel_requested(&mut self) {
self.cancel_requested = true;
self.advance_updated_at();
self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
}
pub fn claim_recovery_lease(&mut self, owner_id: impl Into<String>, queue_snapshot: ManualTransitionQueueSnapshot) {
@@ -389,7 +380,7 @@ impl ManualTransitionJobRecord {
pub fn abandon_recovery_lease(&mut self, lease_id: Uuid) {
if self.state == ManualTransitionJobState::Running && self.lease_id == lease_id {
self.lease_expires_at_unix_nanos = 0;
self.advance_updated_at();
self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
}
}
@@ -407,7 +398,7 @@ impl ManualTransitionJobRecord {
pub fn renew_lease(&mut self, queue_snapshot: ManualTransitionQueueSnapshot) {
let now = OffsetDateTime::now_utc().unix_timestamp_nanos();
self.updated_at_unix_nanos = self.updated_at_unix_nanos.saturating_add(1).max(now);
self.updated_at_unix_nanos = now;
self.lease_expires_at_unix_nanos = manual_transition_job_lease_expires_at(now);
self.queue_snapshot = queue_snapshot;
}
@@ -447,16 +438,11 @@ impl ManualTransitionJobRecord {
pub fn update_running_progress(&mut self, report: ManualTransitionRunReport, queue_snapshot: ManualTransitionQueueSnapshot) {
if self.state == ManualTransitionJobState::Running {
self.merge_scan_report(&report);
self.report.merge_scan_report_preserving_worker(&report);
self.renew_lease(queue_snapshot);
}
}
fn merge_scan_report(&mut self, report: &ManualTransitionRunReport) {
self.report.merge_scan_report_preserving_worker(report);
self.cursor_revision = manual_transition_cursor_revision(&self.report);
}
pub fn mark_unknown_if_unowned(&mut self) {
if self.state == ManualTransitionJobState::Running {
self.state = ManualTransitionJobState::Unknown;
@@ -477,13 +463,9 @@ impl ManualTransitionJobRecord {
}
fn mark_updated_terminal(&mut self) {
self.advance_updated_at();
self.completed_at_unix_nanos = Some(self.updated_at_unix_nanos);
}
fn advance_updated_at(&mut self) {
let now = OffsetDateTime::now_utc().unix_timestamp_nanos();
self.updated_at_unix_nanos = self.updated_at_unix_nanos.saturating_add(1).max(now);
self.updated_at_unix_nanos = now;
self.completed_at_unix_nanos = Some(now);
}
fn mark_terminal_if_worker_drained(&mut self) {
@@ -561,7 +543,6 @@ impl ManualTransitionJobRecord {
if job.state == ManualTransitionJobState::Cancelled && job.cancel_requested {
job.report.cancelled = true;
}
job.cursor_revision = manual_transition_cursor_revision(&job.report);
job.validate()?;
Ok(job)
}
@@ -1128,8 +1109,7 @@ pub fn manual_transition_scope_record_object_name(scope_key: &str) -> Result<Str
pub async fn save_manual_transition_job_record(api: Arc<ECStore>, job: &ManualTransitionJobRecord) -> EcstoreResult<()> {
let object = manual_transition_job_record_object_name(job.job_id).map_err(manual_transition_job_store_error)?;
let data = job.encode().map_err(manual_transition_job_store_error)?;
config_boundary::save_config(api.clone(), &object, data.clone()).await?;
api.record_durable_ilm_decommission_progress(&object, &data).await
config_boundary::save_config(api, &object, data).await
}
pub async fn load_manual_transition_job_record(api: Arc<ECStore>, job_id: Uuid) -> EcstoreResult<ManualTransitionJobRecord> {
@@ -1162,9 +1142,9 @@ pub async fn save_manual_transition_job_record_if_current(
let object = manual_transition_job_record_object_name(job.job_id).map_err(manual_transition_job_store_error)?;
let data = job.encode().map_err(manual_transition_job_store_error)?;
config_boundary::save_config_with_opts_quiet(
api.clone(),
api,
&object,
data.clone(),
data,
&ObjectOptions {
max_parity: true,
http_preconditions: Some(HTTPPreconditions {
@@ -1174,8 +1154,7 @@ pub async fn save_manual_transition_job_record_if_current(
..Default::default()
},
)
.await?;
api.record_durable_ilm_decommission_progress(&object, &data).await
.await
}
/// Applies a job-record mutation with optimistic concurrency control.
@@ -1613,9 +1592,9 @@ pub async fn save_manual_transition_scope_admission_if_absent(
let object = manual_transition_scope_record_object_name(&admission.scope_key).map_err(manual_transition_job_store_error)?;
let data = serde_json::to_vec(admission).map_err(Error::other)?;
config_boundary::save_config_with_opts(
api.clone(),
api,
&object,
data.clone(),
data,
&ObjectOptions {
max_parity: true,
http_preconditions: Some(HTTPPreconditions {
@@ -1625,8 +1604,7 @@ pub async fn save_manual_transition_scope_admission_if_absent(
..Default::default()
},
)
.await?;
api.record_durable_ilm_decommission_progress(&object, &data).await
.await
}
pub async fn load_manual_transition_scope_admission(
@@ -1664,9 +1642,9 @@ pub async fn save_manual_transition_scope_admission_if_current(
let object = manual_transition_scope_record_object_name(&admission.scope_key).map_err(manual_transition_job_store_error)?;
let data = serde_json::to_vec(admission).map_err(Error::other)?;
match config_boundary::save_config_with_opts(
api.clone(),
api,
&object,
data.clone(),
data,
&ObjectOptions {
max_parity: true,
http_preconditions: Some(HTTPPreconditions {
@@ -1682,8 +1660,7 @@ pub async fn save_manual_transition_scope_admission_if_current(
Err(Error::PreconditionFailed)
}
result => result,
}?;
api.record_durable_ilm_decommission_progress(&object, &data).await
}
}
pub async fn claim_manual_transition_scope_admission(
@@ -1976,15 +1953,13 @@ pub async fn delete_manual_transition_scope_admission_if_current(
job_id: Uuid,
lease_id: Uuid,
) -> EcstoreResult<bool> {
let (admission, etag) = match load_manual_transition_scope_admission_with_etag(api.clone(), scope_key).await {
Ok((admission, etag)) if admission.job_id == job_id && admission.lease_id == lease_id => (admission, etag),
let etag = match load_manual_transition_scope_admission_with_etag(api.clone(), scope_key).await {
Ok((admission, etag)) if admission.job_id == job_id && admission.lease_id == lease_id => etag,
Ok(_) => return Ok(false),
Err(Error::ConfigNotFound) => return Ok(true),
Err(err) => return Err(err),
};
let object = manual_transition_scope_record_object_name(scope_key).map_err(manual_transition_job_store_error)?;
let data = serde_json::to_vec(&admission).map_err(Error::other)?;
api.record_durable_ilm_decommission_terminal(&object, &data).await?;
match config_boundary::delete_config_if_match(api, &object, &etag).await {
Ok(()) | Err(Error::ConfigNotFound) => Ok(true),
Err(Error::PreconditionFailed) => Ok(false),
@@ -1996,11 +1971,6 @@ fn manual_transition_job_store_error(err: ManualTransitionJobError) -> Error {
Error::other(err)
}
fn manual_transition_cursor_revision(report: &ManualTransitionRunReport) -> Option<u64> {
report.continuation_token.as_ref()?;
(report.scanned > 0).then_some(report.scanned)
}
pub fn manual_transition_scope_admission_lease_expired(admission: &ManualTransitionScopeAdmission) -> bool {
OffsetDateTime::now_utc().unix_timestamp_nanos() > admission.lease_expires_at_unix_nanos
}
@@ -2240,76 +2210,6 @@ mod tests {
);
}
#[test]
fn manual_transition_job_scan_progress_accumulates_resumed_checkpoint_counters() {
let options = ManualTransitionRunOptions::default();
let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER);
let first_token =
encode_manual_transition_continuation_token(Some("logs/page-a".to_string()), Some("version-a".to_string()));
record.update_running_progress(
ManualTransitionRunReport {
bucket: "bucket".to_string(),
lifecycle_config_found: true,
scanned: 1000,
eligible: 800,
enqueued: 50,
dry_run_eligible: 10,
skipped_not_transition: 2,
skipped_tier: 3,
skipped_delete_marker: 4,
skipped_directory: 5,
skipped_replication: 6,
skipped_already_transitioned: 7,
skipped_already_in_flight: 8,
skipped_queue_full: 9,
skipped_queue_closed: 10,
skipped_queue_timeout: 11,
tier_failure: 12,
truncated_by_duration: true,
continuation_token: first_token,
..Default::default()
},
ManualTransitionQueueSnapshot::default(),
);
let next_token =
encode_manual_transition_continuation_token(Some("logs/page-b".to_string()), Some("version-b".to_string()));
record.update_running_progress(
ManualTransitionRunReport {
bucket: "bucket".to_string(),
scanned: 3,
eligible: 2,
enqueued: 1,
dry_run_eligible: 1,
skipped_not_transition: 1,
tier_failure: 1,
continuation_token: next_token.clone(),
..Default::default()
},
ManualTransitionQueueSnapshot::default(),
);
assert_eq!(record.report.scanned, 1003);
assert_eq!(record.report.eligible, 802);
assert_eq!(record.report.enqueued, 51);
assert_eq!(record.report.dry_run_eligible, 11);
assert_eq!(record.report.skipped_not_transition, 3);
assert_eq!(record.report.skipped_tier, 3);
assert_eq!(record.report.skipped_delete_marker, 4);
assert_eq!(record.report.skipped_directory, 5);
assert_eq!(record.report.skipped_replication, 6);
assert_eq!(record.report.skipped_already_transitioned, 7);
assert_eq!(record.report.skipped_already_in_flight, 8);
assert_eq!(record.report.skipped_queue_full, 9);
assert_eq!(record.report.skipped_queue_closed, 10);
assert_eq!(record.report.skipped_queue_timeout, 11);
assert_eq!(record.report.tier_failure, 13);
assert!(record.report.lifecycle_config_found);
assert!(record.report.truncated_by_duration);
assert_eq!(record.report.continuation_token, next_token);
assert_eq!(record.cursor_revision, Some(1003));
}
#[test]
fn manual_transition_job_apply_worker_result_counts_preserves_existing_failure_reasons() {
let options = ManualTransitionRunOptions::default();
@@ -2677,98 +2577,6 @@ mod tests {
assert!(decoded.report.tier_failure_by_reason.is_empty());
}
#[test]
fn manual_transition_job_record_derives_revision_from_legacy_cursor() {
let options = ManualTransitionRunOptions::default();
let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER);
let continuation_token =
encode_manual_transition_continuation_token(Some("logs/page-a".to_string()), Some("version-a".to_string()));
record.update_running_progress(
ManualTransitionRunReport {
bucket: "bucket".to_string(),
scanned: 9,
continuation_token,
..Default::default()
},
ManualTransitionQueueSnapshot::default(),
);
let encoded = record.encode().expect("job record should encode");
let mut value: serde_json::Value = serde_json::from_slice(&encoded).expect("encoded job should be json");
value["job"]
.as_object_mut()
.expect("job should be object")
.remove("cursor_revision");
let record_bytes = serde_json::to_vec(&value["job"]).expect("legacy job should encode");
value["content_sha256"] = serde_json::Value::String(hex_sha256(&record_bytes, ToOwned::to_owned));
let legacy = serde_json::to_vec(&value).expect("legacy envelope should encode");
let decoded = ManualTransitionJobRecord::decode(record.job_id, &legacy).expect("legacy job should decode");
assert_eq!(decoded.cursor_revision, Some(9));
}
#[test]
fn manual_transition_job_record_omits_cursor_revision_for_old_readers() {
#[allow(dead_code)]
#[derive(serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct LegacyPersistedManualTransitionJobRecord {
schema: String,
content_sha256: String,
job: LegacyManualTransitionJobRecord,
}
#[allow(dead_code)]
#[derive(serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct LegacyManualTransitionJobRecord {
job_id: Uuid,
scope_key: String,
bucket: String,
prefix: String,
tier: Option<String>,
dry_run: bool,
max_objects: Option<u64>,
max_duration: Option<std::time::Duration>,
owner_id: String,
lease_id: Uuid,
lease_expires_at_unix_nanos: i128,
state: ManualTransitionJobState,
scan_completed: bool,
cancel_requested: bool,
created_at_unix_nanos: i128,
updated_at_unix_nanos: i128,
completed_at_unix_nanos: Option<i128>,
report: ManualTransitionRunReport,
queue_snapshot: ManualTransitionQueueSnapshot,
error: Option<String>,
}
let options = ManualTransitionRunOptions::default();
let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER);
let continuation_token =
encode_manual_transition_continuation_token(Some("logs/page-a".to_string()), Some("version-a".to_string()));
record.update_running_progress(
ManualTransitionRunReport {
bucket: "bucket".to_string(),
scanned: 7,
continuation_token: continuation_token.clone(),
..Default::default()
},
ManualTransitionQueueSnapshot::default(),
);
assert_eq!(record.cursor_revision, Some(7));
let encoded = record.encode().expect("job record should encode");
let value: serde_json::Value = serde_json::from_slice(&encoded).expect("encoded job should be json");
assert!(value["job"].get("cursor_revision").is_none());
let legacy: LegacyPersistedManualTransitionJobRecord =
serde_json::from_slice(&encoded).expect("old reader should accept new job record");
assert_eq!(legacy.job.job_id, record.job_id);
assert_eq!(legacy.job.report.continuation_token, continuation_token);
}
#[test]
fn manual_transition_job_record_rejects_unknown_report_fields() {
let options = ManualTransitionRunOptions::default();
@@ -16,7 +16,6 @@ pub mod bucket_lifecycle_audit;
pub mod bucket_lifecycle_ops;
mod config_boundary;
pub mod core;
mod durable_namespace;
pub mod evaluator;
pub mod manual_transition_job;
mod metadata_boundary;
@@ -32,8 +31,3 @@ pub mod tier_free_version_recovery;
pub mod tier_last_day_stats;
pub mod tier_sweeper;
pub mod transition_transaction;
pub(crate) use durable_namespace::{
DurableIlmRecordCheckpoint, ILM_META_PREFIX, ValidatedDurableIlmRecord, classify_durable_ilm_record,
validate_durable_ilm_record,
};
@@ -20,7 +20,6 @@ use tokio_util::sync::CancellationToken;
use tracing::{debug, warn};
use crate::bucket::lifecycle::config_boundary;
use crate::bucket::lifecycle::durable_namespace::TIER_DELETE_JOURNAL_NAMESPACE;
use crate::bucket::lifecycle::runtime_boundary;
use crate::bucket::lifecycle::tier_sweeper::{
Jentry, TierDeleteJournalState, TierDeleteSourceIdentity,
@@ -50,7 +49,7 @@ const TIER_DELETE_JOURNAL_VERSION: u8 = 2;
const TIER_DELETE_JOURNAL_EXACT_VERSION: u8 = 3;
const TIER_DELETE_JOURNAL_STATE_VERSION: u8 = 4;
const TIER_DELETE_JOURNAL_TRANSACTION_VERSION: u8 = 5;
pub(crate) const TIER_DELETE_JOURNAL_PREFIX: &str = TIER_DELETE_JOURNAL_NAMESPACE.prefix;
pub(crate) const TIER_DELETE_JOURNAL_PREFIX: &str = "ilm/tier-delete-journal/";
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
@@ -433,21 +432,6 @@ async fn process_committed_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jen
)
.await?;
}
let path = tier_delete_journal_object_name(je);
let data = encode_tier_delete_journal_entry(je).map_err(std::io::Error::other)?;
let target_pool_indices = api
.record_durable_ilm_decommission_terminal_target_pools(&path, &data)
.await
.map_err(std::io::Error::other)?;
if let Some(target_pool_indices) = target_pool_indices {
for target_pool_idx in target_pool_indices {
match config_boundary::delete_config(api.pools[target_pool_idx].clone(), &path).await {
Ok(()) | Err(Error::ConfigNotFound) => {}
Err(err) => return Err(std::io::Error::other(err)),
}
}
return Ok(());
}
remove_tier_delete_journal_entry(api, je).await
}
@@ -21,7 +21,6 @@ use tracing::{debug, warn};
use uuid::Uuid;
use crate::bucket::lifecycle::config_boundary;
use crate::bucket::lifecycle::durable_namespace::TRANSITION_TRANSACTION_NAMESPACE;
use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
use crate::bucket::lifecycle::tier_sweeper::{
delete_confirmed_transition_candidate_exact_with_lease_idempotent,
@@ -43,7 +42,7 @@ const TRANSITION_TRANSACTION_RECOVERY_INTERVAL: Duration = Duration::from_secs(6
const TRANSITION_TRANSACTION_RECOVERY_TIMEOUT: Duration = Duration::from_secs(300);
pub const TRANSITION_TRANSACTION_SCHEMA: &str = "rustfs-transition-transaction-v1";
pub const TRANSITION_TRANSACTION_PREFIX: &str = "ilm/transition-transactions";
pub const TRANSITION_TRANSACTION_RECORD_PREFIX: &str = TRANSITION_TRANSACTION_NAMESPACE.prefix;
pub const TRANSITION_TRANSACTION_RECORD_PREFIX: &str = "ilm/transition-transactions/records";
pub const MAX_TRANSITION_TRANSACTION_SIZE: usize = 64 * 1024;
pub type Result<T> = std::result::Result<T, TransitionTransactionError>;
@@ -585,8 +584,7 @@ pub(crate) async fn save_transition_transaction_record(
let object =
transition_transaction_record_object_name(transaction.transaction_id).map_err(transition_transaction_store_error)?;
let data = transaction.encode().map_err(transition_transaction_store_error)?;
config_boundary::save_config(api.clone(), &object, data.clone()).await?;
api.record_durable_ilm_decommission_progress(&object, &data).await
config_boundary::save_config(api, &object, data).await
}
pub(crate) async fn load_transition_transaction_record(
@@ -598,14 +596,8 @@ pub(crate) async fn load_transition_transaction_record(
TransitionTransaction::decode(transaction_id, &data).map_err(transition_transaction_store_error)
}
pub(crate) async fn delete_transition_transaction_record(
api: Arc<ECStore>,
transaction: &TransitionTransaction,
) -> EcstoreResult<()> {
let object =
transition_transaction_record_object_name(transaction.transaction_id).map_err(transition_transaction_store_error)?;
let data = transaction.encode().map_err(transition_transaction_store_error)?;
api.record_durable_ilm_decommission_terminal(&object, &data).await?;
pub(crate) async fn delete_transition_transaction_record(api: Arc<ECStore>, transaction_id: Uuid) -> EcstoreResult<()> {
let object = transition_transaction_record_object_name(transaction_id).map_err(transition_transaction_store_error)?;
match config_boundary::delete_config(api, &object).await {
Ok(()) | Err(Error::ConfigNotFound) => Ok(()),
Err(err) => Err(err),
@@ -821,7 +813,7 @@ pub async fn finalize_missing_transition_transaction_for_operator(
if probe != TransitionOperatorProbe::Missing {
return Err(TransitionOperatorError::CandidateNotMissing(probe));
}
delete_transition_transaction_record(api, &transaction)
delete_transition_transaction_record(api, transaction_id)
.await
.map_err(TransitionOperatorError::Store)
}
@@ -857,22 +849,22 @@ pub async fn process_transition_transaction_record(
match transaction.state {
TransitionTransactionState::Uploaded => {
delete_transition_remote_candidate(api.clone(), transaction).await?;
delete_transition_transaction_record(api, transaction).await?;
delete_transition_transaction_record(api, transaction.transaction_id).await?;
Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
}
TransitionTransactionState::CleanupPending => match local_commit_matches_transaction(api.clone(), transaction).await {
Ok(true) => {
delete_transition_transaction_record(api, transaction).await?;
delete_transition_transaction_record(api, transaction.transaction_id).await?;
Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
}
Ok(false) => {
delete_transition_remote_candidate(api.clone(), transaction).await?;
delete_transition_transaction_record(api, transaction).await?;
delete_transition_transaction_record(api, transaction.transaction_id).await?;
Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
}
Err(err) if transition_source_is_missing(&err) => {
delete_transition_remote_candidate(api.clone(), transaction).await?;
delete_transition_transaction_record(api, transaction).await?;
delete_transition_transaction_record(api, transaction.transaction_id).await?;
Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
}
Err(err) => Err(err),
@@ -880,7 +872,7 @@ pub async fn process_transition_transaction_record(
TransitionTransactionState::LocalCommitStarted => {
match local_commit_matches_transaction(api.clone(), transaction).await {
Ok(true) => {
delete_transition_transaction_record(api, transaction).await?;
delete_transition_transaction_record(api, transaction.transaction_id).await?;
Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
}
Ok(false) => Ok(TransitionTransactionRecoveryOutcome::Retained),
@@ -889,7 +881,7 @@ pub async fn process_transition_transaction_record(
}
}
TransitionTransactionState::AbortedNoRemote | TransitionTransactionState::Committed => {
delete_transition_transaction_record(api, transaction).await?;
delete_transition_transaction_record(api, transaction.transaction_id).await?;
Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
}
TransitionTransactionState::UploadOutcomeUnknown => recover_unknown_upload_outcome(api, transaction).await,
@@ -915,7 +907,7 @@ async fn recover_unknown_upload_outcome(
.map_err(Error::other)?
{
TransitionCandidateProbe::Missing => {
delete_transition_transaction_record(api, transaction).await?;
delete_transition_transaction_record(api, transaction.transaction_id).await?;
Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
}
TransitionCandidateProbe::UnversionedPresent => {
@@ -933,7 +925,7 @@ async fn recover_unknown_upload_outcome(
)
.await
.map_err(Error::other)?;
delete_transition_transaction_record(api, transaction).await?;
delete_transition_transaction_record(api, transaction.transaction_id).await?;
Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
}
TransitionCandidateProbe::VersionedPresent(version_id) => {
@@ -966,7 +958,7 @@ async fn cleanup_recovered_unknown_upload_candidate(
.map_err(transition_transaction_store_error)?;
save_transition_transaction_record(api.clone(), &cleanup).await?;
delete_transition_remote_candidate(api.clone(), &cleanup).await?;
delete_transition_transaction_record(api, &cleanup).await?;
delete_transition_transaction_record(api, cleanup.transaction_id).await?;
Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
}
-65
View File
@@ -1076,13 +1076,6 @@ pub async fn get_versioning_config(bucket: &str) -> Result<(VersioningConfigurat
bucket_meta_sys.get_versioning_config(bucket).await
}
pub(crate) async fn has_authoritative_never_versioned_state(bucket: &str) -> Result<bool> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone();
bucket_meta_sys.has_authoritative_never_versioned_state(bucket).await
}
pub async fn get_website_config(bucket: &str) -> Result<(WebsiteConfiguration, OffsetDateTime)> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
@@ -1921,18 +1914,6 @@ impl BucketMetadataSys {
}
}
async fn has_authoritative_never_versioned_state(&self, bucket: &str) -> Result<bool> {
let BucketMetadataAuthority::Authoritative(metadata) = self.get_metadata_authority(bucket).await? else {
return Ok(false);
};
if metadata.versioning_config.is_none() && !metadata.versioning_config_xml.is_empty() {
return Err(Error::other("persisted bucket versioning configuration is invalid"));
}
Ok(metadata.versioning_config.is_none() && metadata.versioning_config_xml.is_empty())
}
pub async fn get_bucket_policy(&self, bucket: &str) -> Result<(BucketPolicy, OffsetDateTime)> {
let bm = match self.get_metadata_authority(bucket).await? {
BucketMetadataAuthority::Authoritative(bm) => bm,
@@ -2488,10 +2469,6 @@ mod tests {
sys.get_versioning_config(bucket).await.is_err(),
"malformed versioning metadata must block destructive requests"
);
assert!(
sys.has_authoritative_never_versioned_state(bucket).await.is_err(),
"malformed versioning metadata must not enable listing shortcuts"
);
assert!(
sys.get_replication_config(bucket).await.is_err(),
"malformed replication metadata must not be reported as ConfigNotFound"
@@ -2515,50 +2492,8 @@ mod tests {
sys.get_object_lock_config_state("authoritative-empty").await.unwrap(),
ObjectLockConfigState::ConfirmedAbsent
));
assert!(
sys.has_authoritative_never_versioned_state("authoritative-empty")
.await
.unwrap(),
"authoritative config absence should identify a never-versioned bucket"
);
assert!(matches!(sys.get_bucket_policy("authoritative-empty").await, Err(Error::ConfigNotFound)));
let mut versioned = BucketMetadata::new("authoritative-versioned");
versioned.versioning_config_xml = b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec();
versioned.versioning_config = Some(VersioningConfiguration {
status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::ENABLED)),
..Default::default()
});
sys.set("authoritative-versioned".to_string(), Arc::new(versioned)).await;
assert!(
!sys.has_authoritative_never_versioned_state("authoritative-versioned")
.await
.unwrap(),
"versioned buckets must retain delete-marker visibility probes"
);
let mut ambiguous = BucketMetadata::new("authoritative-ambiguous-versioning");
ambiguous.versioning_config = Some(VersioningConfiguration::default());
sys.set("authoritative-ambiguous-versioning".to_string(), Arc::new(ambiguous))
.await;
assert!(
!sys.has_authoritative_never_versioned_state("authoritative-ambiguous-versioning")
.await
.unwrap(),
"ambiguous versioning metadata must retain delete-marker visibility probes"
);
sys.fabricated_metadata
.write()
.await
.insert("fabricated-versioning".to_string());
assert!(
!sys.has_authoritative_never_versioned_state("fabricated-versioning")
.await
.unwrap(),
"fabricated metadata must retain delete-marker visibility probes"
);
for dir in &dirs {
std::fs::create_dir_all(dir.path().join("policy-only-legacy")).unwrap();
}
@@ -15,7 +15,7 @@
use crate::bucket::metadata_sys::{ObjectLockConfigState, get_object_lock_config, get_object_lock_config_state};
use crate::bucket::object_lock::objectlock;
use crate::error::{Error, Result, StorageError};
use crate::object_api::{ObjectInfo, ObjectOptions};
use crate::object_api::ObjectInfo;
use s3s::dto::{Date, DefaultRetention, ObjectLockConfiguration, ObjectLockLegalHoldStatus, ObjectLockRetentionMode};
use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE};
use std::sync::Arc;
@@ -136,50 +136,12 @@ pub fn add_years(dt: OffsetDateTime, years: i32) -> OffsetDateTime {
/// Check if an object has legal hold enabled.
/// Returns true if legal hold is ON.
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
fn has_legal_hold(user_defined: &std::collections::HashMap<String, String>) -> bool {
let lhold = objectlock::get_object_legalhold_meta(user_defined);
matches!(lhold.status, Some(ref st) if st.as_str() == ObjectLockLegalHoldStatus::ON)
}
/// Whether an authorized replication write (`ObjectOptions::replication_request`)
/// may overwrite a locked destination version.
///
/// The source's lock state governs a replica (MinIO `checkPutObjectLockAllowed`
/// skips the existing-version check for replicas), and a source-side hold
/// release or retention change reaches this site only through this write. The
/// overwrite is allowed only when the write carries the source timestamp of
/// every category that currently locks the version, so receiver-side LWW
/// (`merge_replication_metadata_lww`) judges each of them: a category locked
/// more recently here is kept, otherwise the source's newer state wins. A write
/// without that timestamp carries no source decision for the category — the
/// metadata replace would lift the lock unjudged — so it stays WORM-rejected.
///
/// The locking categories come from the same authoritative evaluation as the
/// commit-time WORM gate (`check_object_lock_for_deletion_with_state`): the
/// bucket default retention locks a version that carries no explicit
/// retention keys, so it is judged here too rather than read off the keys.
/// Malformed persisted lock metadata or a non-authoritative bucket
/// configuration is an error, never a pass.
pub fn replication_write_may_pass_worm_gate(
state: &ObjectLockConfigState,
obj_info: &ObjectInfo,
opts: &ObjectOptions,
) -> Result<bool> {
if !opts.replication_request {
return Ok(false);
}
if obj_info.delete_marker {
// Delete markers are never locked (same as the WORM gate).
return Ok(true);
}
let config = object_lock_config_from_state(state)?;
if legal_hold_locks(obj_info)? && opts.replication_legalhold_timestamp.is_none() {
return Ok(false);
}
let retention_locked = active_retention(config, obj_info)?.is_some();
Ok(!(retention_locked && opts.replication_retention_timestamp.is_none()))
}
/// Check if an object is locked based on its metadata.
/// This is a common function used by both lifecycle evaluation and deletion checks.
///
@@ -277,101 +239,69 @@ pub(crate) fn check_object_lock_for_deletion_with_config(
return Ok(None);
}
if legal_hold_locks(obj_info)? {
return Ok(Some(ObjectLockBlockReason::LegalHold));
if let Some(status) = obj_info.user_defined.get(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()) {
if status.eq_ignore_ascii_case(ObjectLockLegalHoldStatus::ON) {
return Ok(Some(ObjectLockBlockReason::LegalHold));
}
if !status.eq_ignore_ascii_case(ObjectLockLegalHoldStatus::OFF) {
return Err(Error::other("persisted object legal-hold metadata is invalid"));
}
}
if let Some((mode_str, retain_until)) = active_retention(config, obj_info)?
&& let Some(reason) = check_retention_blocks_deletion(mode_str, Some(retain_until), bypass_governance)
{
return Ok(Some(reason));
}
Ok(None)
}
/// A cleared retention / legal hold is persisted as empty strings (the MinIO
/// on-disk shape, `parse_object_lock_retention`); read it as "no lock" rather
/// than as corrupt metadata.
fn persisted_lock_value<'a>(obj_info: &'a ObjectInfo, key: &str) -> Option<&'a String> {
obj_info.user_defined.get(key).filter(|value| !value.is_empty())
}
/// Whether the version's persisted legal hold is ON. Any other non-empty
/// value than ON/OFF is malformed metadata and fails closed.
fn legal_hold_locks(obj_info: &ObjectInfo) -> Result<bool> {
let Some(status) = persisted_lock_value(obj_info, X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()) else {
return Ok(false);
};
if status.eq_ignore_ascii_case(ObjectLockLegalHoldStatus::ON) {
return Ok(true);
}
if !status.eq_ignore_ascii_case(ObjectLockLegalHoldStatus::OFF) {
return Err(Error::other("persisted object legal-hold metadata is invalid"));
}
Ok(false)
}
/// The retention that currently locks the version, if any: the explicit
/// persisted retention when the keys are present, otherwise the bucket
/// default retention computed from the version's modification time. Returns
/// `(mode, retain_until)` only while the retention is still active.
fn active_retention<'a>(
config: Option<&'a ObjectLockConfiguration>,
obj_info: &ObjectInfo,
) -> Result<Option<(&'a str, OffsetDateTime)>> {
let mode = persisted_lock_value(obj_info, X_AMZ_OBJECT_LOCK_MODE.as_str());
let retain_until = persisted_lock_value(obj_info, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str());
match (mode, retain_until) {
(None, None) => {}
let mode = obj_info.user_defined.get(X_AMZ_OBJECT_LOCK_MODE.as_str());
let retain_until = obj_info.user_defined.get(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str());
let explicit_ret = match (mode, retain_until) {
(None, None) => None,
(Some(mode), Some(retain_until)) => {
let mode =
objectlock::parse_ret_mode(mode).ok_or_else(|| Error::other("persisted object retention mode is invalid"))?;
let retain_until = OffsetDateTime::parse(retain_until, &time::format_description::well_known::Iso8601::DEFAULT)
.map(Date::from)
.map_err(|_| Error::other("persisted object retention date is invalid"))?;
let mode_str = match mode.as_str() {
ObjectLockRetentionMode::COMPLIANCE => ObjectLockRetentionMode::COMPLIANCE,
ObjectLockRetentionMode::GOVERNANCE => ObjectLockRetentionMode::GOVERNANCE,
_ => return Err(Error::other("persisted object retention mode is invalid")),
};
return Ok(is_retention_active(mode_str, Some(&retain_until)).then(|| (mode_str, OffsetDateTime::from(retain_until))));
Some((mode, retain_until))
}
_ => return Err(Error::other("persisted object retention metadata is incomplete")),
};
if let Some((mode, retain_until)) = &explicit_ret {
let mode_str = mode.as_str();
if is_retention_active(mode_str, Some(retain_until))
&& let Some(reason) =
check_retention_blocks_deletion(mode_str, Some(OffsetDateTime::from(retain_until.clone())), bypass_governance)
{
return Ok(Some(reason));
}
}
let Some(default_retention) = config.and_then(|config| config.rule.as_ref()?.default_retention.as_ref()) else {
return Ok(None);
};
let Some(mode) = &default_retention.mode else {
return Ok(None);
};
let mode_str = mode.as_str();
if mode_str != ObjectLockRetentionMode::COMPLIANCE && mode_str != ObjectLockRetentionMode::GOVERNANCE {
return Ok(None);
}
// Calculate retention expiration date from object modification time
let mod_time = obj_info
.mod_time
.ok_or_else(|| Error::other("persisted object modification time is missing"))?;
let now = objectlock::utc_now_ntp();
let retain_until = if let Some(days) = default_retention.days {
mod_time.saturating_add(time::Duration::days(i64::from(days)))
} else {
let years = default_retention
.years
.ok_or_else(|| Error::other("persisted bucket Object Lock retention period is invalid"))?;
add_years(mod_time, years)
};
Ok((retain_until.unix_timestamp() > now.unix_timestamp()).then_some((mode_str, retain_until)))
}
if explicit_ret.is_none()
&& let Some(default_retention) = config.and_then(|config| config.rule.as_ref()?.default_retention.as_ref())
&& let Some(mode) = &default_retention.mode
{
let mode_str = mode.as_str();
if mode_str == ObjectLockRetentionMode::COMPLIANCE || mode_str == ObjectLockRetentionMode::GOVERNANCE {
// Calculate retention expiration date from object modification time
let mod_time = obj_info
.mod_time
.ok_or_else(|| Error::other("persisted object modification time is missing"))?;
let now = objectlock::utc_now_ntp();
let retain_until = if let Some(days) = default_retention.days {
mod_time.saturating_add(time::Duration::days(i64::from(days)))
} else {
let years = default_retention
.years
.ok_or_else(|| Error::other("persisted bucket Object Lock retention period is invalid"))?;
add_years(mod_time, years)
};
fn object_lock_config_from_state(state: &ObjectLockConfigState) -> Result<Option<&ObjectLockConfiguration>> {
match state {
ObjectLockConfigState::Configured { config, .. } => Ok(Some(config)),
ObjectLockConfigState::ConfirmedAbsent => Ok(None),
ObjectLockConfigState::Fabricated => Err(Error::other("bucket Object Lock metadata is not authoritative")),
if retain_until.unix_timestamp() > now.unix_timestamp()
&& let Some(reason) = check_retention_blocks_deletion(mode_str, Some(retain_until), bypass_governance)
{
return Ok(Some(reason));
}
}
}
Ok(None)
}
pub(crate) fn check_object_lock_for_deletion_with_state(
@@ -379,7 +309,13 @@ pub(crate) fn check_object_lock_for_deletion_with_state(
obj_info: &ObjectInfo,
bypass_governance: bool,
) -> Result<Option<ObjectLockBlockReason>> {
check_object_lock_for_deletion_with_config(object_lock_config_from_state(state)?, obj_info, bypass_governance)
match state {
ObjectLockConfigState::Configured { config, .. } => {
check_object_lock_for_deletion_with_config(Some(config), obj_info, bypass_governance)
}
ObjectLockConfigState::ConfirmedAbsent => check_object_lock_for_deletion_with_config(None, obj_info, bypass_governance),
ObjectLockConfigState::Fabricated => Err(Error::other("bucket Object Lock metadata is not authoritative")),
}
}
/// Compatibility wrapper for callers that predate fallible metadata lookup.
@@ -550,210 +486,6 @@ mod tests {
}
}
fn replication_opts(hold_ts: bool, retention_ts: bool) -> ObjectOptions {
ObjectOptions {
replication_request: true,
replication_legalhold_timestamp: hold_ts.then_some(OffsetDateTime::UNIX_EPOCH),
replication_retention_timestamp: retention_ts.then_some(OffsetDateTime::UNIX_EPOCH),
..Default::default()
}
}
fn lock_metadata(entries: &[&[(&str, &str)]]) -> std::collections::HashMap<String, String> {
entries
.iter()
.flat_map(|entries| entries.iter())
.map(|(key, value)| (key.to_string(), value.to_string()))
.collect()
}
fn lock_object_info(user_defined: std::collections::HashMap<String, String>) -> ObjectInfo {
ObjectInfo {
user_defined: Arc::new(user_defined),
mod_time: Some(OffsetDateTime::now_utc()),
..Default::default()
}
}
/// A replication write passes the WORM gate only when it carries the
/// source timestamp of every category that currently locks the version.
#[test]
fn replication_write_passes_worm_gate_only_with_every_locking_category_timestamp() {
use rustfs_utils::http::headers::{
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
};
let hold = [(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, "ON")];
let retention = [
(AMZ_OBJECT_LOCK_MODE_LOWER, "GOVERNANCE"),
(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, "2099-01-01T00:00:00Z"),
];
let expired = [
(AMZ_OBJECT_LOCK_MODE_LOWER, "COMPLIANCE"),
(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, "2000-01-01T00:00:00Z"),
];
let absent = ObjectLockConfigState::ConfirmedAbsent;
let passes = |state: &ObjectLockConfigState, entries: &[&[(&str, &str)]], opts: &ObjectOptions| {
replication_write_may_pass_worm_gate(state, &lock_object_info(lock_metadata(entries)), opts)
.expect("well-formed lock metadata must be judged")
};
assert!(passes(&absent, &[&hold, &retention], &replication_opts(true, true)));
assert!(!passes(&absent, &[&hold, &retention], &replication_opts(true, false)));
assert!(!passes(&absent, &[&hold, &retention], &replication_opts(false, true)));
assert!(passes(&absent, &[&hold], &replication_opts(true, false)));
assert!(!passes(&absent, &[&hold], &replication_opts(false, true)));
assert!(passes(&absent, &[&retention], &replication_opts(false, true)));
assert!(!passes(&absent, &[&retention], &replication_opts(true, false)));
// Expired retention and a released hold no longer lock anything.
assert!(passes(
&absent,
&[&expired, &[(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, "OFF")]],
&replication_opts(false, false)
));
// Never for a non-replication write, whatever it carries.
let local = ObjectOptions {
replication_request: false,
..replication_opts(true, true)
};
assert!(!passes(&absent, &[&hold], &local));
}
/// The bucket default retention locks a version that carries no explicit
/// retention keys (`check_object_lock_for_deletion_with_config` judges it
/// from the modification time), so the replication bypass must demand the
/// retention source timestamp for it too — a tagging-only replication
/// write must not overwrite the default-protected version unjudged.
#[test]
fn replication_write_under_bucket_default_retention_requires_retention_timestamp() {
use rustfs_utils::http::headers::{AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER};
for mode in [ObjectLockRetentionMode::COMPLIANCE, ObjectLockRetentionMode::GOVERNANCE] {
let state = ObjectLockConfigState::Configured {
config: default_retention_config(mode),
updated_at: OffsetDateTime::now_utc(),
};
let no_keys = lock_object_info(std::collections::HashMap::new());
assert!(
check_object_lock_for_deletion_with_state(&state, &no_keys, false)
.expect("default retention must be judged")
.is_some(),
"{mode}: the gate must report the default retention lock"
);
let tagging_only = ObjectOptions {
replication_request: true,
replication_tagging_timestamp: Some(OffsetDateTime::UNIX_EPOCH),
..Default::default()
};
assert!(
!replication_write_may_pass_worm_gate(&state, &no_keys, &tagging_only).expect("judged"),
"{mode}: a tagging-only replication write must not pass the default retention lock"
);
assert!(
replication_write_may_pass_worm_gate(&state, &no_keys, &replication_opts(false, true)).expect("judged"),
"{mode}: the retention source timestamp lets LWW judge the default retention"
);
// Default retention plus a legal hold: both categories need a timestamp.
let held = lock_object_info(lock_metadata(&[&[(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, "ON")]]));
assert!(!replication_write_may_pass_worm_gate(&state, &held, &replication_opts(false, true)).expect("judged"));
assert!(!replication_write_may_pass_worm_gate(&state, &held, &replication_opts(true, false)).expect("judged"));
assert!(replication_write_may_pass_worm_gate(&state, &held, &replication_opts(true, true)).expect("judged"));
// A version whose default retention has already expired (old
// mod_time) is not locked by the default any more.
let expired_default = ObjectInfo {
mod_time: Some(make_datetime(2000, 1, 1)),
..lock_object_info(std::collections::HashMap::new())
};
assert!(replication_write_may_pass_worm_gate(&state, &expired_default, &tagging_only).expect("judged"));
// A delete marker is never locked, so there is nothing to judge.
let delete_marker = ObjectInfo {
delete_marker: true,
..lock_object_info(std::collections::HashMap::new())
};
assert!(replication_write_may_pass_worm_gate(&state, &delete_marker, &tagging_only).expect("judged"));
// Cleared (empty) explicit keys fall back to the bucket default.
let cleared = lock_object_info(lock_metadata(&[&[(AMZ_OBJECT_LOCK_MODE_LOWER, "")]]));
assert!(!replication_write_may_pass_worm_gate(&state, &cleared, &tagging_only).expect("judged"));
}
}
/// The replication bypass never judges from a non-authoritative bucket
/// state or malformed persisted lock metadata; both are errors, not a pass.
#[test]
fn replication_write_worm_gate_fails_closed_on_unverifiable_lock_state() {
use rustfs_utils::http::headers::AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER;
let opts = replication_opts(true, true);
let err = replication_write_may_pass_worm_gate(
&ObjectLockConfigState::Fabricated,
&lock_object_info(std::collections::HashMap::new()),
&opts,
)
.expect_err("fabricated bucket lock metadata must not be judged");
assert!(err.to_string().contains("not authoritative"));
let malformed = lock_object_info(lock_metadata(&[&[(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, "MAYBE")]]));
let err = replication_write_may_pass_worm_gate(&ObjectLockConfigState::ConfirmedAbsent, &malformed, &opts)
.expect_err("malformed legal hold must not be judged");
assert!(err.to_string().contains("legal-hold"));
let state = ObjectLockConfigState::Configured {
config: default_retention_config(ObjectLockRetentionMode::COMPLIANCE),
updated_at: OffsetDateTime::now_utc(),
};
let no_mod_time = ObjectInfo::default();
let err = replication_write_may_pass_worm_gate(&state, &no_mod_time, &opts)
.expect_err("default retention without a modification time must not be judged");
assert!(err.to_string().contains("modification time"));
}
/// A local PutObjectRetention / PutObjectLegalHold "clear" persists the
/// lock keys as empty strings (the MinIO on-disk shape, see
/// `parse_object_lock_retention`); that is "no lock", not corruption, and
/// must not wedge later explicit-version PUTs or deletes
/// (rustfs/backlog#1953).
#[test]
fn deletion_treats_cleared_empty_lock_metadata_as_unlocked() {
use rustfs_utils::http::headers::{
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
};
let cases: [(&str, &[&str]); 3] = [
(
"cleared retention",
&[AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER],
),
("cleared legal hold", &[AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER]),
(
"all cleared",
&[
AMZ_OBJECT_LOCK_MODE_LOWER,
AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER,
],
),
];
for (case, keys) in cases {
let user_defined = keys.iter().map(|key| (key.to_string(), String::new())).collect();
let obj_info = ObjectInfo {
user_defined: Arc::new(user_defined),
..Default::default()
};
let result = check_object_lock_for_deletion_with_config(None, &obj_info, false);
assert!(matches!(result, Ok(None)), "{case}: empty lock keys must read as unlocked: {result:?}");
}
}
#[test]
fn deletion_rejects_invalid_persisted_legal_hold_metadata() {
let mut user_defined = std::collections::HashMap::new();
+5 -8
View File
@@ -44,14 +44,11 @@ mod replication_versioning_boundary;
mod runtime_boundary;
pub use replication_config_boundary::{
ObjectOpts, OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS,
REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS,
REPLICATION_WRITABLE_FIELDS, ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationTargetValidationError,
assign_site_replication_rule_priorities, invalid_replication_config_status_field, is_site_replication_role,
is_site_replication_rule, merge_incoming_replication_config, merge_user_replication_config,
replication_target_arn_deployment_id, replication_target_arns, should_remove_replication_target,
site_replication_rule_deployment_id, unsupported_replication_config_field, validate_replication_config_structure,
validate_replication_config_target_arns,
ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationTargetValidationError,
invalid_replication_config_status_field, replication_target_arns, should_remove_replication_target,
unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
};
pub(crate) use replication_filemeta_boundary::version_purge_statuses_map;
pub use replication_filemeta_boundary::{
@@ -13,12 +13,9 @@
// limitations under the License.
pub use rustfs_replication::{
ObjectOpts, OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS,
REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS,
REPLICATION_WRITABLE_FIELDS, ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationRuleExt,
ReplicationTargetValidationError, assign_site_replication_rule_priorities, invalid_replication_config_status_field,
is_site_replication_role, is_site_replication_rule, merge_incoming_replication_config, merge_user_replication_config,
replication_target_arn_deployment_id, replication_target_arns, should_remove_replication_target,
site_replication_rule_deployment_id, unsupported_replication_config_field, validate_replication_config_structure,
validate_replication_config_target_arns,
ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationRuleExt, ReplicationTargetValidationError,
invalid_replication_config_status_field, replication_target_arns, should_remove_replication_target,
unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
};
@@ -39,7 +39,7 @@ use super::replication_resync_boundary::{
};
use super::replication_resyncer::{
ReplicationResyncer, get_heal_replicate_object_info, replicate_delete, replicate_delete_with_outcome, replicate_object,
replicate_object_with_outcome, update_resync_status_cas,
replicate_object_with_outcome, save_resync_status,
};
use super::replication_state::ReplicationStats;
use super::replication_storage_boundary::{
@@ -1898,7 +1898,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
}
};
let bucket_status = load_bucket_resync_metadata(&opts.bucket, self.storage.clone()).await?;
let mut bucket_status = load_bucket_resync_metadata(&opts.bucket, self.storage.clone()).await?;
if let Some(active) = bucket_status.targets_map.get(&opts.arn) {
if active.resync_id == opts.resync_id {
self.resyncer
@@ -1924,43 +1924,26 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
}
let now = OffsetDateTime::now_utc();
let admitted = TargetReplicationResyncStatus {
start_time: Some(now),
last_update: Some(now),
resync_id: opts.resync_id.clone(),
resync_before_date: opts.resync_before,
resync_status: ResyncStatusType::ResyncPending,
failed_size: 0,
failed_count: 0,
replicated_size: 0,
replicated_count: 0,
bucket: opts.bucket.clone(),
object: String::new(),
error: None,
};
bucket_status.last_update = Some(now);
bucket_status.targets_map.insert(
opts.arn.clone(),
TargetReplicationResyncStatus {
start_time: Some(now),
last_update: Some(now),
resync_id: opts.resync_id.clone(),
resync_before_date: opts.resync_before,
resync_status: ResyncStatusType::ResyncPending,
failed_size: 0,
failed_count: 0,
replicated_size: 0,
replicated_count: 0,
bucket: opts.bucket.clone(),
object: String::new(),
error: None,
},
);
// The admission lock serializes competing admissions, but status
// writers (mark_status, the periodic saver) do not take it — write
// through the CAS so their concurrent updates to other targets are
// never lost, re-checking the conflict gate on each retry.
let (bucket_status, _) = update_resync_status_cas(&opts.bucket, self.storage.clone(), |persisted| {
if let Some(active) = persisted.targets_map.get(&opts.arn) {
if active.resync_id == opts.resync_id {
return Ok(false);
}
if should_auto_resume_resync(active.resync_status) {
return Err(EcstoreError::other(ResyncActiveConflictError {
bucket: opts.bucket.clone(),
arn: opts.arn.clone(),
active_resync_id: active.resync_id.clone(),
}));
}
}
persisted.last_update = Some(now);
persisted.targets_map.insert(opts.arn.clone(), admitted.clone());
Ok(true)
})
.await?;
save_resync_status(&opts.bucket, &bucket_status, self.storage.clone()).await?;
self.resyncer
.status_map
.write()
@@ -1970,83 +1953,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
Ok(true)
}
/// Cancel the pending/started resync intent recorded for `arn`, if any,
/// because its remote target is being removed. Returns the canceled run.
///
/// Runs under the bucket admission lock and reloads `resync.bin` from disk
/// before writing, like admission does: this node's `status_map` entry may
/// be stale relative to intents admitted by other nodes, and persisting it
/// would silently drop their durable restart intents.
pub async fn cancel_bucket_resync_for_removed_target(
self: Arc<Self>,
bucket: &str,
arn: &str,
) -> Result<Option<ResyncOpts>, EcstoreError> {
let bucket = bucket.to_string();
let arn = arn.to_string();
tokio::spawn(async move { self.cancel_bucket_resync_for_removed_target_transaction(bucket, arn).await })
.await
.map_err(|error| EcstoreError::other(format!("replication resync cancellation task failed: {error}")))?
}
async fn cancel_bucket_resync_for_removed_target_transaction(
self: Arc<Self>,
bucket: String,
arn: String,
) -> Result<Option<ResyncOpts>, EcstoreError> {
let admission_lock_key = ReplicationMetadataStore::resync_admission_lock_key(&bucket);
let admission_lock = self
.storage
.new_ns_lock(ReplicationMetadataStore::rustfs_meta_bucket(), &admission_lock_key)
.await?;
// Lock order: bucket resync admission lock -> resync status config-object lock.
let _admission_guard = admission_lock
.get_write_lock(ReplicationLockTiming::acquire_timeout())
.await
.map_err(EcstoreError::from)?;
let mut canceled: Option<ResyncOpts> = None;
let (final_map, _) = update_resync_status_cas(&bucket, self.storage.clone(), |persisted| {
canceled = None;
let Some(intent) = persisted.targets_map.get_mut(&arn) else {
return Ok(false);
};
if !should_auto_resume_resync(intent.resync_status) {
return Ok(false);
}
let now = OffsetDateTime::now_utc();
canceled = Some(ResyncOpts {
bucket: bucket.clone(),
arn: arn.clone(),
resync_id: intent.resync_id.clone(),
resync_before: intent.resync_before_date,
});
intent.resync_status = ResyncStatusType::ResyncCanceled;
intent.last_update = Some(now);
persisted.last_update = Some(now);
Ok(true)
})
.await?;
// Converge only the removed target's cached entry: cached progress
// counters for this node's other running targets stay authoritative.
{
let mut status_map = self.resyncer.status_map.write().await;
let cached = status_map
.entry(bucket.clone())
.or_insert_with(BucketReplicationResyncStatus::new);
if let Some(final_target) = final_map.targets_map.get(&arn) {
cached.targets_map.insert(arn.clone(), final_target.clone());
cached.last_update = final_map.last_update.or(cached.last_update);
}
}
if let Some(opts) = &canceled {
self.resyncer.cancel(opts).await;
}
Ok(canceled)
}
pub async fn activate_bucket_resync(self: Arc<Self>, opts: ResyncOpts, recovering: bool) -> Result<(), EcstoreError> {
let bucket_status = load_bucket_resync_metadata(&opts.bucket, self.storage.clone()).await?;
let Some(target_status) = bucket_status.targets_map.get(&opts.arn) else {
@@ -2804,11 +2710,6 @@ pub trait ReplicationPoolTrait: std::fmt::Debug {
async fn resize(&self, priority: ReplicationPriority, max_workers: usize, max_l_workers: usize);
async fn get_bucket_resync_status(&self, bucket: &str) -> Result<BucketReplicationResyncStatus, EcstoreError>;
async fn cancel_bucket_resync(&self, opts: ResyncOpts) -> Result<(), EcstoreError>;
async fn cancel_bucket_resync_for_removed_target(
self: Arc<Self>,
bucket: &str,
arn: &str,
) -> Result<Option<ResyncOpts>, EcstoreError>;
async fn admit_bucket_resync(self: Arc<Self>, opts: ResyncOpts) -> Result<bool, EcstoreError>;
async fn activate_bucket_resync(self: Arc<Self>, opts: ResyncOpts, recovering: bool) -> Result<(), EcstoreError>;
async fn start_bucket_resync(self: Arc<Self>, opts: ResyncOpts) -> Result<(), EcstoreError>;
@@ -2862,14 +2763,6 @@ impl<S: ReplicationStorage> ReplicationPoolTrait for ReplicationPool<S> {
self.cancel_bucket_resync(opts).await
}
async fn cancel_bucket_resync_for_removed_target(
self: Arc<Self>,
bucket: &str,
arn: &str,
) -> Result<Option<ResyncOpts>, EcstoreError> {
ReplicationPool::<S>::cancel_bucket_resync_for_removed_target(self, bucket, arn).await
}
async fn admit_bucket_resync(self: Arc<Self>, opts: ResyncOpts) -> Result<bool, EcstoreError> {
self.admit_bucket_resync(opts).await
}
@@ -3348,17 +3241,11 @@ mod tests {
async fn put_object(
&self,
bucket: &str,
_bucket: &str,
object: &str,
data: &mut Self::PutObjectReader,
opts: &Self::ObjectOptions,
) -> Result<Self::ObjectInfo, Self::Error> {
let _lock_guard = if opts.no_lock {
None
} else {
let lock = self.new_ns_lock(bucket, object).await?;
Some(lock.get_write_lock(Duration::from_secs(10)).await?)
};
if opts.http_preconditions.is_some()
&& let Some(replacement) = self
.shared
@@ -4004,157 +3891,6 @@ mod tests {
assert_eq!(pool.resyncer.cancel_tokens.read().await.len(), 1);
}
/// Removing a target on node A must cancel only A's intent. Node A's cached
/// status map predates node B's admission, so a cancel that persisted the
/// cache would erase B's durable restart intent for `arn:second`.
#[tokio::test]
async fn removed_target_cancel_preserves_intents_admitted_on_other_nodes() {
let shared = empty_resync_shared_state();
let node_a = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", shared.clone()))).await;
let node_b = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-b", shared.clone()))).await;
let bucket = "removed-target-cancel";
assert!(
node_a
.clone()
.admit_bucket_resync(test_resync_opts(bucket, "arn:first", "run-a"))
.await
.expect("node A admission should persist")
);
assert!(
node_b
.clone()
.admit_bucket_resync(test_resync_opts(bucket, "arn:second", "run-b"))
.await
.expect("node B admission should persist")
);
assert!(
!node_a.resyncer.status_map.read().await[bucket]
.targets_map
.contains_key("arn:second"),
"precondition: node A's cache must be stale relative to node B's admission"
);
let canceled = node_a
.clone()
.cancel_bucket_resync_for_removed_target(bucket, "arn:first")
.await
.expect("cancel should succeed");
assert_eq!(canceled.map(|opts| opts.resync_id), Some("run-a".to_string()));
let persisted = decode_resync_file(&shared.data.lock().expect("test data lock should not be poisoned"))
.expect("persisted status should decode");
assert_eq!(persisted.targets_map["arn:first"].resync_status, ResyncStatusType::ResyncCanceled);
assert_eq!(persisted.targets_map["arn:second"].resync_status, ResyncStatusType::ResyncPending);
assert_eq!(persisted.targets_map["arn:second"].resync_id, "run-b");
// Cache convergence is per-target: only the removed ARN is written
// back (a running target's cached progress counters stay
// authoritative), so node A's cache reflects the cancel while the
// persisted document remains the authority for `arn:second`.
assert_eq!(
node_a.resyncer.status_map.read().await[bucket].targets_map["arn:first"].resync_status,
ResyncStatusType::ResyncCanceled
);
let untouched = node_a
.clone()
.cancel_bucket_resync_for_removed_target(bucket, "arn:first")
.await
.expect("cancel of a terminal intent should be a no-op");
assert!(untouched.is_none());
assert!(
node_a
.clone()
.cancel_bucket_resync_for_removed_target(bucket, "arn:missing")
.await
.expect("cancel of an unknown arn should be a no-op")
.is_none()
);
}
/// The reviewer's resurrect scenario: after node A cancels `arn:first`, a
/// status write from node B — whose cache still holds the pre-cancel map —
/// must not flip `arn:first` back to `Pending` on disk. `mark_status` now
/// persists through the CAS with per-target guards instead of blind-saving
/// its cached whole-bucket map.
#[tokio::test]
async fn stale_peer_status_write_cannot_resurrect_canceled_intent() {
let shared = empty_resync_shared_state();
let node_a = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", shared.clone()))).await;
let node_b = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-b", shared.clone()))).await;
let bucket = "stale-peer-write";
assert!(
node_a
.clone()
.admit_bucket_resync(test_resync_opts(bucket, "arn:first", "run-a"))
.await
.expect("node A admission should persist")
);
assert!(
node_b
.clone()
.admit_bucket_resync(test_resync_opts(bucket, "arn:second", "run-b"))
.await
.expect("node B admission should persist")
);
// Seed node B's stale cache: it saw the map before A's cancel.
let pre_cancel = decode_resync_file(&shared.data.lock().expect("test data lock should not be poisoned"))
.expect("pre-cancel status should decode");
node_b
.resyncer
.status_map
.write()
.await
.insert(bucket.to_string(), pre_cancel);
node_a
.clone()
.cancel_bucket_resync_for_removed_target(bucket, "arn:first")
.await
.expect("cancel should succeed")
.expect("cancel should report the canceled run");
node_b
.resyncer
.mark_status(
ResyncStatusType::ResyncStarted,
test_resync_opts(bucket, "arn:second", "run-b"),
node_b.storage.clone(),
)
.await
.expect("peer status write should succeed");
let persisted = decode_resync_file(&shared.data.lock().expect("test data lock should not be poisoned"))
.expect("persisted status should decode");
assert_eq!(
persisted.targets_map["arn:first"].resync_status,
ResyncStatusType::ResyncCanceled,
"peer's stale cache must not resurrect the canceled intent"
);
assert_eq!(persisted.targets_map["arn:second"].resync_status, ResyncStatusType::ResyncStarted);
// And the reverse guard: a stale write for the canceled target itself
// is refused outright.
node_b
.resyncer
.mark_status(
ResyncStatusType::ResyncStarted,
test_resync_opts(bucket, "arn:first", "run-a"),
node_b.storage.clone(),
)
.await
.expect("guarded status write should be skipped, not fail");
let persisted = decode_resync_file(&shared.data.lock().expect("test data lock should not be poisoned"))
.expect("persisted status should decode");
assert_eq!(persisted.targets_map["arn:first"].resync_status, ResyncStatusType::ResyncCanceled);
assert_eq!(
node_b.resyncer.status_map.read().await[bucket].targets_map["arn:first"].resync_status,
ResyncStatusType::ResyncCanceled,
"the refused writer's cache must converge to the persisted terminal state"
);
}
#[tokio::test]
async fn admitted_resync_waits_for_target_metadata_commit_before_activation() {
let shared = empty_resync_shared_state();
@@ -4294,77 +4030,6 @@ mod tests {
);
}
#[tokio::test]
async fn concurrent_resync_status_cas_preserves_both_mutations() {
let shared = empty_resync_shared_state();
let mut seeded = BucketReplicationResyncStatus::new();
for arn in ["arn:a", "arn:b"] {
seeded.targets_map.insert(
arn.to_string(),
TargetReplicationResyncStatus {
bucket: "cas-race".to_string(),
resync_id: format!("run-{arn}"),
resync_status: ResyncStatusType::ResyncPending,
..Default::default()
},
);
}
*shared.data.lock().expect("test data lock should not be poisoned") =
encode_resync_file(&seeded).expect("seeded resync status should encode");
shared.empty_object_exists.store(true, Ordering::SeqCst);
shared.etag_revision.store(1, Ordering::SeqCst);
shared.block_next_write.store(true, Ordering::SeqCst);
let node_a = Arc::new(LoadResyncNodeStore::new("cas-node-a", shared.clone()));
let node_b = Arc::new(LoadResyncNodeStore::new("cas-node-b", shared.clone()));
let writer_a = tokio::spawn(async move {
update_resync_status_cas("cas-race", node_a, |status| {
status
.targets_map
.get_mut("arn:a")
.expect("seeded target A should exist")
.resync_status = ResyncStatusType::ResyncCanceled;
Ok(true)
})
.await
});
tokio::time::timeout(Duration::from_secs(10), shared.write_started.notified())
.await
.expect("writer A should pause after its precondition check");
let mut writer_b = tokio::spawn(async move {
update_resync_status_cas("cas-race", node_b, |status| {
status
.targets_map
.get_mut("arn:b")
.expect("seeded target B should exist")
.resync_status = ResyncStatusType::ResyncCompleted;
Ok(true)
})
.await
});
let writer_b_before_release = tokio::time::timeout(Duration::from_millis(250), &mut writer_b).await.ok();
shared.allow_write.notify_one();
writer_a
.await
.expect("writer A task should finish")
.expect("writer A should report a successful conditional save");
match writer_b_before_release {
Some(result) => result,
None => writer_b.await,
}
.expect("writer B task should finish")
.expect("writer B should retry and save its mutation");
let persisted = decode_resync_file(&shared.data.lock().expect("test data lock should not be poisoned"))
.expect("persisted resync status should decode");
assert_eq!(persisted.targets_map["arn:a"].resync_status, ResyncStatusType::ResyncCanceled);
assert_eq!(persisted.targets_map["arn:b"].resync_status, ResyncStatusType::ResyncCompleted);
assert!(!shared.last_put_no_lock.load(Ordering::SeqCst));
}
#[test]
fn replication_queue_admission_combines_target_results() {
let mut admission = ReplicationQueueAdmission::Skipped;
@@ -4529,30 +4194,6 @@ mod tests {
assert_eq!(ri.checksum, Some(checksum));
}
#[test]
fn metadata_mrf_roundtrip_preserves_tags_and_admitted_targets() {
let target = "arn:rustfs:replication:target-a";
let object = ObjectInfo {
bucket: "source".to_string(),
name: "object".to_string(),
version_id: Some(Uuid::new_v4()),
user_tags: Arc::new("owner=a3".to_string()),
..Default::default()
};
let live =
replicate_object_info_from_object_info(object.clone(), test_replicate_decision(&[target]), ReplicationType::Metadata);
let persisted = live.to_mrf_entry();
let encoded = encode_mrf_file(std::slice::from_ref(&persisted)).expect("metadata MRF entry should encode");
let decoded = decode_mrf_file(&encoded).expect("metadata MRF entry should decode");
assert_eq!(decoded[0].op, MrfOpKind::Metadata);
assert_eq!(decoded[0].target_arns, vec![target.to_string()]);
let replayed = admitted_mrf_replicate_object(object, &decoded[0], ReplicationType::Metadata);
assert_eq!(replayed.op_type, ReplicationType::Metadata);
assert_eq!(replayed.user_tags, "owner=a3");
assert_eq!(replayed.admitted_target_arns(), vec![target.to_string()]);
}
#[tokio::test]
async fn mrf_save_admission_waits_for_capacity_instead_of_dropping() {
let (tx, mut rx) = mpsc::channel(1);
@@ -40,17 +40,16 @@ use super::replication_resync_boundary::ResyncStatusType;
#[cfg(test)]
use super::replication_resync_boundary::should_count_head_proxy_failure;
use super::replication_resync_boundary::{
BucketReplicationResyncStatus, ResyncOpts, TargetReplicationResyncStatus, decode_resync_file, encode_resync_file,
is_version_id_mismatch, resync_state_accepts_update, resync_status_duration, sanitize_resync_error_detail,
should_auto_resume_resync,
BucketReplicationResyncStatus, ResyncOpts, TargetReplicationResyncStatus, encode_resync_file, is_version_id_mismatch,
resync_state_accepts_update, resync_status_duration, sanitize_resync_error_detail,
};
#[cfg(test)]
use super::replication_resync_boundary::{RESYNC_META_FORMAT, RESYNC_META_VERSION, WIRE_ZERO_TIME_UNIX};
use super::replication_resync_boundary::{RESYNC_META_FORMAT, RESYNC_META_VERSION, WIRE_ZERO_TIME_UNIX, decode_resync_file};
#[cfg(test)]
use super::replication_storage_boundary::ReplicationDeletedObject;
use super::replication_storage_boundary::{
AdvancedGetOptions, EcstoreObjectOperations, GetObjectReader, HTTPPreconditions, HTTPRangeSpec, ObjectInfo, ObjectOptions,
ObjectToDelete, ReplicationObjectIO, ReplicationStorage, StatObjectOptions, StorageObjectInfoOrErr, WalkOptions,
AdvancedGetOptions, EcstoreObjectOperations, GetObjectReader, HTTPRangeSpec, ObjectInfo, ObjectOptions, ObjectToDelete,
ReplicationObjectIO, ReplicationStorage, StatObjectOptions, StorageObjectInfoOrErr, WalkOptions,
};
use super::replication_target_boundary::{
ERR_REPLICATION_SSEC_PASSTHROUGH_UNSUPPORTED, HeadObjectSdkError, PutObjectOptions, PutObjectPartOptions,
@@ -77,8 +76,7 @@ use metrics::counter;
use rmp_serde;
use rustfs_s3_types::EventName;
use rustfs_utils::http::{
AMZ_BUCKET_REPLICATION_STATUS, AMZ_TAGGING_DIRECTIVE, SUFFIX_REPLICATION_RESET, SUFFIX_REPLICATION_STATUS,
has_internal_suffix, insert_str,
AMZ_TAGGING_DIRECTIVE, SUFFIX_REPLICATION_RESET, SUFFIX_REPLICATION_STATUS, has_internal_suffix, insert_str,
};
use rustfs_utils::{DEFAULT_SIP_HASH_KEY, get_env_usize, sip_hash};
#[cfg(test)]
@@ -176,14 +174,6 @@ fn has_raw_status(err: &SdkError<HeadObjectError>, status: u16) -> bool {
err.raw_response().is_some_and(|r| r.status().as_u16() == status)
}
fn metadata_requires_existing_target(op_type: ReplicationType, object_info: &ObjectInfo) -> bool {
op_type == ReplicationType::Metadata
&& object_info
.user_defined
.get(AMZ_BUCKET_REPLICATION_STATUS)
.is_some_and(|status| status.eq_ignore_ascii_case(ReplicationStatusType::Replica.as_str()))
}
const METRIC_VERSION_IDENTITY_DRIFT_TOTAL: &str = "rustfs_replication_version_identity_drift_total";
/// Targets that already produced a version-identity-drift warning this
@@ -429,7 +419,7 @@ impl ReplicationResyncer {
where
S: ReplicationObjectIO,
{
let (updated_target, status_duration) = {
let (bucket_status, status_duration) = {
let mut status_map = self.status_map.write().await;
let now = OffsetDateTime::now_utc();
@@ -500,62 +490,28 @@ impl ReplicationResyncer {
bucket_status.last_update = Some(now);
(state.clone(), status_duration)
(bucket_status.clone(), status_duration)
};
// Persist through the CAS so a stale cached map can never clobber
// states other nodes finalized for other targets; re-run the staleness
// and canceled-is-terminal guards against the freshest persisted entry.
let updated_last_update = updated_target.last_update;
let (final_map, saved) = update_resync_status_cas(&opts.bucket, obj_layer, |persisted| {
if let Some(current) = persisted.targets_map.get(&opts.arn) {
if !resync_state_accepts_update(current, &opts) {
debug!(
event = EVENT_RESYNC_STATUS_UPDATE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %opts.bucket,
arn = %opts.arn,
incoming_resync_id = %opts.resync_id,
current_resync_id = %current.resync_id,
reason = "stale_status_update",
"Skipped persisting stale resync status update"
);
return Ok(false);
}
if current.resync_status == ResyncStatusType::ResyncCanceled && status != ResyncStatusType::ResyncCanceled {
debug!(
event = EVENT_RESYNC_STATUS_UPDATE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %opts.bucket,
arn = %opts.arn,
incoming_status = %status,
reason = "canceled_status_is_terminal",
"Skipped resync status update after cancellation"
);
return Ok(false);
}
}
persisted.targets_map.insert(opts.arn.clone(), updated_target.clone());
persisted.last_update = updated_last_update;
Ok(true)
})
.await?;
// Converge this target's cached entry with what the persisted document
// decided (our update, or the newer/terminal state that outranked it).
{
let mut status_map = self.status_map.write().await;
if let Some(cached) = status_map.get_mut(&opts.bucket)
&& let Some(final_target) = final_map.targets_map.get(&opts.arn)
{
cached.targets_map.insert(opts.arn.clone(), final_target.clone());
cached.last_update = final_map.last_update.or(cached.last_update);
save_resync_status(&opts.bucket, &bucket_status, obj_layer.clone()).await?;
if status != ResyncStatusType::ResyncCanceled {
let canceled_status = self
.status_map
.read()
.await
.get(&opts.bucket)
.filter(|current| {
current.targets_map.get(&opts.arn).is_some_and(|target| {
target.resync_id == opts.resync_id && target.resync_status == ResyncStatusType::ResyncCanceled
})
})
.cloned();
if let Some(canceled_status) = canceled_status {
save_resync_status(&opts.bucket, &canceled_status, obj_layer).await?;
return Ok(());
}
}
if saved && let Some(stats) = runtime_sources::replication_stats() {
if let Some(stats) = runtime_sources::replication_stats() {
stats.record_resync_status(&opts.bucket, status, status_duration).await;
}
@@ -647,16 +603,10 @@ impl ReplicationResyncer {
}
_ = interval.tick() => {
let snapshot: Vec<(String, BucketReplicationResyncStatus)> = self
.status_map
.read()
.await
.iter()
.map(|(bucket, status)| (bucket.clone(), status.clone()))
.collect();
let status_map = self.status_map.read().await;
let mut update = false;
for (bucket, status) in &snapshot {
for (bucket, status) in status_map.iter() {
for target in status.targets_map.values() {
if target.last_update.is_none() {
update = true;
@@ -672,14 +622,7 @@ impl ReplicationResyncer {
}
if update {
// CAS-merge instead of a blind whole-map save: this
// cache may lag other nodes' admissions and
// cancellations, which must not be overwritten.
let result = update_resync_status_cas(bucket, api.clone(), |persisted| {
Ok(merge_local_resync_into_persisted(persisted, status))
})
.await;
if let Err(err) = result {
if let Err(err) = save_resync_status(bucket, status, api.clone()).await {
error!(
event = EVENT_RESYNC_STATUS_UPDATE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
@@ -689,8 +632,8 @@ impl ReplicationResyncer {
error = %err,
"Failed to persist resync status"
);
} else if let Some(last_update) = status.last_update {
last_update_times.insert(bucket.clone(), last_update);
} else {
last_update_times.insert(bucket.clone(), status.last_update.expect("last_update should be set"));
}
}
}
@@ -1499,109 +1442,17 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
})
}
/// Upper bound on optimistic retries for a `resync.bin` compare-and-swap
/// update before giving up; contention on one bucket's status is a handful of
/// writers (status transitions, the periodic saver, admissions), not a crowd.
const RESYNC_STATUS_CAS_MAX_ATTEMPTS: usize = 32;
/// Read-merge-write `resync.bin` under an ETag compare-and-swap.
///
/// Every writer used to persist its node's cached whole-bucket map, so one
/// node's stale cache could silently resurrect a state another node had
/// already finalized (e.g. flip a just-canceled intent back to `Pending`).
/// `apply` receives the freshest persisted map and mutates it in place,
/// returning `Ok(false)` to skip the write. On a concurrent write the load +
/// apply + save cycle is retried against the new document. Returns the final
/// map and whether this call wrote it.
pub(crate) async fn update_resync_status_cas<S, F>(
pub(crate) async fn save_resync_status<S: ReplicationObjectIO>(
bucket: &str,
status: &BucketReplicationResyncStatus,
api: Arc<S>,
mut apply: F,
) -> Result<(BucketReplicationResyncStatus, bool)>
where
S: ReplicationObjectIO,
F: FnMut(&mut BucketReplicationResyncStatus) -> Result<bool>,
{
let config_file = ReplicationMetadataStore::bucket_resync_file_path(bucket);
for _ in 0..RESYNC_STATUS_CAS_MAX_ATTEMPTS {
let (mut status, preconditions) =
match ReplicationConfigStore::read_no_lock_with_metadata(api.clone(), &config_file).await {
Ok((data, object_info)) => {
let etag = object_info
.etag
.filter(|etag| !etag.trim().is_empty())
.ok_or_else(|| Error::other("replication resync status has no ETag for conditional update"))?;
let status = if data.is_empty() {
BucketReplicationResyncStatus::new()
} else {
decode_resync_file(&data)?
};
(
status,
HTTPPreconditions {
if_match: Some(etag),
..Default::default()
},
)
}
Err(Error::ConfigNotFound) => (
BucketReplicationResyncStatus::new(),
HTTPPreconditions {
if_none_match: Some("*".to_string()),
..Default::default()
},
),
Err(err) => return Err(err),
};
if !apply(&mut status)? {
return Ok((status, false));
}
match ReplicationConfigStore::save_conditional(api.clone(), &config_file, encode_resync_file(&status)?, preconditions)
.await
{
Ok(()) => return Ok((status, true)),
Err(Error::PreconditionFailed) => continue,
Err(err) => return Err(err),
}
}
Err(Error::other("replication resync status conditional update did not converge"))
}
) -> Result<()> {
let data = encode_resync_file(status)?;
/// Merge this node's cached bucket resync map into the persisted map for the
/// periodic saver. Per target: same run id overlays the fresher local state
/// unless the persisted state is already terminal and the local one is not
/// (a cancel/completion recorded by another node must stick); a different
/// persisted run id means a newer admission elsewhere and is kept; targets
/// unknown to disk are added. Returns whether `persisted` changed.
pub(crate) fn merge_local_resync_into_persisted(
persisted: &mut BucketReplicationResyncStatus,
local: &BucketReplicationResyncStatus,
) -> bool {
let mut changed = false;
for (arn, local_state) in &local.targets_map {
match persisted.targets_map.get(arn) {
Some(current) if current.resync_id == local_state.resync_id => {
let persisted_terminal = !should_auto_resume_resync(current.resync_status);
let local_terminal = !should_auto_resume_resync(local_state.resync_status);
if persisted_terminal && !local_terminal {
continue;
}
if current != local_state {
persisted.targets_map.insert(arn.clone(), local_state.clone());
changed = true;
}
}
Some(_) => {}
None => {
persisted.targets_map.insert(arn.clone(), local_state.clone());
changed = true;
}
}
}
if changed && local.last_update.is_some() {
persisted.last_update = local.last_update;
}
changed
let config_file = ReplicationMetadataStore::bucket_resync_file_path(bucket);
ReplicationConfigStore::save(api, &config_file, data).await?;
Ok(())
}
pub async fn replicate_delete<S: ReplicationStorage>(dobj: DeletedObjectReplicationInfo, storage: Arc<S>) {
@@ -3643,7 +3494,6 @@ async fn resolve_replicate_all_action(
start_time,
ssec_audit_required,
} = ctx;
let require_existing_target = metadata_requires_existing_target(roi.op_type, &object_info);
let replication_action;
match head_object_for_worker(tgt_client.as_ref(), &tgt_client.bucket, object, roi.version_id.map(|v| v.to_string())).await {
Ok(oi) => {
@@ -3705,13 +3555,7 @@ async fn resolve_replicate_all_action(
// Version-ID format mismatch: retry without versionId and compare ETags.
match head_object_fallback(tgt_client, object).await {
Ok(Some(oi)) => {
let etags_match = replication_etags_match(object_info.etag.as_deref(), oi.e_tag.as_deref());
if require_existing_target && !etags_match {
rinfo.error = Some("replica metadata target does not contain matching object data".to_string());
rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
return None;
}
replication_action = if etags_match {
replication_action = if replication_etags_match(object_info.etag.as_deref(), oi.e_tag.as_deref()) {
if ssec_audit_required
&& !settle_ssec_passthrough_evidence(&oi, tgt_client, bucket, object, rinfo).await
{
@@ -3724,11 +3568,6 @@ async fn resolve_replicate_all_action(
};
}
Ok(None) => {
if require_existing_target {
rinfo.error = Some("replica metadata target does not contain this object version".to_string());
rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
return None;
}
replication_action = ReplicationAction::All;
}
Err(e2) => {
@@ -3754,12 +3593,7 @@ async fn resolve_replicate_all_action(
return None;
}
}
} else if e.as_service_error().is_some_and(|se| se.is_not_found()) || has_raw_status(&e, 404) {
if require_existing_target {
rinfo.error = Some("replica metadata target does not contain this object version".to_string());
rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
return None;
}
} else if e.as_service_error().is_some_and(|se| se.is_not_found()) {
replication_action = ReplicationAction::All;
} else {
rinfo.error = Some(e.to_string());
@@ -4034,7 +3868,6 @@ async fn replicate_object_with_multipart<S: ReplicationObjectIO>(ctx: MultipartR
actual_size,
object_info.etag.clone().unwrap_or_default(),
object_info.mod_time,
&put_opts.internal,
),
)
.await
@@ -4053,87 +3886,6 @@ async fn replicate_object_with_multipart<S: ReplicationObjectIO>(ctx: MultipartR
#[cfg(test)]
mod tests {
use super::super::replication_filemeta_boundary::ReplicateTargetDecision;
fn resync_target_state(resync_id: &str, status: ResyncStatusType, replicated_count: i64) -> TargetReplicationResyncStatus {
TargetReplicationResyncStatus {
resync_id: resync_id.to_string(),
resync_status: status,
replicated_count,
..Default::default()
}
}
/// Periodic-saver merge: fresher local progress overlays the same run,
/// but a terminal state persisted by another node must stick, a newer
/// admission elsewhere is kept, and locally-known targets are added.
#[test]
fn merge_local_resync_keeps_peer_terminal_and_newer_states() {
let mut persisted = BucketReplicationResyncStatus::new();
persisted.targets_map.insert(
"arn:same-run".to_string(),
resync_target_state("run-1", ResyncStatusType::ResyncStarted, 1),
);
persisted.targets_map.insert(
"arn:canceled".to_string(),
resync_target_state("run-1", ResyncStatusType::ResyncCanceled, 0),
);
persisted.targets_map.insert(
"arn:new-run".to_string(),
resync_target_state("run-2", ResyncStatusType::ResyncPending, 0),
);
let mut local = BucketReplicationResyncStatus::new();
local.targets_map.insert(
"arn:same-run".to_string(),
resync_target_state("run-1", ResyncStatusType::ResyncStarted, 9),
);
local.targets_map.insert(
"arn:canceled".to_string(),
resync_target_state("run-1", ResyncStatusType::ResyncPending, 0),
);
local.targets_map.insert(
"arn:new-run".to_string(),
resync_target_state("run-1", ResyncStatusType::ResyncStarted, 3),
);
local.targets_map.insert(
"arn:local-only".to_string(),
resync_target_state("run-1", ResyncStatusType::ResyncPending, 0),
);
local.last_update = Some(OffsetDateTime::now_utc());
assert!(merge_local_resync_into_persisted(&mut persisted, &local));
assert_eq!(persisted.targets_map["arn:same-run"].replicated_count, 9, "fresher local progress wins");
assert_eq!(
persisted.targets_map["arn:canceled"].resync_status,
ResyncStatusType::ResyncCanceled,
"peer terminal state must stick"
);
assert_eq!(
persisted.targets_map["arn:new-run"].resync_id, "run-2",
"newer admission elsewhere is kept"
);
assert!(persisted.targets_map.contains_key("arn:local-only"));
assert_eq!(persisted.last_update, local.last_update);
}
/// A terminal local state for the same run (completion/failure recorded by
/// this node) still overlays a non-terminal persisted state.
#[test]
fn merge_local_resync_reports_no_change_when_maps_agree() {
let mut persisted = BucketReplicationResyncStatus::new();
persisted
.targets_map
.insert("arn:same".to_string(), resync_target_state("run-1", ResyncStatusType::ResyncStarted, 5));
let local = persisted.clone();
assert!(!merge_local_resync_into_persisted(&mut persisted, &local));
let mut local = local.clone();
local
.targets_map
.insert("arn:same".to_string(), resync_target_state("run-1", ResyncStatusType::ResyncCompleted, 5));
assert!(merge_local_resync_into_persisted(&mut persisted, &local));
assert_eq!(persisted.targets_map["arn:same"].resync_status, ResyncStatusType::ResyncCompleted);
}
use super::super::replication_target_boundary::{BucketTarget, BucketTargets};
use super::*;
use s3s::dto::{
@@ -4169,113 +3921,6 @@ mod tests {
})
}
fn spawn_head_status_server(status: u16) -> (String, std::thread::JoinHandle<()>) {
use std::io::{Read, Write};
let listener = std::net::TcpListener::bind(("127.0.0.1", 0)).expect("test HTTP listener should bind");
let endpoint = format!("http://{}", listener.local_addr().expect("test HTTP listener should have an address"));
let handle = std::thread::spawn(move || {
let (mut stream, _) = listener.accept().expect("test HTTP client should connect");
let mut request = [0_u8; 8192];
let bytes_read = stream.read(&mut request).expect("test HTTP request should be read");
assert!(bytes_read > 0, "test HTTP request should not be empty");
assert!(request[..bytes_read].starts_with(b"HEAD "), "replication comparison must use HEAD");
write!(stream, "HTTP/1.1 {status} Test\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
.expect("test HTTP response should be written");
});
(endpoint, handle)
}
#[tokio::test]
async fn replica_metadata_missing_target_stops_before_full_put() {
let (endpoint, server) = spawn_head_status_server(404);
let target = test_target_client(endpoint);
let roi = ReplicateObjectInfo {
bucket: "source".to_string(),
name: "object".to_string(),
version_id: Some(Uuid::new_v4()),
op_type: ReplicationType::Metadata,
// Normal metadata writes replace REPLICA with per-target PENDING
// before constructing the worker request.
replication_status: ReplicationStatusType::Pending,
..Default::default()
};
let object_info = ObjectInfo {
bucket: roi.bucket.clone(),
name: roi.name.clone(),
version_id: roi.version_id,
etag: Some("source-etag".to_string()),
user_defined: Arc::new(HashMap::from([(
AMZ_BUCKET_REPLICATION_STATUS.to_string(),
ReplicationStatusType::Replica.as_str().to_string(),
)])),
..Default::default()
};
let mut rinfo = replicate_all_target_info(&roi, &target);
let action = resolve_replicate_all_action(
ReplicateAllActionContext {
roi: &roi,
tgt_client: &target,
bucket: &roi.bucket,
object: &roi.name,
start_time: OffsetDateTime::now_utc(),
ssec_audit_required: false,
},
object_info,
&mut rinfo,
)
.await;
assert!(action.is_none(), "missing replica metadata targets must not reach the payload PUT path");
assert_eq!(rinfo.replication_status, ReplicationStatusType::Failed);
assert_eq!(
rinfo.error.as_deref(),
Some("replica metadata target does not contain this object version")
);
server.join().expect("test HTTP server should finish");
}
#[tokio::test]
async fn source_metadata_missing_target_rebuilds_object() {
let (endpoint, server) = spawn_head_status_server(404);
let target = test_target_client(endpoint);
let roi = ReplicateObjectInfo {
bucket: "source".to_string(),
name: "object".to_string(),
version_id: Some(Uuid::new_v4()),
op_type: ReplicationType::Metadata,
replication_status: ReplicationStatusType::Pending,
..Default::default()
};
let object_info = ObjectInfo {
bucket: roi.bucket.clone(),
name: roi.name.clone(),
version_id: roi.version_id,
etag: Some("source-etag".to_string()),
..Default::default()
};
let mut rinfo = replicate_all_target_info(&roi, &target);
let action = resolve_replicate_all_action(
ReplicateAllActionContext {
roi: &roi,
tgt_client: &target,
bucket: &roi.bucket,
object: &roi.name,
start_time: OffsetDateTime::now_utc(),
ssec_audit_required: false,
},
object_info,
&mut rinfo,
)
.await;
assert!(matches!(action, Some((ReplicationAction::All, _))));
assert!(rinfo.error.is_none());
server.join().expect("test HTTP server should finish");
}
async fn register_test_target(target: &Arc<TargetClient>) {
ReplicationTargetStore::register_test_target(target).await;
}
@@ -472,7 +472,6 @@ pub(crate) fn replication_complete_multipart_options(
actual_size: String,
source_etag: String,
source_mtime: Option<OffsetDateTime>,
source_internal: &AdvancedPutOptions,
) -> PutObjectOptions {
let mut user_metadata = HashMap::new();
insert_header_map(&mut user_metadata, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, actual_size);
@@ -485,14 +484,6 @@ pub(crate) fn replication_complete_multipart_options(
// mtime must degrade to epoch so header() suppresses the header
// instead of asserting the replication time as the object's mtime.
source_mtime: source_mtime.unwrap_or(OffsetDateTime::UNIX_EPOCH),
// Carry the per-category LWW timestamps on the complete request as
// well: the receiver's CompleteMultipartUpload options builder
// parses the same headers, so the multipart transport gets the
// same receiver-side LWW as the single-PUT transport
// (rustfs/backlog#1953). Epoch values keep the headers suppressed.
tagging_timestamp: source_internal.tagging_timestamp,
retention_timestamp: source_internal.retention_timestamp,
legalhold_timestamp: source_internal.legalhold_timestamp,
replication_status: ReplicationStatusType::Replica,
replication_request: true,
..Default::default()
@@ -672,39 +663,20 @@ mod tests {
#[test]
fn replication_complete_multipart_options_sets_actual_size() {
let source_mtime = OffsetDateTime::from_unix_timestamp(1_716_170_000).expect("valid test timestamp");
let source_internal = AdvancedPutOptions {
tagging_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_100).expect("valid test timestamp"),
retention_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_200).expect("valid test timestamp"),
legalhold_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_300).expect("valid test timestamp"),
..Default::default()
};
let options = replication_complete_multipart_options(
"1024".to_string(),
"0123456789abcdef0123456789abcdef-3".to_string(),
Some(source_mtime),
&source_internal,
);
assert_eq!(options.internal.source_etag, "0123456789abcdef0123456789abcdef-3");
assert_eq!(options.internal.source_mtime, source_mtime);
// The complete request must carry the same per-category LWW timestamps
// as the initiate request; the receiver reads them from the complete
// headers (rustfs/backlog#1953).
assert_eq!(options.internal.tagging_timestamp, source_internal.tagging_timestamp);
assert_eq!(options.internal.retention_timestamp, source_internal.retention_timestamp);
assert_eq!(options.internal.legalhold_timestamp, source_internal.legalhold_timestamp);
// Absent source mtime must degrade to epoch (header suppressed), not
// the AdvancedPutOptions default of now_utc() — that default would
// stamp the replication time as the replica's mtime and break the
// multipart HEAD convergence. Unset category timestamps stay epoch so
// header() keeps suppressing them.
let options_no_mtime =
replication_complete_multipart_options("1024".to_string(), String::new(), None, &AdvancedPutOptions::default());
// multipart HEAD convergence.
let options_no_mtime = replication_complete_multipart_options("1024".to_string(), String::new(), None);
assert_eq!(options_no_mtime.internal.source_mtime.unix_timestamp(), 0);
assert_eq!(options_no_mtime.internal.tagging_timestamp.unix_timestamp(), 0);
assert_eq!(options_no_mtime.internal.retention_timestamp.unix_timestamp(), 0);
assert_eq!(options_no_mtime.internal.legalhold_timestamp.unix_timestamp(), 0);
assert_eq!(
get_header_map(&options.user_metadata, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE).as_deref(),
@@ -32,10 +32,6 @@ pub struct Credentials {
pub access_key: String,
#[serde(rename = "secretKey")]
pub secret_key: String,
// The aliases accept madmin's JSON tags (MinIO-written bucket-targets
// metadata and mc request bodies) without changing the snake_case
// persisted/peer wire format this struct serializes to.
#[serde(alias = "sessionToken")]
pub session_token: Option<String>,
pub expiration: Option<Timestamp>,
}
@@ -206,14 +202,12 @@ pub struct BucketTarget {
#[serde(default)]
pub region: String,
// madmin-go v3.0.109 tags this `bandwidthlimit`; `bandwidth` is a legacy
// alias kept for inputs written before the madmin tag was verified.
#[serde(alias = "bandwidthlimit", alias = "bandwidth", default)]
#[serde(alias = "bandwidth", default)]
pub bandwidth_limit: i64,
#[serde(rename = "replicationSync", default)]
pub replication_sync: bool,
#[serde(alias = "storageclass", default)]
#[serde(default)]
pub storage_class: String,
#[serde(rename = "skipTlsVerify", default)]
pub skip_tls_verify: bool,
@@ -226,7 +220,7 @@ pub struct BucketTarget {
#[serde(rename = "resetBeforeDate", with = "time::serde::rfc3339::option", default)]
pub reset_before_date: Option<OffsetDateTime>,
#[serde(alias = "resetID", default)]
#[serde(default)]
pub reset_id: String,
#[serde(rename = "totalDowntime", with = "duration_seconds", default)]
pub total_downtime: Duration,
@@ -239,7 +233,7 @@ pub struct BucketTarget {
#[serde(default)]
pub latency: LatencyStat,
#[serde(alias = "deploymentID", default)]
#[serde(default)]
pub deployment_id: String,
#[serde(default)]
@@ -537,85 +531,6 @@ mod tests {
assert_eq!(value["totalDowntime"], 90);
}
#[test]
fn bucket_target_persisted_wire_keys_stay_snake_case() {
// bucket-targets.json (persisted via `serde_json::to_vec(&BucketTargets)`
// in the admin set/remove handlers) and the msgpack struct-map form
// (`BucketTargets::marshal_msg`) both come straight from this struct's
// serde field names. madmin naming is applied only in the admin
// response layer (`remote_target_admin_json`); renaming here would
// silently break every existing deployment's persisted metadata.
let targets = BucketTargets {
targets: vec![BucketTarget {
credentials: Some(Credentials {
access_key: "ak".to_string(),
secret_key: "sk".to_string(),
session_token: Some("token".to_string()),
expiration: None,
}),
bandwidth_limit: 5,
storage_class: "STANDARD".to_string(),
reset_id: "reset-1".to_string(),
deployment_id: "deploy-1".to_string(),
..Default::default()
}],
};
let json = serde_json::to_value(&targets).expect("targets should serialize to JSON");
let msgpack: serde_json::Value =
rmp_serde::from_slice(&targets.marshal_msg().expect("targets should marshal to msgpack"))
.expect("msgpack struct map should decode into a JSON value");
for (wire, entry) in [("JSON", &json["targets"][0]), ("msgpack", &msgpack["targets"][0])] {
assert_eq!(entry["bandwidth_limit"], 5, "{wire} key `bandwidth_limit` must stay");
assert_eq!(entry["storage_class"], "STANDARD", "{wire} key `storage_class` must stay");
assert_eq!(entry["reset_id"], "reset-1", "{wire} key `reset_id` must stay");
assert_eq!(entry["deployment_id"], "deploy-1", "{wire} key `deployment_id` must stay");
assert_eq!(entry["credentials"]["session_token"], "token", "{wire} key `session_token` must stay");
}
}
#[test]
fn minio_written_bucket_targets_json_populates_madmin_named_fields() {
// A MinIO-written bucket-targets.json carries madmin's JSON tags
// (`bandwidthlimit`, `storageclass`, `resetID`, `deploymentID`,
// `credentials.sessionToken` — madmin-go v3.0.109 bucket-targets.go).
// On migration these must land in the matching fields instead of
// silently defaulting (backlog#1951).
let targets: BucketTargets = serde_json::from_value(serde_json::json!({
"targets": [{
"sourcebucket": "src",
"endpoint": "minio.example:9000",
"credentials": {
"accessKey": "ak",
"secretKey": "sk",
"sessionToken": "minio-session-token"
},
"targetbucket": "dst",
"type": "replication",
"replicationSync": true,
"bandwidthlimit": 107374182400i64,
"storageclass": "STANDARD",
"resetID": "reset-789",
"deploymentID": "deploy-123"
}]
}))
.expect("MinIO-written bucket-targets.json must deserialize");
let target = &targets.targets[0];
assert_eq!(target.bandwidth_limit, 107374182400);
assert_eq!(target.storage_class, "STANDARD");
assert_eq!(target.reset_id, "reset-789");
assert_eq!(target.deployment_id, "deploy-123");
assert_eq!(
target
.credentials
.as_ref()
.and_then(|credentials| credentials.session_token.as_deref()),
Some("minio-session-token")
);
}
#[test]
fn test_bucket_target_debug_redacts_credentials() {
let target = BucketTarget {
@@ -218,7 +218,6 @@ pub struct ListPathRawOptions {
pub path: String,
pub recursive: bool,
pub incl_deleted: bool,
pub skip_hidden_prefix_check: bool,
pub filter_prefix: Option<String>,
pub forward_to: Option<String>,
pub min_disks: usize,
@@ -250,7 +249,6 @@ impl Clone for ListPathRawOptions {
path: self.path.clone(),
recursive: self.recursive,
incl_deleted: self.incl_deleted,
skip_hidden_prefix_check: self.skip_hidden_prefix_check,
filter_prefix: self.filter_prefix.clone(),
forward_to: self.forward_to.clone(),
min_disks: self.min_disks,
@@ -276,7 +274,6 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
base_dir: opts.path.clone(),
recursive: opts.recursive,
incl_deleted: opts.incl_deleted,
skip_hidden_prefix_check: opts.skip_hidden_prefix_check,
report_notfound: opts.report_not_found,
filter_prefix: opts.filter_prefix.clone(),
forward_to: opts.forward_to.clone(),
@@ -1101,13 +1098,11 @@ mod tests {
#[test]
fn walk_dir_options_preserve_zero_total_and_inherited_stall_timeouts() {
let options = walk_dir_options(&ListPathRawOptions {
skip_hidden_prefix_check: true,
walkdir_timeout: Some(Duration::ZERO),
walkdir_stall_timeout: None,
..Default::default()
});
assert!(options.skip_hidden_prefix_check);
assert_eq!(options.timeout_ms, Some(0));
assert_eq!(options.stall_timeout_ms, None);
assert!(!options.skip_total_timeout);
+13 -69
View File
@@ -94,7 +94,6 @@ const REPLAY_CACHE_AUTO_MEMORY_PERCENT: u64 = 13;
const REPLAY_CACHE_AUTO_RPC_RPS_PER_CPU: usize = 4096;
const REPLAY_CACHE_AUTO_MAX_CAPACITY: usize = 33_554_432;
const NS_SCANNER_CAPABILITY_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-capability-v3";
const NS_SCANNER_TIER_REGISTRY_GENERATION_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-tier-registry-generation-v1";
pub const TONIC_RPC_PREFIX: &str = "/node_service.NodeService";
static INTERNODE_RPC_SIGNATURE_STRICT: LazyLock<bool> = LazyLock::new(|| {
get_env_bool(
@@ -637,79 +636,40 @@ pub fn verify_put_file_capability(challenge: Uuid, server_epoch: Uuid, version:
.map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid put_file capability proof"))
}
fn update_ns_scanner_capability_mac(
mac: &mut HmacSha256,
challenge: Uuid,
server_epoch: Uuid,
supports_tier_registry_generation: bool,
) {
fn update_ns_scanner_capability_mac(mac: &mut HmacSha256, challenge: Uuid, server_epoch: Uuid) {
mac.update(NS_SCANNER_CAPABILITY_AUTH_DOMAIN);
mac.update(&NS_SCANNER_PROTOCOL_VERSION.to_be_bytes());
mac.update(challenge.as_bytes());
mac.update(server_epoch.as_bytes());
if supports_tier_registry_generation {
// The optional response capability is part of the authenticated
// scope. A proxy cannot turn an old/unsupported peer into a worker
// that receives generation-fenced scanner work.
mac.update(NS_SCANNER_TIER_REGISTRY_GENERATION_AUTH_DOMAIN);
}
}
fn generate_ns_scanner_capability_proof(
secret: &str,
challenge: Uuid,
server_epoch: Uuid,
supports_tier_registry_generation: bool,
) -> std::io::Result<Vec<u8>> {
fn generate_ns_scanner_capability_proof(secret: &str, challenge: Uuid, server_epoch: Uuid) -> std::io::Result<Vec<u8>> {
if challenge.is_nil() || server_epoch.is_nil() {
return Err(std::io::Error::other("Invalid namespace scanner capability scope"));
}
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch, supports_tier_registry_generation);
update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch);
Ok(mac.finalize().into_bytes().to_vec())
}
fn verify_ns_scanner_capability_proof(
secret: &str,
challenge: Uuid,
server_epoch: Uuid,
proof: &[u8],
supports_tier_registry_generation: bool,
) -> std::io::Result<()> {
fn verify_ns_scanner_capability_proof(secret: &str, challenge: Uuid, server_epoch: Uuid, proof: &[u8]) -> std::io::Result<()> {
if challenge.is_nil() || server_epoch.is_nil() {
return Err(std::io::Error::other("Invalid namespace scanner capability scope"));
}
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch, supports_tier_registry_generation);
update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch);
mac.verify_slice(proof)
.map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid namespace scanner capability proof"))
}
pub fn sign_ns_scanner_capability(challenge: Uuid, server_epoch: Uuid) -> std::io::Result<Vec<u8>> {
sign_ns_scanner_capability_with_tier_registry_generation(challenge, server_epoch, false)
generate_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch)
}
pub fn verify_ns_scanner_capability(challenge: Uuid, server_epoch: Uuid, proof: &[u8]) -> std::io::Result<()> {
verify_ns_scanner_capability_with_tier_registry_generation(challenge, server_epoch, proof, false)
}
pub fn sign_ns_scanner_capability_with_tier_registry_generation(
challenge: Uuid,
server_epoch: Uuid,
supports_tier_registry_generation: bool,
) -> std::io::Result<Vec<u8>> {
generate_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch, supports_tier_registry_generation)
}
pub fn verify_ns_scanner_capability_with_tier_registry_generation(
challenge: Uuid,
server_epoch: Uuid,
proof: &[u8],
supports_tier_registry_generation: bool,
) -> std::io::Result<()> {
verify_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch, proof, supports_tier_registry_generation)
verify_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch, proof)
}
#[derive(Clone, Copy)]
@@ -1749,28 +1709,13 @@ mod tests {
let secret = "test-scanner-capability-secret";
let challenge = Uuid::new_v4();
let server_epoch = Uuid::new_v4();
let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false)
.expect("capability proof should be generated");
let proof =
generate_ns_scanner_capability_proof(secret, challenge, server_epoch).expect("capability proof should be generated");
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false).is_ok());
assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof, false).is_err());
assert!(verify_ns_scanner_capability_proof(secret, challenge, Uuid::new_v4(), &proof, false).is_err());
assert!(verify_ns_scanner_capability_proof("different-secret", challenge, server_epoch, &proof, false).is_err());
}
#[test]
fn namespace_scanner_capability_proof_binds_tier_registry_generation_support() {
let secret = "test-scanner-capability-secret";
let challenge = Uuid::new_v4();
let server_epoch = Uuid::new_v4();
let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, true)
.expect("generation capability proof should be generated");
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, true).is_ok());
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false).is_err());
let legacy = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false)
.expect("legacy capability proof should be generated");
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &legacy, true).is_err());
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof).is_ok());
assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof).is_err());
assert!(verify_ns_scanner_capability_proof(secret, challenge, Uuid::new_v4(), &proof).is_err());
assert!(verify_ns_scanner_capability_proof("different-secret", challenge, server_epoch, &proof).is_err());
}
/// Security regression for GHSA-r5qv-rc46-hv8q (internode RPC fail-closed,
@@ -2994,7 +2939,6 @@ mod tests {
dst_volume: "bucket".to_string(),
dst_path: "object".to_string(),
file_info_bin: vec![0x81, 0xA1, 0x76, 0x01].into(),
scanner_publication_lease_token: Vec::new().into(),
};
let body = rustfs_protos::canonical_rename_data_request_body(&message).expect("small request should encode");
let mut request = tonic::Request::new(());
@@ -13,18 +13,17 @@
// limitations under the License.
use crate::cluster::rpc::{
build_auth_headers, build_put_file_auth_trailer, verify_ns_scanner_capability_with_tier_registry_generation,
verify_put_file_capability,
build_auth_headers, build_put_file_auth_trailer, verify_ns_scanner_capability, verify_put_file_capability,
};
use crate::disk::error::{Error, Result};
use crate::disk::{FileReader, FileWriter};
use crate::storage_api_contracts::internode::{
NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY, NS_SCANNER_LEADER_EPOCH_QUERY,
NS_SCANNER_PROTOCOL_VERSION, NS_SCANNER_PROTOCOL_VERSION_QUERY, NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY,
NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY,
NsScannerCapabilityResponse, PUT_FILE_AUTH_QUERY, PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_CHALLENGE_QUERY,
PUT_FILE_CAPABILITY_QUERY, PUT_FILE_CAPABILITY_VERSION, PUT_FILE_NONCE_QUERY, PUT_FILE_SERVER_EPOCH_QUERY,
PutFileCapabilityResponse, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, NsScannerCapabilityResponse, PUT_FILE_AUTH_QUERY,
PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_CHALLENGE_QUERY, PUT_FILE_CAPABILITY_QUERY, PUT_FILE_CAPABILITY_VERSION,
PUT_FILE_NONCE_QUERY, PUT_FILE_SERVER_EPOCH_QUERY, PutFileCapabilityResponse, WALK_DIR_BODY_SHA256_QUERY,
WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
};
use async_trait::async_trait;
use http::{HeaderMap, HeaderValue, Method, header::CONTENT_TYPE};
@@ -138,12 +137,6 @@ fn put_file_capability_status_is_legacy(status: u16) -> bool {
status == 404
}
fn ns_scanner_capability_error_allows_legacy(error: &Error) -> bool {
[400, 404, 405, 426]
.into_iter()
.any(|status| error.is_internode_http_status(status))
}
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
#[allow(
dead_code,
@@ -227,7 +220,6 @@ pub struct NsScannerStreamRequest {
#[derive(Debug, Clone)]
pub struct NsScannerCapabilityRequest {
pub endpoint: String,
pub supports_tier_registry_generation: bool,
}
/// Data-plane stream opener used by `RemoteDisk`.
@@ -260,15 +252,6 @@ pub trait InternodeDataTransport: Send + Sync + std::fmt::Debug {
async fn probe_ns_scanner(&self, _request: NsScannerCapabilityRequest) -> Result<Uuid> {
Err(Error::MethodNotAllowed)
}
async fn probe_ns_scanner_capability(&self, request: NsScannerCapabilityRequest) -> Result<NsScannerCapabilityResponse> {
let server_epoch = self.probe_ns_scanner(request).await?;
Ok(NsScannerCapabilityResponse {
version: NS_SCANNER_PROTOCOL_VERSION,
server_epoch,
proof: Vec::new(),
supports_tier_registry_generation: None,
})
}
// Interface facet nobody calls yet: every transport implements both, but no
// caller negotiates on them. Kept for the internode transport split
// (backlog#1350); deleting them would delete the seam and six impls.
@@ -352,44 +335,27 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
}
async fn probe_ns_scanner(&self, request: NsScannerCapabilityRequest) -> Result<Uuid> {
Ok(self.probe_ns_scanner_capability(request).await?.server_epoch)
}
async fn probe_ns_scanner_capability(&self, request: NsScannerCapabilityRequest) -> Result<NsScannerCapabilityResponse> {
if request.supports_tier_registry_generation {
return match self.probe_ns_scanner_capability_once(&request).await {
Ok(response) => Ok(response),
Err(marked_error) if ns_scanner_capability_error_allows_legacy(&marked_error) => {
// A v3 peer may reject the additive query marker, ignore
// it, or return its legacy proof. Retry once without the
// marker and only downgrade after that legacy response is
// authenticated; an unverified epoch is never trusted.
let legacy_request = NsScannerCapabilityRequest {
endpoint: request.endpoint.clone(),
supports_tier_registry_generation: false,
};
match self.probe_ns_scanner_capability_once(&legacy_request).await {
Ok(mut response) => {
response.supports_tier_registry_generation = None;
Ok(response)
}
Err(legacy_error) if ns_scanner_capability_error_allows_legacy(&legacy_error) => {
// Some old deployments expose only the legacy
// protocol response (or advertise 426). Treat
// the pair as an explicit unsupported result so
// the scanner can use its coordinator fallback.
Err(Error::MethodNotAllowed)
}
Err(_) => Err(marked_error),
}
}
// A server failure, network failure, or authentication error
// is not evidence of an old parser. Do not issue an
// unauthenticated legacy probe or silently downgrade.
Err(marked_error) => Err(marked_error),
};
let challenge = Uuid::new_v4();
let url = build_ns_scanner_capability_url(&request, challenge);
let mut headers = msgpack_headers();
build_auth_headers(&url, &Method::GET, &mut headers)?;
let reader = HttpReader::new(url, Method::GET, headers, None).await?;
let mut body = Vec::new();
reader
.take(u64::try_from(NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE + 1).unwrap_or(u64::MAX))
.read_to_end(&mut body)
.await?;
if body.is_empty() || body.len() > NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE {
return Err(Error::other("invalid remote namespace scanner capability response size"));
}
self.probe_ns_scanner_capability_once(&request).await
let response: NsScannerCapabilityResponse =
rmp_serde::from_slice(&body).map_err(|_| Error::other("invalid remote namespace scanner capability response"))?;
if response.version != NS_SCANNER_PROTOCOL_VERSION || response.server_epoch.is_nil() {
return Err(Error::other("incompatible remote namespace scanner capability response"));
}
verify_ns_scanner_capability(challenge, response.server_epoch, &response.proof)
.map_err(|err| Error::other(format!("remote namespace scanner capability authentication failed: {err}")))?;
Ok(response.server_epoch)
}
fn name(&self) -> &'static str {
@@ -402,53 +368,6 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
}
impl TcpHttpInternodeDataTransport {
async fn probe_ns_scanner_capability_once(
&self,
request: &NsScannerCapabilityRequest,
) -> Result<NsScannerCapabilityResponse> {
let challenge = Uuid::new_v4();
let url = build_ns_scanner_capability_url(request, challenge);
let mut headers = msgpack_headers();
build_auth_headers(&url, &Method::GET, &mut headers)?;
let reader = HttpReader::new(url, Method::GET, headers, None).await?;
let mut body = Vec::new();
reader
.take(u64::try_from(NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE + 1).unwrap_or(u64::MAX))
.read_to_end(&mut body)
.await?;
if body.is_empty() || body.len() > NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE {
return Err(Error::other("invalid remote namespace scanner capability response size"));
}
let mut response: NsScannerCapabilityResponse =
rmp_serde::from_slice(&body).map_err(|_| Error::other("invalid remote namespace scanner capability response"))?;
if response.version != NS_SCANNER_PROTOCOL_VERSION || response.server_epoch.is_nil() {
return Err(Error::other("incompatible remote namespace scanner capability response"));
}
if let Err(err) = verify_ns_scanner_capability_with_tier_registry_generation(
challenge,
response.server_epoch,
&response.proof,
request.supports_tier_registry_generation,
) {
// A permissive older peer can ignore the additive marker and
// return a valid legacy-scope proof with HTTP 200. Accept that
// response only after independently authenticating the legacy
// scope; all other verification failures remain fail-closed.
if request.supports_tier_registry_generation && ns_scanner_capability_legacy_proof_is_valid(challenge, &response) {
response.supports_tier_registry_generation = None;
return Ok(response);
}
return Err(Error::other(format!("remote namespace scanner capability authentication failed: {err}")));
}
// The proof authenticates the requested capability scope, not the
// optional response field. Derive the client-facing bit from that
// verified scope so an intermediary cannot strip or rewrite the field
// and force a silent downgrade after a successful generation-bound
// handshake.
normalize_ns_scanner_capability_response(&mut response, request.supports_tier_registry_generation);
Ok(response)
}
async fn put_file_auth_capability(&self, endpoint: &str) -> Result<Option<Uuid>> {
resolve_put_file_auth_capability(endpoint, || async {
tokio::time::timeout(PUT_FILE_CAPABILITY_PROBE_TIMEOUT, self.probe_put_file_auth(endpoint))
@@ -730,14 +649,6 @@ fn build_walk_dir_url(request: &WalkDirStreamRequest) -> String {
)
}
fn normalize_ns_scanner_capability_response(response: &mut NsScannerCapabilityResponse, requested_generation_support: bool) {
response.supports_tier_registry_generation = requested_generation_support.then_some(true);
}
fn ns_scanner_capability_legacy_proof_is_valid(challenge: Uuid, response: &NsScannerCapabilityResponse) -> bool {
verify_ns_scanner_capability_with_tier_registry_generation(challenge, response.server_epoch, &response.proof, false).is_ok()
}
fn build_ns_scanner_url(request: &NsScannerStreamRequest) -> String {
let body_sha256 = hex_simd::encode_to_string(Sha256::digest(&request.body), hex_simd::AsciiCase::Lower);
format!(
@@ -764,18 +675,13 @@ fn build_ns_scanner_url(request: &NsScannerStreamRequest) -> String {
fn build_ns_scanner_capability_url(request: &NsScannerCapabilityRequest, challenge: Uuid) -> String {
format!(
"{}{}?{}={}&{}={}{}",
"{}{}?{}={}&{}={}",
request.endpoint,
NS_SCANNER_PATH,
NS_SCANNER_PROTOCOL_VERSION_QUERY,
NS_SCANNER_PROTOCOL_VERSION,
NS_SCANNER_CAPABILITY_CHALLENGE_QUERY,
challenge,
if request.supports_tier_registry_generation {
format!("&{}=true", NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY)
} else {
String::new()
}
challenge
)
}
@@ -888,7 +794,6 @@ mod tests {
let probe_err = transport
.probe_ns_scanner(NsScannerCapabilityRequest {
endpoint: "http://node1:9000".to_string(),
supports_tier_registry_generation: false,
})
.await
.expect_err("legacy transport should report namespace scanner as unsupported");
@@ -1482,7 +1387,6 @@ mod tests {
let url = build_ns_scanner_capability_url(
&NsScannerCapabilityRequest {
endpoint: "http://node1:9000".to_string(),
supports_tier_registry_generation: false,
},
challenge,
);
@@ -1495,85 +1399,6 @@ mod tests {
);
}
#[test]
fn ns_scanner_capability_url_marks_generation_support_only_when_requested() {
let challenge = Uuid::new_v4();
let url = build_ns_scanner_capability_url(
&NsScannerCapabilityRequest {
endpoint: "http://node1:9000".to_string(),
supports_tier_registry_generation: true,
},
challenge,
);
assert!(url.contains(&format!("&{}=true", NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY)));
}
#[test]
fn ns_scanner_capability_legacy_fallback_requires_explicit_compatibility_status() {
for status in [400, 404, 405, 426] {
let error = Error::from(rustfs_rio::new_test_internode_http_io_error(
rustfs_rio::InternodeHttpErrorKind::HttpStatus(http::StatusCode::from_u16(status).expect("test status")),
));
assert!(
ns_scanner_capability_error_allows_legacy(&error),
"status {status} should permit legacy retry"
);
}
let marked_server_error = Error::from(rustfs_rio::new_test_internode_http_io_error(
rustfs_rio::InternodeHttpErrorKind::HttpStatus(http::StatusCode::INTERNAL_SERVER_ERROR),
));
let network_error = Error::from(rustfs_rio::new_test_internode_http_io_error(
rustfs_rio::InternodeHttpErrorKind::ConnectionRefused,
));
let authentication_error = Error::other("remote namespace scanner capability authentication failed");
assert!(!ns_scanner_capability_error_allows_legacy(&marked_server_error));
assert!(!ns_scanner_capability_error_allows_legacy(&network_error));
assert!(!ns_scanner_capability_error_allows_legacy(&authentication_error));
}
#[test]
fn authenticated_ns_scanner_capability_ignores_unprotected_response_bit() {
let mut response = NsScannerCapabilityResponse {
version: NS_SCANNER_PROTOCOL_VERSION,
server_epoch: Uuid::new_v4(),
proof: Vec::new(),
supports_tier_registry_generation: None,
};
normalize_ns_scanner_capability_response(&mut response, true);
assert_eq!(response.supports_tier_registry_generation, Some(true));
response.supports_tier_registry_generation = Some(false);
normalize_ns_scanner_capability_response(&mut response, false);
assert_eq!(response.supports_tier_registry_generation, None);
}
#[test]
fn ns_scanner_capability_accepts_only_authenticated_legacy_scope_after_marker_mismatch() {
crate::runtime::sources::ensure_test_rpc_secret();
let challenge = Uuid::new_v4();
let response = NsScannerCapabilityResponse {
version: NS_SCANNER_PROTOCOL_VERSION,
server_epoch: Uuid::new_v4(),
proof: crate::cluster::rpc::sign_ns_scanner_capability(challenge, Uuid::new_v4())
.expect("placeholder proof should be generated"),
supports_tier_registry_generation: None,
};
// A proof bound to a different challenge cannot authorize the legacy
// fallback, even though the response has the expected shape.
assert!(!ns_scanner_capability_legacy_proof_is_valid(challenge, &response));
let server_epoch = response.server_epoch;
let valid_response = NsScannerCapabilityResponse {
proof: crate::cluster::rpc::sign_ns_scanner_capability(challenge, server_epoch)
.expect("legacy proof should be generated"),
..response
};
assert!(ns_scanner_capability_legacy_proof_is_valid(challenge, &valid_response));
}
#[test]
fn transport_config_defaults_to_tcp_http() {
let transport = build_internode_data_transport(None).unwrap();
+4 -8
View File
@@ -35,9 +35,8 @@ pub use http_auth::{
TONIC_RPC_PREFIX, build_auth_headers, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, gen_signature_headers,
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
set_tonic_mutation_body_digest, set_tonic_rolling_canonical_body_digest, set_tonic_rolling_mutation_body_digest,
sign_ns_scanner_capability, sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability,
sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer,
sign_ns_scanner_capability, sign_put_file_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge,
tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_put_file_auth_trailer,
verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
verify_tonic_rpc_signature_with_bootstrap,
@@ -48,12 +47,9 @@ pub use internode_data_transport::build_internode_data_transport_from_env;
pub(crate) use peer_rest_client::TierConfigReloadOutcome;
pub use peer_rest_client::{
KMS_SIGNAL_SUBSYSTEM, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG,
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, ScannerPublicationLease,
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity,
};
pub(crate) use peer_s3_client::heal_bucket_local_on_disks;
pub use peer_s3_client::{
LocalPeerS3Client, PeerS3Client, S3PeerSys, ScannerBucketListing, ScannerSetBucketListing, decode_heal_bucket_rpc_options,
encode_heal_bucket_rpc_options,
};
pub use peer_s3_client::{LocalPeerS3Client, PeerS3Client, S3PeerSys, ScannerBucketListing, ScannerSetBucketListing};
pub use remote_disk::RemoteDisk;
pub use remote_locker::RemoteClient;
@@ -20,7 +20,6 @@ use crate::cluster::rpc::{set_tonic_canonical_body_digest, set_tonic_mutation_bo
use crate::error::{Error, Result};
use crate::storage_api_contracts::internode::{
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION,
};
use crate::{
bucket::replication::BucketStats,
@@ -46,9 +45,8 @@ use rustfs_protos::proto_gen::node_service::{
HealControlRequest, LoadBucketMetadataRequest, LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest,
LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest,
LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ReplacementRecoveryStatusRequest,
ScannerActivityRequest, ScannerActivityResponse, ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest,
ScannerPublicationLeaseResponse, ServerInfoRequest, SignalServiceRequest, SignalServiceResponse, StartDecommissionRequest,
StartProfilingRequest, StopRebalanceRequest, TierMutationAbortRequest, TierMutationCommitRequest,
ScannerActivityRequest, ScannerActivityResponse, ServerInfoRequest, SignalServiceRequest, SignalServiceResponse,
StartDecommissionRequest, StartProfilingRequest, StopRebalanceRequest, TierMutationAbortRequest, TierMutationCommitRequest,
TierMutationControlResponse, TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
tier_mutation_control_service_client::TierMutationControlServiceClient,
};
@@ -86,13 +84,7 @@ const HEAL_CONTROL_PAYLOAD_MAX_SIZE: usize = 64 * 1024;
const PEER_REST_RECOVERY_MAX_ATTEMPTS: u32 = 60;
const PEER_REST_RECOVERY_MAX_BACKOFF: Duration = Duration::from_secs(30);
const SCANNER_ACTIVITY_MAX_MESSAGE_SIZE: usize = 1024;
/// Reserve time for the acquire response's network/clock uncertainty. The
/// server owns the real expiry; this local deadline is intentionally earlier
/// so a coordinator never starts a bounded persistence operation at the edge
/// of a remote lease.
const SCANNER_PUBLICATION_LEASE_SAFETY_MARGIN: Duration = Duration::from_secs(5);
const REPLICATION_STATS_MAX_MESSAGE_SIZE: usize = 8 * 1024 * 1024;
const BUCKET_METADATA_RELOAD_TIMEOUT: Duration = Duration::from_secs(5);
/// Error for a peer that reported `success = false` without an `error_info` payload.
///
@@ -157,8 +149,6 @@ pub struct ScannerPeerActivity {
pub data_movement_active: Option<bool>,
pub dirty_usage_generation: Option<u64>,
pub dirty_usage_pending: Option<bool>,
pub movement_generation: Option<u64>,
pub publication_blocked: Option<bool>,
}
fn decode_scanner_activity_with_verifier(
@@ -175,14 +165,7 @@ fn decode_scanner_activity_with_verifier(
{
return Err(Error::other("peer returned an invalid scanner activity instance ID"));
}
let (
topology_digest,
data_movement_active,
dirty_usage_generation,
dirty_usage_pending,
movement_generation,
publication_blocked,
) = match response.protocol_version {
let (topology_digest, data_movement_active, dirty_usage_generation, dirty_usage_pending) = match response.protocol_version {
// RUSTFS_COMPAT_TODO(ns-scanner-rpc-v3): legacy response fields are unauthenticated. Remove after protocol v0 peers are unsupported.
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION
if response.topology_digest.is_empty()
@@ -191,7 +174,7 @@ fn decode_scanner_activity_with_verifier(
&& response.dirty_usage_generation == 0
&& !response.dirty_usage_pending =>
{
(None, None, None, None, None, None)
(None, None, None, None)
}
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION => {
return Err(Error::other("legacy scanner activity peer returned unexpected extended fields"));
@@ -214,11 +197,9 @@ fn decode_scanner_activity_with_verifier(
Some(response.data_movement_active),
None,
None,
None,
None,
)
}
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION => {
SCANNER_ACTIVITY_PROTOCOL_VERSION => {
if response.dirty_usage_pending && response.dirty_usage_generation == 0 {
return Err(Error::other("scanner activity peer returned pending dirty usage without a generation"));
}
@@ -236,42 +217,11 @@ fn decode_scanner_activity_with_verifier(
Some(response.data_movement_active),
Some(response.dirty_usage_generation),
Some(response.dirty_usage_pending),
None,
None,
)
}
SCANNER_ACTIVITY_PROTOCOL_VERSION => {
if response.dirty_usage_pending && response.dirty_usage_generation == 0 {
return Err(Error::other("scanner activity peer returned pending dirty usage without a generation"));
}
let movement_generation = response
.movement_generation
.ok_or_else(|| Error::other("scanner activity peer omitted its movement generation"))?;
let publication_blocked = response
.publication_blocked
.ok_or_else(|| Error::other("scanner activity peer omitted its publication blocked state"))?;
if movement_generation == u64::MAX {
return Err(Error::other("scanner activity peer exhausted its movement generation"));
}
let canonical = rustfs_protos::canonical_scanner_activity_v7_response_body(challenge, &response)
.map_err(|_| Error::other("scanner activity peer response is too large to authenticate"))?;
verify_proof(&canonical, &response.response_proof)?;
(
Some(
response
.topology_digest
.as_ref()
.try_into()
.map_err(|_| Error::other("peer returned an invalid scanner topology digest"))?,
),
Some(response.data_movement_active),
Some(response.dirty_usage_generation),
Some(response.dirty_usage_pending),
Some(movement_generation),
Some(publication_blocked),
)
version => {
return Err(Error::other(format!("peer returned unsupported scanner activity protocol {version}")));
}
version => return Err(Error::other(format!("peer returned unsupported scanner activity protocol {version}"))),
};
Ok(ScannerPeerActivity {
instance_id: response.instance_id,
@@ -282,8 +232,6 @@ fn decode_scanner_activity_with_verifier(
data_movement_active,
dirty_usage_generation,
dirty_usage_pending,
movement_generation,
publication_blocked,
})
}
@@ -294,17 +242,6 @@ fn decode_scanner_activity(response: ScannerActivityResponse, challenge: &[u8; 1
})
}
fn scanner_activity_protocol_unsupported(err: &Error) -> bool {
matches!(
err,
Error::Io(io_err)
if embedded_tonic_status(io_err).is_some_and(|status| {
status.code() == tonic::Code::FailedPrecondition
&& status.message().starts_with("unsupported scanner activity request protocol")
})
)
}
fn validate_heal_control_capability_proof(canonical_ack: &[u8], proof: &[u8]) -> Result<()> {
verify_tonic_rpc_response_proof(canonical_ack, proof)
.map_err(|_| Error::other("peer returned an invalid heal control capability proof"))
@@ -347,76 +284,6 @@ pub struct PeerLiveEventsBatch {
pub truncated: bool,
}
#[derive(Clone, Debug)]
pub struct ScannerPublicationLease {
pub token: Uuid,
pub movement_generation: u64,
/// Stable storage owner identity. This is distinct from the activity
/// session and is bound into both acquire and release proofs.
pub owner_id: String,
/// Process/session nonce observed by the final activity probe.
pub session_id: String,
pub expires_at: std::time::Instant,
}
impl ScannerPublicationLease {
pub fn is_valid(&self) -> bool {
std::time::Instant::now() < self.expires_at
}
}
fn validate_scanner_publication_lease_response_fields(
response: &ScannerPublicationLeaseResponse,
expected_session_id: &str,
expected_generation: u64,
) -> Result<(Uuid, String)> {
if !response.success {
return Err(Error::other(
response
.error
.as_ref()
.map(|error| error.error_info.clone())
.unwrap_or_else(|| "peer rejected scanner publication lease".to_string()),
));
}
if response.movement_generation != expected_generation {
return Err(Error::other("peer returned a different scanner publication lease generation"));
}
if response.session_id != expected_session_id {
return Err(Error::other("peer returned a different scanner publication lease session"));
}
let owner_id = Uuid::parse_str(&response.owner_id)
.ok()
.filter(|owner_id| !owner_id.is_nil())
.map(|owner_id| owner_id.to_string())
.ok_or_else(|| Error::other("peer returned an invalid scanner publication lease owner"))?;
if response.lease_ttl_ms != crate::store::SCANNER_PUBLICATION_LEASE_TTL_MS {
return Err(Error::other("peer returned an unsupported scanner publication lease TTL"));
}
let token = Uuid::from_slice(response.token.as_ref())
.map_err(|_| Error::other("peer returned an invalid scanner publication lease token"))?;
Ok((token, owner_id))
}
fn scanner_publication_lease_deadline(
request_started: std::time::Instant,
response_received: std::time::Instant,
lease_ttl_ms: u64,
) -> Result<std::time::Instant> {
let lease_window = Duration::from_millis(lease_ttl_ms)
.checked_sub(SCANNER_PUBLICATION_LEASE_SAFETY_MARGIN)
.ok_or_else(|| Error::other("scanner publication lease TTL is shorter than its safety margin"))?;
let elapsed = response_received
.checked_duration_since(request_started)
.ok_or_else(|| Error::other("scanner publication lease response clock moved backwards"))?;
if elapsed >= lease_window {
return Err(Error::other("scanner publication lease response arrived after its safety window"));
}
request_started
.checked_add(lease_window)
.ok_or_else(|| Error::other("scanner publication lease deadline overflowed"))
}
#[derive(Clone, Debug)]
pub struct PeerRestClient {
pub host: XHost,
@@ -1222,9 +1089,7 @@ impl PeerRestClient {
.await?
.max_decoding_message_size(BACKGROUND_HEAL_STATUS_MAX_MESSAGE_SIZE);
let response = match client
.background_heal_status(Request::new(BackgroundHealStatusRequest {
protocol_version: rustfs_protos::BACKGROUND_HEAL_STATUS_PROTOCOL_VERSION,
}))
.background_heal_status(Request::new(BackgroundHealStatusRequest::default()))
.await
{
Ok(response) => response.into_inner(),
@@ -1463,38 +1328,27 @@ impl PeerRestClient {
}
pub async fn load_bucket_metadata(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
let result = tokio::time::timeout(BUCKET_METADATA_RELOAD_TIMEOUT, async {
let result = self.load_bucket_metadata_once(bucket, scanner_maintenance_change).await;
if let Err(err) = &result
&& Self::is_network_like_error(err)
{
self.prepare_retry().await;
return self.load_bucket_metadata_once(bucket, scanner_maintenance_change).await;
self.finalize_result(
async {
let mut client = self.get_client().await?;
let mut request = Request::new(LoadBucketMetadataRequest {
bucket: bucket.to_string(),
scanner_maintenance_change,
});
set_tonic_mutation_body_digest(&mut request)?;
let response = client.load_bucket_metadata(request).await?.into_inner();
if !response.success {
if let Some(msg) = response.error_info {
return Err(Error::other(msg));
}
return Err(peer_failure_without_details("load_bucket_metadata", Some(bucket)));
}
Ok(())
}
result
})
.await,
)
.await
.unwrap_or_else(|_| Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))));
self.finalize_result(result).await
}
async fn load_bucket_metadata_once(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
let mut client = self.get_client().await?;
let mut request = Request::new(LoadBucketMetadataRequest {
bucket: bucket.to_string(),
scanner_maintenance_change,
});
set_tonic_mutation_body_digest(&mut request)?;
request.set_timeout(BUCKET_METADATA_RELOAD_TIMEOUT);
let response = client.load_bucket_metadata(request).await?.into_inner();
if !response.success {
if let Some(msg) = response.error_info {
return Err(Error::other(msg));
}
return Err(peer_failure_without_details("load_bucket_metadata", Some(bucket)));
}
Ok(())
}
pub async fn delete_bucket_metadata(&self, bucket: &str) -> Result<()> {
@@ -1772,11 +1626,10 @@ impl PeerRestClient {
.await
}
async fn scanner_activity_request_with_protocol(
async fn scanner_activity_request(
&self,
acknowledge_instance_id: String,
acknowledge_dirty_usage_generation: u64,
protocol_version: u32,
) -> Result<ScannerPeerActivity> {
self.finalize_result(
async {
@@ -1788,7 +1641,7 @@ impl PeerRestClient {
.max_encoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE);
let mut request = Request::new(ScannerActivityRequest {
challenge: challenge.as_bytes().to_vec().into(),
protocol_version,
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
acknowledge_instance_id,
acknowledge_dirty_usage_generation,
});
@@ -1804,168 +1657,11 @@ impl PeerRestClient {
}
pub async fn scanner_activity(&self) -> Result<ScannerPeerActivity> {
let result = self
.scanner_activity_request_with_protocol(String::new(), 0, SCANNER_ACTIVITY_PROTOCOL_VERSION)
.await;
if result.as_ref().err().is_some_and(scanner_activity_protocol_unsupported) {
// A v6 peer cannot parse the v7 marker. Its authenticated
// response is still decoded as untrusted terminal state, so the
// scanner will defer publication until every peer is v7.
self.scanner_activity_request_with_protocol(String::new(), 0, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION)
.await
} else {
result
}
self.scanner_activity_request(String::new(), 0).await
}
pub async fn acknowledge_scanner_dirty_usage(&self, instance_id: String, generation: u64) -> Result<ScannerPeerActivity> {
let result = self
.scanner_activity_request_with_protocol(instance_id.clone(), generation, SCANNER_ACTIVITY_PROTOCOL_VERSION)
.await;
if result.as_ref().err().is_some_and(scanner_activity_protocol_unsupported) {
self.scanner_activity_request_with_protocol(instance_id, generation, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION)
.await
} else {
result
}
}
/// Acquire a bounded, storage-owned read admission on the peer that
/// produced the final activity generation. Older peers do not implement
/// the lease form and are rejected rather than downgraded.
pub async fn acquire_scanner_publication_lease(
&self,
expected_session_id: &str,
expected_generation: u64,
) -> Result<ScannerPublicationLease> {
let request_started = std::time::Instant::now();
self.finalize_result(
async {
let challenge = Uuid::new_v4();
let mut client = self
.get_client()
.await?
.max_decoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE)
.max_encoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE);
let mut request = Request::new(ScannerPublicationLeaseRequest {
challenge: challenge.as_bytes().to_vec().into(),
expected_movement_generation: expected_generation,
ttl_ms: crate::store::SCANNER_PUBLICATION_LEASE_TTL_MS,
expected_session_id: expected_session_id.to_string(),
token: Bytes::new(),
});
let canonical = rustfs_protos::canonical_scanner_publication_lease_request_body(request.get_ref())
.map_err(|_| Error::other("scanner publication lease request is too large to authenticate"))?;
set_tonic_canonical_body_digest(&mut request, &canonical)?;
let response = client.acquire_scanner_publication_lease(request).await?.into_inner();
let response_body =
rustfs_protos::canonical_scanner_publication_lease_response_body(challenge.as_bytes(), &response)
.map_err(|_| Error::other("scanner publication lease response is too large to authenticate"))?;
verify_tonic_rpc_response_proof(&response_body, &response.response_proof)
.map_err(|_| Error::other("peer returned an invalid scanner publication lease proof"))?;
let (token, owner_id) =
validate_scanner_publication_lease_response_fields(&response, expected_session_id, expected_generation)?;
Ok(ScannerPublicationLease {
token,
movement_generation: response.movement_generation,
owner_id,
session_id: response.session_id,
expires_at: scanner_publication_lease_deadline(
request_started,
std::time::Instant::now(),
response.lease_ttl_ms,
)?,
})
}
.await,
)
.await
}
/// Revalidate the exact token immediately before the coordinator commits
/// its final publication. The peer keeps the original movement read
/// guard in its token table; a restart drops that table and changes the
/// activity session, so this proof fails closed instead of accepting an
/// ABA generation value.
pub async fn validate_scanner_publication_lease(&self, lease: &ScannerPublicationLease) -> Result<()> {
self.finalize_result(
async {
let challenge = Uuid::new_v4();
let mut client = self
.get_client()
.await?
.max_decoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE)
.max_encoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE);
let mut request = Request::new(ScannerPublicationLeaseRequest {
challenge: challenge.as_bytes().to_vec().into(),
expected_movement_generation: lease.movement_generation,
ttl_ms: crate::store::SCANNER_PUBLICATION_LEASE_TTL_MS,
expected_session_id: lease.session_id.clone(),
token: lease.token.as_bytes().to_vec().into(),
});
let canonical = rustfs_protos::canonical_scanner_publication_lease_request_body(request.get_ref())
.map_err(|_| Error::other("scanner publication lease validation request is too large to authenticate"))?;
set_tonic_canonical_body_digest(&mut request, &canonical)?;
let response = client.acquire_scanner_publication_lease(request).await?.into_inner();
let response_body =
rustfs_protos::canonical_scanner_publication_lease_response_body(challenge.as_bytes(), &response).map_err(
|_| Error::other("scanner publication lease validation response is too large to authenticate"),
)?;
verify_tonic_rpc_response_proof(&response_body, &response.response_proof)
.map_err(|_| Error::other("peer returned an invalid scanner publication lease validation proof"))?;
let (token, owner_id) =
validate_scanner_publication_lease_response_fields(&response, &lease.session_id, lease.movement_generation)?;
if token != lease.token {
return Err(Error::other("peer returned a different scanner publication lease token"));
}
if owner_id != lease.owner_id {
return Err(Error::other("peer returned a different scanner publication lease owner"));
}
Ok(())
}
.await,
)
.await
}
pub async fn release_scanner_publication_lease(&self, lease: &ScannerPublicationLease) -> Result<()> {
self.finalize_result(
async {
let challenge = Uuid::new_v4();
let mut client = self.get_client().await?;
let mut request = Request::new(ScannerPublicationLeaseReleaseRequest {
challenge: challenge.as_bytes().to_vec().into(),
token: lease.token.as_bytes().to_vec().into(),
owner_id: lease.owner_id.clone(),
session_id: lease.session_id.clone(),
});
let canonical = rustfs_protos::canonical_scanner_publication_lease_release_request_body(request.get_ref())
.map_err(|_| Error::other("scanner publication lease release request is too large to authenticate"))?;
set_tonic_canonical_body_digest(&mut request, &canonical)?;
let request_body = request.get_ref().clone();
let response = client.release_scanner_publication_lease(request).await?.into_inner();
let response_body = rustfs_protos::canonical_scanner_publication_lease_release_response_body(
challenge.as_bytes(),
&request_body,
&response,
)
.map_err(|_| Error::other("scanner publication lease release response is too large to authenticate"))?;
verify_tonic_rpc_response_proof(&response_body, &response.response_proof)
.map_err(|_| Error::other("peer returned an invalid scanner publication lease release proof"))?;
if response.success {
Ok(())
} else {
Err(Error::other(
response
.error
.map(|error| error.error_info)
.unwrap_or_else(|| "peer rejected scanner publication lease release".to_string()),
))
}
}
.await,
)
.await
self.scanner_activity_request(instance_id, generation).await
}
pub async fn get_metacache_listing(&self) -> Result<()> {
@@ -2281,52 +1977,6 @@ mod tests {
use temp_env::async_with_vars;
use tracing_subscriber::{Registry, fmt::MakeWriter, layer::SubscriberExt};
#[test]
fn scanner_publication_lease_response_rejects_stale_generation_and_session() {
let token = Uuid::new_v4();
let response = ScannerPublicationLeaseResponse {
success: true,
token: token.as_bytes().to_vec().into(),
movement_generation: 7,
lease_ttl_ms: crate::store::SCANNER_PUBLICATION_LEASE_TTL_MS,
error: None,
response_proof: Bytes::new(),
owner_id: Uuid::new_v4().to_string(),
session_id: "session-a".to_string(),
};
assert!(validate_scanner_publication_lease_response_fields(&response, "session-a", 7).is_ok());
let stale_generation = ScannerPublicationLeaseResponse {
movement_generation: 6,
..response.clone()
};
let error = validate_scanner_publication_lease_response_fields(&stale_generation, "session-a", 7)
.expect_err("a response from an older movement generation must be rejected");
assert!(error.to_string().contains("different scanner publication lease generation"));
let stale_session = ScannerPublicationLeaseResponse {
session_id: "session-b".to_string(),
..response
};
let error = validate_scanner_publication_lease_response_fields(&stale_session, "session-a", 7)
.expect_err("a response from an older scanner session must be rejected");
assert!(error.to_string().contains("different scanner publication lease session"));
}
#[test]
fn scanner_publication_lease_deadline_accounts_for_delayed_rpc_response() {
let started = std::time::Instant::now();
let expected_deadline = started + Duration::from_secs(55);
let deadline = scanner_publication_lease_deadline(started, started + Duration::from_secs(10), 60_000)
.expect("a response inside the safety window should retain the original deadline");
assert_eq!(deadline, expected_deadline);
let error = scanner_publication_lease_deadline(started, started + Duration::from_secs(55), 60_000)
.expect_err("a response arriving at the safety boundary must fail closed");
assert!(error.to_string().contains("after its safety window"));
}
#[test]
fn replication_stats_response_decodes_valid_empty_provider() {
let mut stats = BucketStats::default();
@@ -2565,8 +2215,6 @@ mod tests {
response_proof: Vec::new().into(),
dirty_usage_generation: 0,
dirty_usage_pending: false,
movement_generation: None,
publication_blocked: None,
})
.expect("legacy peers should retain their activity generations during a rolling upgrade");
assert_eq!(
@@ -2580,8 +2228,6 @@ mod tests {
data_movement_active: None,
dirty_usage_generation: None,
dirty_usage_pending: None,
movement_generation: None,
publication_blocked: None,
}
);
@@ -2595,8 +2241,6 @@ mod tests {
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 0,
dirty_usage_pending: false,
movement_generation: None,
publication_blocked: None,
})
.expect("protocol v4 peers should remain observable during a rolling upgrade");
assert_eq!(
@@ -2610,29 +2254,9 @@ mod tests {
data_movement_active: Some(true),
dirty_usage_generation: None,
dirty_usage_pending: None,
movement_generation: None,
publication_blocked: None,
}
);
let v6 = decode_test_scanner_activity(ScannerActivityResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: SCANNER_ACTIVITY_V6_PROTOCOL_VERSION,
topology_digest: vec![7; 32].into(),
data_movement_active: true,
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
movement_generation: None,
publication_blocked: None,
})
.expect("v6 peers should remain readable without a v7 publication proof");
assert_eq!(v6.movement_generation, None);
assert_eq!(v6.publication_blocked, None);
assert_eq!(v6.dirty_usage_generation, Some(11));
let malformed_topology = ScannerActivityResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
@@ -2643,8 +2267,6 @@ mod tests {
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
movement_generation: Some(19),
publication_blocked: Some(false),
};
assert!(
decode_test_scanner_activity(malformed_topology)
@@ -2663,8 +2285,6 @@ mod tests {
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
movement_generation: Some(19),
publication_blocked: Some(false),
};
assert!(
decode_test_scanner_activity(missing_instance)
@@ -2683,8 +2303,6 @@ mod tests {
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
movement_generation: Some(19),
publication_blocked: Some(false),
};
assert!(
decode_test_scanner_activity(malformed_instance)
@@ -2703,8 +2321,6 @@ mod tests {
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
movement_generation: Some(19),
publication_blocked: Some(false),
})
.expect("complete activity responses should be accepted");
assert_eq!(
@@ -2718,31 +2334,9 @@ mod tests {
data_movement_active: Some(true),
dirty_usage_generation: Some(11),
dirty_usage_pending: Some(true),
movement_generation: Some(19),
publication_blocked: Some(false),
}
);
let missing_movement_generation = ScannerActivityResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
topology_digest: vec![7; 32].into(),
data_movement_active: false,
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: false,
movement_generation: None,
publication_blocked: Some(false),
};
assert!(
decode_test_scanner_activity(missing_movement_generation)
.expect_err("v7 activity must carry movement generation")
.to_string()
.contains("movement generation")
);
let pending_without_generation = ScannerActivityResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
@@ -2753,8 +2347,6 @@ mod tests {
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 0,
dirty_usage_pending: true,
movement_generation: Some(19),
publication_blocked: Some(false),
};
assert!(
decode_test_scanner_activity(pending_without_generation)
@@ -2773,8 +2365,6 @@ mod tests {
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
movement_generation: None,
publication_blocked: None,
};
assert!(
decode_test_scanner_activity(previous_with_dirty_usage)
@@ -2793,8 +2383,6 @@ mod tests {
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 0,
dirty_usage_pending: false,
movement_generation: None,
publication_blocked: None,
};
assert!(
decode_test_scanner_activity(legacy_with_topology)
@@ -2813,8 +2401,6 @@ mod tests {
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
movement_generation: None,
publication_blocked: None,
};
assert!(
decode_test_scanner_activity(unsupported_protocol)
@@ -2833,8 +2419,6 @@ mod tests {
response_proof: Vec::new().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
movement_generation: Some(19),
publication_blocked: Some(false),
};
assert!(
decode_test_scanner_activity(missing_proof)
+24 -562
View File
@@ -18,7 +18,6 @@ use crate::cluster::rpc::client::{
node_service_time_out_client,
};
use crate::cluster::rpc::set_tonic_mutation_body_digest;
use crate::core::pools::{PoolMeta, PoolMetaWriteState};
use crate::disk::error::DiskError;
use crate::disk::error::{Error, Result};
use crate::disk::error_reduce::{BUCKET_OP_IGNORED_ERRS, is_all_buckets_not_found, reduce_write_quorum_errs};
@@ -47,12 +46,7 @@ use std::sync::{
Mutex as StdMutex,
atomic::{AtomicBool, Ordering},
};
use std::{
collections::{BTreeSet, HashMap},
fmt::Debug,
sync::Arc,
time::Duration,
};
use std::{collections::HashMap, fmt::Debug, sync::Arc, time::Duration};
#[cfg(test)]
use tokio::sync::Notify;
use tokio::{net::TcpStream, sync::RwLock, time};
@@ -105,9 +99,6 @@ impl DeleteBucketEmptyScanBarrier {
#[cfg(test)]
static DELETE_BUCKET_EMPTY_SCAN_BARRIER: StdMutex<Option<Arc<DeleteBucketEmptyScanBarrier>>> = StdMutex::new(None);
#[cfg(test)]
static HEAL_BUCKET_PRE_MUTATION_BARRIER: StdMutex<Option<Arc<DeleteBucketEmptyScanBarrier>>> = StdMutex::new(None);
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
enum HealBucketOperation {
Make,
@@ -180,15 +171,6 @@ pub(crate) fn install_delete_bucket_empty_scan_barrier() -> Arc<DeleteBucketEmpt
barrier
}
#[cfg(test)]
fn install_heal_bucket_pre_mutation_barrier() -> Arc<DeleteBucketEmptyScanBarrier> {
let barrier = Arc::new(DeleteBucketEmptyScanBarrier::default());
*HEAL_BUCKET_PRE_MUTATION_BARRIER
.lock()
.expect("heal bucket mutation barrier lock should not be poisoned") = Some(barrier.clone());
barrier
}
#[cfg(test)]
async fn pause_after_delete_bucket_empty_scan() {
let barrier = DELETE_BUCKET_EMPTY_SCAN_BARRIER
@@ -200,20 +182,6 @@ async fn pause_after_delete_bucket_empty_scan() {
}
}
#[cfg(test)]
async fn pause_before_heal_bucket_volume_mutation() {
let barrier = HEAL_BUCKET_PRE_MUTATION_BARRIER
.lock()
.expect("heal bucket mutation barrier lock should not be poisoned")
.take();
if let Some(barrier) = barrier {
barrier.pause().await;
}
}
#[cfg(not(test))]
async fn pause_before_heal_bucket_volume_mutation() {}
#[derive(Clone, Debug)]
pub struct ScannerBucketListing {
pub buckets: Vec<BucketInfo>,
@@ -285,53 +253,9 @@ fn resolve_heal_bucket_mode(opts: &mut HealOpts, pool_errs: &[Option<Error>]) ->
Ok(())
}
#[derive(serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct HealBucketRpcEnvelope {
options: HealOpts,
#[serde(rename = "fencedPools", default)]
fenced_pools: Vec<usize>,
}
pub fn encode_heal_bucket_rpc_options(opts: HealOpts, fenced_pools: &[usize]) -> Result<String> {
if fenced_pools.is_empty() {
return serde_json::to_string(&opts).map_err(Into::into);
}
serde_json::to_string(&HealBucketRpcEnvelope {
options: opts,
fenced_pools: fenced_pools.to_vec(),
})
.map_err(Into::into)
}
pub fn decode_heal_bucket_rpc_options(payload: &str) -> Result<(HealOpts, Vec<usize>)> {
match serde_json::from_str::<HealBucketRpcEnvelope>(payload) {
Ok(envelope) => Ok((envelope.options, envelope.fenced_pools)),
Err(envelope_err) => serde_json::from_str::<HealOpts>(payload)
.map(|options| (options, Vec::new()))
.map_err(|legacy_err| {
Error::other(format!(
"decode heal bucket RPC options failed: envelope={envelope_err}; legacy={legacy_err}"
))
}),
}
}
#[async_trait]
pub trait PeerS3Client: Debug + Sync + Send + 'static {
async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem>;
async fn heal_bucket_with_fence(&self, bucket: &str, opts: &HealOpts, _fenced_pools: &[usize]) -> Result<HealResultItem> {
self.heal_bucket(bucket, opts).await
}
async fn heal_bucket_with_fence_from_movement_guarded_coordinator(
&self,
bucket: &str,
opts: &HealOpts,
fenced_pools: &[usize],
) -> Result<HealResultItem> {
self.heal_bucket_with_fence(bucket, opts, fenced_pools).await
}
async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()>;
async fn list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>>;
async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()>;
@@ -385,29 +309,6 @@ impl S3PeerSys {
impl S3PeerSys {
pub async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
self.heal_bucket_with_fence(bucket, opts, &[]).await
}
pub async fn heal_bucket_with_fence(&self, bucket: &str, opts: &HealOpts, fenced_pools: &[usize]) -> Result<HealResultItem> {
self.heal_bucket_with_fence_inner(bucket, opts, fenced_pools, false).await
}
pub async fn heal_bucket_with_fence_from_movement_guarded_coordinator(
&self,
bucket: &str,
opts: &HealOpts,
fenced_pools: &[usize],
) -> Result<HealResultItem> {
self.heal_bucket_with_fence_inner(bucket, opts, fenced_pools, true).await
}
async fn heal_bucket_with_fence_inner(
&self,
bucket: &str,
opts: &HealOpts,
fenced_pools: &[usize],
movement_guard_held: bool,
) -> Result<HealResultItem> {
let mut opts = *opts;
let mut futures = Vec::with_capacity(self.clients.len());
for client in self.clients.iter() {
@@ -430,14 +331,7 @@ impl S3PeerSys {
let opts_clone = opts;
let heal_bucket_results_clone = heal_bucket_results.clone();
futures.push(async move {
let result = if movement_guard_held {
client
.heal_bucket_with_fence_from_movement_guarded_coordinator(bucket, &opts_clone, fenced_pools)
.await
} else {
client.heal_bucket_with_fence(bucket, &opts_clone, fenced_pools).await
};
match result {
match client.heal_bucket(bucket, &opts_clone).await {
Ok(res) => {
heal_bucket_results_clone.write().await[idx] = res;
None
@@ -732,63 +626,6 @@ impl LocalPeerS3Client {
.filter(|disk| usize::try_from(disk.endpoint().pool_idx).is_ok_and(|pool_idx| pools.contains(&pool_idx)))
.collect()
}
async fn heal_bucket_with_fence_inner(
&self,
bucket: &str,
opts: &HealOpts,
fenced_pools: &[usize],
movement_guard_held: bool,
) -> Result<HealResultItem> {
let disks = self.local_disks_for_pools().await.into_iter().map(Some).collect();
let store = runtime_sources::object_store_handle().filter(|store| Arc::ptr_eq(&store.ctx, &self.instance_ctx));
#[cfg(not(test))]
if store.is_none() {
return Err(Error::other("bucket heal refused: pool metadata is unavailable for this instance"));
}
let movement_gate = store.as_ref().map(|store| store.ctx.data_movement_operation_gate());
let movement_guard = try_acquire_bucket_heal_movement_guard(movement_gate.as_ref(), movement_guard_held)?;
let save_guard = acquire_bucket_heal_write_guard(store.as_ref().map(|store| &store.pool_meta_save_gate)).await?;
let result = heal_bucket_local_on_disks_with_pool_meta(
bucket,
opts,
disks,
store.as_ref().map(|store| &store.pool_meta),
fenced_pools,
)
.await;
drop(save_guard);
drop(movement_guard);
result
}
}
fn try_acquire_bucket_heal_movement_guard<'a>(
gate: Option<&'a Arc<tokio::sync::RwLock<()>>>,
movement_guard_held: bool,
) -> Result<Option<tokio::sync::RwLockReadGuard<'a, ()>>> {
if movement_guard_held {
return Ok(None);
}
let Some(gate) = gate else {
return Ok(None);
};
// Do not queue a receiver behind a movement writer while its coordinator
// holds another node's read guard; failing fast breaks that cross-node cycle.
gate.try_read()
.map(Some)
.map_err(|_| crate::error::StorageError::SlowDown.into())
}
async fn acquire_bucket_heal_write_guard<'a>(
gate: Option<&'a tokio::sync::Mutex<PoolMetaWriteState>>,
) -> Result<Option<tokio::sync::MutexGuard<'a, PoolMetaWriteState>>> {
let Some(gate) = gate else {
return Ok(None);
};
let guard = gate.lock().await;
guard.ensure_write_safe("bucket heal cannot run while pool metadata requires recovery")?;
Ok(Some(guard))
}
#[async_trait]
@@ -798,20 +635,8 @@ impl PeerS3Client for LocalPeerS3Client {
}
async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
self.heal_bucket_with_fence(bucket, opts, &[]).await
}
async fn heal_bucket_with_fence(&self, bucket: &str, opts: &HealOpts, fenced_pools: &[usize]) -> Result<HealResultItem> {
self.heal_bucket_with_fence_inner(bucket, opts, fenced_pools, false).await
}
async fn heal_bucket_with_fence_from_movement_guarded_coordinator(
&self,
bucket: &str,
opts: &HealOpts,
fenced_pools: &[usize],
) -> Result<HealResultItem> {
self.heal_bucket_with_fence_inner(bucket, opts, fenced_pools, true).await
let disks = self.local_disks_for_pools().await.into_iter().map(Some).collect();
heal_bucket_local_on_disks(bucket, opts, disks).await
}
async fn list_bucket(&self, _opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
@@ -1254,13 +1079,9 @@ impl PeerS3Client for RemotePeerS3Client {
}
async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
self.heal_bucket_with_fence(bucket, opts, &[]).await
}
async fn heal_bucket_with_fence(&self, bucket: &str, opts: &HealOpts, fenced_pools: &[usize]) -> Result<HealResultItem> {
self.execute_with_timeout(
|| async {
let options = encode_heal_bucket_rpc_options(*opts, fenced_pools)?;
let options: String = serde_json::to_string(opts)?;
let mut client = self.get_client().await?;
let mut request = Request::new(HealBucketRequest {
bucket: bucket.to_string(),
@@ -1408,117 +1229,6 @@ pub(crate) async fn heal_bucket_local_on_disks(
opts: &HealOpts,
disks: Vec<Option<DiskStore>>,
) -> Result<HealResultItem> {
if let Some(store) = runtime_sources::object_store_handle() {
return heal_bucket_local_on_disks_with_pool_meta(bucket, opts, disks, Some(&store.pool_meta), &[]).await;
}
#[cfg(test)]
return heal_bucket_local_on_disks_with_pool_meta(bucket, opts, disks, None, &[]).await;
#[cfg(not(test))]
Err(Error::other("bucket heal refused: pool metadata is unavailable"))
}
fn disk_pool_index(disk: &DiskStore) -> Result<usize> {
usize::try_from(disk.endpoint().pool_idx)
.map_err(|_| Error::other(format!("invalid bucket-heal pool index {}", disk.endpoint().pool_idx)))
}
fn fenced_decommission_drive_state() -> DriveState {
DriveState::Unknown("skipped-decommission-suspended".to_string())
}
fn heal_bucket_fence_detail(fenced_pools: &BTreeSet<usize>) -> Option<String> {
if fenced_pools.is_empty() {
return None;
}
let pools = fenced_pools.iter().map(usize::to_string).collect::<Vec<_>>().join(", ");
Some(format!("skipped: bucket-volume heal fenced on decommission-suspended pool(s): {pools}"))
}
async fn snapshot_heal_bucket_fence(
disks: &[Option<DiskStore>],
pool_meta: Option<&RwLock<PoolMeta>>,
dispatch_fenced_pools: &[usize],
) -> Result<(Vec<bool>, BTreeSet<usize>)> {
let mut fenced_disks = vec![false; disks.len()];
let mut fenced_pools = dispatch_fenced_pools.iter().copied().collect::<BTreeSet<_>>();
let pool_meta = match pool_meta {
Some(pool_meta) => Some(pool_meta.read().await),
None => None,
};
if let Some(pool_meta) = pool_meta.as_ref()
&& let Some(pool_idx) = fenced_pools.iter().find(|pool_idx| **pool_idx >= pool_meta.pools.len())
{
return Err(Error::other(format!(
"bucket-heal dispatch fence pool index {pool_idx} is absent from {} pool metadata entries",
pool_meta.pools.len()
)));
}
for (disk_index, disk) in disks.iter().enumerate() {
let Some(disk) = disk else {
continue;
};
let pool_idx = disk_pool_index(disk)?;
if let Some(pool_meta) = pool_meta.as_ref() {
if pool_idx >= pool_meta.pools.len() {
return Err(Error::other(format!(
"bucket-heal pool index {pool_idx} is absent from {} pool metadata entries",
pool_meta.pools.len()
)));
}
if pool_meta.is_suspended(pool_idx) {
fenced_pools.insert(pool_idx);
}
}
if fenced_pools.contains(&pool_idx) {
fenced_disks[disk_index] = true;
}
}
Ok((fenced_disks, fenced_pools))
}
async fn run_heal_bucket_volume_mutation<F, Fut>(
disk: &DiskStore,
pool_meta: Option<&RwLock<PoolMeta>>,
operation: F,
) -> Result<Option<usize>>
where
F: FnOnce() -> Fut,
Fut: std::future::Future<Output = Result<()>>,
{
let Some(pool_meta) = pool_meta else {
operation().await?;
return Ok(None);
};
let pool_idx = disk_pool_index(disk)?;
let pool_meta = pool_meta.read().await;
if pool_idx >= pool_meta.pools.len() {
return Err(Error::other(format!(
"bucket-heal pool index {pool_idx} is absent from {} pool metadata entries",
pool_meta.pools.len()
)));
}
if pool_meta.is_suspended(pool_idx) {
return Ok(Some(pool_idx));
}
// Keep the metadata read guard through the disk mutation so a decommission
// transition cannot pass between this state check and the destructive action.
operation().await?;
Ok(None)
}
async fn heal_bucket_local_on_disks_with_pool_meta(
bucket: &str,
opts: &HealOpts,
disks: Vec<Option<DiskStore>>,
pool_meta: Option<&RwLock<PoolMeta>>,
dispatch_fenced_pools: &[usize],
) -> Result<HealResultItem> {
let (fenced_disks, mut fenced_pool_idxs) = snapshot_heal_bucket_fence(&disks, pool_meta, dispatch_fenced_pools).await?;
let fenced_disks = Arc::new(fenced_disks);
let before_state = Arc::new(RwLock::new(vec![String::new(); disks.len()]));
let after_state = Arc::new(RwLock::new(vec![String::new(); disks.len()]));
@@ -1528,14 +1238,7 @@ async fn heal_bucket_local_on_disks_with_pool_meta(
let bucket = bucket.to_string();
let bs_clone = before_state.clone();
let as_clone = after_state.clone();
let fenced_disks = fenced_disks.clone();
futures.push(async move {
if fenced_disks[index] {
let skipped = fenced_decommission_drive_state().to_string();
bs_clone.write().await[index] = skipped.clone();
as_clone.write().await[index] = skipped;
return None;
}
let disk = match disk {
Some(disk) => disk,
None => {
@@ -1598,14 +1301,9 @@ async fn heal_bucket_local_on_disks_with_pool_meta(
state: state.to_string(),
});
}
if let Some(detail) = heal_bucket_fence_detail(&fenced_pool_idxs) {
res.detail = detail;
}
return Ok(res);
}
pause_before_heal_bucket_volume_mutation().await;
let mut operation_error = errs
.iter()
.filter_map(|err| match err {
@@ -1617,35 +1315,26 @@ async fn heal_bucket_local_on_disks_with_pool_meta(
if opts.remove && !bucket.starts_with(disk::RUSTFS_META_BUCKET) && !is_all_buckets_not_found(&errs) {
let mut futures = Vec::new();
for (index, disk) in disks.iter().enumerate() {
if fenced_disks[index] || matches!(errs[index].as_ref(), Some(Error::DiskNotFound | Error::VolumeNotFound)) {
if matches!(errs[index].as_ref(), Some(Error::DiskNotFound | Error::VolumeNotFound)) {
continue;
}
let Some(disk) = disk.clone() else {
continue;
};
let bucket = bucket.to_string();
let mutation_disk = disk.clone();
futures.push(async move {
let result = run_heal_bucket_volume_mutation(&disk, pool_meta, || async move {
if let Some(err) = injected_heal_bucket_operation_error(&bucket, index, HealBucketOperation::Delete) {
return Err(err);
}
mutation_disk.delete_volume(&bucket, false).await
})
.await;
(index, result)
if let Some(err) = injected_heal_bucket_operation_error(&bucket, index, HealBucketOperation::Delete) {
return (index, Err(err));
}
(index, disk.delete_volume(&bucket, false).await)
});
}
for (index, result) in join_all(futures).await {
match result {
Ok(None) | Err(Error::VolumeNotFound) => {
Ok(()) | Err(Error::VolumeNotFound) => {
after_state.write().await[index] = DriveState::Missing.to_string();
}
Ok(Some(pool_idx)) => {
fenced_pool_idxs.insert(pool_idx);
after_state.write().await[index] = fenced_decommission_drive_state().to_string();
}
Err(Error::VolumeNotEmpty) => {
warn!(
bucket,
@@ -1676,38 +1365,30 @@ async fn heal_bucket_local_on_disks_with_pool_meta(
let bs_clone = before_state.clone();
futures.push(async move {
if bs_clone.read().await[idx] == DriveState::Missing.to_string() {
let Some(disk) = disk else {
return (idx, Err(Error::DiskNotFound));
let Some(disk) = disk.as_ref() else {
return (idx, Some(Error::DiskNotFound));
};
let mutation_disk = disk.clone();
let result = run_heal_bucket_volume_mutation(&disk, pool_meta, || async move {
if let Some(err) = injected_heal_bucket_operation_error(&bucket, idx, HealBucketOperation::Make) {
return Err(err);
}
match mutation_disk.make_volume(&bucket).await {
Ok(()) | Err(Error::VolumeExists) => Ok(()),
Err(err) => Err(err),
}
})
.await;
return (idx, result);
if let Some(err) = injected_heal_bucket_operation_error(&bucket, idx, HealBucketOperation::Make) {
return (idx, Some(err));
}
match disk.make_volume(&bucket).await {
Ok(()) | Err(Error::VolumeExists) => return (idx, None),
Err(err) => return (idx, Some(err)),
}
}
(idx, Ok(None))
(idx, None)
});
}
for (index, result) in join_all(futures).await {
match result {
Ok(None) => {
None => {
if before_state.read().await[index] == DriveState::Missing.to_string() {
after_state.write().await[index] = DriveState::Ok.to_string();
}
}
Ok(Some(pool_idx)) => {
fenced_pool_idxs.insert(pool_idx);
after_state.write().await[index] = fenced_decommission_drive_state().to_string();
}
Err(err) => {
Some(err) => {
after_state.write().await[index] = match &err {
Error::DiskNotFound => DriveState::Offline.to_string(),
_ => DriveState::Corrupt.to_string(),
@@ -1728,10 +1409,6 @@ async fn heal_bucket_local_on_disks_with_pool_meta(
});
}
if let Some(detail) = heal_bucket_fence_detail(&fenced_pool_idxs) {
res.detail = detail;
}
match operation_error {
Some(err) => Err(err),
None => Ok(res),
@@ -1749,7 +1426,6 @@ async fn clone_drives() -> Vec<Option<DiskStore>> {
#[cfg(test)]
mod tests {
use super::*;
use crate::core::pools::{PoolDecommissionInfo, PoolMetaReplicaState, PoolStatus};
use crate::disk::WalkDirOptions;
use crate::disk::disk_store::LocalDiskWrapper;
use crate::disk::endpoint::Endpoint;
@@ -1923,23 +1599,6 @@ mod tests {
disks
}
fn heal_bucket_pool_meta(suspended_pool: Option<usize>) -> PoolMeta {
PoolMeta {
pools: (0..2)
.map(|pool_idx| PoolStatus {
id: pool_idx,
cmd_line: format!("pool-{pool_idx}"),
last_update: ::time::OffsetDateTime::UNIX_EPOCH,
decommission: (suspended_pool == Some(pool_idx)).then(|| PoolDecommissionInfo {
start_time: Some(::time::OffsetDateTime::UNIX_EPOCH),
..Default::default()
}),
})
.collect(),
..Default::default()
}
}
fn test_remote_peer(addr: &str) -> RemotePeerS3Client {
RemotePeerS3Client {
pools: Some(vec![0]),
@@ -2251,173 +1910,6 @@ mod tests {
reset_local_disk_test_state().await;
}
#[tokio::test]
#[serial]
async fn heal_bucket_rechecks_decommission_before_recreating_volume() {
reset_local_disk_test_state().await;
let temp_dir = TempDir::new().expect("create temp dir for bucket-heal fence regression");
let disks = init_test_local_disks_for_pools(&temp_dir, &[(0, 1), (1, 1)], "heal-bucket-mutation-fence").await;
let bucket = "fenced-recreate-bucket";
disks[0]
.make_volume(bucket)
.await
.expect("active pool should start with the bucket volume");
let pool_meta = Arc::new(RwLock::new(heal_bucket_pool_meta(None)));
let barrier = install_heal_bucket_pre_mutation_barrier();
let heal = tokio::spawn({
let disks = disks.clone();
let pool_meta = pool_meta.clone();
async move {
heal_bucket_local_on_disks_with_pool_meta(
bucket,
&HealOpts {
recreate: true,
..Default::default()
},
disks.into_iter().map(Some).collect(),
Some(pool_meta.as_ref()),
&[],
)
.await
}
});
barrier.wait_until_paused().await;
pool_meta.write().await.pools[1].decommission = Some(PoolDecommissionInfo {
start_time: Some(::time::OffsetDateTime::UNIX_EPOCH),
..Default::default()
});
barrier.release();
let result = heal
.await
.expect("bucket-heal task should join")
.expect("suspended pool should be reported as skipped");
assert!(result.detail.contains("skipped") && result.detail.contains('1'));
assert!(matches!(disks[1].stat_volume(bucket).await, Err(Error::VolumeNotFound)));
reset_local_disk_test_state().await;
}
#[tokio::test]
#[serial]
async fn heal_bucket_dispatch_fence_blocks_stale_active_peer_state() {
reset_local_disk_test_state().await;
let temp_dir = TempDir::new().expect("create temp dir for stale bucket-heal peer regression");
let disks = init_test_local_disks_for_pools(&temp_dir, &[(0, 1), (1, 1)], "heal-bucket-dispatch-fence").await;
let bucket = "dispatch-fenced-bucket";
disks[0]
.make_volume(bucket)
.await
.expect("active pool should start with the bucket volume");
let stale_pool_meta = RwLock::new(heal_bucket_pool_meta(None));
let result = heal_bucket_local_on_disks_with_pool_meta(
bucket,
&HealOpts {
recreate: true,
..Default::default()
},
disks.iter().cloned().map(Some).collect(),
Some(&stale_pool_meta),
&[1],
)
.await
.expect("dispatch fence should override stale active peer metadata");
assert!(result.detail.contains("skipped") && result.detail.contains('1'));
assert!(matches!(disks[1].stat_volume(bucket).await, Err(Error::VolumeNotFound)));
reset_local_disk_test_state().await;
}
#[tokio::test]
async fn local_bucket_heal_refuses_receiver_with_unsafe_pool_metadata() {
let gate = tokio::sync::Mutex::new(PoolMetaWriteState::default());
gate.lock().await.observe_replicas(PoolMetaReplicaState {
needs_repair: true,
repair_write_safe: false,
});
let err = acquire_bucket_heal_write_guard(Some(&gate))
.await
.expect_err("receiver-side bucket heal must honor the local pool metadata gate");
assert!(
err.to_string()
.contains("bucket heal cannot run while pool metadata requires recovery")
);
}
#[tokio::test]
async fn receiver_bucket_heal_fails_fast_behind_queued_movement_writer() {
let gate = Arc::new(tokio::sync::RwLock::new(()));
let coordinator_guard = gate.read().await;
let writer_gate = gate.clone();
let mut writer = tokio::spawn(async move {
let _writer_guard = writer_gate.write().await;
});
while gate.try_read().is_ok() {
tokio::task::yield_now().await;
}
let err = try_acquire_bucket_heal_movement_guard(Some(&gate), false)
.expect_err("receiver must not wait behind a queued movement writer");
assert_eq!(err, crate::error::StorageError::SlowDown.into());
assert!(
try_acquire_bucket_heal_movement_guard(Some(&gate), true)
.expect("coordinator-owned movement guard should be reused")
.is_none()
);
drop(coordinator_guard);
tokio::time::timeout(std::time::Duration::from_secs(1), &mut writer)
.await
.expect("queued movement writer should proceed after the coordinator guard is released")
.expect("movement writer task should not panic");
}
#[tokio::test]
#[serial]
async fn heal_bucket_keeps_suspended_pool_volume_on_remove() {
reset_local_disk_test_state().await;
let temp_dir = TempDir::new().expect("create temp dir for bucket-heal delete fence regression");
let disks = init_test_local_disks_for_pools(&temp_dir, &[(0, 1), (1, 1)], "heal-bucket-delete-fence").await;
let bucket = "fenced-remove-bucket";
for disk in &disks {
disk.make_volume(bucket)
.await
.expect("bucket volume should exist before heal");
}
let pool_meta = RwLock::new(heal_bucket_pool_meta(Some(1)));
let result = heal_bucket_local_on_disks_with_pool_meta(
bucket,
&HealOpts {
remove: true,
..Default::default()
},
disks.iter().cloned().map(Some).collect(),
Some(&pool_meta),
&[],
)
.await
.expect("suspended pool should be skipped during bucket-volume removal");
assert!(result.detail.contains("skipped") && result.detail.contains('1'));
assert!(matches!(disks[0].stat_volume(bucket).await, Err(Error::VolumeNotFound)));
disks[1]
.stat_volume(bucket)
.await
.expect("suspended pool bucket volume must not be deleted");
reset_local_disk_test_state().await;
}
#[tokio::test]
#[serial]
async fn heal_bucket_local_dry_run_reports_discovered_drive_states() {
@@ -2631,36 +2123,6 @@ mod tests {
assert!(partial.recreate);
}
#[test]
fn heal_bucket_rpc_envelope_preserves_legacy_compatibility_fail_closed() {
let opts = HealOpts {
recreate: true,
pool: Some(2),
..Default::default()
};
let encoded = encode_heal_bucket_rpc_options(opts, &[1, 2]).expect("encode bucket-heal RPC envelope");
assert!(
serde_json::from_str::<HealOpts>(&encoded).is_err(),
"an old peer must reject the nested request instead of ignoring its dispatch fence"
);
let (decoded, fenced_pools) =
decode_heal_bucket_rpc_options(&encoded).expect("new peer should decode bucket-heal RPC envelope");
assert!(decoded.recreate);
assert_eq!(decoded.pool, Some(2));
assert_eq!(fenced_pools, vec![1, 2]);
let legacy = encode_heal_bucket_rpc_options(opts, &[]).expect("encode legacy HealOpts for an unfenced heal");
let old_peer_opts = serde_json::from_str::<HealOpts>(&legacy).expect("old peer should decode an unfenced heal request");
assert!(old_peer_opts.recreate);
assert_eq!(old_peer_opts.pool, Some(2));
let (decoded, fenced_pools) = decode_heal_bucket_rpc_options(&legacy).expect("new peer should accept a legacy request");
assert!(decoded.recreate);
assert_eq!(decoded.pool, Some(2));
assert!(fenced_pools.is_empty());
}
#[tokio::test]
async fn test_make_bucket_reduces_quorum_by_pool_participants() {
let peer_sys = S3PeerSys {
+46 -150
View File
@@ -781,16 +781,14 @@ impl RemoteDisk {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
let probe = self.data_transport.probe_ns_scanner_capability(NsScannerCapabilityRequest {
let probe = self.data_transport.probe_ns_scanner(NsScannerCapabilityRequest {
endpoint: self.endpoint.grid_host(),
supports_tier_registry_generation: true,
});
let result = timeout(NS_SCANNER_CAPABILITY_PROBE_TIMEOUT, probe)
.await
.map_err(|_| DiskError::other("remote namespace scanner capability probe timed out"))?;
match result {
Ok(response) if response.supports_tier_registry_generation == Some(true) => Ok(Some(response.server_epoch)),
Ok(_) => Ok(None),
Ok(server_epoch) => Ok(Some(server_epoch)),
// RUSTFS_COMPAT_TODO(ns-scanner-rpc-v3): old peers and legacy transports lack the authenticated startup-epoch handshake. Remove after every supported peer implements namespace scanner protocol v3.
Err(DiskError::MethodNotAllowed) => Ok(None),
Err(err)
@@ -1981,20 +1979,6 @@ impl RemoteDisk {
fi: &FileInfo,
dst_volume: &str,
dst_path: &str,
) -> Result<RenameDataResp> {
self.rename_data_borrowed_with_fence(src_volume, src_path, fi, dst_volume, dst_path, None)
.await
}
#[tracing::instrument(level = "trace", skip_all)]
pub(crate) async fn rename_data_borrowed_with_fence(
&self,
src_volume: &str,
src_path: &str,
fi: &FileInfo,
dst_volume: &str,
dst_path: &str,
scanner_publication_lease_token: Option<Uuid>,
) -> Result<RenameDataResp> {
trace!(
event = EVENT_REMOTE_DISK_RPC,
@@ -2027,18 +2011,9 @@ impl RemoteDisk {
dst_volume: dst_volume.to_string(),
dst_path: dst_path.to_string(),
file_info_bin: file_info_bin.into(),
scanner_publication_lease_token: scanner_publication_lease_token
.map(|token| token.as_bytes().to_vec().into())
.unwrap_or_default(),
});
let canonical_body = rustfs_protos::canonical_rename_data_request_body(request.get_ref());
if scanner_publication_lease_token.is_some() {
let canonical_body =
canonical_body.map_err(|_| Error::other("rename_data request length cannot be represented"))?;
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical_body).map_err(Error::other)?;
} else {
attach_mutation_body_digest(&mut request, canonical_body, "rename_data")?;
}
attach_mutation_body_digest(&mut request, canonical_body, "rename_data")?;
let response = client.rename_data(request).await?.into_inner();
@@ -2058,70 +2033,6 @@ impl RemoteDisk {
)
.await
}
/// Delete a path while binding the target-side operation to a scanner
/// publication lease. The ordinary `DiskAPI::delete` path keeps the
/// legacy digest/compatibility behavior by passing no token.
#[tracing::instrument(level = "trace", skip_all)]
pub(crate) async fn delete_with_scanner_publication_lease(
&self,
volume: &str,
path: &str,
opt: DeleteOptions,
scanner_publication_lease_token: Option<Uuid>,
) -> Result<()> {
trace!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
recursive = opt.recursive,
immediate = opt.immediate,
fenced = scanner_publication_lease_token.is_some(),
op = "delete",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
let options = serde_json::to_string(&opt)?;
let mut client = self
.get_client()
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
let mut request = Request::new(DeleteRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
options,
scanner_publication_lease_token: scanner_publication_lease_token
.map(|token| token.as_bytes().to_vec().into())
.unwrap_or_default(),
});
let canonical_body = rustfs_protos::canonical_delete_request_body(request.get_ref());
if scanner_publication_lease_token.is_some() {
let canonical_body =
canonical_body.map_err(|_| Error::other("delete request length cannot be represented"))?;
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical_body).map_err(Error::other)?;
} else {
attach_mutation_body_digest(&mut request, canonical_body, "delete")?;
}
let response = client.delete(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
},
get_max_timeout_duration(),
)
.await
}
}
#[async_trait::async_trait]
@@ -3533,7 +3444,47 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(level = "trace", skip_all)]
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
self.delete_with_scanner_publication_lease(volume, path, opt, None).await
trace!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
recursive = opt.recursive,
immediate = opt.immediate,
op = "delete",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
let options = serde_json::to_string(&opt)?;
let mut client = self
.get_client()
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
let mut request = Request::new(DeleteRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
options,
});
let canonical_body = rustfs_protos::canonical_delete_request_body(request.get_ref());
attach_mutation_body_digest(&mut request, canonical_body, "delete")?;
let response = client.delete(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
},
get_max_timeout_duration(),
)
.await
}
#[tracing::instrument(level = "trace", skip_all)]
@@ -4089,21 +4040,10 @@ mod tests {
NsScannerProbe(NsScannerCapabilityRequest),
}
#[derive(Debug, Clone)]
#[derive(Debug, Clone, Default)]
struct RecordingInternodeDataTransport {
calls: Arc<StdMutex<Vec<RecordedTransportCall>>>,
ns_scanner_probe_status: Arc<StdMutex<Option<u16>>>,
ns_scanner_generation_support: Arc<StdMutex<Option<bool>>>,
}
impl Default for RecordingInternodeDataTransport {
fn default() -> Self {
Self {
calls: Arc::default(),
ns_scanner_probe_status: Arc::default(),
ns_scanner_generation_support: Arc::new(StdMutex::new(Some(true))),
}
}
}
#[derive(Clone, Debug)]
@@ -4323,15 +4263,6 @@ mod tests {
Self {
calls: Arc::default(),
ns_scanner_probe_status: Arc::new(StdMutex::new(Some(status))),
ns_scanner_generation_support: Arc::new(StdMutex::new(Some(true))),
}
}
fn with_ns_scanner_generation_support(support: Option<bool>) -> Self {
Self {
calls: Arc::default(),
ns_scanner_probe_status: Arc::default(),
ns_scanner_generation_support: Arc::new(StdMutex::new(support)),
}
}
@@ -5014,23 +4945,6 @@ mod tests {
Ok(Uuid::from_u128(1))
}
async fn probe_ns_scanner_capability(
&self,
request: NsScannerCapabilityRequest,
) -> Result<crate::storage_api_contracts::internode::NsScannerCapabilityResponse> {
let server_epoch = self.probe_ns_scanner(request).await?;
let supports_tier_registry_generation = *self
.ns_scanner_generation_support
.lock()
.expect("namespace scanner generation support lock poisoned");
Ok(crate::storage_api_contracts::internode::NsScannerCapabilityResponse {
version: crate::storage_api_contracts::internode::NS_SCANNER_PROTOCOL_VERSION,
server_epoch,
proof: Vec::new(),
supports_tier_registry_generation,
})
}
fn name(&self) -> &'static str {
"recording"
}
@@ -6843,22 +6757,6 @@ mod tests {
}
}
#[tokio::test]
async fn test_remote_disk_namespace_scanner_capability_falls_back_without_generation_support() {
for support in [None, Some(false)] {
let transport = RecordingInternodeDataTransport::with_ns_scanner_generation_support(support);
let remote_disk = new_remote_disk_with_transport(Arc::new(transport)).await;
assert_eq!(
remote_disk
.ns_scanner_server_epoch()
.await
.expect("missing generation support should be classified as unsupported"),
None
);
}
}
#[tokio::test]
async fn test_remote_disk_namespace_scanner_capability_rejects_legacy_transport() {
let remote_disk = new_remote_disk_with_transport(Arc::new(RetryingOpenReadInternodeDataTransport::default())).await;
@@ -6911,7 +6809,7 @@ mod tests {
}
#[tokio::test]
async fn test_remote_disk_walk_dir_preserves_control_options() {
async fn test_remote_disk_walk_dir_preserves_skip_total_timeout_option() {
let transport = RecordingInternodeDataTransport::default();
let remote_disk = new_remote_disk_with_transport(Arc::new(transport.clone())).await;
let opts = WalkDirOptions {
@@ -6919,7 +6817,6 @@ mod tests {
base_dir: "prefix".to_string(),
recursive: true,
skip_total_timeout: true,
skip_hidden_prefix_check: true,
..Default::default()
};
let mut writer = Vec::new();
@@ -6936,7 +6833,6 @@ mod tests {
let sent_opts: WalkDirOptions =
serde_json::from_slice(&request.body).expect("walk_dir request body should deserialize");
assert!(sent_opts.skip_total_timeout);
assert!(sent_opts.skip_hidden_prefix_check);
assert_eq!(request.stall_timeout, Some(get_drive_walkdir_stall_timeout()));
}
other => panic!("expected walk-dir transport call, got {other:?}"),
-19
View File
@@ -406,25 +406,6 @@ where
Ok(data)
}
pub(crate) async fn read_config_limited_preserve_empty<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<Vec<u8>>
where
S: EcstoreObjectIO,
{
let (data, _obj) = read_config_limited_preserve_empty_with_metadata(api, file, max_bytes).await?;
Ok(data)
}
pub(crate) async fn read_config_limited_preserve_empty_with_metadata<S>(
api: Arc<S>,
file: &str,
max_bytes: usize,
) -> Result<(Vec<u8>, ObjectInfo)>
where
S: EcstoreObjectIO,
{
read_config_with_metadata_inner(api, file, &ObjectOptions::default(), true, Some(max_bytes)).await
}
/// Read an existing config object without treating an empty payload as absent.
/// Callers that validate their own payload format need to distinguish corruption
/// from `ConfigNotFound`.
File diff suppressed because it is too large Load Diff
+4 -12
View File
@@ -1241,16 +1241,8 @@ pub(crate) async fn make_local_two_set_sets() -> (Vec<tempfile::TempDir>, Arc<Se
make_local_two_set_sets_with_ctx(bootstrap_ctx()).await
}
#[cfg(any(test, feature = "test-util"))]
#[cfg(test)]
pub(crate) async fn make_local_two_set_sets_with_ctx(ctx: Arc<InstanceContext>) -> (Vec<tempfile::TempDir>, Arc<Sets>) {
make_local_two_set_sets_for_pool_with_ctx(ctx, 0).await
}
#[cfg(any(test, feature = "test-util"))]
pub(crate) async fn make_local_two_set_sets_for_pool_with_ctx(
ctx: Arc<InstanceContext>,
pool_idx: usize,
) -> (Vec<tempfile::TempDir>, Arc<Sets>) {
use crate::layout::endpoint::Endpoint;
use rustfs_lock::client::local::LocalClient;
@@ -1266,7 +1258,7 @@ pub(crate) async fn make_local_two_set_sets_for_pool_with_ctx(
let temp_dir = tempfile::tempdir().expect("tempdir should be created");
let mut endpoint = Endpoint::try_from(temp_dir.path().to_str().expect("tempdir path should be utf8"))
.expect("endpoint should parse");
endpoint.set_pool_index(pool_idx);
endpoint.set_pool_index(0);
endpoint.set_set_index(set_index);
endpoint.set_disk_index(disk_index);
let disk = new_disk(
@@ -1302,7 +1294,7 @@ pub(crate) async fn make_local_two_set_sets_for_pool_with_ctx(
2,
1,
set_index,
pool_idx,
0,
endpoints,
format.clone(),
lockers,
@@ -1315,7 +1307,7 @@ pub(crate) async fn make_local_two_set_sets_for_pool_with_ctx(
let sets = Arc::new(Sets {
id: format.id,
disk_set: disk_sets,
pool_idx,
pool_idx: 0,
endpoints: PoolEndpoints {
legacy: false,
set_count: 2,
+27 -83
View File
@@ -813,7 +813,7 @@ pub(crate) fn is_equivalent_data_movement_metadata(
.all(|(key, value)| source.user_defined.get(key) == Some(value))
}
pub(crate) fn is_equivalent_data_movement_object_identity(
fn is_equivalent_data_movement_object_identity(
source: &ObjectInfo,
target: &ObjectInfo,
compare_mod_time: bool,
@@ -1023,11 +1023,10 @@ pub(crate) enum SourceCleanupError {
Storage(#[from] Error),
}
#[derive(Clone, Default)]
#[derive(Clone, Copy, Default)]
pub(crate) struct SourceCleanupBucketFence<'a> {
pub(crate) expected_incarnation_id: Option<uuid::Uuid>,
pub(crate) lifecycle_guard: Option<&'a rustfs_lock::NamespaceLockGuard>,
pub(crate) namespace_lock_lost_signal: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
pub(crate) object_mutation_fence: Option<&'a SourceCleanupMutationFence>,
}
@@ -1062,7 +1061,7 @@ pub(crate) async fn ensure_source_cleanup_versions_unchanged(
ensure_source_cleanup_versions_match(expected, &current, allowed_missing)
}
#[cfg(any(test, feature = "test-util"))]
#[cfg(test)]
struct SourceCleanupDeleteBarrierState {
bucket: String,
object: String,
@@ -1072,7 +1071,7 @@ struct SourceCleanupDeleteBarrierState {
release: tokio::sync::Notify,
}
#[cfg(any(test, feature = "test-util"))]
#[cfg(test)]
#[allow(
dead_code,
reason = "installed by set_disk object tests behind `--features test-util` (backlog#1823)"
@@ -1081,11 +1080,11 @@ pub(crate) struct SourceCleanupDeleteBarrier {
state: Arc<SourceCleanupDeleteBarrierState>,
}
#[cfg(any(test, feature = "test-util"))]
#[cfg(test)]
static SOURCE_CLEANUP_DELETE_BARRIERS: std::sync::OnceLock<std::sync::Mutex<Vec<Arc<SourceCleanupDeleteBarrierState>>>> =
std::sync::OnceLock::new();
#[cfg(any(test, feature = "test-util"))]
#[cfg(test)]
#[allow(
dead_code,
reason = "installed by set_disk object tests behind `--features test-util` (backlog#1823)"
@@ -1149,7 +1148,7 @@ pub(crate) fn notify_source_cleanup_mutation_fence_pending(bucket: &str, object:
}
}
#[cfg(any(test, feature = "test-util"))]
#[cfg(test)]
impl Drop for SourceCleanupDeleteBarrier {
fn drop(&mut self) {
self.state.release.notify_one();
@@ -1161,7 +1160,7 @@ impl Drop for SourceCleanupDeleteBarrier {
}
}
#[cfg(any(test, feature = "test-util"))]
#[cfg(test)]
async fn pause_source_cleanup_before_delete(bucket: &str, object: &str) {
let barrier = SOURCE_CLEANUP_DELETE_BARRIERS
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
@@ -1221,7 +1220,7 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
ensure_source_cleanup_versions_unchanged(set.clone(), bucket, object, expected, allowed_missing, op_label).await?;
#[cfg(any(test, feature = "test-util"))]
#[cfg(test)]
pause_source_cleanup_before_delete(bucket, object).await;
let mut opts = ObjectOptions {
@@ -1241,9 +1240,6 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
if let Some(bucket_lifecycle_guard) = bucket_fence.lifecycle_guard {
opts.add_bucket_lifecycle_lock_guard(bucket_lifecycle_guard);
}
if let Some(signal) = bucket_fence.namespace_lock_lost_signal {
opts.add_namespace_lock_lost_signal(signal);
}
let result = set.delete_object(bucket, cleanup_key.as_str(), opts).await;
if result.is_ok() {
crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
@@ -1414,13 +1410,11 @@ pub(crate) async fn migrate_decommission_object(
rd,
source_bucket_incarnation_id,
op_label,
None,
Some(&_mutation_fence),
)
.await
}
#[cfg(test)]
pub(crate) async fn migrate_object(
store: Arc<ECStore>,
pool_idx: usize,
@@ -1429,33 +1423,9 @@ pub(crate) async fn migrate_object(
source_bucket_incarnation_id: Option<uuid::Uuid>,
op_label: &str,
) -> Result<()> {
migrate_object_with_lock_lost_signal(store, pool_idx, bucket, rd, source_bucket_incarnation_id, op_label, None).await
migrate_object_inner(store, pool_idx, bucket, rd, source_bucket_incarnation_id, op_label, None).await
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn migrate_object_with_lock_lost_signal(
store: Arc<ECStore>,
pool_idx: usize,
bucket: String,
rd: GetObjectReader,
source_bucket_incarnation_id: Option<uuid::Uuid>,
op_label: &str,
lock_lost_signal: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
) -> Result<()> {
migrate_object_inner(
store,
pool_idx,
bucket,
rd,
source_bucket_incarnation_id,
op_label,
lock_lost_signal,
None,
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn migrate_object_inner(
store: Arc<ECStore>,
pool_idx: usize,
@@ -1463,7 +1433,6 @@ async fn migrate_object_inner(
rd: GetObjectReader,
source_bucket_incarnation_id: Option<uuid::Uuid>,
op_label: &str,
lock_lost_signal: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
mutation_fence: Option<&ObjectLockDiagGuard>,
) -> Result<()> {
let object_info = rd.object_info.clone();
@@ -1477,9 +1446,6 @@ async fn migrate_object_inner(
if should_use_multipart_data_movement(&object_info, has_part_checksums) {
let mut new_multipart_opts = data_movement_new_multipart_opts(&object_info, pool_idx);
new_multipart_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id;
if let Some(signal) = lock_lost_signal.as_ref() {
new_multipart_opts.add_namespace_lock_lost_signal(Arc::clone(signal));
}
let (res, target_pool_idx, expected_bucket_incarnation_id) = match store
.handle_new_multipart_upload_with_pool_idx(&bucket, &object_info.name, &new_multipart_opts, mutation_fence)
.await
@@ -1524,7 +1490,7 @@ async fn migrate_object_inner(
err,
)
})?;
let mut part_opts = ObjectOptions {
let part_opts = ObjectOptions {
part_number: Some(part.number),
preserve_etag: Some(part.etag.clone()),
data_movement: true,
@@ -1532,9 +1498,6 @@ async fn migrate_object_inner(
expected_bucket_incarnation_id,
..Default::default()
};
if let Some(signal) = lock_lost_signal.as_ref() {
part_opts.add_namespace_lock_lost_signal(Arc::clone(signal));
}
let pi = match store
.put_object_part_for_data_movement(
target_pool_idx,
@@ -1579,9 +1542,6 @@ async fn migrate_object_inner(
)
})?;
complete_multipart_opts.expected_bucket_incarnation_id = expected_bucket_incarnation_id;
if let Some(signal) = lock_lost_signal.as_ref() {
complete_multipart_opts.add_namespace_lock_lost_signal(Arc::clone(signal));
}
if let Err(err) = store
.clone()
.complete_multipart_upload_for_data_movement(
@@ -1630,18 +1590,18 @@ async fn migrate_object_inner(
if multipart_result.is_ok() && should_abort_multipart_upload(&abort_multipart_flag) {
let abort_result = store
.abort_multipart_upload_for_data_movement(target_pool_idx, &bucket, &object_info.name, &res.upload_id, &{
let mut opts = ObjectOptions {
.abort_multipart_upload_for_data_movement(
target_pool_idx,
&bucket,
&object_info.name,
&res.upload_id,
&ObjectOptions {
data_movement: true,
src_pool_idx: pool_idx,
expected_bucket_incarnation_id,
..Default::default()
};
if let Some(signal) = lock_lost_signal.as_ref() {
opts.add_namespace_lock_lost_signal(Arc::clone(signal));
}
opts
})
},
)
.await;
match abort_result {
Ok(()) => return Ok(()),
@@ -1699,18 +1659,18 @@ async fn migrate_object_inner(
if let Err(primary_err) = multipart_result {
if should_abort_multipart_upload(&abort_multipart_flag) {
return match store
.abort_multipart_upload_for_data_movement(target_pool_idx, &bucket, &object_info.name, &res.upload_id, &{
let mut opts = ObjectOptions {
.abort_multipart_upload_for_data_movement(
target_pool_idx,
&bucket,
&object_info.name,
&res.upload_id,
&ObjectOptions {
data_movement: true,
src_pool_idx: pool_idx,
expected_bucket_incarnation_id,
..Default::default()
};
if let Some(signal) = lock_lost_signal.as_ref() {
opts.add_namespace_lock_lost_signal(Arc::clone(signal));
}
opts
})
},
)
.await
{
Ok(()) => Err(primary_err),
@@ -1745,9 +1705,6 @@ async fn migrate_object_inner(
let mut put_opts = data_movement_put_object_opts(&object_info, pool_idx);
put_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id;
if let Some(signal) = lock_lost_signal {
put_opts.add_namespace_lock_lost_signal(signal);
}
let (target_pool_idx, put_result) = store
.put_object_for_data_movement(&bucket, &object_info.name, &mut data, &put_opts, mutation_fence)
.await
@@ -1993,19 +1950,6 @@ mod tests {
assert!(source_cleanup_versions_match_with_allowed_missing(&expected, &current, &allowed_missing));
}
#[test]
fn test_decommission_cleanup_preflight_accepts_migrated_free_version_consumed_from_source() {
let migrated = cleanup_test_file_info("object.txt", Uuid::from_u128(1), "migrated");
let mut free_version = cleanup_test_file_info("object.txt", Uuid::from_u128(2), "tier-cleanup");
free_version.deleted = true;
free_version.set_tier_free_version();
let expected = cleanup_test_versions(vec![migrated.clone(), free_version.clone()]);
let current = cleanup_test_versions(vec![migrated]);
let allowed_missing = vec![source_cleanup_version_identity(&free_version)];
assert!(source_cleanup_versions_match_with_allowed_missing(&expected, &current, &allowed_missing));
}
#[test]
fn test_decommission_cleanup_preflight_rejects_unexpected_missing_version() {
let migrated = cleanup_test_file_info("object.txt", Uuid::from_u128(1), "migrated");
+33 -400
View File
@@ -73,16 +73,6 @@ struct CachedBucketUsage {
// mutation. A strictly later generation is required before the mutation
// evidence can be discarded.
pending_scanner_position: Option<(u64, u64)>,
// Deletes are visible to admin immediately, but quota admission keeps
// them pending until a complete scanner generation reconciles the set.
// This marker intentionally remains process-local: the delete request
// updates this overlay before the scanner writes a durable snapshot. If
// the process restarts first, loading the persisted complete snapshot
// restores the pre-reconciliation (larger) baseline, which is
// conservative for quota admission. A persisted post-delete snapshot is
// necessarily a complete scanner reconciliation and therefore creates a
// fresh cache entry with no pending hold.
pending_negative_delta: u64,
}
type UsageMemoryCache = Arc<RwLock<HashMap<String, CachedBucketUsage>>>;
@@ -388,12 +378,8 @@ pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store:
"nonconverged data usage observations cannot replace the quota-authoritative snapshot",
));
}
let Some(expected_publication_epoch) = store.scanner_data_usage_publication_epoch().await else {
return Err(Error::other("data usage publication is blocked by data movement"));
};
// Prevent older data from overwriting newer persisted stats
let existing_snapshot = load_data_usage_snapshot(store.clone()).await;
if let Ok((existing, source)) = existing_snapshot
if let Ok((existing, source)) = load_data_usage_snapshot(store.clone()).await
&& source.is_authoritative()
&& let Some(reason) = stale_data_usage_persist_reason_for_source(&data_usage_info, &existing, source, SystemTime::now())
{
@@ -404,31 +390,19 @@ pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store:
return Ok(());
}
save_data_usage_in_backend(data_usage_info, store, expected_publication_epoch).await
save_data_usage_in_backend(data_usage_info, store).await
}
async fn save_data_usage_in_backend(
data_usage_info: DataUsageInfo,
store: Arc<ECStore>,
expected_publication_epoch: u64,
) -> Result<(), Error> {
async fn save_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<ECStore>) -> Result<(), Error> {
let data =
serde_json::to_vec(&data_usage_info).map_err(|e| Error::other(format!("Failed to serialize data usage info: {e}")))?;
// Save to backend using the same mechanism as original code
let Some((publication_guard, publication_epoch)) = store.scanner_data_usage_publication_admission_guard().await else {
return Err(Error::other("data usage publication is blocked by data movement"));
};
if publication_epoch != expected_publication_epoch {
return Err(Error::other("data usage publication epoch changed before save"));
}
crate::config::com::save_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH, data)
.await
.map_err(Error::other)?;
drop(publication_guard);
cleanup_observed_data_usage_after_authoritative_save_with_publication(store.as_ref(), &data_usage_info, Some(store.as_ref()))
.await;
cleanup_observed_data_usage_after_authoritative_save(store.as_ref(), &data_usage_info).await;
// Invalidate the cached snapshot so readers observe the new save on their
// next request instead of waiting out the remaining TTL. The next cached
@@ -465,24 +439,11 @@ impl ObservedDataUsageSnapshotCleanup for ECStore {
}
}
async fn cleanup_observed_data_usage_after_authoritative_save_with_publication<S>(
store: &S,
authoritative: &DataUsageInfo,
publication_store: Option<&ECStore>,
) where
async fn cleanup_observed_data_usage_after_authoritative_save<S>(store: &S, authoritative: &DataUsageInfo)
where
S: EcstoreObjectIO + ObservedDataUsageSnapshotCleanup + ?Sized,
{
let observed_read_epoch = match publication_store {
Some(publication_store) => {
let Some(epoch) = publication_store.scanner_data_usage_publication_epoch().await else {
return;
};
Some(epoch)
}
None => None,
};
let observed_snapshot = load_data_usage_for_bucket_removal(store, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await;
let (observed, revision) = match observed_snapshot {
let (observed, revision) = match load_data_usage_for_bucket_removal(store, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await {
Ok(Some(snapshot)) => snapshot,
Ok(None) => return,
Err(err) => {
@@ -498,19 +459,6 @@ async fn cleanup_observed_data_usage_after_authoritative_save_with_publication<S
return;
}
let publication_guard = match publication_store {
Some(publication_store) => {
let Some((guard, publication_epoch)) = publication_store.scanner_data_usage_publication_admission_guard().await
else {
return;
};
if observed_read_epoch.is_some_and(|expected| expected != publication_epoch) {
return;
}
Some(guard)
}
None => None,
};
match store.delete_observed_data_usage_snapshot(&revision).await {
Ok(()) | Err(Error::ConfigNotFound | Error::FileNotFound | Error::ObjectNotFound(_, _) | Error::PreconditionFailed) => {}
Err(err) => {
@@ -521,15 +469,6 @@ async fn cleanup_observed_data_usage_after_authoritative_save_with_publication<S
);
}
}
drop(publication_guard);
}
#[cfg(test)]
async fn cleanup_observed_data_usage_after_authoritative_save<S>(store: &S, authoritative: &DataUsageInfo)
where
S: EcstoreObjectIO + ObservedDataUsageSnapshotCleanup + ?Sized,
{
cleanup_observed_data_usage_after_authoritative_save_with_publication(store, authoritative, None).await;
}
fn set_buckets_count_from_usage(data_usage_info: &mut DataUsageInfo) {
@@ -570,7 +509,7 @@ pub async fn remove_bucket_usage_from_backend(store: Arc<ECStore>, bucket: &str)
pub(crate) async fn remove_bucket_usage_for_namespace_change(store: &ECStore, bucket: &str) -> Result<(), Error> {
prepare_bucket_usage_for_namespace_change(bucket, None).await?;
remove_bucket_usage_from_backend_with_guard_fenced(store, bucket, None).await
remove_bucket_usage_from_backend_with_guard(store, bucket, None).await
}
pub(crate) async fn prepare_bucket_usage_for_namespace_change(
@@ -593,7 +532,6 @@ pub(crate) async fn prepare_bucket_usage_for_namespace_change(
Ok(())
}
#[cfg(test)]
pub(crate) async fn remove_bucket_usage_from_backend_with_guard<S>(
store: &S,
bucket: &str,
@@ -603,24 +541,6 @@ where
S: EcstoreObjectIO + ?Sized,
{
let result = remove_bucket_usage_from_backend_with_store_and_guard(store, bucket, guard).await;
invalidate_bucket_usage_snapshot_caches(guard, bucket).await?;
result
}
pub(crate) async fn remove_bucket_usage_from_backend_with_guard_fenced(
store: &ECStore,
bucket: &str,
guard: Option<&rustfs_lock::NamespaceLockGuard>,
) -> Result<(), Error> {
let result = remove_bucket_usage_from_backend_with_store_and_guard_and_publication(store, bucket, guard, Some(store)).await;
invalidate_bucket_usage_snapshot_caches(guard, bucket).await?;
result
}
async fn invalidate_bucket_usage_snapshot_caches(
guard: Option<&rustfs_lock::NamespaceLockGuard>,
bucket: &str,
) -> Result<(), Error> {
let mut snapshot_cache = data_usage_snapshot_cache().write().await;
ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cache invalidation")?;
clear_data_usage_snapshot_cache(&mut snapshot_cache);
@@ -628,7 +548,7 @@ async fn invalidate_bucket_usage_snapshot_caches(
let mut admin_snapshot_cache = admin_data_usage_snapshot_cache().write().await;
ensure_bucket_namespace_guard(guard, bucket, "admin data usage snapshot cache invalidation")?;
clear_admin_data_usage_snapshot_cache(&mut admin_snapshot_cache);
Ok(())
result
}
async fn load_data_usage_for_bucket_removal<S>(store: &S, object: &str) -> Result<Option<(DataUsageInfo, String)>, Error>
@@ -687,88 +607,48 @@ fn ensure_bucket_namespace_guard(
Ok(())
}
#[cfg(test)]
async fn remove_bucket_usage_from_backend_with_store_and_guard<S>(
store: &S,
bucket: &str,
guard: Option<&rustfs_lock::NamespaceLockGuard>,
) -> Result<(), Error>
where
S: EcstoreObjectIO + ?Sized,
{
remove_bucket_usage_from_backend_with_store_and_guard_and_publication(store, bucket, guard, None).await
}
async fn remove_bucket_usage_from_backend_with_store_and_guard_and_publication<S>(
store: &S,
bucket: &str,
guard: Option<&rustfs_lock::NamespaceLockGuard>,
publication_store: Option<&ECStore>,
) -> Result<(), Error>
where
S: EcstoreObjectIO + ?Sized,
{
ensure_bucket_namespace_guard(guard, bucket, "data usage primary cleanup")?;
let primary_seed_epoch = match publication_store {
Some(publication_store) => Some(
publication_store
.scanner_data_usage_publication_epoch()
.await
.ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?,
),
None => None,
};
let primary_seed = load_data_usage_seed_for_missing_primary(store).await?;
remove_bucket_usage_from_object_with_retries_and_publication(
remove_bucket_usage_from_object_with_retries(
store,
DATA_USAGE_OBJ_NAME_PATH.as_str(),
bucket,
DATA_USAGE_REMOVE_CAS_RETRIES,
primary_seed.as_ref(),
Some(&primary_seed),
guard,
publication_store.map(|store| (store, primary_seed_epoch)),
)
.await?;
ensure_bucket_namespace_guard(guard, bucket, "data usage backup cleanup")?;
let backup_seed_epoch = match publication_store {
Some(publication_store) => Some(
publication_store
.scanner_data_usage_publication_epoch()
.await
.ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?,
),
None => None,
};
let backup_seed = load_data_usage_for_bucket_removal(store, DATA_USAGE_OBJ_NAME_PATH.as_str())
.await?
.map(|(data_usage_info, _)| data_usage_info)
.filter(|data_usage_info| !data_usage_info.usage_snapshot_bootstrap_pending)
.or_else(|| {
primary_seed
.clone()
.filter(|data_usage_info| !data_usage_info.usage_snapshot_bootstrap_pending)
});
remove_bucket_usage_from_object_with_retries_and_publication(
.map_or(primary_seed, |(data_usage_info, _)| data_usage_info);
remove_bucket_usage_from_object_with_retries(
store,
DATA_USAGE_OBJ_BACKUP_PATH.as_str(),
bucket,
DATA_USAGE_REMOVE_CAS_RETRIES,
backup_seed.as_ref(),
Some(&backup_seed),
guard,
publication_store.map(|store| (store, backup_seed_epoch)),
)
.await?;
ensure_bucket_namespace_guard(guard, bucket, "observed data usage cleanup")?;
if let Err(err) = remove_bucket_usage_from_object_with_retries_and_publication(
if let Err(err) = remove_bucket_usage_from_object_with_retries(
store,
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
bucket,
DATA_USAGE_REMOVE_CAS_RETRIES,
None,
guard,
publication_store.map(|store| (store, None)),
)
.await
{
@@ -782,21 +662,12 @@ where
LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(),
LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str(),
] {
remove_bucket_usage_from_object_with_retries_and_publication(
store,
object,
bucket,
DATA_USAGE_REMOVE_CAS_RETRIES,
None,
guard,
publication_store.map(|store| (store, None)),
)
.await?;
remove_bucket_usage_from_object_with_retries(store, object, bucket, DATA_USAGE_REMOVE_CAS_RETRIES, None, guard).await?;
}
Ok(())
}
async fn load_data_usage_seed_for_missing_primary<S>(store: &S) -> Result<Option<DataUsageInfo>, Error>
async fn load_data_usage_seed_for_missing_primary<S>(store: &S) -> Result<DataUsageInfo, Error>
where
S: EcstoreObjectIO + ?Sized,
{
@@ -809,13 +680,12 @@ where
if !authoritative {
data_usage_info.usage_snapshot_complete = false;
}
return Ok(Some(data_usage_info));
return Ok(data_usage_info);
}
}
Ok(None)
Ok(DataUsageInfo::default())
}
#[cfg(test)]
async fn remove_bucket_usage_from_object_with_retries<S>(
store: &S,
object: &str,
@@ -824,42 +694,12 @@ async fn remove_bucket_usage_from_object_with_retries<S>(
missing_seed: Option<&DataUsageInfo>,
guard: Option<&rustfs_lock::NamespaceLockGuard>,
) -> Result<(), Error>
where
S: EcstoreObjectIO + ?Sized,
{
remove_bucket_usage_from_object_with_retries_and_publication(store, object, bucket, cas_retries, missing_seed, guard, None)
.await
}
async fn remove_bucket_usage_from_object_with_retries_and_publication<S>(
store: &S,
object: &str,
bucket: &str,
cas_retries: usize,
missing_seed: Option<&DataUsageInfo>,
guard: Option<&rustfs_lock::NamespaceLockGuard>,
publication: Option<(&ECStore, Option<u64>)>,
) -> Result<(), Error>
where
S: EcstoreObjectIO + ?Sized,
{
for attempt in 0..=cas_retries {
ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cleanup")?;
let read_epoch = match publication {
Some((publication_store, expected_publication_epoch)) => {
let epoch = publication_store
.scanner_data_usage_publication_epoch()
.await
.ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?;
if expected_publication_epoch.is_some_and(|expected| expected != epoch) {
return Err(Error::other("data usage publication epoch changed before snapshot read"));
}
Some(epoch)
}
None => None,
};
let loaded_snapshot = load_data_usage_for_bucket_removal(store, object).await?;
let (mut data_usage_info, revision) = match loaded_snapshot {
let (mut data_usage_info, revision) = match load_data_usage_for_bucket_removal(store, object).await? {
Some((data_usage_info, revision)) => (data_usage_info, Some(revision)),
None => match missing_seed {
Some(data_usage_info) => (data_usage_info.clone(), None),
@@ -883,22 +723,6 @@ where
},
};
ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot commit")?;
let publication_guard = match publication {
Some((publication_store, expected_publication_epoch)) => {
let Some((guard, publication_epoch)) = publication_store.scanner_data_usage_publication_admission_guard().await
else {
return Err(Error::other("data usage publication is blocked by data movement"));
};
if expected_publication_epoch
.or(read_epoch)
.is_some_and(|expected| expected != publication_epoch)
{
return Err(Error::other("data usage publication epoch changed before snapshot commit"));
}
Some(guard)
}
None => None,
};
let save_result = store
.put_object(
RUSTFS_META_BUCKET,
@@ -911,7 +735,6 @@ where
},
)
.await;
drop(publication_guard);
match save_result {
Ok(_) => return Ok(()),
Err(err) => {
@@ -1125,12 +948,7 @@ async fn load_observed_data_usage_snapshot(store: Arc<ECStore>) -> Option<DataUs
};
match parse_usage_snapshot(&data) {
Ok(info)
if info.usage_snapshot_converged == Some(false)
&& (info.is_complete_bucket_usage_snapshot() || info.is_valid_partial_snapshot()) =>
{
Some(info)
}
Ok(info) if info.usage_snapshot_converged == Some(false) && info.is_complete_bucket_usage_snapshot() => Some(info),
Ok(_) => {
error!(
event = "data_usage_snapshot_load_failed",
@@ -1175,7 +993,7 @@ async fn load_admin_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataU
}
fn discard_incomplete_bucket_usage(data_usage_info: &mut DataUsageInfo) {
if !data_usage_info.is_complete_bucket_usage_snapshot() && !data_usage_info.usage_snapshot_partial {
if !data_usage_info.is_complete_bucket_usage_snapshot() {
data_usage_info.usage_snapshot_complete = false;
data_usage_info.buckets_usage.clear();
data_usage_info.bucket_sizes.clear();
@@ -1825,7 +1643,6 @@ fn cached_bucket_usage_from_backend(usage: BucketUsageInfo, updated_at: SystemTi
dirty: false,
stale_snapshot_pending: false,
pending_scanner_position: None,
pending_negative_delta: 0,
}
}
@@ -1839,7 +1656,6 @@ fn cached_bucket_usage_now(usage: BucketUsageInfo) -> CachedBucketUsage {
dirty: false,
stale_snapshot_pending: false,
pending_scanner_position: None,
pending_negative_delta: 0,
}
}
@@ -1992,7 +1808,6 @@ pub async fn record_bucket_object_delete_memory(bucket: &str, deleted_size: u64,
.or_insert_with(|| cached_bucket_usage_now(BucketUsageInfo::default()));
entry.usage.size = entry.usage.size.saturating_sub(deleted_size);
entry.pending_negative_delta = entry.pending_negative_delta.saturating_add(deleted_size);
if removed_current_object {
entry.usage.objects_count = entry.usage.objects_count.saturating_sub(1);
entry.usage.versions_count = entry.usage.versions_count.saturating_sub(1);
@@ -2048,7 +1863,7 @@ pub async fn get_bucket_usage_memory(bucket: &str) -> Option<u64> {
cache
.get(bucket)
.filter(|cached| cached.authoritative)
.map(|cached| cached.usage.size.saturating_add(cached.pending_negative_delta))
.map(|cached| cached.usage.size)
}
async fn update_usage_cache_if_needed() {
@@ -2453,8 +2268,6 @@ pub async fn init_compression_total_memory_from_backend(store: Arc<ECStore>) {
#[cfg(test)]
mod tests {
use super::*;
use crate::layout::endpoints::EndpointServerPools;
use crate::runtime::instance::InstanceContext;
use crate::storage_api_contracts::object::ObjectIO as _;
use rustfs_data_usage::BucketUsageInfo;
use rustfs_lock::{LocalClient, LockRequest, LockType, NamespaceLock, ObjectKey};
@@ -2477,7 +2290,6 @@ mod tests {
error_after_commit_put: Option<usize>,
advance_time_on_put: Option<Duration>,
advance_time_after_get: Option<(UsageObjectSlot, Duration)>,
advance_publication_epoch_after_get: Option<(UsageObjectSlot, Arc<InstanceContext>)>,
advance_time_before_put: Option<(usize, Duration)>,
advance_time_after_put: Option<(usize, Duration)>,
put_count: usize,
@@ -2555,21 +2367,10 @@ mod tests {
}
_ => None,
};
let advance_publication_epoch = match state.advance_publication_epoch_after_get {
Some((expected_slot, ref ctx)) if expected_slot == slot => {
let ctx = Arc::clone(ctx);
state.advance_publication_epoch_after_get = None;
Some(ctx)
}
_ => None,
};
drop(state);
if let Some(duration) = advance {
tokio::time::advance(duration).await;
}
if let Some(ctx) = advance_publication_epoch {
ctx.advance_data_movement_operation_epoch();
}
Ok(crate::object_api::GetObjectReader {
stream: Box::new(Cursor::new(data)),
object_info: ObjectInfo {
@@ -2770,23 +2571,6 @@ mod tests {
.to_string()
}
fn build_publication_store(ctx: Arc<InstanceContext>) -> Arc<ECStore> {
let endpoint_pools = EndpointServerPools::default();
Arc::new(ECStore {
id: uuid::Uuid::new_v4(),
disk_map: HashMap::new(),
pools: Vec::new(),
peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoint_pools, ctx.clone()),
pool_meta: RwLock::new(crate::core::pools::PoolMeta::default()),
rebalance_meta: RwLock::new(None),
decommission_cancelers: RwLock::new(Vec::new()),
start_gate: TokioMutex::new(()),
pool_meta_save_gate: TokioMutex::default(),
ctx,
bucket_fence_registry: Arc::default(),
})
}
#[test]
fn data_usage_cache_absence_covers_the_variants_that_actually_arrive() {
// `to_object_err` rewrites the raw storage variants before they reach
@@ -3159,45 +2943,6 @@ mod tests {
assert_eq!(selected.usage_snapshot_converged, Some(true));
}
#[test]
fn persisted_authoritative_stalls_but_memory_overlay_remains_visible() {
let authoritative = DataUsageInfo {
last_update: Some(SystemTime::UNIX_EPOCH),
scanner_epoch: Some(4),
scanner_cycle: Some(10),
usage_snapshot_complete: true,
..Default::default()
};
let mut partial = authoritative.clone();
partial.last_update = Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1));
partial.scanner_cycle = Some(11);
partial.usage_snapshot_complete = false;
partial.usage_snapshot_partial = true;
partial.usage_snapshot_converged = Some(false);
partial.usage_snapshot_authoritative_baseline = Some(authoritative.snapshot_identity());
partial.usage_snapshot_set_states = vec![rustfs_data_usage::DataUsageSnapshotSetState {
pool_index: 0,
set_index: 0,
scanner_cycle: Some(10),
scanner_epoch: Some(4),
scan_plan_digest: Some([1; 32]),
complete: false,
tombstone: false,
}];
partial.buckets_usage.insert(
"bucket".to_string(),
BucketUsageInfo {
size: 100,
..Default::default()
},
);
partial.buckets_count = 1;
let (selected, _) = select_admin_data_usage_snapshot(authoritative, true, Some(partial));
assert!(selected.usage_snapshot_partial);
assert_eq!(selected.buckets_usage.get("bucket").map(|usage| usage.size), Some(100));
}
#[tokio::test]
async fn authoritative_save_cleanup_removes_observed_snapshot_best_effort() {
let store = UsageCasStore::default();
@@ -4531,15 +4276,7 @@ mod tests {
.expect("namespace lock acquisition should not fail")
.expect("namespace lock should be acquired"),
);
let snapshot = data_usage_info_for_test(BUCKET, 2, 84, SystemTime::now());
let encoded = serde_json::to_vec(&snapshot).expect("usage snapshot should encode");
let store = Arc::new(UsageCasStore {
state: Mutex::new(UsageCasState {
object: Some((encoded.clone(), 1)),
backup_object: Some((encoded, 1)),
..Default::default()
}),
});
let store = Arc::new(UsageCasStore::default());
let successor = data_usage_info_for_test(BUCKET, 7, 294, SystemTime::now());
let mut snapshot_cache = data_usage_snapshot_cache().write().await;
*snapshot_cache = Some(CachedDataUsageSnapshot {
@@ -4638,43 +4375,21 @@ mod tests {
}
#[tokio::test]
async fn remove_bucket_usage_does_not_synthesize_authoritative_snapshot_when_all_missing() {
async fn remove_bucket_usage_creates_primary_and_backup_fences_when_missing() {
let store = Arc::new(UsageCasStore::default());
remove_bucket_usage_from_backend_with_store(store.as_ref(), "bucket-a")
.await
.expect("bucket removal should remain a no-op without a usage baseline");
.expect("bucket removal should create both usage fences");
let state = store.state.lock().await;
assert_eq!(state.put_count, 0);
assert!(state.object.is_none());
assert!(state.backup_object.is_none());
}
#[tokio::test]
async fn remove_bucket_usage_does_not_seed_backup_from_pristine_bootstrap_marker() {
let marker = DataUsageInfo {
last_update: Some(SystemTime::now()),
usage_snapshot_converged: Some(false),
usage_snapshot_bootstrap_pending: true,
..Default::default()
};
let store = Arc::new(UsageCasStore {
state: Mutex::new(UsageCasState {
object: Some((serde_json::to_vec(&marker).expect("bootstrap marker should encode"), 1)),
..Default::default()
}),
});
remove_bucket_usage_from_backend_with_store(store.as_ref(), "bucket-a")
.await
.expect("bucket removal should preserve the pending primary without creating a backup");
let state = store.state.lock().await;
assert!(state.backup_object.is_none());
let saved = serde_json::from_slice::<DataUsageInfo>(&state.object.as_ref().expect("pending primary should remain").0)
.expect("pending primary should decode");
assert!(saved.usage_snapshot_bootstrap_pending);
assert_eq!(state.put_count, 2);
for (data, revision) in [state.object.as_ref(), state.backup_object.as_ref()].into_iter().flatten() {
let saved = serde_json::from_slice::<DataUsageInfo>(data).expect("saved usage snapshot should decode");
assert_eq!(*revision, 1);
assert!(saved.last_update.is_some());
assert!(!data_usage_contains_bucket(&saved, "bucket-a"));
}
}
#[tokio::test]
@@ -4826,39 +4541,6 @@ mod tests {
assert_eq!(backup_err, Error::PreconditionFailed);
}
#[tokio::test]
async fn remove_bucket_usage_rejects_movement_epoch_flip_between_read_and_commit() {
let ctx = Arc::new(InstanceContext::new());
let publication_store = build_publication_store(ctx.clone());
let snapshot = data_usage_info_for_test("bucket-a", 2, 84, SystemTime::now());
let store = Arc::new(UsageCasStore {
state: Mutex::new(UsageCasState {
object: Some((serde_json::to_vec(&snapshot).expect("usage snapshot should encode"), 1)),
advance_publication_epoch_after_get: Some((UsageObjectSlot::Primary, ctx)),
..Default::default()
}),
});
let expected_epoch = publication_store
.scanner_data_usage_publication_epoch()
.await
.expect("idle publication store should admit the initial read");
let err = remove_bucket_usage_from_object_with_retries_and_publication(
store.as_ref(),
DATA_USAGE_OBJ_NAME_PATH.as_str(),
"bucket-a",
0,
None,
None,
Some((publication_store.as_ref(), Some(expected_epoch))),
)
.await
.expect_err("a movement epoch flip during the read must fence the commit");
assert!(err.to_string().contains("epoch changed"));
assert_eq!(store.state.lock().await.put_count, 0);
}
#[tokio::test]
async fn remove_bucket_usage_confirms_ambiguous_committed_final_attempt() {
let initial = data_usage_info_for_test("bucket-a", 2, 84, SystemTime::now());
@@ -4983,55 +4665,6 @@ mod tests {
);
}
#[tokio::test]
#[serial]
async fn partial_usage_is_observational_not_authoritative_for_quota() {
clear_usage_memory_cache_for_test().await;
let mut partial = data_usage_info_for_test("bucket-a", 10, 100, SystemTime::now());
partial.usage_snapshot_complete = false;
partial.usage_snapshot_partial = true;
replace_bucket_usage_memory_from_info(&partial).await;
assert_eq!(get_bucket_usage_memory("bucket-a").await, None);
}
#[tokio::test]
#[serial]
async fn stale_quota_uses_complete_baseline_plus_positive_deltas() {
clear_usage_memory_cache_for_test().await;
let baseline = data_usage_info_for_test("bucket-a", 1, 100, SystemTime::now());
replace_bucket_usage_memory_from_info(&baseline).await;
record_bucket_object_write_memory("bucket-a", None, 25).await;
assert_eq!(get_bucket_usage_memory("bucket-a").await, Some(125));
}
#[tokio::test]
#[serial]
async fn negative_delta_waits_for_set_reconciliation() {
clear_usage_memory_cache_for_test().await;
let baseline = data_usage_info_for_test("bucket-a", 1, 100, SystemTime::UNIX_EPOCH + Duration::from_secs(100));
replace_bucket_usage_memory_from_info(&baseline).await;
record_bucket_object_delete_memory("bucket-a", 25, true).await;
assert_eq!(get_bucket_usage_memory("bucket-a").await, Some(100));
// Simulate a process restart: the request-path overlay is gone, but
// the persisted authoritative snapshot is still the pre-reconciliation
// baseline. Quota must remain conservative until a complete scanner
// result proves the delete.
clear_usage_memory_cache_for_test().await;
replace_bucket_usage_memory_from_info(&baseline).await;
assert_eq!(get_bucket_usage_memory("bucket-a").await, Some(100));
let reconciled = data_usage_info_for_test("bucket-a", 0, 75, SystemTime::UNIX_EPOCH + Duration::from_secs(101));
replace_bucket_usage_memory_from_info(&reconciled).await;
assert_eq!(get_bucket_usage_memory("bucket-a").await, Some(75));
}
#[tokio::test]
#[serial]
async fn memory_overlay_counts_versioned_overwrite_as_new_version() {
+2 -76
View File
@@ -13,11 +13,9 @@
// limitations under the License.
use std::{
collections::HashMap,
fs::Metadata,
path::{Path, PathBuf},
sync::{Arc, Mutex, OnceLock},
time::{Duration, Instant},
path::Path,
sync::{Arc, OnceLock},
};
use tokio::{
fs::{self, File},
@@ -227,78 +225,6 @@ pub async fn read_file(path: impl AsRef<Path>) -> io::Result<Vec<u8>> {
fs::read(path.as_ref()).await
}
// Bucket existence cache - reduces statx syscalls for repeated bucket checks
/// Cache for bucket directory existence checks.
struct BucketExistenceCache {
cache: Mutex<HashMap<PathBuf, (Instant, bool)>>,
ttl: Duration,
}
impl BucketExistenceCache {
fn new(ttl: Duration) -> Self {
Self {
cache: Mutex::new(HashMap::new()),
ttl,
}
}
fn check_exists(&self, path: &PathBuf) -> Option<bool> {
let mut cache = self.cache.lock().ok()?;
if let Some((timestamp, exists)) = cache.get(path) {
if timestamp.elapsed() < self.ttl {
return Some(*exists);
}
cache.remove(path);
}
None
}
fn record(&self, path: PathBuf, exists: bool) {
if let Ok(mut cache) = self.cache.lock() {
cache.insert(path, (Instant::now(), exists));
}
}
fn invalidate(&self, path: &PathBuf) {
if let Ok(mut cache) = self.cache.lock() {
cache.remove(path);
}
}
}
static BUCKET_EXISTENCE_CACHE: std::sync::LazyLock<BucketExistenceCache> =
std::sync::LazyLock::new(|| BucketExistenceCache::new(Duration::from_secs(60)));
/// Cached access check - reduces statx syscalls
pub async fn cached_access(path: impl AsRef<Path>) -> io::Result<()> {
let path_buf = path.as_ref().to_path_buf();
if let Some(exists) = BUCKET_EXISTENCE_CACHE.check_exists(&path_buf) {
if exists {
return Ok(());
}
return Err(io::Error::new(io::ErrorKind::NotFound, "bucket not found (cached)"));
}
let result = fs::metadata(&path_buf).await;
match &result {
Ok(_) => BUCKET_EXISTENCE_CACHE.record(path_buf, true),
Err(e) if e.kind() == io::ErrorKind::NotFound => {
BUCKET_EXISTENCE_CACHE.record(path_buf, false);
}
_ => {}
}
result?;
Ok(())
}
pub fn invalidate_bucket_cache(path: impl AsRef<Path>) {
BUCKET_EXISTENCE_CACHE.invalidate(&path.as_ref().to_path_buf());
}
#[cfg(test)]
mod tests {
use super::*;
+18 -209
View File
@@ -35,10 +35,7 @@ use crate::disk::{
error::{DiskError, Error, FileAccessDeniedWithContext, Result},
error_conv::{to_access_error, to_file_error, to_unformatted_disk_error, to_volume_error},
format::FormatV3,
fs::{
O_APPEND, O_CREATE, O_RDONLY, O_TRUNC, O_WRONLY, access, cached_access, invalidate_bucket_cache, lstat, lstat_std,
remove, remove_all_std, remove_std, rename,
},
fs::{O_APPEND, O_CREATE, O_RDONLY, O_TRUNC, O_WRONLY, access, lstat, lstat_std, remove, remove_all_std, remove_std, rename},
is_quota_mutation_fence_path, os,
os::{check_path_length, is_dir_not_empty_error, is_empty_dir, is_root_disk, rename_all, rename_all_ignore_missing_source},
quota_mutation_fence_path,
@@ -83,11 +80,6 @@ use uuid::Uuid;
const DELETED_OBJECTS_CLEANUP_INTERVAL: Duration = Duration::from_secs(60 * 5);
const STALE_TMP_OBJECT_EXPIRY: Duration = Duration::from_secs(24 * 60 * 60);
#[cfg(test)]
tokio::task_local! {
static DIRECTORY_LISTING_ENTRY_PROBE_COUNT: Arc<AtomicUsize>;
}
const RUSTFS_META_TMP_OLD_BUCKET: &str = ".rustfs.sys/tmp-old";
const INLINE_METADATA_ROLLBACK_DIR_XOR: u128 = 0x7275737466735f696e6c696e655f7262;
const DELETE_MARKER_ROLLBACK_FILE: &str = "xl.meta.delete-marker.rollback";
@@ -3561,7 +3553,7 @@ impl LocalIoBackend for StdBackend {
let access_check_start = metrics_enabled.then(std::time::Instant::now);
let volume_dir = local_disk_bucket_path(self.io_root(), volume)?;
if !skip_access_checks(volume) {
cached_access(&volume_dir)
access(&volume_dir)
.await
.map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?;
}
@@ -3626,7 +3618,7 @@ impl LocalIoBackend for StdBackend {
async fn open_read_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<FileReader> {
let volume_dir = local_disk_bucket_path(self.io_root(), volume)?;
if !skip_access_checks(volume) {
cached_access(&volume_dir)
access(&volume_dir)
.await
.map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?;
}
@@ -3666,7 +3658,7 @@ impl LocalIoBackend for StdBackend {
async fn open_full_read(&self, volume: &str, path: &str) -> Result<FileReader> {
let volume_dir = local_disk_bucket_path(self.io_root(), volume)?;
if !skip_access_checks(volume) {
cached_access(&volume_dir)
access(&volume_dir)
.await
.map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?;
}
@@ -3720,7 +3712,7 @@ impl LocalIoBackend for StdBackend {
WriteMode::Append => {
let volume_dir = local_disk_bucket_path(self.io_root(), volume)?;
if !skip_access_checks(volume) {
cached_access(&volume_dir)
access(&volume_dir)
.await
.map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?;
}
@@ -5825,7 +5817,7 @@ impl LocalDisk {
async fn delete_unleased(&self, volume: &str, path: &str, opt: &DeleteOptions) -> Result<()> {
let volume_dir = self.io_get_bucket_path(volume)?;
if !skip_access_checks(volume)
&& let Err(e) = cached_access(&volume_dir).await
&& let Err(e) = access(&volume_dir).await
{
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
}
@@ -6747,7 +6739,7 @@ impl LocalDisk {
let read_dir_result = match read_dir_entries_with_walk_stall(&dir_path_abs, -1, stall).await {
Err(err) if err == Error::FileNotFound && !skip_access_checks(&opts.bucket) => {
let volume_dir = self.io_get_bucket_path(&opts.bucket)?;
if let Err(access_err) = cached_access(&volume_dir).await {
if let Err(access_err) = access(&volume_dir).await {
Err(to_access_error(access_err, DiskError::VolumeAccessDenied).into())
} else {
Err(err)
@@ -7065,7 +7057,6 @@ impl LocalDisk {
meta.name.push_str(SLASH_SEPARATOR);
if opts.recursive
|| opts.incl_deleted
|| opts.skip_hidden_prefix_check
|| self
.directory_has_listing_entry(&opts.bucket, &meta.name, opts.incl_deleted, stall)
.await?
@@ -7211,10 +7202,6 @@ impl LocalDisk {
continue;
}
#[cfg(test)]
let _previous_probe_count =
DIRECTORY_LISTING_ENTRY_PROBE_COUNT.try_with(|probe_count| probe_count.fetch_add(1, Ordering::Relaxed));
let entries = match with_walk_stall_timeout(stall, self.list_dir("", bucket, &current, -1)).await {
Ok(entries) => entries,
Err(err) => {
@@ -8013,15 +8000,10 @@ impl DiskAPI for LocalDisk {
use std::io::Write as _;
let file_path = self.io_get_object_path(volume, path)?;
let lock_path = file_path.with_extension("rustfs-cas.lock");
let path = path.to_string();
let sync_metadata = effective_durability(volume).syncs_commit_metadata();
return Ok(tokio::task::spawn_blocking(move || {
// A persistent directory lock bounds metadata growth. Removing
// per-target lock files can split flock ownership across inodes.
let lock_path = file_path
.parent()
.ok_or_else(|| std::io::Error::new(ErrorKind::InvalidInput, "conditional file has no parent"))?
.join(".rustfs-cas.lock");
let lock = std::fs::OpenOptions::new()
.create(true)
.truncate(false)
@@ -8086,25 +8068,7 @@ impl DiskAPI for LocalDisk {
.map_err(DiskError::from)??);
}
#[cfg(windows)]
{
let file_path = self.io_get_object_path(volume, path)?;
let sync_metadata = effective_durability(volume).syncs_commit_metadata();
let publication_root = self.publication_root.clone();
return Ok(tokio::task::spawn_blocking(move || {
os::compare_and_update_control_file(
&file_path,
expected.as_deref(),
replacement.as_deref(),
sync_metadata,
&publication_root,
)
})
.await
.map_err(DiskError::from)??);
}
#[cfg(not(any(unix, windows)))]
#[cfg(not(unix))]
{
let _ = (volume, path, expected, replacement);
Err(DiskError::MethodNotAllowed)
@@ -8137,7 +8101,7 @@ impl DiskAPI for LocalDisk {
async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
let volume_dir = self.io_get_bucket_path(volume)?;
if !skip_access_checks(volume)
&& let Err(e) = cached_access(&volume_dir).await
&& let Err(e) = access(&volume_dir).await
{
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
}
@@ -8345,7 +8309,7 @@ impl DiskAPI for LocalDisk {
if e == DiskError::FileNotFound {
if !skip_access_checks(volume)
&& let Err(err) = cached_access(&volume_dir).await
&& let Err(err) = access(&volume_dir).await
&& err.kind() == ErrorKind::NotFound
{
resp.results[i] = CHECK_PART_VOLUME_NOT_FOUND;
@@ -8871,7 +8835,7 @@ impl DiskAPI for LocalDisk {
Err(e) => {
if e.kind() == ErrorKind::NotFound
&& !skip_access_checks(volume)
&& let Err(e) = cached_access(&volume_dir).await
&& let Err(e) = access(&volume_dir).await
{
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
}
@@ -8900,7 +8864,7 @@ impl DiskAPI for LocalDisk {
let volume_dir = self.io_get_bucket_path(&opts.bucket)?;
if !skip_access_checks(&opts.bucket)
&& let Err(e) = with_walk_stall_deadline(stall, cached_access(&volume_dir)).await?
&& let Err(e) = with_walk_stall_deadline(stall, access(&volume_dir)).await?
{
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
}
@@ -8976,7 +8940,6 @@ impl DiskAPI for LocalDisk {
)
.await?;
out.close().await?;
Ok(())
}
@@ -9981,12 +9944,9 @@ impl DiskAPI for LocalDisk {
let volume_dir = self.io_get_bucket_path(volume)?;
// Volume creation is a mutation boundary, so it must observe the live
// filesystem rather than a potentially stale existence-cache entry.
if let Err(e) = access(&volume_dir).await {
if e.kind() == ErrorKind::NotFound {
os::make_dir_all(&volume_dir, self.io_root()).await?;
invalidate_bucket_cache(&volume_dir);
return Ok(());
}
error!(
@@ -10044,7 +10004,7 @@ impl DiskAPI for LocalDisk {
async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()> {
let volume_dir = self.io_get_bucket_path(volume)?;
if !skip_access_checks(volume) {
cached_access(&volume_dir)
access(&volume_dir)
.await
.map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?;
}
@@ -10877,7 +10837,6 @@ impl DiskAPI for LocalDisk {
// hit path skips the volume-access check, so nothing else would notice)
// (rustfs/backlog#1177).
self.io_backend.invalidate_cached_fds_for_volume(volume);
invalidate_bucket_cache(&p);
Ok(())
}
@@ -17528,86 +17487,6 @@ mod test {
assert_eq!(objs_returned, 1);
}
#[tokio::test]
async fn test_scan_dir_nonrecursive_visible_prefix_probe_cost() {
use rustfs_filemeta::MetacacheReader;
use tempfile::tempdir;
const PREFIX_COUNT: usize = 64;
let dir = tempdir().expect("tempdir should be created");
let bucket = "test-bucket";
let bucket_dir = dir.path().join(bucket);
let mut expected_names = Vec::with_capacity(PREFIX_COUNT);
for index in 0..PREFIX_COUNT {
let prefix = format!("prefix-{index:04}");
let object_name = format!("{prefix}/nested/object");
let object_dir = bucket_dir.join(&object_name);
fs::create_dir_all(&object_dir)
.await
.expect("visible object directory should be created");
let mut metadata = FileMeta::default();
let mut file_info = FileInfo::new(&object_name, 1, 1);
file_info.mod_time = Some(OffsetDateTime::now_utc());
metadata.add_version(file_info).expect("visible metadata should be valid");
fs::write(
object_dir.join(STORAGE_FORMAT_FILE),
metadata.marshal_msg().expect("visible metadata should encode"),
)
.await
.expect("visible object metadata should be written");
expected_names.push(format!("{prefix}/"));
}
async fn scan_prefixes(disk: &LocalDisk, bucket: &str, skip_hidden_prefix_check: bool) -> (Vec<String>, usize) {
let probe_count = Arc::new(AtomicUsize::new(0));
let (reader, mut writer) = tokio::io::duplex(64 * 1024);
let mut output = MetacacheWriter::new(&mut writer);
let opts = WalkDirOptions {
bucket: bucket.to_string(),
skip_hidden_prefix_check,
..Default::default()
};
let mut objects_returned = 0;
DIRECTORY_LISTING_ENTRY_PROBE_COUNT
.scope(
Arc::clone(&probe_count),
disk.scan_dir("".to_string(), "".to_string(), &opts, &mut output, &mut objects_returned, false, None),
)
.await
.expect("scan_dir should succeed");
output.close().await.expect("metacache writer should close");
drop(output);
drop(writer);
let mut reader = MetacacheReader::new(reader);
let names = reader
.read_all()
.await
.expect("scan output should decode")
.into_iter()
.map(|entry| entry.name)
.collect::<Vec<_>>();
(names, probe_count.load(Ordering::Relaxed))
}
let endpoint =
Endpoint::try_from(dir.path().to_str().expect("tempdir path should be UTF-8")).expect("endpoint should parse");
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should initialize");
let (conservative_names, conservative_probes) = scan_prefixes(&disk, bucket, false).await;
let (fast_path_names, fast_path_probes) = scan_prefixes(&disk, bucket, true).await;
assert_eq!(conservative_names, expected_names);
assert_eq!(fast_path_names, expected_names);
assert_eq!(conservative_probes, PREFIX_COUNT * 3);
assert_eq!(fast_path_probes, 0);
}
#[tokio::test]
async fn test_scan_dir_nonrecursive_skips_dirs_with_only_hidden_delete_markers() {
use rustfs_filemeta::MetacacheReader;
@@ -18430,29 +18309,6 @@ mod test {
let _ = fs::remove_dir_all(&p).await;
}
#[tokio::test]
async fn make_volume_rechecks_stale_positive_existence_cache() {
let root_dir = tempfile::tempdir().expect("temporary disk root should be created");
let endpoint = Endpoint::try_from(root_dir.path().to_string_lossy().as_ref()).expect("endpoint should parse");
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should initialize");
let volume_dir = disk.io_get_bucket_path("bucket").expect("bucket path should resolve");
fs::create_dir_all(&volume_dir)
.await
.expect("bucket directory should be created");
cached_access(&volume_dir)
.await
.expect("existing bucket should populate the cache");
fs::remove_dir(&volume_dir)
.await
.expect("bucket directory should be removed outside the cache");
disk.make_volume("bucket")
.await
.expect("volume creation should recheck the live filesystem");
assert!(fs::metadata(volume_dir).await.is_ok(), "volume directory should be recreated");
}
#[tokio::test]
async fn test_delete_volume() {
let p = "./testv1";
@@ -21967,9 +21823,9 @@ mod test {
assert!(matches!(results[1].as_ref().unwrap_err(), DiskError::Io(_)));
}
#[cfg(any(unix, windows))]
#[cfg(unix)]
#[tokio::test]
async fn windows_and_unix_conditional_file_update_never_deletes_a_new_owner() {
async fn conditional_file_update_never_deletes_a_new_owner() {
use tempfile::tempdir;
let dir = tempdir().expect("temp dir should be created");
@@ -22000,18 +21856,8 @@ mod test {
disk.read_all(RUSTFS_META_BUCKET, HEALING_MARKER_PATH)
.await
.expect("new owner marker should remain"),
owner_b.clone()
owner_b
);
assert_eq!(
disk.compare_and_update_file(RUSTFS_META_BUCKET, HEALING_MARKER_PATH, Some(owner_b), None)
.await
.expect("current owner should remove marker"),
ConditionalFileUpdate::Updated
);
assert!(matches!(
disk.read_all(RUSTFS_META_BUCKET, HEALING_MARKER_PATH).await,
Err(DiskError::FileNotFound)
));
}
#[cfg(unix)]
@@ -22026,10 +21872,7 @@ mod test {
let marker_path = disk
.get_object_path(RUSTFS_META_BUCKET, HEALING_MARKER_PATH)
.expect("marker path should resolve");
let lock_path = marker_path
.parent()
.expect("marker path should have a parent")
.join(".rustfs-cas.lock");
let lock_path = marker_path.with_extension("rustfs-cas.lock");
let lock = std::fs::OpenOptions::new()
.create(true)
.truncate(false)
@@ -22050,40 +21893,6 @@ mod test {
assert!(matches!(err, DiskError::Io(ref err) if err.kind() == ErrorKind::WouldBlock));
}
#[cfg(windows)]
#[tokio::test]
async fn windows_conditional_file_update_returns_would_block_when_marker_lock_is_contended() {
let dir = tempfile::tempdir().expect("temp dir should be created");
let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
ensure_test_volume(&disk, RUSTFS_META_BUCKET).await;
let marker_path = disk
.get_object_path(RUSTFS_META_BUCKET, HEALING_MARKER_PATH)
.expect("marker path should resolve");
let lock_path = marker_path
.parent()
.expect("marker path should have a parent")
.join(".rustfs-cas.lock");
let lock = std::fs::OpenOptions::new()
.create(true)
.truncate(false)
.read(true)
.write(true)
.open(lock_path)
.expect("marker lock should open");
lock.try_lock().expect("marker lock should be held");
let err = tokio::time::timeout(
Duration::from_secs(1),
disk.compare_and_update_file(RUSTFS_META_BUCKET, HEALING_MARKER_PATH, None, Some(Bytes::from_static(b"owner"))),
)
.await
.expect("contended conditional update must not block")
.expect_err("contended conditional update must retry");
assert!(matches!(err, DiskError::Io(ref err) if err.kind() == ErrorKind::WouldBlock));
}
#[cfg(target_os = "linux")]
#[tokio::test]
async fn replacement_io_paths_stay_under_the_mount_lease() {
+1 -62
View File
@@ -677,23 +677,6 @@ impl DiskAPI for Disk {
}
impl Disk {
pub(crate) async fn delete_with_scanner_publication_lease(
&self,
volume: &str,
path: &str,
opts: DeleteOptions,
scanner_publication_lease_token: Option<Uuid>,
) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.delete(volume, path, opts).await,
Disk::Remote(remote_disk) => {
remote_disk
.delete_with_scanner_publication_lease(volume, path, opts, scanner_publication_lease_token)
.await
}
}
}
pub(crate) async fn rename_data_borrowed(
&self,
src_volume: &str,
@@ -701,19 +684,6 @@ impl Disk {
fi: &FileInfo,
dst_volume: &str,
dst_path: &str,
) -> Result<RenameDataResp> {
self.rename_data_borrowed_with_fence(src_volume, src_path, fi, dst_volume, dst_path, None)
.await
}
pub(crate) async fn rename_data_borrowed_with_fence(
&self,
src_volume: &str,
src_path: &str,
fi: &FileInfo,
dst_volume: &str,
dst_path: &str,
scanner_publication_lease_token: Option<Uuid>,
) -> Result<RenameDataResp> {
match self {
Disk::Local(local_disk) => {
@@ -723,14 +693,7 @@ impl Disk {
}
Disk::Remote(remote_disk) => {
remote_disk
.rename_data_borrowed_with_fence(
src_volume,
src_path,
fi,
dst_volume,
dst_path,
scanner_publication_lease_token,
)
.rename_data_borrowed(src_volume, src_path, fi, dst_volume, dst_path)
.await
}
}
@@ -1231,11 +1194,6 @@ pub struct WalkDirOptions {
#[serde(default)]
pub incl_deleted: bool,
// Skip recursive prefix visibility probes only when authoritative bucket
// metadata proves versioning was never enabled.
#[serde(default)]
pub skip_hidden_prefix_check: bool,
// ReportNotFound will return errFileNotFound if all disks reports the BaseDir cannot be found.
pub report_notfound: bool,
@@ -1574,7 +1532,6 @@ mod tests {
base_dir: "/path/to/dir".to_string(),
recursive: true,
incl_deleted: false,
skip_hidden_prefix_check: false,
report_notfound: false,
filter_prefix: Some("prefix_".to_string()),
forward_to: Some("object/path".to_string()),
@@ -1589,7 +1546,6 @@ mod tests {
assert_eq!(opts.base_dir, "/path/to/dir");
assert!(opts.recursive);
assert!(!opts.incl_deleted);
assert!(!opts.skip_hidden_prefix_check);
assert!(!opts.report_notfound);
assert_eq!(opts.filter_prefix, Some("prefix_".to_string()));
assert_eq!(opts.forward_to, Some("object/path".to_string()));
@@ -1600,23 +1556,6 @@ mod tests {
assert_eq!(opts.stall_timeout_duration(), Some(std::time::Duration::from_secs(20)));
}
#[test]
fn test_walk_dir_options_default_hidden_prefix_check_for_old_peers() {
let mut encoded = serde_json::to_value(WalkDirOptions {
skip_hidden_prefix_check: true,
..Default::default()
})
.expect("walk options should serialize");
encoded
.as_object_mut()
.expect("walk options should serialize as an object")
.remove("skip_hidden_prefix_check");
let decoded: WalkDirOptions = serde_json::from_value(encoded).expect("old peer options should deserialize");
assert!(!decoded.skip_hidden_prefix_check);
}
/// Test DeleteOptions structure
#[test]
fn test_delete_options() {
-85
View File
@@ -12,8 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#[cfg(windows)]
use crate::disk::ConditionalFileUpdate;
use crate::disk::error::DiskError;
use crate::disk::error::Result;
use crate::disk::error_conv::to_file_error;
@@ -3460,89 +3458,6 @@ fn read_windows_relative_file(file_path: &Path, parent_guard: &ExistingBaseDirec
Ok(Some(data))
}
#[cfg(windows)]
pub(crate) fn compare_and_update_control_file(
file_path: &Path,
expected: Option<&[u8]>,
replacement: Option<&[u8]>,
sync_metadata: bool,
publication_root: &PublicationRoot,
) -> io::Result<ConditionalFileUpdate> {
use windows_sys::{
Wdk::Storage::FileSystem::{
FILE_NON_DIRECTORY_FILE, FILE_OPEN, FILE_OPEN_IF, FILE_OPEN_REPARSE_POINT, FILE_SYNCHRONOUS_IO_NONALERT,
},
Win32::Storage::FileSystem::{
DELETE, FILE_ATTRIBUTE_NORMAL, FILE_READ_ATTRIBUTES, FILE_SHARE_READ, FILE_SHARE_WRITE, FILE_WRITE_DATA, SYNCHRONIZE,
},
};
let parent = file_path
.parent()
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "conditional file has no parent"))?;
let parent_guard = lock_windows_directory_tree(parent, Some(parent), publication_root)?;
let lock = open_windows_relative(
parent_guard.last_handle()?,
std::ffi::OsStr::new(".rustfs-cas.lock"),
SYNCHRONIZE | FILE_READ_ATTRIBUTES | FILE_WRITE_DATA,
FILE_SHARE_READ | FILE_SHARE_WRITE,
FILE_OPEN_IF,
FILE_NON_DIRECTORY_FILE | FILE_OPEN_REPARSE_POINT | FILE_SYNCHRONOUS_IO_NONALERT,
FILE_ATTRIBUTE_NORMAL,
true,
)?;
validate_windows_owned_file(&lock)?;
match lock.as_file().try_lock() {
Ok(()) => {}
Err(std::fs::TryLockError::WouldBlock) => return Err(io::Error::from(io::ErrorKind::WouldBlock)),
Err(std::fs::TryLockError::Error(err)) => return Err(err),
}
let current = read_windows_relative_file(file_path, &parent_guard)?;
let matches = match (&current, expected) {
(None, None) => true,
(Some(current), Some(expected)) => current.as_slice() == expected,
_ => false,
};
if !matches {
return Ok(match current {
None => ConditionalFileUpdate::Missing,
Some(_) => ConditionalFileUpdate::Mismatch,
});
}
match replacement {
Some(replacement) => RenameDestinationPathGuard {
directory: parent.to_path_buf(),
_directory_guard: parent_guard,
}
.write_file_for_path_access(file_path, replacement, sync_metadata, sync_metadata)?,
None => {
let file_name = file_path
.file_name()
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "conditional file must have a name"))?;
let file = open_windows_relative(
parent_guard.last_handle()?,
file_name,
DELETE | SYNCHRONIZE | FILE_READ_ATTRIBUTES,
FILE_SHARE_READ,
FILE_OPEN,
FILE_NON_DIRECTORY_FILE | FILE_OPEN_REPARSE_POINT | FILE_SYNCHRONOUS_IO_NONALERT,
0,
true,
)?;
validate_windows_owned_file(&file)?;
set_windows_file_delete_on_close(&file, true)?;
drop(file);
if sync_metadata {
fsync_dir_std(parent)?;
}
}
}
Ok(ConditionalFileUpdate::Updated)
}
#[cfg(windows)]
fn open_windows_directory_component(
parent: &WindowsDirectoryHandle,
+6 -38
View File
@@ -784,24 +784,6 @@ pub(crate) fn create_deferred_bitrot_reader_with_stripe_handle(
///
/// # Returns
/// A Result containing the BitrotWriterWrapper or an error
/// Size hint handed to `DiskAPI::create_file` for a bitrot-wrapped shard.
///
/// A known length is grown by one checksum per shard so the on-disk file size
/// matches what the bitrot writer emits. A negative length is the
/// unknown-size sentinel (`HashReader::SIZE_PRESERVE_LAYER`, used by SSE and
/// compression) and must be preserved: `RemoteDisk::create_file` forwards it
/// in the `put_file_stream` query, and the receiver only treats `size > 0` as
/// a fixed body length when locating the authenticated trailer. Clamping it
/// to `0` would claim an empty body and misframe the stream. `0` stays `0`
/// because a genuinely empty object still means an empty body.
fn bitrot_create_file_size(length: i64, shard_size: usize, checksum_algo: &HashAlgorithm) -> i64 {
if length <= 0 {
return length;
}
let length = length as usize;
(length.div_ceil(shard_size) * checksum_algo.size() + length) as i64
}
pub async fn create_bitrot_writer(
is_inline_buffer: bool,
disk: Option<&DiskStore>,
@@ -814,7 +796,12 @@ pub async fn create_bitrot_writer(
let writer = if is_inline_buffer {
CustomWriter::new_inline_buffer()
} else if let Some(disk) = disk {
let length = bitrot_create_file_size(length, shard_size, &checksum_algo);
let length = if length > 0 {
let length = length as usize;
(length.div_ceil(shard_size) * checksum_algo.size() + length) as i64
} else {
0
};
let file = disk.create_file("", volume, path, length).await?;
#[cfg(feature = "hotpath")]
@@ -833,25 +820,6 @@ mod tests {
use rustfs_rio::ChunkReader;
use std::collections::VecDeque;
#[test]
fn bitrot_create_file_size_grows_known_length_by_checksums() {
// 10 bytes over 4-byte shards = 3 shards, each followed by a 32-byte hash.
assert_eq!(bitrot_create_file_size(10, 4, &HashAlgorithm::HighwayHash256), 10 + 3 * 32);
assert_eq!(bitrot_create_file_size(10, 4, &HashAlgorithm::None), 10);
}
#[test]
fn bitrot_create_file_size_keeps_empty_and_unknown_distinct() {
assert_eq!(bitrot_create_file_size(0, 4, &HashAlgorithm::HighwayHash256), 0);
// SSE/compression streams advertise SIZE_PRESERVE_LAYER (-1); the remote
// put_file_stream receiver relies on a non-positive size to parse the auth
// trailer from the stream tail, so the sentinel must survive untouched.
assert_eq!(
bitrot_create_file_size(rustfs_rio::HashReader::SIZE_PRESERVE_LAYER, 4, &HashAlgorithm::HighwayHash256),
rustfs_rio::HashReader::SIZE_PRESERVE_LAYER
);
}
struct TestChunkReader {
chunks: VecDeque<Bytes>,
}
-74
View File
@@ -84,61 +84,6 @@ impl NamespaceLockFence {
}
}
#[cfg(test)]
static NAMESPACE_LOCK_SIGNAL_TEST_FENCES: std::sync::OnceLock<std::sync::Mutex<Vec<(usize, NamespaceLockFence)>>> =
std::sync::OnceLock::new();
#[cfg(test)]
pub(crate) struct NamespaceLockSignalTestFence {
signal_key: usize,
}
#[cfg(test)]
impl NamespaceLockSignalTestFence {
pub(crate) fn install_with_loss_handle(
signal: &Arc<rustfs_lock::distributed_lock::LockLostSignal>,
loss_handle: Arc<std::sync::atomic::AtomicBool>,
) -> Self {
let fence = NamespaceLockFence {
signals: Arc::default(),
forced_lost: Arc::new(vec![loss_handle]),
};
let signal_key = Arc::as_ptr(signal) as usize;
let mut fences = NAMESPACE_LOCK_SIGNAL_TEST_FENCES
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
.lock()
.expect("namespace lock signal test fence should not be poisoned");
assert!(
!fences.iter().any(|(key, _)| *key == signal_key),
"namespace lock signal test fence must be unique"
);
fences.push((signal_key, fence));
Self { signal_key }
}
}
#[cfg(test)]
impl Drop for NamespaceLockSignalTestFence {
fn drop(&mut self) {
let mut fences = NAMESPACE_LOCK_SIGNAL_TEST_FENCES
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
.lock()
.expect("namespace lock signal test fence should not be poisoned");
fences.retain(|(key, _)| *key != self.signal_key);
}
}
#[cfg(test)]
pub(crate) fn namespace_lock_signal_test_fence_is_lost(signal: &Arc<rustfs_lock::distributed_lock::LockLostSignal>) -> bool {
NAMESPACE_LOCK_SIGNAL_TEST_FENCES
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
.lock()
.expect("namespace lock signal test fence should not be poisoned")
.iter()
.find(|(key, _)| *key == Arc::as_ptr(signal) as usize)
.is_some_and(|(_, fence)| fence.is_lock_lost())
}
#[derive(Debug)]
pub struct ObjectLockConfigSnapshot {
store_id: Option<Uuid>,
@@ -446,11 +391,6 @@ pub struct ObjectOptions {
pub tier_delete_journal_api: Option<Arc<crate::store::ECStore>>,
}
/// Transient scanner-only carrier for target-side publication lease tokens.
/// SetDisks consumes and removes this key before constructing durable
/// FileInfo metadata; it must never appear in an S3-visible object.
pub const SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY: &str = "x-rustfs-internal-scanner-publication-lease-fence-v1";
impl ObjectOptions {
pub fn set_quota_admission(&mut self, current_usage: u64, quota_limit: u64) -> bool {
self.quota_admission = (current_usage <= quota_limit).then_some(QuotaAdmission {
@@ -465,23 +405,9 @@ impl ObjectOptions {
}
pub(crate) fn add_namespace_lock_lost_signal(&mut self, signal: Arc<rustfs_lock::distributed_lock::LockLostSignal>) {
#[cfg(test)]
let test_fence = NAMESPACE_LOCK_SIGNAL_TEST_FENCES
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
.lock()
.expect("namespace lock signal test fence should not be poisoned")
.iter()
.find(|(key, _)| *key == Arc::as_ptr(&signal) as usize)
.map(|(_, fence)| fence.clone());
self.namespace_lock_fence
.get_or_insert_with(NamespaceLockFence::new)
.add_signal(signal);
#[cfg(test)]
if let Some(test_fence) = test_fence {
self.namespace_lock_fence
.get_or_insert_with(NamespaceLockFence::new)
.extend(&test_fence);
}
}
pub(crate) fn ensure_namespace_lock_fence(&mut self) {
+9 -230
View File
@@ -52,32 +52,11 @@ use crate::services::tier::tier::TierConfigMgr;
use rustfs_lock::{GlobalLockManager, get_global_lock_manager};
use s3s::region::Region;
use std::collections::{HashMap, HashSet};
use std::sync::{
Arc, OnceLock,
atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering},
};
use tokio::sync::{Mutex, Notify, OnceCell, OwnedRwLockReadGuard, RwLock};
use tokio::time::Instant;
use std::sync::{Arc, OnceLock};
use tokio::sync::{OnceCell, RwLock};
use tokio_util::sync::CancellationToken;
use uuid::Uuid;
const SCANNER_PUBLICATION_STATE_UNKNOWN: u8 = 0;
const SCANNER_PUBLICATION_STATE_ALLOWED: u8 = 1;
const SCANNER_PUBLICATION_STATE_BLOCKED: u8 = 2;
pub(crate) const SCANNER_PUBLICATION_LEASE_MAX_ENTRIES: usize = 256;
/// A lease is deliberately short-lived. The coordinator treats expiry as a
/// failed publication rather than silently continuing with a peer that may
/// have started movement after the lease was abandoned.
pub(crate) const SCANNER_PUBLICATION_LEASE_TTL: std::time::Duration = std::time::Duration::from_secs(60);
pub(crate) struct ScannerPublicationLeaseEntry {
pub(crate) expires_at: Instant,
pub(crate) movement_generation: u64,
pub(crate) _operation_guard: OwnedRwLockReadGuard<()>,
}
/// Runtime state owned by a single `ECStore` instance.
///
/// This is intentionally minimal in the first migration slice; subsequent
@@ -181,34 +160,10 @@ pub struct InstanceContext {
/// workers (scanner/heal/tier/lifecycle) without touching another instance.
/// Replaces the process-global cancel-token static.
background_cancel_token: OnceLock<CancellationToken>,
/// Serializes data-movement transitions with scanner publication commits.
/// Readers are held across one publication commit; movement transitions
/// take the writer at their durable state commit boundary.
data_movement_operation_gate: Arc<RwLock<()>>,
/// Remote scanner publication leases own a read guard until explicit
/// release or bounded expiry. Keeping the guard in storage-owned state
/// makes a remote movement transition wait on the same fence as a local
/// scanner commit.
scanner_publication_leases: Arc<Mutex<HashMap<Uuid, ScannerPublicationLeaseEntry>>>,
/// Monotonic admission epoch paired with the operation gate. A
/// publication admitted before a movement transition must never be
/// mistaken for one admitted after the transition.
data_movement_operation_epoch: AtomicU64,
/// Once the admission epoch reaches its reserved terminal value, no new
/// publication may be admitted. Keeping this state separate from the
/// saturating counter prevents an unchanged `u64::MAX` value from being
/// mistaken for a fresh epoch after overflow.
data_movement_operation_epoch_exhausted: AtomicBool,
/// Storage-owned generation for movement state changes. This is separate
/// from the publication admission epoch so scanners can wait for a
/// terminal/clear transition without treating the wake as a publication
/// permit.
data_movement_generation: AtomicU64,
data_movement_generation_exhausted: AtomicBool,
data_movement_generation_notify: Arc<Notify>,
/// Last storage-owned movement snapshot observed under the operation
/// gate. SetDisks cache writers fail closed until ECStore refreshes it.
scanner_publication_state: AtomicU8,
/// Serializes decommission data-movement operations with cancellation and
/// a subsequent restart. Readers are held across one object side effect;
/// the transition path takes the writer after cancelling the routine.
decommission_operation_gate: Arc<RwLock<()>>,
/// Resolves object-encryption material at the application boundary.
object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
@@ -249,14 +204,7 @@ impl InstanceContext {
local_disk_set_drives: Arc::new(RwLock::new(Vec::new())),
bucket_metadata_sys: std::sync::Mutex::new(None),
background_cancel_token: OnceLock::new(),
data_movement_operation_gate: Arc::new(RwLock::new(())),
scanner_publication_leases: Arc::new(Mutex::new(HashMap::new())),
data_movement_operation_epoch: AtomicU64::new(0),
data_movement_operation_epoch_exhausted: AtomicBool::new(false),
data_movement_generation: AtomicU64::new(0),
data_movement_generation_exhausted: AtomicBool::new(false),
data_movement_generation_notify: Arc::new(Notify::new()),
scanner_publication_state: AtomicU8::new(SCANNER_PUBLICATION_STATE_UNKNOWN),
decommission_operation_gate: Arc::new(RwLock::new(())),
object_encryption_resolver: OnceLock::new(),
tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()),
@@ -275,177 +223,8 @@ impl InstanceContext {
self.lock_manager.clone()
}
pub(crate) fn data_movement_operation_gate(&self) -> Arc<RwLock<()>> {
Arc::clone(&self.data_movement_operation_gate)
}
pub(crate) async fn install_scanner_publication_lease(
&self,
token: Uuid,
expires_at: Instant,
movement_generation: u64,
operation_guard: OwnedRwLockReadGuard<()>,
) -> bool {
let mut leases = self.scanner_publication_leases.lock().await;
if leases.len() >= SCANNER_PUBLICATION_LEASE_MAX_ENTRIES {
return false;
}
leases.insert(
token,
ScannerPublicationLeaseEntry {
expires_at,
movement_generation,
_operation_guard: operation_guard,
},
);
true
}
pub(crate) async fn remove_scanner_publication_lease(&self, token: Uuid) -> bool {
self.scanner_publication_leases.lock().await.remove(&token).is_some()
}
/// Check a lease token while the caller holds the movement read guard.
///
/// The token table is deliberately process-owned and non-persistent: a
/// restarted instance has no entries from the previous process, so an old
/// coordinator proof cannot become valid again merely because the
/// movement generation counter restarted at zero.
pub(crate) async fn scanner_publication_lease_is_active(&self, token: Uuid) -> bool {
let mut leases = self.scanner_publication_leases.lock().await;
let now = Instant::now();
let Some(expires_at) = leases.get(&token).map(|entry| entry.expires_at) else {
return false;
};
if expires_at <= now {
leases.remove(&token);
return false;
}
true
}
/// Return the generation bound to a live lease. The lease entry owns the
/// movement read guard, so a successful lookup remains valid for the
/// caller's guard-protected operation; expiry is still fail-closed.
pub(crate) async fn scanner_publication_lease_generation(&self, token: Uuid) -> Option<u64> {
let mut leases = self.scanner_publication_leases.lock().await;
let now = Instant::now();
let (expires_at, movement_generation) = leases
.get(&token)
.map(|entry| (entry.expires_at, entry.movement_generation))?;
if expires_at <= now {
leases.remove(&token);
return None;
}
Some(movement_generation)
}
pub(crate) async fn expire_scanner_publication_lease(&self, token: Uuid, expires_at: Instant) {
let mut leases = self.scanner_publication_leases.lock().await;
let should_remove = leases.get(&token).is_some_and(|entry| entry.expires_at <= expires_at);
if should_remove {
leases.remove(&token);
}
}
pub(crate) fn data_movement_operation_epoch(&self) -> u64 {
self.data_movement_operation_epoch.load(Ordering::Acquire)
}
pub(crate) fn data_movement_operation_epoch_exhausted(&self) -> bool {
self.data_movement_operation_epoch_exhausted.load(Ordering::Acquire)
}
pub(crate) fn data_movement_generation(&self) -> u64 {
self.data_movement_generation.load(Ordering::Acquire)
}
pub(crate) fn data_movement_generation_exhausted(&self) -> bool {
self.data_movement_generation_exhausted.load(Ordering::Acquire)
}
pub(crate) fn data_movement_generation_notify(&self) -> Arc<Notify> {
Arc::clone(&self.data_movement_generation_notify)
}
pub(crate) fn scanner_publication_state_allowed(&self) -> bool {
!self.data_movement_operation_epoch_exhausted()
&& !self.data_movement_generation_exhausted()
&& self.scanner_publication_state.load(Ordering::Acquire) == SCANNER_PUBLICATION_STATE_ALLOWED
}
pub(crate) fn set_scanner_publication_state(&self, blocked: bool) {
self.scanner_publication_state.store(
if blocked {
SCANNER_PUBLICATION_STATE_BLOCKED
} else {
SCANNER_PUBLICATION_STATE_ALLOWED
},
Ordering::Release,
);
}
pub(crate) fn advance_data_movement_operation_epoch(&self) -> u64 {
self.scanner_publication_state
.store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release);
let previous = self.data_movement_operation_epoch.load(Ordering::Acquire);
let _ = self
.data_movement_operation_epoch
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |epoch| Some(epoch.saturating_add(1)));
let result = self.data_movement_operation_epoch.load(Ordering::Acquire);
if result == u64::MAX {
self.data_movement_operation_epoch_exhausted.store(true, Ordering::Release);
}
if result != previous {
let _ = self.advance_data_movement_generation();
}
result
}
/// Advance the movement generation after a durable movement transition.
/// The generation is deliberately bounded: once it reaches `u64::MAX`,
/// publication and generation-based waits fail closed rather than reusing
/// an indistinguishable saturated value.
pub(crate) fn advance_data_movement_generation(&self) -> Option<u64> {
if self.data_movement_generation_exhausted.load(Ordering::Acquire) {
return None;
}
let updated = self
.data_movement_generation
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |generation| generation.checked_add(1));
match updated {
Ok(previous) => {
let Some(generation) = previous.checked_add(1) else {
self.data_movement_generation_exhausted.store(true, Ordering::Release);
return None;
};
if generation == u64::MAX {
self.data_movement_generation_exhausted.store(true, Ordering::Release);
}
self.data_movement_generation_notify.notify_waiters();
Some(generation)
}
Err(_) => {
self.data_movement_generation_exhausted.store(true, Ordering::Release);
None
}
}
}
#[cfg(test)]
pub(crate) fn set_data_movement_operation_epoch_for_test(&self, epoch: u64) {
self.data_movement_operation_epoch.store(epoch, Ordering::Release);
self.data_movement_operation_epoch_exhausted
.store(epoch == u64::MAX, Ordering::Release);
self.scanner_publication_state
.store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release);
}
#[cfg(test)]
pub(crate) fn set_data_movement_generation_for_test(&self, generation: u64) {
self.data_movement_generation.store(generation, Ordering::Release);
self.data_movement_generation_exhausted
.store(generation == u64::MAX, Ordering::Release);
pub(crate) fn decommission_operation_gate(&self) -> Arc<RwLock<()>> {
Arc::clone(&self.decommission_operation_gate)
}
/// Install the application-owned object-encryption resolver once.
+2 -2
View File
@@ -119,8 +119,8 @@ pub(crate) fn endpoint_erasure_set_count() -> Option<usize> {
endpoint_pools().map(|endpoints| endpoints.es_count())
}
pub(crate) fn endpoint_pool_is_local(endpoints: &EndpointServerPools, pool_index: usize) -> bool {
endpoints
pub(crate) fn endpoint_pool_is_local(pool_index: usize) -> bool {
get_global_endpoints()
.as_ref()
.get(pool_index)
.is_some_and(|pool| pool.endpoints.as_ref().first().is_some_and(|endpoint| endpoint.is_local))
@@ -220,14 +220,6 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo
last_usage_save_unix_secs: metrics.usage_freshness.last_usage_save_unix_secs,
last_usage_save_result: metrics.usage_freshness.last_usage_save_result,
last_usage_save_result_code: metrics.usage_freshness.last_usage_save_result_code,
last_durable_success_unix_secs: metrics.usage_freshness.last_durable_success_unix_secs,
last_publication_unix_secs: metrics.usage_freshness.last_publication_unix_secs,
last_publication_state: metrics.usage_freshness.last_publication_state,
last_publication_reason: metrics.usage_freshness.last_publication_reason,
deferred_pending: metrics.usage_freshness.deferred_pending,
deferred_total: metrics.usage_freshness.deferred_total,
last_deferred_unix_secs: metrics.usage_freshness.last_deferred_unix_secs,
last_deferred_reason: metrics.usage_freshness.last_deferred_reason,
},
maintenance_control: MadminScannerMaintenanceControlSnapshot {
primary_control: metrics.maintenance_control.primary_control,
@@ -824,14 +816,6 @@ mod test {
last_usage_save_unix_secs: 12,
last_usage_save_result: "success".to_string(),
last_usage_save_result_code: 1,
last_durable_success_unix_secs: 13,
last_publication_unix_secs: 14,
last_publication_state: "published".to_string(),
last_publication_reason: "complete".to_string(),
deferred_pending: true,
deferred_total: 15,
last_deferred_unix_secs: 16,
last_deferred_reason: "data_movement".to_string(),
},
..Default::default()
});
@@ -844,14 +828,6 @@ mod test {
assert_eq!(scanner.usage_freshness.last_usage_save_unix_secs, 12);
assert_eq!(scanner.usage_freshness.last_usage_save_result, "success");
assert_eq!(scanner.usage_freshness.last_usage_save_result_code, 1);
assert_eq!(scanner.usage_freshness.last_durable_success_unix_secs, 13);
assert_eq!(scanner.usage_freshness.last_publication_unix_secs, 14);
assert_eq!(scanner.usage_freshness.last_publication_state, "published");
assert_eq!(scanner.usage_freshness.last_publication_reason, "complete");
assert!(scanner.usage_freshness.deferred_pending);
assert_eq!(scanner.usage_freshness.deferred_total, 15);
assert_eq!(scanner.usage_freshness.last_deferred_unix_secs, 16);
assert_eq!(scanner.usage_freshness.last_deferred_reason, "data_movement");
}
#[test]
+5 -171
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::cluster::rpc::{PeerRestClient, ScannerPeerActivity, ScannerPublicationLease, TierConfigReloadOutcome};
use crate::cluster::rpc::{PeerRestClient, ScannerPeerActivity, TierConfigReloadOutcome};
use crate::diagnostics::admin_server_info::get_commit_id;
use crate::disk::DiskAPI;
use crate::error::{Error, Result};
@@ -51,13 +51,7 @@ const TIER_CONFIG_RELOAD_RETRY_CAP: Duration = Duration::from_secs(5);
const REMOTE_VERSION_STATE_PROBE_INTERVAL: Duration = Duration::from_secs(10);
const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30);
const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 2;
#[derive(Clone, Debug)]
pub struct ScannerPublicationLeaseGrant {
pub host: String,
pub lease: ScannerPublicationLease,
}
const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 1;
/// Cached result from the last successful admin call to a peer.
struct PeerAdminCache {
@@ -216,24 +210,6 @@ pub fn cross_pool_fence_fleet_proof_matches(proof: &CrossPoolFenceFleetProofToke
fleet_capability_proof_matches(cross_pool_fence_fleet_proof_slot(), &proof.0)
}
#[cfg(test)]
pub(crate) fn install_cross_pool_fence_fleet_proof_for_test() {
let topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY
.get()
.cloned()
.unwrap_or_else(|| "pool-activation-test-topology".to_string());
let _ = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.set(topology.clone());
let mut state = cross_pool_fence_fleet_proof_slot()
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
state.topology_conflict = false;
state.proof = Some(FleetCapabilityProof {
topology_fingerprint: topology,
peer_epochs: Arc::new(BTreeMap::new()),
expires_at: Instant::now() + Duration::from_secs(60 * 60),
});
}
#[cfg(any(test, feature = "test-util"))]
pub fn rotate_cross_pool_fence_fleet_proof_for_test() -> bool {
let mut state = cross_pool_fence_fleet_proof_slot()
@@ -376,7 +352,7 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
capability = "cross_pool_fence_v2",
capability = "cross_pool_fence_v1",
state = "failed_closed",
error = %err,
"notification capability probe"
@@ -1145,21 +1121,9 @@ impl NotificationSys {
}
}
let local_rebalance_id = match expected_rebalance_id {
Some(expected_id) => Some(expected_id.to_owned()),
None => store.current_rebalance_id().await,
};
match store.stop_rebalance_for_id(local_rebalance_id.as_deref()).await {
match store.stop_rebalance_for_id(expected_rebalance_id).await {
Ok(_) => {
let save_result = match local_rebalance_id.as_deref() {
Some(expected_id) => {
store
.save_rebalance_stats_for_id(usize::MAX, RebalSaveOpt::StoppedAt, expected_id)
.await
}
None => Ok(()),
};
if let Err(err) = save_result {
if let Err(err) = store.save_rebalance_stats(usize::MAX, RebalSaveOpt::StoppedAt).await {
error!(
event = EVENT_NOTIFICATION_PEER_PROPAGATION,
component = LOG_COMPONENT_ECSTORE,
@@ -1497,102 +1461,6 @@ impl NotificationSys {
aggregate_scanner_dirty_usage_acknowledgement_results(join_all(futures).await, failures)
}
/// Acquire remote publication leases in a deterministic host order. A
/// missing/legacy peer is a hard publication deferral; already acquired
/// leases are released before returning so a partial acquisition cannot
/// pin movement on one peer.
pub async fn acquire_scanner_publication_leases(
&self,
mut targets: Vec<(String, String, u64)>,
) -> Result<Vec<ScannerPublicationLeaseGrant>> {
targets.sort_by(|left, right| left.0.cmp(&right.0));
for pair in targets.windows(2) {
if pair[0].0 == pair[1].0 {
return Err(Error::other(format!("duplicate scanner publication lease target: {}", pair[0].0)));
}
}
let mut grants = Vec::with_capacity(targets.len());
for (host, session_id, generation) in targets {
let Some(client) = self
.peer_clients
.iter()
.flatten()
.find(|client| client.grid_host == host)
.cloned()
else {
let _ = self.release_scanner_publication_leases(grants).await;
return Err(Error::other(format!("scanner publication lease peer {host} is unavailable")));
};
match client.acquire_scanner_publication_lease(&session_id, generation).await {
Ok(lease) => grants.push(ScannerPublicationLeaseGrant { host, lease }),
Err(err) => {
let _ = self.release_scanner_publication_leases(grants).await;
return Err(Error::other(format!("scanner publication lease acquisition failed: {err}")));
}
}
}
Ok(grants)
}
pub async fn release_scanner_publication_leases(&self, mut grants: Vec<ScannerPublicationLeaseGrant>) -> Result<()> {
grants.sort_by(|left, right| right.host.cmp(&left.host));
let mut failures = Vec::new();
for grant in grants {
let Some(client) = self
.peer_clients
.iter()
.flatten()
.find(|client| client.grid_host == grant.host)
else {
failures.push(format!("peer {} is unavailable", grant.host));
continue;
};
if let Err(err) = client.release_scanner_publication_lease(&grant.lease).await {
failures.push(format!("peer {} release failed: {err}", grant.host));
}
}
if failures.is_empty() {
Ok(())
} else {
Err(Error::other(format!(
"scanner publication lease release failures: {}",
failures.join("; ")
)))
}
}
/// Revalidate every remote lease in deterministic host order immediately
/// before a final scanner metadata write. A peer restart removes its
/// process-owned token table and changes its activity session, so an old
/// generation cannot pass this proof even when the numeric generation is
/// reused.
pub async fn validate_scanner_publication_leases(&self, grants: &[ScannerPublicationLeaseGrant]) -> Result<()> {
let mut grants = grants.to_vec();
grants.sort_by(|left, right| left.host.cmp(&right.host));
for pair in grants.windows(2) {
if pair[0].host == pair[1].host {
return Err(Error::other(format!("duplicate scanner publication lease target: {}", pair[0].host)));
}
}
for grant in grants {
let Some(client) = self
.peer_clients
.iter()
.flatten()
.find(|client| client.grid_host == grant.host)
.cloned()
else {
return Err(Error::other(format!("scanner publication lease peer {} is unavailable", grant.host)));
};
client
.validate_scanner_publication_lease(&grant.lease)
.await
.map_err(|err| Error::other(format!("scanner publication lease validation failed for {}: {err}", grant.host)))?;
}
Ok(())
}
pub async fn reload_site_replication_config(&self) -> Vec<NotificationPeerErr> {
let mut futures = Vec::with_capacity(self.peer_clients.len());
for client in self.peer_clients.iter() {
@@ -2735,38 +2603,6 @@ mod tests {
assert!(err.to_string().contains("no remote peers"));
}
#[tokio::test]
async fn scanner_publication_lease_release_reports_all_unavailable_peers() {
let sys = NotificationSys {
peer_clients: Vec::new(),
all_peer_clients: Vec::new(),
peer_topology_hosts: Vec::new(),
peer_admin_caches: Vec::new(),
tier_config_reload_workers: Default::default(),
};
let grants = ["peer-a", "peer-b"]
.into_iter()
.map(|host| ScannerPublicationLeaseGrant {
host: host.to_string(),
lease: ScannerPublicationLease {
token: Uuid::new_v4(),
movement_generation: 3,
owner_id: Uuid::new_v4().to_string(),
session_id: "session-a".to_string(),
expires_at: Instant::now() + Duration::from_secs(30),
},
})
.collect();
let error = sys
.release_scanner_publication_leases(grants)
.await
.expect_err("an unavailable peer must not silently release a remote lease");
let message = error.to_string();
assert!(message.contains("peer-a"));
assert!(message.contains("peer-b"));
}
#[tokio::test]
async fn scanner_activity_probe_rejects_an_incomplete_peer_topology() {
let client = PeerRestClient::new(
@@ -2889,8 +2725,6 @@ mod tests {
data_movement_active: Some(false),
dirty_usage_generation: Some(2),
dirty_usage_pending,
movement_generation: Some(1),
publication_blocked: Some(false),
};
let pending = aggregate_scanner_dirty_usage_acknowledgement_results(
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+5 -12
View File
@@ -155,7 +155,7 @@ impl RebalanceMeta {
self.save_with_opts(store, ObjectOptions::default()).await
}
pub async fn save_with_opts<S>(&self, store: Arc<S>, mut opts: ObjectOptions) -> Result<()>
pub async fn save_with_opts<S>(&self, store: Arc<S>, opts: ObjectOptions) -> Result<()>
where
S: ObjectIO<
Error = Error,
@@ -188,14 +188,6 @@ impl RebalanceMeta {
let msg = rmp_serde::to_vec(self)?;
data.extend(msg);
if self.stopped_at.is_none() && is_rebalance_conflicting_with_decommission(self) {
rustfs_utils::http::metadata_compat::insert_str(
&mut opts.user_defined,
rustfs_utils::http::metadata_compat::SUFFIX_REBALANCE_RUN_ID,
self.id.clone(),
);
}
save_config_with_opts(store, REBAL_META_NAME, data, &opts).await?;
Ok(())
@@ -872,6 +864,10 @@ pub(super) fn merge_rebalance_meta(remote: &mut RebalanceMeta, local: &Rebalance
RebalanceMetaMergeOutcome::Merged
}
#[allow(
dead_code,
reason = "stop-transition helper retained beside stop_rebalance_meta_snapshot; no caller yet (backlog#1823)"
)]
pub(super) fn mark_started_rebalance_pools_stopped(meta: &mut RebalanceMeta, stop_time: OffsetDateTime) {
for pool_stat in meta.pool_stats.iter_mut() {
if pool_stat.info.status == RebalStatus::Started {
@@ -939,9 +935,6 @@ pub(super) fn stop_rebalance_meta_snapshot_for_id(
}
stop_rebalance_state(meta, now);
// The caller holds the activation writer after admission was cancelled,
// so all entry readers have drained and no later entry can be admitted.
mark_started_rebalance_pools_stopped(meta, now);
meta.last_refreshed_at = Some(now);
Ok(Some(meta.clone()))
}
@@ -102,20 +102,11 @@ pub(crate) trait MigrationBackend: Send + Sync {
pub(crate) struct RebalanceMigrationBackend<'a> {
source: &'a SetDisks,
store: &'a ECStore,
lock_lost_signal: Option<std::sync::Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
}
impl<'a> RebalanceMigrationBackend<'a> {
pub(crate) fn new(
source: &'a SetDisks,
store: &'a ECStore,
lock_lost_signal: Option<std::sync::Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
) -> Self {
Self {
source,
store,
lock_lost_signal,
}
pub(crate) fn new(source: &'a SetDisks, store: &'a ECStore) -> Self {
Self { source, store }
}
}
@@ -139,11 +130,7 @@ impl MigrationBackend for RebalanceMigrationBackend<'_> {
fi: &FileInfo,
opts: &ObjectOptions,
) -> Result<()> {
let mut opts = opts.clone();
if let Some(signal) = self.lock_lost_signal.as_ref() {
opts.add_namespace_lock_lost_signal(std::sync::Arc::clone(signal));
}
self.store.decommission_tiered_object(bucket, object, fi, &opts).await
self.store.decommission_tiered_object(bucket, object, fi, opts).await
}
}
@@ -12,8 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#[cfg(test)]
use crate::disk::DiskAPI;
use crate::error::{Error, Result};
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use tokio::time::Duration;
@@ -47,8 +45,6 @@ mod runtime;
mod types;
mod worker;
#[cfg(feature = "test-util")]
pub use entry::test_util::PausedRebalanceEntryTestFixture;
pub(crate) use meta::is_rebalance_conflicting_with_decommission;
pub use meta::{decode_rebalance_stop_propagation_record, encode_rebalance_stop_propagation_record};
pub use types::{
@@ -57,130 +53,5 @@ pub use types::{
};
use types::{RebalanceBucketConfigs, RebalanceBucketOutcome, RebalanceEntryOutcome};
#[cfg(any(test, feature = "test-util"))]
pub async fn test_store_with_persisted_rebalance_meta(
meta: RebalanceMeta,
) -> (Vec<tempfile::TempDir>, std::sync::Arc<crate::store::ECStore>) {
let ctx = std::sync::Arc::new(crate::runtime::instance::InstanceContext::new());
let (temp_dirs, pool) = crate::core::sets::make_local_two_set_sets_with_ctx(ctx.clone()).await;
meta.save(pool.clone())
.await
.expect("rebalance test metadata should be persisted");
let endpoint_pools: crate::layout::endpoints::EndpointServerPools = vec![pool.endpoints.clone()].into();
let store = std::sync::Arc::new(crate::store::ECStore {
id: uuid::Uuid::new_v4(),
disk_map: std::collections::HashMap::new(),
pools: vec![pool],
peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoint_pools, ctx.clone()),
pool_meta: tokio::sync::RwLock::new(crate::core::pools::PoolMeta::default()),
rebalance_meta: tokio::sync::RwLock::new(Some(meta)),
decommission_cancelers: tokio::sync::RwLock::new(vec![None]),
start_gate: tokio::sync::Mutex::new(()),
pool_meta_save_gate: tokio::sync::Mutex::default(),
ctx,
bucket_fence_registry: std::sync::Arc::default(),
});
(temp_dirs, store)
}
#[cfg(test)]
pub(crate) async fn test_two_pool_stores(
rebalance_meta: Option<RebalanceMeta>,
) -> (
Vec<tempfile::TempDir>,
std::sync::Arc<crate::store::ECStore>,
std::sync::Arc<crate::store::ECStore>,
) {
test_two_pool_stores_with_contexts(rebalance_meta, false).await
}
#[cfg(test)]
pub(crate) async fn test_two_pool_stores_with_isolated_node_contexts(
rebalance_meta: Option<RebalanceMeta>,
) -> (
Vec<tempfile::TempDir>,
std::sync::Arc<crate::store::ECStore>,
std::sync::Arc<crate::store::ECStore>,
) {
test_two_pool_stores_with_contexts(rebalance_meta, true).await
}
#[cfg(test)]
async fn test_two_pool_stores_with_contexts(
rebalance_meta: Option<RebalanceMeta>,
isolate_node_contexts: bool,
) -> (
Vec<tempfile::TempDir>,
std::sync::Arc<crate::store::ECStore>,
std::sync::Arc<crate::store::ECStore>,
) {
crate::services::notification_sys::install_cross_pool_fence_fleet_proof_for_test();
use crate::core::pools::PoolMeta;
use crate::layout::endpoints::{EndpointServerPools, SetupType};
let ctx = std::sync::Arc::new(crate::runtime::instance::InstanceContext::new());
ctx.update_erasure_type(SetupType::DistErasure).await;
let (mut temp_dirs, first_pool) =
crate::core::sets::make_local_two_set_sets_for_pool_with_ctx(std::sync::Arc::clone(&ctx), 0).await;
let (second_temp_dirs, second_pool) =
crate::core::sets::make_local_two_set_sets_for_pool_with_ctx(std::sync::Arc::clone(&ctx), 1).await;
temp_dirs.extend(second_temp_dirs);
let pools = vec![first_pool, second_pool];
{
let local_disk_map = ctx.local_disk_map();
let mut local_disk_map = local_disk_map.write().await;
for pool in &pools {
for set in &pool.disk_set {
for disk in set.disks.read().await.iter().flatten() {
local_disk_map.insert(disk.endpoint().to_string(), Some(disk.clone()));
}
}
}
}
let pool_meta = PoolMeta::new(&pools, &PoolMeta::default());
pool_meta
.save(pools.clone())
.await
.expect("baseline pool metadata should be persisted");
if let Some(meta) = rebalance_meta.as_ref() {
meta.save(pools[0].clone())
.await
.expect("active rebalance metadata should be persisted");
}
let endpoint_pools: EndpointServerPools = pools.iter().map(|pool| pool.endpoints.clone()).collect::<Vec<_>>().into();
ctx.set_endpoints(endpoint_pools.clone());
let other_ctx = if isolate_node_contexts {
let other_ctx = std::sync::Arc::new(crate::runtime::instance::InstanceContext::new());
other_ctx.update_erasure_type(SetupType::DistErasure).await;
*other_ctx.local_disk_map().write().await = ctx.local_disk_map().read().await.clone();
other_ctx.set_endpoints(endpoint_pools.clone());
other_ctx
} else {
std::sync::Arc::clone(&ctx)
};
let make_store = |store_ctx: std::sync::Arc<crate::runtime::instance::InstanceContext>| {
std::sync::Arc::new(crate::store::ECStore {
id: uuid::Uuid::new_v4(),
disk_map: std::collections::HashMap::new(),
pools: pools.clone(),
peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoint_pools, std::sync::Arc::clone(&store_ctx)),
pool_meta: tokio::sync::RwLock::new(pool_meta.clone()),
rebalance_meta: tokio::sync::RwLock::new(rebalance_meta.clone()),
decommission_cancelers: tokio::sync::RwLock::new(vec![None, None]),
start_gate: tokio::sync::Mutex::new(()),
pool_meta_save_gate: tokio::sync::Mutex::default(),
ctx: store_ctx,
bucket_fence_registry: std::sync::Arc::default(),
})
};
let store = make_store(ctx);
let other_store = make_store(other_ctx);
if isolate_node_contexts {
crate::bucket::metadata_sys::init_bucket_metadata_sys(std::sync::Arc::clone(&store), Vec::new()).await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(std::sync::Arc::clone(&other_store), Vec::new()).await;
}
(temp_dirs, store, other_store)
}
#[cfg(test)]
mod rebalance_unit_tests;

Some files were not shown because too many files have changed in this diff Show More