Compare commits

..

8 Commits

Author SHA1 Message Date
overtrue a8cecf6462 test(e2e): register verified Darwin test membership 2026-09-05 20:18:52 +08:00
overtrue 980f3abbd3 chore: merge main PR evidence guidance 2026-09-05 19:00:05 +08:00
overtrue e36650827b chore: integrate shared quick checks for E2E validation 2026-09-05 18:59:07 +08:00
overtrue 09c8e10d5e feat(test): verify the E2E server build and source identity 2026-09-05 18:58:01 +08:00
overtrue 0ff03c596c chore: merge main after ECStore compile repair 2026-09-05 18:46:20 +08:00
overtrue a74919db8e fix(ci): reject dependencies on required quick checks 2026-09-05 18:17:13 +08:00
overtrue e77c6f0ca5 fix(ci): install actionlint from its verified release 2026-09-05 17:42:14 +08:00
overtrue 3149411943 fix(ci): share quick checks and lint workflows 2026-09-05 17:37:20 +08:00
126 changed files with 1394 additions and 13161 deletions
-2
View File
@@ -1,2 +0,0 @@
sha256-linux=9785867929047dfd8c6f768e0d2b1e0a8fdba85216f4a4139093b1619d03ff07
sha256-darwin=9785867929047dfd8c6f768e0d2b1e0a8fdba85216f4a4139093b1619d03ff07
+1 -1
View File
@@ -1,2 +1,2 @@
sha256-darwin=a881fd7d3f5cb94654221ca85b8b30cce1b95e608824a55a15339cbc294e6d34
sha256-darwin=85ee9b9e66e916dbf51857298637b6bcd1c23c2b066aaa4c171ce961c2d002b5
sha256-linux=a2933d83dfe74ffa03410a0959333a1c48288b8469ca9f17273d449d7510c24b
+2 -3
View File
@@ -3,10 +3,9 @@
.NOTPARALLEL: pre-commit pre-pr dev-check
.PHONY: setup-hooks
setup-hooks: ## Install the configured pre-commit hooks
setup-hooks: ## Set up git hooks
@echo "🔧 Setting up git hooks..."
pre-commit validate-config
pre-commit install
chmod +x .git/hooks/pre-commit
@echo "✅ Git hooks setup complete!"
.PHONY: doc-paths-check
+1
View File
@@ -38,6 +38,7 @@ script-tests: ## Run shell script tests
./scripts/test_python_bin.sh
./scripts/check_embedded_secrets.sh --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/test_e2e_binary.py
$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/test_security_workflow.py
-33
View File
@@ -183,13 +183,6 @@ test-group = 'e2e-reliability'
filter = 'package(e2e_test) & test(/^inline_fast_path_cluster_test::/)'
test-group = 'e2e-inline-boundaries'
# 4-node 4-drive distributed Actions suite: each case starts four rustfs
# processes and up to sixteen data directories. Serialize across nextest's
# process boundary so several 4x4 clusters never overlap.
[[profile.default.overrides]]
filter = 'package(e2e_test) & test(/^distributed::/)'
test-group = 'e2e-cluster-nightly'
# Vault KMS tests share the fixed dev-server port 8200. serial_test's #[serial]
# does not cross nextest process boundaries, so keep every Vault-backed test in
# one group.
@@ -533,27 +526,6 @@ path = "junit.xml"
filter = 'package(e2e_test)'
test-group = 'e2e-cluster-nightly'
# ---------------------------------------------------------------------------
# e2e-distributed profile — 4-node 4-disk Actions suite
# ---------------------------------------------------------------------------
# Storage-sensitive PR / nightly / dispatch lane owned by
# .github/workflows/e2e-distributed.yml.
# Each case starts four rustfs processes (and for site replication, two
# clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY.
# Serialized via e2e-cluster-nightly with no retries.
[profile.e2e-distributed]
default-filter = 'package(e2e_test) & test(/^distributed::/)'
fail-fast = false
# Decommission / rebalance cases poll for up to 180s with little stdout.
slow-timeout = { period = "120s", terminate-after = 6 }
[profile.e2e-distributed.junit]
path = "junit.xml"
[[profile.e2e-distributed.overrides]]
filter = 'package(e2e_test)'
test-group = 'e2e-cluster-nightly'
# ---------------------------------------------------------------------------
# e2e-odm-interop profile — on-demand migration provider interop lane (ODM-20)
# ---------------------------------------------------------------------------
@@ -614,10 +586,6 @@ path = "junit.xml"
# cluster-fault lane. heal_erasure_disk_rebuild is intentionally not
# excluded here because backlog#2213 promotes core heal rebuild coverage to
# this merge/main lane while retaining nightly coverage.
# * distributed:: — 4-node 4-disk Actions suite (S3, lock, versioning,
# replication, quota, observability, expand/decommission/rebalance, site
# replication, chaos, upgrade history/IAM). Owns [profile.e2e-distributed] and
# .github/workflows/e2e-distributed.yml.
# * on_demand_migration::interop_test — the ODM-20 provider interoperability
# cases, which are meaningless without a source: they run in the dedicated
# [profile.e2e-odm-interop] lane below, where the workflow points them at a
@@ -639,7 +607,6 @@ default-filter = """
package(e2e_test)
& !test(/^protocols::/)
& !test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|degraded_listing_availability_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
& !test(/^distributed::/)
& !test(/^replication_extension_test::/)
& !test(/^replication_target_matrix_test::/)
& !test(/^on_demand_migration::(concurrency_test|fault_test|interop_test|real_source_test)::/)
+1
View File
@@ -98,6 +98,7 @@ runs:
shell: bash
run: |
python3 ./scripts/check_test_wiring.py --self-test
python3 ./scripts/test_e2e_binary.py
python3 ./scripts/check_scheduled_validation_freshness.py --self-test
python3 ./scripts/test_security_workflow.py
python3 ./scripts/check_test_wiring.py
-5
View File
@@ -4,11 +4,6 @@
{ "workflow": ".github/workflows/ci.yml", "max_age_hours": 192 },
{ "workflow": ".github/workflows/coverage.yml", "max_age_hours": 192 },
{ "workflow": ".github/workflows/e2e-replication-nightly.yml", "max_age_hours": 36 },
{
"workflow": ".github/workflows/e2e-distributed.yml",
"max_age_hours": 36,
"never_ran_grace_until": "2026-09-18T00:00:00Z"
},
{ "workflow": ".github/workflows/e2e-s3tests.yml", "max_age_hours": 192 },
{ "workflow": ".github/workflows/fuzz.yml", "max_age_hours": 36 },
{ "workflow": ".github/workflows/mint.yml", "max_age_hours": 192 },
+14 -10
View File
@@ -582,13 +582,15 @@ jobs:
install-build-packaging-tools: 'false'
- name: Build debug binary
run: cargo build -p rustfs --bins --features e2e-test-hooks
run: python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks
- name: Upload debug binary
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: rustfs-debug-binary
path: target/debug/rustfs
path: |
target/debug/rustfs
target/debug/rustfs.e2e.json
if-no-files-found: error
retention-days: 1
@@ -620,13 +622,15 @@ jobs:
install-build-packaging-tools: 'false'
- name: Build debug binary with rio-v2
run: cargo build -p rustfs --bins --features rio-v2,e2e-test-hooks
run: python3 scripts/e2e_binary.py build --bins --features rio-v2,e2e-test-hooks
- name: Upload debug binary
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: rustfs-debug-binary-rio-v2
path: target/debug/rustfs
path: |
target/debug/rustfs
target/debug/rustfs.e2e.json
if-no-files-found: error
retention-days: 1
@@ -775,7 +779,7 @@ jobs:
NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-smoke-logs
run: |
cargo nextest run --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" \
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" \
--status-level all --final-status-level all --failure-output final
- name: Upload e2e smoke diagnostics
@@ -811,7 +815,7 @@ jobs:
RUSTFS_TEST_PORT="$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()')"
RUSTFS_TEST_PORT="${RUSTFS_TEST_PORT}" \
RUSTFS_TEST_LOG="${RUN_ROOT}/rustfs.log" \
./scripts/e2e-run.sh ./target/debug/rustfs "${RUN_ROOT}/data"
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- ./scripts/e2e-run.sh ./target/debug/rustfs "${RUN_ROOT}/data"
- name: Upload test logs
if: failure()
@@ -913,7 +917,7 @@ jobs:
# extend that filter, never add ad-hoc e2e jobs here. Reuses the downloaded
# debug binary; each test spawns its own rustfs server on a random port.
- name: Run e2e full suite
run: cargo nextest run --profile e2e-full -p e2e_test
run: python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-full -p e2e_test
- name: Upload junit
if: always()
@@ -974,7 +978,7 @@ jobs:
- name: Run end-to-end tests
run: |
s3s-e2e --version
./scripts/e2e-run.sh ./target/debug/rustfs /tmp/rustfs
python3 scripts/e2e_binary.py run --features rio-v2,e2e-test-hooks -- ./scripts/e2e-run.sh ./target/debug/rustfs /tmp/rustfs
- name: Upload test logs
if: failure()
@@ -1017,7 +1021,7 @@ jobs:
S3_PORT="${S3_PORT}" \
DATA_ROOT="${RUN_ROOT}" \
S3TESTS_CONF=artifacts/s3tests-single/s3tests.conf \
./scripts/s3-tests/run.sh
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- ./scripts/s3-tests/run.sh
- name: Upload s3 test artifacts
if: always()
@@ -1099,7 +1103,7 @@ jobs:
S3_PORT="${S3_PORT}" \
DATA_ROOT="${RUN_ROOT}" \
S3TESTS_CONF=artifacts/s3tests-single/s3tests.conf \
./scripts/s3-tests/run.sh
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- ./scripts/s3-tests/run.sh
- name: Upload s3 test artifacts
if: always()
-206
View File
@@ -1,206 +0,0 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# 4-node 4-disk distributed e2e lane.
#
# Each selected test starts a real localhost cluster via
# `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
# case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
# nightly runs, and manual dispatches all execute the same fail-closed suite.
# Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
#
# Isolated pool filesystems: expand/decommission/rebalance cases require
# independent `statfs` capacity. `sm-standard-4` is an ARC pod
# (`scripts/ci/check_runner_ephemerality.sh`) and usually has no
# `/dev/loop-control`, so `mount -o loop` fails with ENOENT ("mount failed:
# No such file or directory"). The prepare step therefore mounts four 1 GiB
# tmpfs instances and exports them as `RUSTFS_E2E_POOL_ROOTS`.
name: e2e-distributed
on:
pull_request:
paths:
- "Cargo.lock"
- "Cargo.toml"
- ".config/nextest.toml"
- ".github/workflows/e2e-distributed.yml"
- "crates/audit/**"
- "crates/common/**"
- "crates/config/**"
- "crates/e2e_test/**"
- "crates/ecstore/**"
- "crates/filemeta/**"
- "crates/heal/**"
- "crates/iam/**"
- "crates/lock/**"
- "crates/madmin/**"
- "crates/notify/**"
- "crates/replication/**"
- "crates/s3-client/**"
- "crates/s3-ops/**"
- "crates/s3-types/**"
- "crates/scanner/**"
- "crates/storage-api/**"
- "crates/utils/**"
- "rustfs/**"
workflow_dispatch:
inputs:
filter:
description: "Optional nextest -E filter (default: the whole e2e-distributed profile)"
required: false
default: ""
schedule:
# 05:53 UTC nightly — clear of e2e-nightly (04:29) and ODM interop (05:23).
- cron: "53 5 * * *"
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ github.event_name != 'schedule' }}
jobs:
distributed:
name: Distributed 4-node 4-disk e2e
runs-on: sm-standard-4
timeout-minutes: 180
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
NO_PROXY: 127.0.0.1,localhost
HTTP_PROXY: ""
HTTPS_PROXY: ""
# Pinned previous release used by distributed::upgrade_test (same pin as e2e-upgrade.yml).
UPGRADE_SOURCE_VERSION: 1.0.0-rc.2
UPGRADE_SOURCE_ASSET: rustfs-linux-x86_64-gnu-v1.0.0-rc.2.zip
UPGRADE_SOURCE_SHA256: 7c789386bf85278f865b8e0d359bf4edb84d5aa408cc3fa54a18c25ca74cd6e7
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Setup Rust environment
uses: ./.github/actions/setup
with:
rust-version: stable
cache-shared-key: ci-e2e-distributed
cache-save-if: ${{ github.ref == 'refs/heads/main' }}
install-build-packaging-tools: 'false'
- name: Prepare isolated filesystems for pool movement
run: |
set -euo pipefail
mount_base="${RUNNER_TEMP}/rustfs-e2e-pools"
mkdir -p "${mount_base}"
roots=()
for pool in 0 1 2 3; do
mountpoint="${mount_base}/pool-${pool}"
mkdir -p "${mountpoint}"
# sm-standard-4 is an ARC pod without usable loop devices, so
# `mount -o loop` fails with ENOENT. Sized tmpfs still reports a
# distinct st_dev and independent 1G statfs capacity.
sudo mount -t tmpfs -o size=1G,nosuid,nodev,mode=1777 tmpfs "${mountpoint}"
sudo chmod 1777 "${mountpoint}"
roots+=("${mountpoint}")
done
printf -v joined_roots '%s:' "${roots[@]}"
echo "RUSTFS_E2E_POOL_ROOTS=${joined_roots%:}" >> "${GITHUB_ENV}"
findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[0]}"
findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[1]}"
findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[2]}"
findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[3]}"
- name: Download pinned previous release
env:
SOURCE_DIR: ${{ runner.temp }}/rustfs-upgrade-source
run: |
set -euo pipefail
mkdir -p "$SOURCE_DIR"
archive="$SOURCE_DIR/$UPGRADE_SOURCE_ASSET"
curl --fail --location --retry 3 --output "$archive" \
"https://github.com/${GITHUB_REPOSITORY}/releases/download/${UPGRADE_SOURCE_VERSION}/${UPGRADE_SOURCE_ASSET}"
echo "$UPGRADE_SOURCE_SHA256 $archive" | sha256sum --check --strict
unzip -q "$archive" -d "$SOURCE_DIR"
chmod +x "$SOURCE_DIR/rustfs"
test -x "$SOURCE_DIR/rustfs"
echo "RUSTFS_UPGRADE_SOURCE_BINARY=$SOURCE_DIR/rustfs" >> "$GITHUB_ENV"
- name: Build rustfs binary
run: |
cargo build -p rustfs --bins
: > target/debug/rustfs.features
- name: Verify distributed e2e membership
env:
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-distributed-list.json
run: |
cargo nextest list --profile e2e-distributed -p e2e_test --message-format json > "${NEXTEST_LISTING}"
python3 ./scripts/check_test_wiring.py --check-profile e2e-distributed "${NEXTEST_LISTING}"
- name: Run distributed 4-node e2e suite
env:
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-distributed-logs
FILTER: ${{ inputs.filter }}
run: |
set -euo pipefail
if [ -n "${FILTER}" ]; then
cargo nextest run --profile e2e-distributed -p e2e_test -E "${FILTER}"
else
cargo nextest run --profile e2e-distributed -p e2e_test --no-tests=fail
fi
- name: Upload distributed e2e diagnostics
if: always()
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: e2e-distributed-${{ github.run_number }}
path: |
target/nextest/e2e-distributed/junit.xml
${{ runner.temp }}/rustfs-e2e-distributed-list.json
${{ runner.temp }}/rustfs-e2e-distributed-logs/
retention-days: 7
if-no-files-found: warn
- name: Unmount isolated pool filesystems
if: always()
run: |
set -euo pipefail
mount_base="${RUNNER_TEMP}/rustfs-e2e-pools"
for pool in 0 1 2 3; do
mountpoint="${mount_base}/pool-${pool}"
if mountpoint --quiet "${mountpoint}"; then
sudo umount "${mountpoint}"
fi
done
alert-on-failure:
name: Alert on scheduled failure
needs: [distributed]
if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
issues: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Open or update failure-tracking issue
uses: ./.github/actions/schedule-failure-issue
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
+9 -11
View File
@@ -89,14 +89,10 @@ jobs:
- name: Verify awscurl
run: test -x "$AWSCURL_PATH"
# Build the rustfs binary once up front. The e2e tests spawn it as a
# child process (crates/e2e_test/src/common.rs) and will build it on
# demand otherwise, but a single explicit build avoids several parallel
# nextest test processes racing to build it at once.
# Build once and carry its source/binary identity into the test invocation.
- name: Build rustfs binary
run: |
cargo build -p rustfs --bins
: > target/debug/rustfs.features
python3 scripts/e2e_binary.py build --bins
- name: Verify replication e2e membership
env:
@@ -108,7 +104,7 @@ jobs:
- name: Run replication e2e nightly suite
env:
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-repl-nightly-logs
run: cargo nextest run --profile e2e-repl-nightly -p e2e_test
run: python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-repl-nightly -p e2e_test
- name: Upload nextest junit report
if: always()
@@ -144,8 +140,7 @@ jobs:
- name: Build rustfs binary
run: |
cargo build -p rustfs --bins --features e2e-test-hooks
: > target/debug/rustfs.features
python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks
- name: Verify cluster fault e2e membership
env:
@@ -157,7 +152,7 @@ jobs:
- name: Run cluster fault e2e nightly suite
env:
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-nightly-logs
run: cargo nextest run --profile e2e-nightly -p e2e_test
run: python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-nightly -p e2e_test
- name: Upload cluster fault diagnostics
if: always()
@@ -198,6 +193,9 @@ jobs:
sudo apt-get install -y -qq iproute2
ss -tn state CLOSE-WAIT >/dev/null
- name: Build protocol server
run: python3 scripts/e2e_binary.py build --features "$RUSTFS_BUILD_FEATURES"
# The suite owns fixed protocol ports and serializes its internal cases.
- name: Verify protocol e2e membership
env:
@@ -210,7 +208,7 @@ jobs:
env:
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-protocol-e2e-logs
run: >-
cargo nextest run -j 1 --profile e2e-protocols -p e2e_test --no-capture
python3 scripts/e2e_binary.py run --features "$RUSTFS_BUILD_FEATURES" -- cargo nextest run -j 1 --profile e2e-protocols -p e2e_test --no-capture
- name: Upload protocol diagnostics
if: always()
+2 -3
View File
@@ -98,12 +98,11 @@ jobs:
- name: Build current RustFS binary
run: |
cargo build --locked -p rustfs --bin rustfs
: > target/debug/rustfs.features
python3 scripts/e2e_binary.py build
- name: Run upgrade compatibility test
run: |
cargo test --locked -p e2e_test \
python3 scripts/e2e_binary.py run -- cargo test --locked -p e2e_test \
"upgrade_compatibility_test::${{ matrix.test }}" \
-- --ignored --exact --nocapture
@@ -132,7 +132,7 @@ jobs:
s3api create-bucket --bucket "${RUSTFS_ODM_INTEROP_BUCKET}"
- name: Build the RustFS binary under test
run: cargo build --locked -p rustfs --bins
run: python3 scripts/e2e_binary.py build --bins
# The lane selects tests by module, so a rename would quietly shrink it.
# The committed digest in .config/e2e-odm-interop-selection.txt fails
@@ -143,7 +143,7 @@ jobs:
python3 ./scripts/check_test_wiring.py --check-profile e2e-odm-interop "${NEXTEST_LISTING}"
- name: Run the interop cases against MinIO
run: cargo nextest run --profile e2e-odm-interop -p e2e_test --no-tests=fail
run: python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-odm-interop -p e2e_test --no-tests=fail
- name: Build the MinIO interop report
if: always()
@@ -251,7 +251,7 @@ jobs:
- name: Build the RustFS binary under test
if: steps.credentials.outputs.present == 'true'
run: cargo build --locked -p rustfs --bins
run: python3 scripts/e2e_binary.py build --bins
# A filterset that matches nothing is valid, so the count is asserted
# rather than inferred from a green run.
@@ -272,7 +272,7 @@ jobs:
- name: Run the three-case minimum
if: steps.credentials.outputs.present == 'true'
run: |
cargo nextest run --profile e2e-odm-interop -p e2e_test \
python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-odm-interop -p e2e_test \
-E "${CLOUD_CASE_FILTER}" --no-tests=fail
- name: Build the ${{ matrix.provider }} interop report
+3
View File
@@ -54,6 +54,9 @@ env:
jobs:
heal-test:
runs-on: smoke-testing
# Requirement: a failing suite must not fail the workflow; failures
# are filed to rustfs/backlog and the chain continues.
continue-on-error: true
timeout-minutes: 480
# Standalone manual run, or one link of the nightly functional chain
# (storage -> heal -> pool). Pool expansion no longer re-runs heal.
+2
View File
@@ -49,6 +49,7 @@ env:
jobs:
kms-test:
runs-on: smoke-testing
continue-on-error: true
timeout-minutes: 420
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps:
@@ -108,6 +109,7 @@ jobs:
- name: Run KMS suite
id: test
continue-on-error: true
env:
LOG_FILE: /tmp/rustfs-kms.log
run: |
@@ -84,6 +84,9 @@ env:
jobs:
performance-test:
runs-on: pf-testing
# Requirement: a failing benchmark must not fail the workflow;
# failures are filed to rustfs/backlog.
continue-on-error: true
timeout-minutes: 900
# Run on manual dispatch, or when the nightly build completed successfully.
# Skipped when nightly failed.
@@ -76,6 +76,9 @@ jobs:
pool-expansion-test:
name: Pool expansion / decommission test
runs-on: smoke-testing
# Requirement: a failing suite must not fail the workflow; failures
# are filed to rustfs/backlog and the chain continues.
continue-on-error: true
timeout-minutes: 360
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
env:
@@ -62,6 +62,9 @@ env:
jobs:
replication-test:
runs-on: smoke-testing
# A failed replication run must not break the chain or the workflow: the
# failure is reported to rustfs/backlog instead (see the issue step).
continue-on-error: true
timeout-minutes: 360
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps:
@@ -113,6 +116,7 @@ jobs:
- name: Run replication suite
id: test
continue-on-error: true
env:
LOG_FILE: /tmp/rustfs-replication.log
run: |
@@ -37,6 +37,7 @@ env:
jobs:
s3-compat-test:
runs-on: smoke-testing
continue-on-error: true
timeout-minutes: 360
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps:
@@ -87,6 +88,7 @@ jobs:
- name: Run S3 compatibility suite
id: test
continue-on-error: true
env:
LOG_FILE: /tmp/rustfs-s3-compat.log
run: |
@@ -46,6 +46,7 @@ env:
jobs:
storage-test:
runs-on: smoke-testing
continue-on-error: true
timeout-minutes: 360
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps:
@@ -96,6 +97,7 @@ jobs:
- name: Run storage engine suite
id: test
continue-on-error: true
env:
LOG_FILE: /tmp/rustfs-storage.log
run: |
+3
View File
@@ -61,6 +61,9 @@ env:
jobs:
tier-test:
runs-on: smoke-testing
# Requirement: a failing suite must not fail the workflow; failures
# are filed to rustfs/backlog and the chain continues.
continue-on-error: true
timeout-minutes: 420
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps:
@@ -79,6 +79,7 @@ env:
jobs:
upgrade-test:
runs-on: smoke-testing
continue-on-error: true
timeout-minutes: 420
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps:
@@ -141,6 +142,7 @@ jobs:
- name: Run upgrade compatibility suite
id: test
continue-on-error: true
env:
LOG_FILE: /tmp/rustfs-upgrade.log
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
@@ -22,7 +22,6 @@ on:
- "Continuous Integration"
- "coverage"
- "e2e-nightly"
- "e2e-distributed"
- "e2e-s3tests"
- "Fuzz"
- "mint"
+3 -3
View File
@@ -3,9 +3,9 @@
repos:
- repo: local
hooks:
- id: rustfs-fmt-check
name: Rust formatting
entry: cargo fmt --all --check
- id: rustfs-dev-check
name: rustfs dev-check
entry: make dev-check
language: system
types: [rust]
pass_filenames: false
+37 -11
View File
@@ -109,17 +109,24 @@ affected boundaries and risks. CI still runs its configured repository gates.
### 🔒 Git Pre-commit Hooks (optional)
The optional hook uses the checked-in `.pre-commit-config.yaml`. Install [pre-commit](https://pre-commit.com/#installation), then run this from the checkout or a linked worktree:
Git hooks are **not** versioned in this repository, so a fresh clone has no
active pre-commit hook. If you add your own `.git/hooks/pre-commit` (a good
choice is a one-liner that runs `make pre-commit`), you can mark it executable
with:
```bash
make setup-hooks
```
The hook runs `cargo fmt --all --check` when staged files include Rust source. It does not compile the workspace or run tests. Fix formatting with `cargo fmt --all`, inspect and stage the result, then commit again.
Or manually:
`pre-commit install` resolves Git's hook directory for linked worktrees and preserves an existing hook in migration mode. If you use `core.hooksPath`, keep that hook manager and integrate `pre-commit run` there; the installer refuses to silently replace that configuration.
```bash
chmod +x .git/hooks/pre-commit
```
A local hook provides early formatting feedback. With or without it, follow the verification tiers in `AGENTS.md`, run relevant behavioral tests, and satisfy the CI merge gates. `make pre-commit` and `make dev-check` remain explicit broader commands.
With or without a hook, follow the verification tiers in `AGENTS.md`. Run the
applicable scoped checks, and reserve `make pre-pr` for broad cross-module
changes whose impact cannot be bounded by those checks.
### 📝 Formatting Configuration
@@ -131,11 +138,31 @@ fn_call_width = 90
single_line_let_else_max_width = 100
```
### 🚫 Commit Prevention
If you set up a pre-commit hook and your code doesn't meet the formatting requirements, the hook will:
1. **Block the commit** and show clear error messages
2. **Provide exact commands** to fix the issues
3. **Guide you through** the resolution process
Example output when formatting fails:
```
❌ Code formatting check failed!
💡 Please run 'cargo fmt --all' to format your code before committing.
🔧 Quick fix:
cargo fmt --all
git add .
git commit
```
### 🔄 Development Workflow
1. **Make your changes**
2. **Format your code**: `make fmt` or `cargo fmt --all`
3. **Select relevant checks** using the validation tier in `AGENTS.md`; use `make pre-commit` when its broader fast gate adds useful coverage
3. **Run the fast gate**: `make pre-commit` (no clippy, no tests)
4. **Commit your changes**: `git commit -m "your message"`
5. **Complete the applicable multi-role adversarial review** for non-exempt changes (see `AGENTS.md`)
6. **Run applicable scoped checks before opening/updating a PR**; consider
@@ -179,12 +206,11 @@ Configure your IDE to:
#### Pre-commit hook not running?
```bash
pre-commit validate-config
pre-commit run --all-files
# Inspect any configured hook manager; do not overwrite it.
git config --get core.hooksPath
# Install if no separate hook manager is configured.
make setup-hooks
# Check if hook is executable
ls -la .git/hooks/pre-commit
# Make it executable if needed
chmod +x .git/hooks/pre-commit
```
#### Formatting issues?
Generated
+13 -32
View File
@@ -315,7 +315,7 @@ dependencies = [
"strum",
"thiserror 2.0.20",
"uuid",
"zstd 0.13.3",
"zstd",
]
[[package]]
@@ -508,7 +508,7 @@ dependencies = [
"arrow-select",
"flatbuffers",
"lz4_flex",
"zstd 0.13.3",
"zstd",
]
[[package]]
@@ -2249,8 +2249,8 @@ dependencies = [
"liblzma",
"lz4",
"memchr",
"zstd 0.13.3",
"zstd-safe 7.3.0",
"zstd",
"zstd-safe",
]
[[package]]
@@ -4067,7 +4067,7 @@ dependencies = [
"uuid",
"walkdir",
"zip",
"zstd 0.14.0",
"zstd",
]
[[package]]
@@ -5971,7 +5971,7 @@ dependencies = [
"lz4",
"snap",
"uuid",
"zstd 0.13.3",
"zstd",
]
[[package]]
@@ -7658,7 +7658,7 @@ dependencies = [
"snap",
"tokio",
"twox-hash",
"zstd 0.13.3",
"zstd",
]
[[package]]
@@ -9618,7 +9618,7 @@ dependencies = [
"x509-parser",
"zeroize",
"zip",
"zstd 0.14.0",
"zstd",
]
[[package]]
@@ -9792,7 +9792,6 @@ dependencies = [
"path-absolutize",
"pin-project-lite",
"proptest",
"quick-xml",
"rand 0.10.2",
"ratelimit",
"rcgen",
@@ -10229,7 +10228,7 @@ dependencies = [
"thiserror 2.0.20",
"walkdir",
"zip",
"zstd 0.14.0",
"zstd",
]
[[package]]
@@ -10396,7 +10395,7 @@ dependencies = [
"tracing-opentelemetry",
"tracing-subscriber",
"url",
"zstd 0.14.0",
"zstd",
]
[[package]]
@@ -10986,7 +10985,7 @@ dependencies = [
"transform-stream",
"url",
"windows",
"zstd 0.14.0",
"zstd",
]
[[package]]
@@ -14096,7 +14095,7 @@ dependencies = [
"typed-path",
"zeroize",
"zopfli",
"zstd 0.13.3",
"zstd",
]
[[package]]
@@ -14129,16 +14128,7 @@ version = "0.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
dependencies = [
"zstd-safe 7.3.0",
]
[[package]]
name = "zstd"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf06bd8162af0734b344780deb55b42a2429ae430870d13fcc12f238e880fe6e"
dependencies = [
"zstd-safe 8.0.0",
"zstd-safe",
]
[[package]]
@@ -14150,15 +14140,6 @@ dependencies = [
"zstd-sys",
]
[[package]]
name = "zstd-safe"
version = "8.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae42c0555055784c70058d19ba8e275528e8a99a706684868ace5da4e716a4ab"
dependencies = [
"zstd-sys",
]
[[package]]
name = "zstd-sys"
version = "2.1.0+zstd.1.5.7"
+5 -5
View File
@@ -199,10 +199,10 @@ serde_urlencoded = "0.7.1"
# matching stable releases are not available yet, while previous stable lines
# have incompatible APIs. Keep them exact-pinned and monitor upstream for stable
# releases.
aes-gcm = { version = "0.11.1" }
argon2 = { version = "0.6.0" }
blake2 = "0.11.0"
chacha20poly1305 = { version = "0.11.0" }
aes-gcm = { version = "=0.11.1" }
argon2 = { version = "=0.6.0" }
blake2 = "=0.11.0"
chacha20poly1305 = { version = "=0.11.0" }
crc-fast = "1.10.0"
hmac = { version = "0.13.0" }
jsonwebtoken = { version = "11.0.0" }
@@ -343,7 +343,7 @@ windows = { version = "0.62.2" }
windows-sys = "0.61.2"
xxhash-rust = { version = "0.8.18" }
zip = "8.6.0"
zstd = "0.14.0"
zstd = "0.13.3"
# Observability and Metrics
metrics = "0.24.6"
-15
View File
@@ -130,21 +130,6 @@ Scanner cycle budget controls:
- timeout returns S3 `SlowDown`, so clients should use normal SDK retry handling.
- this is not a fdatasync or group-commit switch. Track fdatasync batching separately with `rustfs_s3_put_object_rename_fdatasync_batch_files`.
## Remote tier timeout environment variables
- `RUSTFS_TIER_REMOTE_CONNECT_TIMEOUT_SECS`
- remote tier TCP connect timeout.
- default is `10`.
- must be positive; zero fails tier client initialization, while an invalid integer is logged and falls back to the default.
- `RUSTFS_TIER_REMOTE_REQUEST_TIMEOUT_SECS`
- remote tier request timeout through response headers.
- default is `86400` so large transition uploads keep a production-safe budget.
- must be positive; zero fails tier client initialization, while an invalid integer is logged and falls back to the default. Very large values are accepted and act as a correspondingly long budget.
- `RUSTFS_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS`
- maximum idle time between remote tier response-body chunks.
- default is `60`; the timer resets only when non-empty body data keeps progressing.
- must be positive; zero fails tier client initialization, while an invalid integer is logged and falls back to the default.
## Drive timeout environment variables
- `RUSTFS_DRIVE_METADATA_TIMEOUT_SECS`
-32
View File
@@ -137,28 +137,6 @@ pub const DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED: bool = false;
const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE);
const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED);
/// Environment variable for remote tier TCP connect timeout in seconds.
pub const ENV_TIER_REMOTE_CONNECT_TIMEOUT_SECS: &str = "RUSTFS_TIER_REMOTE_CONNECT_TIMEOUT_SECS";
/// Default remote tier TCP connect timeout in seconds.
pub const DEFAULT_TIER_REMOTE_CONNECT_TIMEOUT_SECS: u64 = 10;
/// Environment variable for the remote tier request timeout in seconds.
///
/// This bounds upload/download request progress through response headers. The
/// default is intentionally large so multi-TiB transition uploads keep their
/// previous production budget while black-hole remotes no longer wait forever.
pub const ENV_TIER_REMOTE_REQUEST_TIMEOUT_SECS: &str = "RUSTFS_TIER_REMOTE_REQUEST_TIMEOUT_SECS";
/// Default remote tier request timeout in seconds.
pub const DEFAULT_TIER_REMOTE_REQUEST_TIMEOUT_SECS: u64 = 24 * 60 * 60;
/// Environment variable for remote tier response-body idle timeout in seconds.
///
/// The timer is re-armed on every non-empty response-body chunk, so slow but
/// progressing remotes can continue while silent response bodies are cancelled.
pub const ENV_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS: &str = "RUSTFS_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS";
/// Default remote tier response-body idle timeout in seconds.
pub const DEFAULT_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS: u64 = 60;
/// Request the object-transaction fencing contract used by storage-owned
/// cleanup receipts and lock-window optimizations.
///
@@ -834,16 +812,6 @@ mod remote_version_state_tests {
);
}
#[test]
fn remote_tier_timeout_env_names_are_stable() {
assert_eq!(super::ENV_TIER_REMOTE_CONNECT_TIMEOUT_SECS, "RUSTFS_TIER_REMOTE_CONNECT_TIMEOUT_SECS");
assert_eq!(super::ENV_TIER_REMOTE_REQUEST_TIMEOUT_SECS, "RUSTFS_TIER_REMOTE_REQUEST_TIMEOUT_SECS");
assert_eq!(
super::ENV_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS,
"RUSTFS_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS"
);
}
#[test]
fn data_movement_part_checksum_gate_uses_stable_environment_names() {
assert_eq!(super::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, "RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_WRITE");
+33 -48
View File
@@ -1,7 +1,7 @@
# e2e_test
End-to-end test suite for RustFS. Each test spawns a **real `rustfs` binary**
(built on demand from the workspace) and drives it over the network with the
(built and identified before the test invocation) and drives it over the network with the
AWS SDK (`aws-sdk-s3`), raw HTTP (`reqwest` / `awscurl`), or a protocol client
(FTPS / WebDAV / SFTP). This is the black-box integration layer: exhaustive
end-to-end behavior lives here, unit behavior stays in the source crates
@@ -26,38 +26,33 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
| **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
| **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
| **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
| **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
| **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
## How to run
All commands assume repo root. `cargo test` triggers an on-demand build of the
`rustfs` binary from [`src/common.rs`](src/common.rs) (`rustfs_binary_path`) on
first use — the first invocation is slow, later ones reuse the binary.
All commands assume repo root and Python 3.9 or newer on Linux or macOS. Build the server once through the provenance entry point, then run the test command through the same script:
```bash
# Whole crate (default = ignored tests skipped)
cargo nextest run -p e2e_test
python3 scripts/e2e_binary.py build --features e2e-test-hooks
# Whole crate (ignored tests remain skipped)
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run -p e2e_test
# One module
cargo nextest run -p e2e_test -E 'test(list_objects_v2_pagination_test)'
# PR smoke subset (see "CI smoke subset" below)
cargo nextest run --profile e2e-smoke -p e2e_test
# ILM serial lane — ignored lifecycle tests, single-threaded (mirrors CI)
cargo nextest run -j1 --run-ignored ignored-only -p rustfs-scanner -p rustfs \
-E 'binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))'
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run -p e2e_test -E 'test(list_objects_v2_pagination_test)'
# PR smoke subset
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke -p e2e_test
```
The protocols suite has its own contract (fixed bind ports 90229301,
single-worker execution, feature-gated scheduling) documented in
[`src/protocols/README.md`](src/protocols/README.md). `RUSTFS_BUILD_FEATURES`
selects which features the spawned binary is built with; leave it unset to run
every protocol entry. Use the exact profile command under
[Troubleshooting](#troubleshooting) for CI-equivalent execution.
`build` records the source contents, HEAD, resolved Cargo features, profile, toolchain, and binary SHA-256 beside the executable in `rustfs.e2e.json`. `run` validates that identity before and after the command, preserves command failures, and removes its temporary run receipt on completion. The Rust harness checks that receipt before starting each server; it never compiles a server inside a test process. Source or binary changes during a run invalidate the result, even when the test command succeeds. Use an isolated worktree and keep it unchanged until the command finishes.
The additional `--features` arguments must match between `build` and `run`; Cargo defaults remain enabled. The wrapper supplies `RUSTFS_BUILD_FEATURES` from Cargo's resolved feature list, including features enabled by `full`. Protocol helpers require a subset of that list. `CARGO_TARGET_DIR` and `--profile release` are supported. An in-workspace target directory must be Git-ignored; tracked files are always included in the source identity. `build --bins` preserves CI lanes that compile all RustFS binary targets. For a downloaded artifact, copy both the executable and its sidecar, then use `run`; do not generate a new identity for an arbitrary prebuilt binary. `CARGO_BIN_EXE_rustfs` cannot override the verified executable.
Each build/run holds an exclusive `rustfs.e2e.lock` marker beside the binary; concurrent wrappers fail immediately. Use a private target directory and do not run ordinary Cargo builds against it while tests are active: Cargo does not honor this marker. Interrupted runs fail and terminate their command group. After an uncatchable kill, inspect the PID recorded in a leftover marker and remove it only after confirming its owner has stopped. Embedded file symlinks are hashed through their target; embedded directory symlinks are rejected because their contents cannot be enumerated safely by this entry point.
The protocols suite has its own fixed-port and single-worker contract in [`src/protocols/README.md`](src/protocols/README.md). Use its command under [Troubleshooting](#troubleshooting).
### `#[ignore]` semantics
@@ -123,7 +118,7 @@ via `create_s3_client(idx)` / `create_all_clients()`. See
| `wait_for_server_ready` | Poll readiness before issuing requests |
| `create_s3_client` / `create_test_bucket` / `delete_test_bucket` | aws-sdk-s3 client + bucket lifecycle |
| `find_available_port` | Random free port (isolation primitive) |
| `rustfs_binary_path` / `_with_features` | Locate/build the binary; honors `RUSTFS_BUILD_FEATURES` |
| `rustfs_binary_path` / `_with_features` | Verify this run's binary receipt and required feature subset |
| `requested_rustfs_build_features` / `rustfs_build_feature_enabled` | Feature-gate a test to what the binary was built with |
| `execute_awscurl` / `awscurl_post` / `_get` / `_put` / `_delete` / `awscurl_post_sts_form_urlencoded` | Admin/STS API calls via `awscurl`; missing binaries are test failures |
| `replication_fast_env` | Env vars that shrink replication timers (from repl-4); pass to `start_rustfs_server_with_env` |
@@ -172,7 +167,6 @@ the same profile for membership and execution with one nightly worker.
| KMS suite | `e2e-full` job, merge queue + main | **Active** |
| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
| Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) |
| Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
| Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
| Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |
@@ -187,35 +181,26 @@ the wiring source of truth. Committed test-ID digests under
**Reproduce a CI failure locally** — run the exact profile/lane:
```bash
# Smoke (e2e-tests job) — includes the 20 fast replication tests
cargo nextest run --profile e2e-smoke -p e2e_test
# Full single-node merge/main lane
cargo nextest run --profile e2e-full -p e2e_test
# Cluster fault nightly lane
cargo nextest run --profile e2e-nightly -p e2e_test
# 4-node 4-disk distributed lane (S3 / lock / versioning / replication / decommission / chaos / upgrade)
# Upgrade cases need RUSTFS_UPGRADE_SOURCE_BINARY; without it they fail closed.
cargo nextest run --profile e2e-distributed -p e2e_test
# Replication nightly lane; awscurl is required for STS paths
cargo nextest run --profile e2e-repl-nightly -p e2e_test
# Fixed-port protocol nightly lane
RUSTFS_BUILD_FEATURES=ftps,webdav,sftp \
cargo nextest run -j 1 --profile e2e-protocols -p e2e_test --no-capture
# ILM serial lane
# Smoke, full, and cluster lanes share a server with fault-test hooks.
python3 scripts/e2e_binary.py build --features e2e-test-hooks
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke -p e2e_test
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-full -p e2e_test
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-nightly -p e2e_test
# Replication nightly uses the default server; awscurl is required for STS.
python3 scripts/e2e_binary.py build
python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-repl-nightly -p e2e_test
# Protocol nightly owns fixed ports.
python3 scripts/e2e_binary.py build --features ftps,webdav,sftp
python3 scripts/e2e_binary.py run --features ftps,webdav,sftp -- cargo nextest run -j 1 --profile e2e-protocols -p e2e_test --no-capture
# The ILM serial lane does not use this server harness.
cargo nextest run -j1 --run-ignored ignored-only -p rustfs-scanner -p rustfs \
-E 'binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))'
# s3s-e2e black box
./scripts/e2e-run.sh ./target/debug/rustfs /tmp/rustfs-e2e-data
```
**Stale binary.** Tests build the `rustfs` binary once and reuse it. To avoid
rebuilding while iterating on tests, `common.rs` reuses an existing binary when
running *inside* the e2e test process even if sources changed
(`can_reuse_inside_e2e`, [`src/common.rs`](src/common.rs) line 98). Downside: if
you changed **server** code, force a rebuild with
`cargo build -p rustfs` (or `touch` a source file outside the reuse window)
before re-running, or CI's freshly built artifact will diverge from your local
one.
**Stale or unverified binary.** Re-run the matching `build` command after changing source or features, then invoke tests through `run`. A missing receipt, copied old executable, or mismatched build identity is a prerequisite failure. Bare Cargo invocations that start a server deliberately fail; unit tests that do not start a server can still run directly.
**Port already in use / orphan processes.** A hard-killed run can leak a
`rustfs` child holding its port. Find and kill it:
+123 -212
View File
@@ -31,7 +31,6 @@ use rustfs_signer::constants::UNSIGNED_PAYLOAD;
use rustfs_signer::sign_v4;
use s3s::Body;
use serde_json;
use std::ffi::OsStr;
use std::fs as stdfs;
use std::io::ErrorKind;
use std::net::SocketAddr;
@@ -44,7 +43,6 @@ use tokio::net::TcpStream;
use tokio::time::sleep;
use tracing::{error, info, warn};
use uuid::Uuid;
use walkdir::WalkDir;
// Common constants for all E2E tests
pub const DEFAULT_ACCESS_KEY: &str = "rustfsadmin";
@@ -365,59 +363,75 @@ fn resolve_rustfs_binary_path(workspace: &Path, configured_target_dir: Option<&P
path
}
/// Resolve the RustFS binary relative to the workspace, optionally requesting build features.
/// Resolve the server verified by `scripts/e2e_binary.py run` for this test invocation.
/// Requested features are a required subset of the server's resolved Cargo features.
pub fn rustfs_binary_path_with_features(requested_features: Option<&str>) -> PathBuf {
if let Some(path) = std::env::var_os("CARGO_BIN_EXE_rustfs") {
return PathBuf::from(path);
}
let requested_features = requested_features.and_then(normalize_rustfs_build_features);
let workspace = workspace_root();
let configured_target_dir = std::env::var_os("CARGO_TARGET_DIR").map(PathBuf::from);
let binary_path = resolve_rustfs_binary_path(&workspace, configured_target_dir.as_deref());
let binary_path = std::env::var_os("CARGO_BIN_EXE_rustfs")
.map(PathBuf::from)
.unwrap_or_else(|| resolve_rustfs_binary_path(&workspace, configured_target_dir.as_deref()));
let receipt_path = std::env::var_os("RUSTFS_E2E_BINARY_RECEIPT").map(PathBuf::from);
receipt_path
.ok_or_else(|| std::io::Error::new(ErrorKind::NotFound, "missing E2E run receipt"))
.and_then(|receipt| verify_e2e_binary_receipt(&receipt, &workspace, &binary_path, requested_features))
.unwrap_or_else(|error| {
panic!(
"E2E server prerequisite failed: {error}. Build with `python3 scripts/e2e_binary.py build --features <features>` and run tests with `python3 scripts/e2e_binary.py run --features <features> -- cargo nextest run ...`"
)
})
}
let features_match = binary_features_match(&binary_path, requested_features.as_deref());
let source_is_newer = workspace_sources_newer_than_binary(&binary_path);
let can_reuse_inside_e2e = running_inside_e2e_test_binary() && requested_features.is_none() && features_match;
if binary_path.is_file() && features_match && (!source_is_newer || can_reuse_inside_e2e) {
if source_is_newer {
warn!(
"RustFS binary at {:?} appears older than workspace sources; reusing it inside cargo test to avoid nested builds",
binary_path
);
}
info!("Using existing RustFS binary at {:?}", binary_path);
return binary_path;
#[derive(serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct E2eBinaryReceipt {
schema: u32,
workspace: PathBuf,
binary: PathBuf,
size: u64,
modified_ns: u128,
features: Vec<String>,
}
fn verify_e2e_binary_receipt(
receipt_path: &Path,
workspace: &Path,
binary_path: &Path,
requested_features: Option<&str>,
) -> std::io::Result<PathBuf> {
let receipt: E2eBinaryReceipt = serde_json::from_slice(&stdfs::read(receipt_path)?)?;
let binary = binary_path.canonicalize()?;
let metadata = binary.metadata()?;
let modified_ns = metadata
.modified()?
.duration_since(std::time::UNIX_EPOCH)
.map_err(std::io::Error::other)?
.as_nanos();
// The runner hashes source and binary before/after the entire suite. Each
// nextest process checks only this invocation's path, features, and file stat.
if receipt.schema != 1
|| receipt.workspace != workspace.canonicalize()?
|| receipt.binary != binary
|| !metadata.is_file()
|| receipt.size != metadata.len()
|| receipt.modified_ns != modified_ns
{
return Err(std::io::Error::new(
ErrorKind::InvalidData,
"E2E server differs from this run's verified binary",
));
}
info!("Building RustFS binary to ensure it's up to date...");
build_rustfs_binary(requested_features.as_deref(), &binary_path);
info!("Using RustFS binary at {:?}", binary_path);
binary_path
}
fn workspace_sources_newer_than_binary(binary_path: &PathBuf) -> bool {
let Ok(binary_meta) = std::fs::metadata(binary_path) else {
return true;
};
let Ok(binary_modified) = binary_meta.modified() else {
return true;
};
let workspace = workspace_root();
let watch_roots = [
workspace.join("Cargo.toml"),
workspace.join("Cargo.lock"),
workspace.join("rustfs"),
workspace.join("crates"),
];
watch_roots.iter().any(|path| path_is_newer_than(binary_modified, path))
}
fn running_inside_e2e_test_binary() -> bool {
std::env::var("CARGO_PKG_NAME").is_ok_and(|value| value == "e2e_test")
if let Some(requested) = requested_features.and_then(normalize_rustfs_build_features)
&& requested
.split(',')
.any(|feature| !receipt.features.iter().any(|actual| actual == feature))
{
return Err(std::io::Error::new(
ErrorKind::InvalidInput,
"E2E server is missing a requested build feature",
));
}
Ok(binary)
}
pub fn requested_rustfs_build_features() -> Option<String> {
@@ -447,96 +461,6 @@ pub fn rustfs_build_feature_enabled(requested_features: Option<&str>, required_f
.any(|feature| feature.eq_ignore_ascii_case(RUSTFS_FULL_FEATURE) || feature.eq_ignore_ascii_case(required_feature))
}
fn rustfs_binary_features_stamp_path(binary_path: &Path) -> PathBuf {
binary_path.with_extension("features")
}
fn binary_features_match(binary_path: &Path, requested_features: Option<&str>) -> bool {
let stamp_path = rustfs_binary_features_stamp_path(binary_path);
let recorded = stdfs::read_to_string(stamp_path)
.ok()
.and_then(|value| normalize_rustfs_build_features(&value));
let requested = requested_features.and_then(normalize_rustfs_build_features);
match requested.as_deref() {
Some(features) => recorded.as_deref() == Some(features),
None => recorded.is_none(),
}
}
fn path_is_newer_than(binary_modified: std::time::SystemTime, path: &Path) -> bool {
if path.is_file() {
return std::fs::metadata(path)
.and_then(|meta| meta.modified())
.map(|modified| modified > binary_modified)
.unwrap_or(false);
}
if !path.is_dir() {
return false;
}
WalkDir::new(path)
.into_iter()
.filter_entry(|entry| {
let name = entry.file_name();
name != OsStr::new("target") && name != OsStr::new(".git")
})
.filter_map(Result::ok)
.filter(|entry| entry.file_type().is_file())
.any(|entry| {
std::fs::metadata(entry.path())
.and_then(|meta| meta.modified())
.map(|modified| modified > binary_modified)
.unwrap_or(false)
})
}
/// Build the RustFS binary using cargo
fn build_rustfs_binary(requested_features: Option<&str>, binary_path: &Path) {
let workspace = workspace_root();
info!("Building RustFS binary from workspace: {:?}", workspace);
let _profile = if cfg!(debug_assertions) {
info!("Building in debug mode");
"dev"
} else {
info!("Building in release mode");
"release"
};
let mut cmd = Command::new("cargo");
cmd.current_dir(&workspace).args(["build", "--bin", "rustfs"]);
if let Some(features) = requested_features {
cmd.arg("--features").arg(features);
info!("Building with features: {}", features);
}
if !cfg!(debug_assertions) {
cmd.arg("--release");
}
info!(
"Executing: cargo build --bin rustfs {}",
if cfg!(debug_assertions) { "" } else { "--release" }
);
let output = cmd.output().expect("Failed to execute cargo build command");
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
panic!("Failed to build RustFS binary. Error: {stderr}");
}
let stamp_path = rustfs_binary_features_stamp_path(binary_path);
if let Err(err) = stdfs::write(&stamp_path, requested_features.unwrap_or_default()) {
warn!("Failed to write RustFS feature stamp {:?}: {}", stamp_path, err);
}
info!("✅ RustFS binary built successfully");
}
fn awscurl_binary_path() -> PathBuf {
std::env::var_os("AWSCURL_PATH")
.map(PathBuf::from)
@@ -1700,69 +1624,6 @@ impl RustFSTestClusterEnvironment {
Ok(())
}
/// Append a new single-node erasure pool to a stopped multi-pool cluster.
///
/// Used to simulate pool expansion on localhost: every pool already owns
/// exactly one node with `drives_per_node >= 2` (the only multi-pool layout
/// the single-host `RUSTFS_VOLUMES` syntax can express). The new node is
/// allocated a fresh port and empty drive directories; callers must
/// [`Self::start`] afterwards so every process picks up the extended
/// volumes argument. Existing data directories are left untouched.
pub async fn append_single_node_pool(&mut self) -> Result<usize, Box<dyn std::error::Error + Send + Sync>> {
if self.nodes.iter().any(|node| node.process.is_some()) {
return Err("stop the cluster before appending a pool".into());
}
if self.topology.drives_per_node < 2 {
return Err(
"append_single_node_pool requires drives_per_node >= 2 (the server parser rejects a single-drive ellipses pool)"
.into(),
);
}
let mut pools = self.topology.normalized_pools();
for (pool_idx, nodes) in pools.iter().enumerate() {
if nodes.len() != 1 {
return Err(format!(
"pool {pool_idx} spans {} nodes; append_single_node_pool requires one node per pool",
nodes.len()
)
.into());
}
}
let new_idx = self.nodes.len();
let port = RustFSTestEnvironment::find_available_port().await?;
let address = format!("127.0.0.1:{port}");
let data_dirs: Vec<String> = (0..self.topology.drives_per_node)
.map(|drive| format!("{}/node{}/drive{}", self.temp_dir, new_idx, drive))
.collect();
for dir in &data_dirs {
fs::create_dir_all(dir).await?;
}
self.nodes.push(ClusterNode {
url: format!("http://{address}"),
address,
data_dir: data_dirs[0].clone(),
data_dirs,
pool_idx: pools.len(),
process: None,
});
pools.push(vec![new_idx]);
self.topology.node_count = self.nodes.len();
self.topology.pools = pools;
self.node_extra_env.push(Vec::new());
self.node_capture_log_paths.push(None);
self.volume_proxy_addresses.push(None);
if !self.extra_env.iter().any(|(key, _)| key == "RUSTFS_UNSAFE_BYPASS_DISK_CHECK") {
self.extra_env
.push(("RUSTFS_UNSAFE_BYPASS_DISK_CHECK".to_string(), "true".to_string()));
}
Ok(new_idx)
}
/// Gracefully stop one cluster node and wait for its process to exit.
///
/// This is intentionally separate from [`Self::stop_node`]: the latter is
@@ -2136,16 +1997,66 @@ mod tests {
}
#[test]
fn binary_feature_stamp_matching_uses_normalized_features() {
let binary_path = std::env::temp_dir().join(format!("rustfs-feature-stamp-test-{}", Uuid::new_v4()));
let stamp_path = rustfs_binary_features_stamp_path(&binary_path);
fn explicit_binary_without_run_receipt_is_rejected() {
const CHILD_ENV: &str = "RUSTFS_E2E_RECEIPT_TEST_CHILD";
if std::env::var_os(CHILD_ENV).is_some() {
rustfs_binary_path_with_features(None);
return;
}
let executable = std::env::current_exe().expect("locate isolated test process");
let output = Command::new(&executable)
.args([
"--exact",
"common::tests::explicit_binary_without_run_receipt_is_rejected",
"--nocapture",
])
.env(CHILD_ENV, "1")
.env("CARGO_BIN_EXE_rustfs", &executable)
.env_remove("RUSTFS_E2E_BINARY_RECEIPT")
.output()
.expect("run the missing-receipt scenario with isolated environment variables");
assert!(!output.status.success(), "an explicit binary must not bypass run verification");
assert!(String::from_utf8_lossy(&output.stderr).contains("missing E2E run receipt"));
}
stdfs::write(&stamp_path, " SFTP, ftps ").expect("write feature stamp");
assert!(binary_features_match(&binary_path, Some("sftp,ftps")));
assert!(binary_features_match(&binary_path, Some(" SFTP, FTPS ")));
assert!(!binary_features_match(&binary_path, Some("sftp")));
stdfs::remove_file(stamp_path).ok();
#[test]
fn e2e_run_receipt_rejects_replaced_binary_and_missing_features() {
let directory = std::env::temp_dir().join(format!("rustfs-e2e-receipt-test-{}", Uuid::new_v4()));
stdfs::create_dir(&directory).expect("create receipt fixture");
let binary = directory.join("rustfs");
let receipt = directory.join("receipt.json");
stdfs::write(&binary, "server").expect("write fixture binary");
let metadata = binary.metadata().expect("stat fixture binary");
let record = serde_json::json!({
"schema": 1,
"workspace": directory.canonicalize().expect("canonical workspace"),
"binary": binary.canonicalize().expect("canonical binary"),
"size": metadata.len(),
"modified_ns": metadata.modified().expect("modified time").duration_since(std::time::UNIX_EPOCH).expect("positive timestamp").as_nanos(),
"features": ["default", "full", "ftps", "webdav", "sftp"]
});
stdfs::write(&receipt, serde_json::to_vec(&record).expect("serialize receipt")).expect("write receipt");
verify_e2e_binary_receipt(&receipt, &directory, &binary, Some("sftp,webdav")).expect("resolved feature subset");
verify_e2e_binary_receipt(&receipt, &directory, &binary, Some("full")).expect("full was actually requested");
assert_eq!(
verify_e2e_binary_receipt(&receipt, &directory, &binary, Some("rio-v2"))
.expect_err("full does not enable rio-v2")
.kind(),
ErrorKind::InvalidInput
);
let other = directory.join("old-server");
stdfs::write(&other, "server").expect("write alternate binary");
assert!(verify_e2e_binary_receipt(&receipt, &directory, &other, None).is_err());
stdfs::write(&binary, "different server").expect("replace fixture binary");
assert!(verify_e2e_binary_receipt(&receipt, &directory, &binary, None).is_err());
stdfs::remove_file(&receipt).expect("remove expired receipt");
assert_eq!(
verify_e2e_binary_receipt(&receipt, &directory, &binary, None)
.expect_err("expired receipt")
.kind(),
ErrorKind::NotFound
);
stdfs::remove_dir_all(directory).expect("remove receipt fixture");
}
/// Build a cluster environment struct in-memory (no ports, no processes) so
@@ -1,222 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::harness::{
DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, retrying_get_equals, unique_bucket,
wait_for_ready, wait_until,
};
use crate::chaos::{census_object_version_on_disk, signed_admin_post};
use crate::common::{build_test_s3_config, init_logging};
use crate::fault_proxy::FaultMode;
use aws_sdk_s3::Client;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::{Barrier, mpsc};
use tokio::time::timeout;
#[tokio::test]
async fn kill_and_restart_node_preserves_objects() -> TestResult {
init_logging();
let mut dist = DistCluster::start(DistLayout::FourByFour).await?;
let bucket = unique_bucket("killnode");
dist.create_bucket(&bucket).await?;
let body = vec![0x11u8; 128 * 1024];
put_object(&dist.client(0)?, &bucket, "keep.bin", body.clone()).await?;
dist.cluster.stop_node(3)?;
retrying_get_equals(&dist.client(0)?, &bucket, "keep.bin", &body, Duration::from_secs(20)).await?;
dist.cluster.start_node(3).await?;
wait_for_ready(&dist.cluster).await?;
assert_object_bytes(&dist.client(3)?, &bucket, "keep.bin", &body).await?;
Ok(())
}
#[tokio::test]
async fn full_cluster_restart_preserves_objects() -> TestResult {
init_logging();
let mut dist = DistCluster::start(DistLayout::FourByFour).await?;
let bucket = unique_bucket("pwr");
dist.create_bucket(&bucket).await?;
let body = vec![0x44u8; 64 * 1024];
put_object(&dist.client(1)?, &bucket, "survive.bin", body.clone()).await?;
dist.cluster.stop();
dist.cluster.start().await?;
wait_for_ready(&dist.cluster).await?;
for node_idx in 0..dist.cluster.nodes.len() {
assert_object_bytes(&dist.client(node_idx)?, &bucket, "survive.bin", &body).await?;
}
Ok(())
}
#[tokio::test]
async fn fresh_drive_replacement_is_physically_healed_without_data_change() -> TestResult {
init_logging();
let mut dist = DistCluster::start_with_env(DistLayout::FourByFour, &[("RUSTFS_HEAL_ENABLED", "true")]).await?;
let bucket = unique_bucket("baddrive");
dist.create_bucket(&bucket).await?;
let body = payload_for("fresh-drive/durable.bin", 8 * 1024 * 1024);
put_object(&dist.client(1)?, &bucket, "durable.bin", body.clone()).await?;
let replaced_drive = PathBuf::from(&dist.cluster.nodes[0].data_dirs[0]);
let baseline = census_object_version_on_disk(&replaced_drive, &bucket, "durable.bin", None)?;
assert!(
baseline.is_complete(),
"replacement target did not hold a complete baseline shard: {baseline:?}"
);
assert!(
!baseline.expected_part_numbers.is_empty(),
"replacement witness must use physical part shards: {baseline:?}"
);
dist.cluster.stop_node(0)?;
let format_path = replaced_drive.join(".rustfs.sys/format.json");
let format = std::fs::read(&format_path)?;
let retired_drive = PathBuf::from(format!("{}.retired", replaced_drive.display()));
std::fs::rename(&replaced_drive, &retired_drive)?;
std::fs::create_dir_all(format_path.parent().ok_or("replacement format path omitted parent")?)?;
std::fs::write(&format_path, format)?;
let empty = census_object_version_on_disk(&replaced_drive, &bucket, "durable.bin", None)?;
assert!(!empty.has_xl_meta, "fresh replacement unexpectedly retained object metadata: {empty:?}");
dist.cluster.start_node(0).await?;
wait_for_ready(&dist.cluster).await?;
let heal_body =
r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
let heal_url = format!("{}/rustfs/admin/v3/heal/{bucket}?forceStart=true", dist.cluster.nodes[1].url);
signed_admin_post(&heal_url, Some(heal_body), &dist.cluster.access_key, &dist.cluster.secret_key).await?;
wait_until(
Duration::from_secs(90),
|| async {
let healed = census_object_version_on_disk(&replaced_drive, &bucket, "durable.bin", None)?;
Ok(healed.matches_manifest(&baseline))
},
"fresh replacement contains the original complete shard manifest",
)
.await?;
for node_idx in 0..dist.cluster.nodes.len() {
assert_object_bytes(&dist.client(node_idx)?, &bucket, "durable.bin", &body).await?;
}
Ok(())
}
#[tokio::test]
async fn concurrent_gets_survive_peer_node_kill() -> TestResult {
init_logging();
let mut dist = DistCluster::start(DistLayout::FourByFour).await?;
let bucket = unique_bucket("getkill");
dist.create_bucket(&bucket).await?;
let body = payload_for("inflight/steady.bin", 8 * 1024 * 1024);
put_object(&dist.client(0)?, &bucket, "steady.bin", body.clone()).await?;
let live: Vec<_> = (0..3).map(|idx| dist.client(idx)).collect::<Result<Vec<_>, _>>()?;
let worker_count = 12;
let release = Arc::new(Barrier::new(worker_count + 1));
let (started_tx, mut started_rx) = mpsc::unbounded_channel();
let mut handles = Vec::new();
for idx in 0..worker_count {
let client = live[idx % live.len()].clone();
let bucket = bucket.clone();
let body = body.clone();
let release = release.clone();
let started_tx = started_tx.clone();
handles.push(tokio::spawn(async move {
let response = client.get_object().bucket(&bucket).key("steady.bin").send().await?;
if response.content_length() != Some(body.len() as i64) {
return Err::<(), Box<dyn std::error::Error + Send + Sync>>(
format!("worker {idx} received a wrong content length").into(),
);
}
started_tx.send(idx)?;
release.wait().await;
let actual = response.body.collect().await?.into_bytes();
if actual.as_ref() != body.as_slice() {
return Err(format!("worker {idx} received corrupted bytes after peer kill").into());
}
Ok(())
}));
}
drop(started_tx);
for _ in 0..worker_count {
timeout(Duration::from_secs(30), started_rx.recv())
.await?
.ok_or("a streaming GET exited before reaching the kill barrier")?;
}
dist.cluster.stop_node(3)?;
release.wait().await;
for handle in handles {
handle.await??;
}
dist.cluster.start_node(3).await?;
wait_for_ready(&dist.cluster).await?;
assert_object_bytes(&dist.client(3)?, &bucket, "steady.bin", &body).await?;
Ok(())
}
#[tokio::test]
async fn blackholed_node_client_network_preserves_cluster_availability_and_recovers() -> TestResult {
init_logging();
let dist = DistCluster::start(DistLayout::FourByFour).await?;
let proxy = crate::fault_proxy::FaultProxy::start(dist.cluster.nodes[3].address.parse()?).await?;
let proxied_url = format!("http://{}", proxy.local_addr());
let proxied_client = Client::from_conf(build_test_s3_config(
&proxied_url,
&dist.cluster.access_key,
&dist.cluster.secret_key,
None,
"distributed-network-chaos",
));
let result: TestResult = async {
let bucket = unique_bucket("netfault");
dist.create_bucket(&bucket).await?;
let baseline = payload_for("network/baseline.bin", 1024 * 1024);
put_object(&dist.client(0)?, &bucket, "baseline.bin", baseline.clone()).await?;
assert_object_bytes(&proxied_client, &bucket, "baseline.bin", &baseline).await?;
proxy.set_mode(FaultMode::Blackhole);
assert_eq!(proxy.mode(), FaultMode::Blackhole);
if let Ok(Ok(_)) = timeout(
Duration::from_secs(5),
proxied_client.get_object().bucket(&bucket).key("baseline.bin").send(),
)
.await
{
return Err("blackholed node endpoint unexpectedly completed a GET".into());
}
let during = payload_for("network/during.bin", 1024 * 1024);
timeout(Duration::from_secs(30), async {
put_object(&dist.client(1)?, &bucket, "during-blackhole.bin", during.clone()).await?;
assert_object_bytes(&dist.client(2)?, &bucket, "baseline.bin", &baseline).await?;
assert_object_bytes(&dist.client(0)?, &bucket, "during-blackhole.bin", &during).await?;
Ok::<_, Box<dyn std::error::Error + Send + Sync>>(())
})
.await??;
proxy.set_mode(FaultMode::Pass);
retrying_get_equals(&proxied_client, &bucket, "during-blackhole.bin", &during, Duration::from_secs(30)).await?;
Ok(())
}
.await;
proxy.set_mode(FaultMode::Pass);
proxy.shutdown().await;
result
}
@@ -1,98 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
use crate::common::init_logging;
use std::collections::BTreeSet;
use std::sync::Arc;
use tokio::sync::Barrier;
#[tokio::test]
async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult {
init_logging();
let dist = DistCluster::start(DistLayout::FourByFour).await?;
let bucket = unique_bucket("conc");
dist.create_bucket(&bucket).await?;
let clients = Arc::new(dist.clients()?);
let worker_count = 24;
let rounds = 4;
let barrier = Arc::new(Barrier::new(worker_count));
let mut handles = Vec::new();
for idx in 0..worker_count {
let clients = clients.clone();
let barrier = barrier.clone();
let bucket = bucket.clone();
handles.push(tokio::spawn(async move {
barrier.wait().await;
let writer = &clients[idx % clients.len()];
let reader = &clients[(idx + 1) % clients.len()];
let copier = &clients[(idx + 2) % clients.len()];
let mut retained = Vec::with_capacity(rounds);
for round in 0..rounds {
let key = format!("source/worker-{idx:02}-round-{round}.bin");
let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin");
let body = payload_for(&key, 64 * 1024);
put_object(writer, &bucket, &key, body.clone()).await?;
let head = reader.head_object().bucket(&bucket).key(&key).send().await?;
if head.content_length() != Some(body.len() as i64) {
return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into());
}
assert_object_bytes(reader, &bucket, &key, &body).await?;
copier
.copy_object()
.bucket(&bucket)
.key(&copy_key)
.copy_source(format!("{bucket}/{key}"))
.send()
.await?;
assert_object_bytes(writer, &bucket, &copy_key, &body).await?;
writer.delete_object().bucket(&bucket).key(&key).send().await?;
let missing = reader
.head_object()
.bucket(&bucket)
.key(&key)
.send()
.await
.expect_err("deleted source key must not remain visible");
if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) {
return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into());
}
retained.push((copy_key, body));
}
Ok::<_, Box<dyn std::error::Error + Send + Sync>>(retained)
}));
}
let mut inventory = Vec::new();
for handle in handles {
inventory.extend(handle.await??);
}
let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect();
for (node_idx, client) in clients.iter().enumerate() {
let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?;
let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing");
for (key, body) in &inventory {
assert_object_bytes(client, &bucket, key, body)
.await
.map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
}
}
Ok(())
}
@@ -1,74 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::harness::{
DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, decommission_running_with_progress,
decommission_status_json, payload_for, put_inventory_retrying, retrying_get_equals, retrying_put, start_decommission,
unique_bucket, wait_for_decommission_complete, wait_for_decommission_running_with_progress,
};
use crate::common::init_logging;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::Barrier;
#[tokio::test]
async fn concurrent_puts_during_decommission_do_not_lose_baseline_or_new_objects() -> TestResult {
init_logging();
let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
let bucket = unique_bucket("concdecom");
dist.create_bucket(&bucket).await?;
let baseline_client = dist.client(0)?;
let inventory = put_inventory_retrying(&baseline_client, &bucket, 96, 256 * 1024, Duration::from_secs(30)).await?;
dist.expand_to_four_pools().await?;
start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
let clients = Arc::new(dist.clients()?);
let barrier = Arc::new(Barrier::new(17));
let mut handles = Vec::new();
for idx in 0..16 {
let clients = clients.clone();
let barrier = barrier.clone();
let bucket = bucket.clone();
handles.push(tokio::spawn(async move {
barrier.wait().await;
let client = &clients[idx % clients.len()];
let key = format!("live/{idx:02}.bin");
let body = payload_for(&key, 8 * 1024);
retrying_put(client, &bucket, &key, body.clone(), Duration::from_secs(45)).await?;
Ok::<_, Box<dyn std::error::Error + Send + Sync>>((key, body))
}));
}
wait_for_decommission_running_with_progress(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
barrier.wait().await;
let mut live_objects = Vec::new();
for handle in handles {
live_objects.push(handle.await??);
}
let status = decommission_status_json(&dist.cluster).await?;
if !decommission_running_with_progress(&status, DECOMMISSION_POOL_ID)? {
return Err(format!("decommission did not remain active across concurrent PUTs: {status}").into());
}
wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
let checker = dist.client(2)?;
assert_inventory(&checker, &bucket, &inventory).await?;
for (key, body) in live_objects {
retrying_get_equals(&checker, &bucket, &key, &body, Duration::from_secs(30)).await?;
}
Ok(())
}
@@ -1,156 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::harness::{
DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, enable_versioning, put_inventory_retrying,
sha256_hex, start_decommission, unique_bucket, wait_for_decommission_active, wait_for_decommission_complete,
};
use crate::common::init_logging;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
use std::time::Duration;
#[tokio::test]
async fn decommission_does_not_alter_object_sha256_across_pools() -> TestResult {
init_logging();
let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
let bucket = unique_bucket("integrity");
dist.create_bucket(&bucket).await?;
let client = dist.client(0)?;
enable_versioning(&client, &bucket).await?;
let inventory = put_inventory_retrying(&client, &bucket, 96, 256 * 1024, Duration::from_secs(30)).await?;
let before: Vec<(String, String)> = inventory.iter().map(|(key, body)| (key.clone(), sha256_hex(body))).collect();
let versioned_key = "history/versioned.bin";
let version_one = b"historical bytes before data movement".to_vec();
let version_two = b"current bytes before data movement".to_vec();
let version_one_id = client
.put_object()
.bucket(&bucket)
.key(versioned_key)
.body(ByteStream::from(version_one.clone()))
.send()
.await?
.version_id()
.ok_or("historical PUT omitted version ID")?
.to_string();
let version_two_id = client
.put_object()
.bucket(&bucket)
.key(versioned_key)
.body(ByteStream::from(version_two.clone()))
.send()
.await?
.version_id()
.ok_or("current PUT omitted version ID")?
.to_string();
let multipart_key = "multipart/moved.bin";
let first_part = vec![0x31; 5 * 1024 * 1024];
let second_part = vec![0x72; 1024 * 1024];
let upload = client
.create_multipart_upload()
.bucket(&bucket)
.key(multipart_key)
.send()
.await?;
let upload_id = upload.upload_id().ok_or("movement multipart upload omitted upload ID")?;
let uploaded_one = client
.upload_part()
.bucket(&bucket)
.key(multipart_key)
.upload_id(upload_id)
.part_number(1)
.body(ByteStream::from(first_part.clone()))
.send()
.await?;
let uploaded_two = client
.upload_part()
.bucket(&bucket)
.key(multipart_key)
.upload_id(upload_id)
.part_number(2)
.body(ByteStream::from(second_part.clone()))
.send()
.await?;
client
.complete_multipart_upload()
.bucket(&bucket)
.key(multipart_key)
.upload_id(upload_id)
.multipart_upload(
CompletedMultipartUpload::builder()
.parts(
CompletedPart::builder()
.part_number(1)
.e_tag(uploaded_one.e_tag().ok_or("movement part 1 omitted ETag")?)
.build(),
)
.parts(
CompletedPart::builder()
.part_number(2)
.e_tag(uploaded_two.e_tag().ok_or("movement part 2 omitted ETag")?)
.build(),
)
.build(),
)
.send()
.await?;
dist.expand_to_four_pools().await?;
start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
wait_for_decommission_active(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
let after_client = dist.client(2)?;
assert_inventory(&after_client, &bucket, &inventory).await?;
for (key, expected_hash) in before {
let got = after_client.get_object().bucket(&bucket).key(&key).send().await?;
let body = got.body.collect().await?.into_bytes();
assert_eq!(sha256_hex(body.as_ref()), expected_hash, "checksum changed for {key} after decommission");
}
for (version_id, expected) in [(&version_one_id, &version_one), (&version_two_id, &version_two)] {
let got = after_client
.get_object()
.bucket(&bucket)
.key(versioned_key)
.version_id(version_id)
.send()
.await?
.body
.collect()
.await?
.into_bytes();
assert_eq!(got.as_ref(), expected.as_slice(), "version {version_id} changed after decommission");
}
let mut expected_multipart = first_part;
expected_multipart.extend_from_slice(&second_part);
let got_multipart = after_client
.get_object()
.bucket(&bucket)
.key(multipart_key)
.send()
.await?
.body
.collect()
.await?
.into_bytes();
assert_eq!(
sha256_hex(got_multipart.as_ref()),
sha256_hex(&expected_multipart),
"multipart checksum changed after decommission"
);
Ok(())
}
@@ -1,81 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::harness::{
DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, list_pools_json, put_inventory,
put_inventory_retrying, start_decommission, start_rebalance, unique_bucket, wait_for_decommission_active,
wait_for_decommission_complete, wait_for_rebalance_active, wait_for_rebalance_complete,
};
use crate::common::init_logging;
use std::time::Duration;
#[tokio::test]
async fn four_node_pool_expand_preserves_objects_then_rebalance() -> TestResult {
init_logging();
let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
let bucket = unique_bucket("expand");
dist.create_bucket(&bucket).await?;
let client = dist.client(0)?;
let inventory = put_inventory(&client, &bucket, 64, 256 * 1024).await?;
assert_inventory(&client, &bucket, &inventory).await?;
for expected_nodes in 2..=4 {
let new_node = dist.append_pool_and_restart().await?;
assert_eq!(new_node + 1, expected_nodes);
assert_inventory(&dist.client(new_node)?, &bucket, &inventory).await?;
}
assert_eq!(dist.cluster.nodes.len(), 4);
// Prove that the expanded pool map is durable, and clear any recovery
// latch raised while the newly-added pool replicas converged.
dist.restart_current_binary_gracefully().await?;
let after_expand = dist.client(0)?;
assert_inventory(&after_expand, &bucket, &inventory).await?;
let peer = dist.client(3)?;
assert_inventory(&peer, &bucket, &inventory).await?;
let rebalance_id = start_rebalance(&dist.cluster).await?;
wait_for_rebalance_active(&dist.cluster, &rebalance_id, Duration::from_secs(30)).await?;
wait_for_rebalance_complete(&dist.cluster, &rebalance_id, Duration::from_secs(180)).await?;
assert_inventory(&peer, &bucket, &inventory).await?;
Ok(())
}
#[tokio::test]
async fn four_pool_decommission_moves_objects_without_loss() -> TestResult {
init_logging();
let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
let bucket = unique_bucket("decom");
dist.create_bucket(&bucket).await?;
let client = dist.client(0)?;
let inventory = put_inventory_retrying(&client, &bucket, 96, 128 * 1024, Duration::from_secs(30)).await?;
dist.expand_to_four_pools().await?;
let pools_before = list_pools_json(&dist.cluster).await?;
let pool_count = pools_before
.as_array()
.map(Vec::len)
.or_else(|| pools_before.get("pools").and_then(serde_json::Value::as_array).map(Vec::len))
.ok_or_else(|| format!("pool list omitted an array: {pools_before}"))?;
assert_eq!(pool_count, 4, "expected exactly four pools before decommission: {pools_before}");
start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
wait_for_decommission_active(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
let after = dist.client(2)?;
assert_inventory(&after, &bucket, &inventory).await?;
Ok(())
}
@@ -1,149 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::harness::{
DistCluster, DistLayout, TestResult, assert_object_bytes, get_object_bytes, put_object, unique_bucket, wait_until,
};
use crate::common::init_logging;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
use std::time::Duration;
#[tokio::test]
async fn four_node_four_drive_multipart_and_cross_node_listing_agree() -> TestResult {
init_logging();
let dist = DistCluster::start(DistLayout::FourByFour).await?;
let bucket = unique_bucket("extra");
dist.create_bucket(&bucket).await?;
let client = dist.client(0)?;
let key = "multipart.bin";
let part1 = vec![0x41u8; 5 * 1024 * 1024];
let part2 = vec![0x42u8; 5 * 1024 * 1024];
let upload = client.create_multipart_upload().bucket(&bucket).key(key).send().await?;
let upload_id = upload.upload_id().ok_or("missing upload id")?.to_string();
let uploaded1 = client
.upload_part()
.bucket(&bucket)
.key(key)
.upload_id(&upload_id)
.part_number(1)
.body(ByteStream::from(part1.clone()))
.send()
.await?;
let uploaded2 = client
.upload_part()
.bucket(&bucket)
.key(key)
.upload_id(&upload_id)
.part_number(2)
.body(ByteStream::from(part2.clone()))
.send()
.await?;
client
.complete_multipart_upload()
.bucket(&bucket)
.key(key)
.upload_id(&upload_id)
.multipart_upload(
CompletedMultipartUpload::builder()
.parts(
CompletedPart::builder()
.part_number(1)
.e_tag(uploaded1.e_tag().unwrap_or_default())
.build(),
)
.parts(
CompletedPart::builder()
.part_number(2)
.e_tag(uploaded2.e_tag().unwrap_or_default())
.build(),
)
.build(),
)
.send()
.await?;
let mut expected = part1;
expected.extend_from_slice(&part2);
for node_idx in 0..dist.cluster.nodes.len() {
assert_object_bytes(&dist.client(node_idx)?, &bucket, key, &expected).await?;
}
put_object(&client, &bucket, "list/a", b"a".to_vec()).await?;
put_object(&dist.client(2)?, &bucket, "list/b", b"b".to_vec()).await?;
let mut seen = Vec::new();
for node_idx in 0..dist.cluster.nodes.len() {
let listed = dist
.client(node_idx)?
.list_objects_v2()
.bucket(&bucket)
.prefix("list/")
.send()
.await?;
let keys: Vec<String> = listed
.contents()
.iter()
.filter_map(|object| object.key().map(str::to_string))
.collect();
seen.push(keys);
}
for keys in &seen[1..] {
assert_eq!(&seen[0], keys, "list results diverged across nodes: {seen:?}");
}
let got = get_object_bytes(&dist.client(3)?, &bucket, "list/a").await?;
assert_eq!(got, b"a");
Ok(())
}
#[tokio::test]
async fn four_node_list_buckets_agree_across_all_nodes() -> TestResult {
init_logging();
let dist = DistCluster::start(DistLayout::FourByFour).await?;
let bucket = unique_bucket("listed");
dist.create_bucket(&bucket).await?;
put_object(&dist.client(0)?, &bucket, "seed.bin", b"seed".to_vec()).await?;
for node_idx in 0..dist.cluster.nodes.len() {
let client = dist.client(node_idx)?;
let name = bucket.clone();
wait_until(
Duration::from_secs(20),
|| {
let client = client.clone();
let name = name.clone();
async move {
let listed = client.list_buckets().send().await?;
Ok(listed.buckets().iter().any(|entry| entry.name() == Some(name.as_str())))
}
},
&format!("node {node_idx} lists {bucket}"),
)
.await?;
wait_until(
Duration::from_secs(20),
|| {
let client = dist.client(node_idx).expect("client");
let name = bucket.clone();
async move { Ok(get_object_bytes(&client, &name, "seed.bin").await.ok() == Some(b"seed".to_vec())) }
},
&format!("node {node_idx} reads seed.bin"),
)
.await?;
}
Ok(())
}
File diff suppressed because it is too large Load Diff
-35
View File
@@ -1,35 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! 4-node 4-drive distributed e2e coverage.
//!
//! Selected by `[profile.e2e-distributed]` and run from
//! `.github/workflows/e2e-distributed.yml`. Excluded from `e2e-full` because
//! each case starts four real `rustfs` processes.
mod chaos_test;
mod concurrency_stability_test;
mod concurrent_data_movement_test;
mod data_integrity_movement_test;
mod expand_decommission_rebalance_test;
mod extra_test;
mod harness;
mod object_lock_test;
mod observability_test;
mod replication_quota_test;
mod s3_basic_test;
mod s3_during_data_movement_test;
mod site_replication_test;
mod upgrade_test;
mod versioning_test;
@@ -1,219 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::harness::{DistCluster, DistLayout, TestResult, unique_bucket};
use crate::common::init_logging;
use crate::object_lock::common::{
delete_object_with_bypass, put_object_lock_configuration, put_object_with_legal_hold, put_object_with_retention,
};
use aws_sdk_s3::Client;
use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::error::SdkError;
use aws_sdk_s3::operation::delete_object::DeleteObjectError;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{
DefaultRetention, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
ObjectLockRule,
};
use chrono::{Duration as ChronoDuration, Utc};
fn delete_denied(error: &SdkError<DeleteObjectError>, context: &str) -> TestResult {
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
if code == Some("AccessDenied") {
Ok(())
} else {
Err(format!("{context}: expected AccessDenied, got {error:?}").into())
}
}
async fn expect_versioned_delete_denied(
client: &Client,
bucket: &str,
key: &str,
version_id: &str,
bypass: bool,
context: &str,
) -> TestResult {
match delete_object_with_bypass(client, bucket, key, Some(version_id), bypass).await {
Ok(_) => Err(format!("{context}: DeleteObject of retained version must be denied").into()),
Err(error) => delete_denied(error.as_ref(), context),
}
}
#[tokio::test]
async fn four_node_four_drive_object_lock_worm_blocks_delete() -> TestResult {
init_logging();
let dist = DistCluster::start(DistLayout::FourByFour).await?;
let client = dist.client(0)?;
let peer = dist.client(2)?;
let bucket = unique_bucket("objlock");
client
.create_bucket()
.bucket(&bucket)
.object_lock_enabled_for_bucket(true)
.send()
.await?;
let retain_until = Utc::now() + ChronoDuration::days(1);
let compliance_key = "compliance.bin";
let compliance_version = put_object_with_retention(
&client,
&bucket,
compliance_key,
b"locked-compliance",
ObjectLockRetentionMode::Compliance,
retain_until,
)
.await?;
// Unversioned DELETE is allowed: it only creates a delete marker. WORM
// applies to a specific version id.
let marker = peer.delete_object().bucket(&bucket).key(compliance_key).send().await?;
assert_eq!(
marker.delete_marker(),
Some(true),
"unversioned DELETE on a locked object must create a delete marker"
);
expect_versioned_delete_denied(&peer, &bucket, compliance_key, &compliance_version, false, "COMPLIANCE without bypass")
.await?;
expect_versioned_delete_denied(&peer, &bucket, compliance_key, &compliance_version, true, "COMPLIANCE with bypass").await?;
let governance_key = "governance.bin";
let governance_version = put_object_with_retention(
&client,
&bucket,
governance_key,
b"locked-governance",
ObjectLockRetentionMode::Governance,
retain_until,
)
.await?;
expect_versioned_delete_denied(&peer, &bucket, governance_key, &governance_version, false, "GOVERNANCE without bypass")
.await?;
delete_object_with_bypass(&peer, &bucket, governance_key, Some(&governance_version), true).await?;
let deleted_governance = peer
.head_object()
.bucket(&bucket)
.key(governance_key)
.version_id(&governance_version)
.send()
.await
.expect_err("GOVERNANCE bypass must remove the retained version");
assert_eq!(
deleted_governance.raw_response().map(|response| response.status().as_u16()),
Some(404),
"deleted GOVERNANCE version returned an unexpected HEAD result: {deleted_governance:?}"
);
let hold_key = "legal-hold.bin";
let hold_version =
put_object_with_legal_hold(&client, &bucket, hold_key, b"legal-hold", ObjectLockLegalHoldStatus::On).await?;
expect_versioned_delete_denied(&peer, &bucket, hold_key, &hold_version, false, "legal hold without bypass").await?;
expect_versioned_delete_denied(&peer, &bucket, hold_key, &hold_version, true, "legal hold with bypass").await?;
Ok(())
}
#[tokio::test]
async fn four_node_default_retention_is_visible_and_non_lock_bucket_rejects_configuration() -> TestResult {
init_logging();
let dist = DistCluster::start(DistLayout::FourByFour).await?;
let writer = dist.client(0)?;
let reader = dist.client(3)?;
let bucket = unique_bucket("default-lock");
writer
.create_bucket()
.bucket(&bucket)
.object_lock_enabled_for_bucket(true)
.send()
.await?;
put_object_lock_configuration(&writer, &bucket, ObjectLockRetentionMode::Governance, Some(1), None).await?;
let key = "default-governance.bin";
let put = writer
.put_object()
.bucket(&bucket)
.key(key)
.body(ByteStream::from_static(b"default retention payload"))
.send()
.await?;
let version_id = put.version_id().ok_or("default-retained PUT omitted version ID")?;
let config = reader.get_object_lock_configuration().bucket(&bucket).send().await?;
let default_retention = config
.object_lock_configuration()
.and_then(|configuration| configuration.rule())
.and_then(|rule| rule.default_retention())
.ok_or("GetObjectLockConfiguration omitted default retention")?;
assert_eq!(default_retention.mode().map(|mode| mode.as_str()), Some("GOVERNANCE"));
assert_eq!(default_retention.days(), Some(1));
let retention = reader
.get_object_retention()
.bucket(&bucket)
.key(key)
.version_id(version_id)
.send()
.await?;
let retention = retention.retention().ok_or("GetObjectRetention omitted applied retention")?;
assert_eq!(retention.mode().map(|mode| mode.as_str()), Some("GOVERNANCE"));
let retain_until = retention
.retain_until_date()
.ok_or("default retention omitted retain-until date")?;
assert!(retain_until.secs() > Utc::now().timestamp(), "default retention is not in the future");
let versioning = reader.get_bucket_versioning().bucket(&bucket).send().await?;
assert_eq!(versioning.status().map(|status| status.as_str()), Some("Enabled"));
expect_versioned_delete_denied(&reader, &bucket, key, version_id, false, "default GOVERNANCE retention without bypass")
.await?;
let plain_bucket = unique_bucket("no-lock");
dist.create_bucket(&plain_bucket).await?;
let configuration = ObjectLockConfiguration::builder()
.object_lock_enabled(ObjectLockEnabled::Enabled)
.rule(
ObjectLockRule::builder()
.default_retention(
DefaultRetention::builder()
.mode(ObjectLockRetentionMode::Governance)
.days(1)
.build(),
)
.build(),
)
.build();
let error = writer
.put_object_lock_configuration()
.bucket(&plain_bucket)
.object_lock_configuration(configuration)
.send()
.await
.expect_err("an unversioned bucket must reject Object Lock enablement");
let service_error = error
.as_service_error()
.ok_or("non-lock bucket rejection was not an S3 service error")?;
assert_eq!(service_error.code(), Some("InvalidBucketState"), "unexpected error: {error:?}");
assert_eq!(
service_error.message(),
Some("Object Lock configuration cannot be enabled on existing buckets"),
"unexpected error: {error:?}"
);
Ok(())
}
@@ -1,236 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::harness::{DistCluster, DistLayout, TestResult, cluster_admin_ok, unique_bucket, wait_for_ready};
use crate::common::{admin_request, init_logging, local_http_client};
use aws_sdk_s3::operation::RequestId;
use aws_sdk_s3::primitives::ByteStream;
use bytes::Bytes;
use http::Method;
use http_body_util::{BodyExt, Empty};
use hyper::body::Incoming;
use hyper::service::service_fn;
use hyper::{Request, Response};
use hyper_util::rt::TokioIo;
use local_ip_address::local_ip;
use rustfs_madmin::metrics::RealtimeMetrics;
use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
use serde_json::Value;
use std::convert::Infallible;
use std::time::Duration;
use tokio::net::TcpListener;
use tokio::sync::mpsc;
use tokio::task::JoinHandle;
use tokio::time::{Instant, timeout};
async fn spawn_audit_collector() -> TestResult<(String, mpsc::UnboundedReceiver<Value>, JoinHandle<()>)> {
let listener = TcpListener::bind("0.0.0.0:0").await?;
let endpoint = format!("http://{}/audit", std::net::SocketAddr::new(local_ip()?, listener.local_addr()?.port()));
let (tx, rx) = mpsc::unbounded_channel();
let handle = tokio::spawn(async move {
loop {
let Ok((stream, _)) = listener.accept().await else {
return;
};
let tx = tx.clone();
tokio::spawn(async move {
let service = service_fn(move |request: Request<Incoming>| {
let tx = tx.clone();
async move {
let method = request.method().clone();
if let Ok(body) = request.into_body().collect().await
&& method == Method::POST
&& let Ok(payload) = serde_json::from_slice::<Value>(&body.to_bytes())
{
if let Some(records) = payload["Records"].as_array() {
for entry in records {
let _ = tx.send(entry.clone());
}
} else {
let _ = tx.send(payload);
}
}
Ok::<_, Infallible>(Response::new(Empty::<Bytes>::new()))
}
});
let _ = hyper::server::conn::http1::Builder::new()
.serve_connection(TokioIo::new(stream), service)
.await;
});
}
});
Ok((endpoint, rx, handle))
}
async fn wait_for_audit_entry(
rx: &mut mpsc::UnboundedReceiver<Value>,
bucket: &str,
key: &str,
request_id: &str,
) -> TestResult<Value> {
let deadline = Instant::now() + Duration::from_secs(30);
let mut seen = Vec::new();
loop {
let remaining = deadline.saturating_duration_since(Instant::now());
if remaining.is_zero() {
return Err(format!(
"audit webhook did not receive PutObject for {bucket}/{key}; received {} other records: {seen:?}",
seen.len()
)
.into());
}
let entry = match timeout(remaining, rx.recv()).await {
Ok(Some(entry)) => entry,
Ok(None) => return Err("audit collector stopped before the expected entry arrived".into()),
Err(_) => {
return Err(format!(
"audit webhook did not receive PutObject for {bucket}/{key}; received {} other records: {seen:?}",
seen.len()
)
.into());
}
};
if entry["api"]["name"].as_str() == Some("s3:PutObject")
&& entry["api"]["bucket"].as_str() == Some(bucket)
&& entry["api"]["object"].as_str() == Some(key)
&& entry["requestID"].as_str() == Some(request_id)
{
return Ok(entry);
}
if seen.len() < 8 {
seen.push(format!(
"api={:?} bucket={:?} object={:?} requestID={:?}",
entry["api"]["name"].as_str(),
entry["api"]["bucket"].as_str(),
entry["api"]["object"].as_str(),
entry["requestID"].as_str()
));
}
}
}
#[tokio::test]
async fn four_node_health_inventory_metrics_and_audit_delivery_are_consistent() -> TestResult {
init_logging();
let (audit_endpoint, mut audit_entries, collector) = spawn_audit_collector().await?;
let audit_origin = reqwest::Url::parse(&audit_endpoint)?.origin().ascii_serialization();
let audit_env = [
("RUSTFS_AUDIT_ENABLE", "true"),
("RUSTFS_AUDIT_WEBHOOK_ENABLE_DISTRIBUTED", "on"),
("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_DISTRIBUTED", audit_endpoint.as_str()),
(ENV_OUTBOUND_ALLOW_ORIGINS, audit_origin.as_str()),
];
let mut dist = DistCluster::new_stopped_with_env(DistLayout::FourByFour, &audit_env).await?;
for node_idx in 0..dist.cluster.nodes.len() {
let queue_dir = format!("{}/audit-queue-node-{node_idx}", dist.cluster.temp_dir);
tokio::fs::create_dir_all(&queue_dir).await?;
dist.cluster
.set_node_env(node_idx, "RUSTFS_AUDIT_WEBHOOK_QUEUE_DIR_DISTRIBUTED", queue_dir)?;
}
dist.cluster.start().await?;
wait_for_ready(&dist.cluster).await?;
let http = local_http_client();
for node in &dist.cluster.nodes {
for probe in ["ready", "live"] {
let response = http.get(format!("{}/health/{probe}", node.url)).send().await?;
assert!(
response.status().is_success(),
"node {} {probe} probe failed: {}",
node.address,
response.status()
);
}
}
let info_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/info", None).await?;
let info: Value = serde_json::from_str(&info_body)?;
let servers = info["info"]["servers"]
.as_array()
.ok_or_else(|| format!("admin info omitted servers: {info}"))?;
assert_eq!(servers.len(), 4, "admin info did not report all four nodes: {info}");
let storage_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/storageinfo", None).await?;
let storage: Value = serde_json::from_str(&storage_body)?;
let disks = storage["info"]["disks"]
.as_array()
.ok_or_else(|| format!("storageinfo omitted disks: {storage}"))?;
assert_eq!(disks.len(), 16, "storageinfo did not report all sixteen drives: {storage}");
assert!(
disks.iter().all(|disk| {
disk["state"].as_str().is_some_and(|state| state.eq_ignore_ascii_case("ok"))
&& disk["runtimeState"]
.as_str()
.is_some_and(|state| state.eq_ignore_ascii_case("online"))
}),
"storageinfo reported a drive that was not healthy and online: {storage}"
);
for (node_idx, node) in dist.cluster.nodes.iter().enumerate() {
let (status, metrics_body) = admin_request(
&node.url,
Method::GET,
"/rustfs/admin/v3/metrics?n=1&by-host=true&by-disk=true",
None,
&dist.cluster.access_key,
&dist.cluster.secret_key,
)
.await?;
assert!(status.is_success(), "node {node_idx} metrics failed: {status} {metrics_body}");
let sample: RealtimeMetrics = serde_json::from_str(
metrics_body
.lines()
.next()
.ok_or_else(|| format!("node {node_idx} returned empty metrics"))?,
)?;
assert!(sample.finally, "node {node_idx} metrics sample was not terminal");
assert!(sample.errors.is_empty(), "node {node_idx} metrics reported errors: {:?}", sample.errors);
assert!(!sample.hosts.is_empty(), "node {node_idx} metrics omitted hosts");
}
let targets_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/audit/target/list", None).await?;
let targets: Value = serde_json::from_str(&targets_body)?;
let configured = targets["audit_endpoints"]
.as_array()
.ok_or_else(|| format!("audit target list omitted audit_endpoints: {targets}"))?
.iter()
.any(|target| target["account_id"].as_str() == Some("distributed") && target["service"].as_str() == Some("webhook"));
assert!(configured, "configured audit webhook was missing: {targets}");
let bucket = unique_bucket("audit");
dist.create_bucket(&bucket).await?;
let key = "correlated/audit-object.bin";
let put = dist
.client(2)?
.put_object()
.bucket(&bucket)
.key(key)
.body(ByteStream::from_static(b"distributed audit payload"))
.send()
.await?;
let request_id = put.request_id().ok_or("PutObject response omitted request ID")?;
let audit = wait_for_audit_entry(&mut audit_entries, &bucket, key, request_id).await?;
assert_eq!(
audit["api"]["status_code"].as_i64(),
Some(200),
"audit entry did not report success: {audit}"
);
assert!(
!audit.to_string().contains(&dist.cluster.secret_key),
"audit entry leaked the root secret key"
);
collector.abort();
Ok(())
}
@@ -1,191 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::harness::{
DistCluster, DistLayout, TestResult, enable_versioning, put_bucket_replication, put_object, retrying_put, set_bucket_quota,
set_remote_target, unique_bucket, wait_for_ready, wait_for_replicated_bytes, wait_until,
};
use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, init_logging};
use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::primitives::ByteStream;
use http::Method;
use std::time::Duration;
async fn wait_for_replication_status(
client: &aws_sdk_s3::Client,
bucket: &str,
key: &str,
expected: &[&str],
timeout: Duration,
) -> TestResult {
wait_until(
timeout,
|| async {
let head = client.head_object().bucket(bucket).key(key).send().await?;
Ok(head
.replication_status()
.is_some_and(|status| expected.contains(&status.as_str())))
},
&format!("replication status for {bucket}/{key} in {expected:?}"),
)
.await
}
#[tokio::test]
async fn four_node_bucket_replication_converges_to_peer_cluster() -> TestResult {
init_logging();
let (source, mut target) = DistCluster::start_replication_pair().await?;
let source_bucket = unique_bucket("replsrc");
let target_bucket = unique_bucket("repldst");
source.create_bucket(&source_bucket).await?;
target.create_bucket(&target_bucket).await?;
let source_client = source.client(0)?;
let target_client = target.client(0)?;
enable_versioning(&source_client, &source_bucket).await?;
enable_versioning(&target_client, &target_bucket).await?;
let arn = set_remote_target(&source.cluster, &source_bucket, &target.cluster, &target_bucket).await?;
put_bucket_replication(&source.cluster, &source_bucket, &arn).await?;
let key = "replicated/metadata-and-tags.bin";
let body = b"distributed-bucket-replication".to_vec();
source_client
.put_object()
.bucket(&source_bucket)
.key(key)
.metadata("origin", "four-node-source")
.tagging("suite=distributed&shape=metadata")
.body(ByteStream::from(body.clone()))
.send()
.await?;
wait_for_replicated_bytes(&target_client, &target_bucket, key, &body, Duration::from_secs(45)).await?;
wait_for_replication_status(&source_client, &source_bucket, key, &["COMPLETED"], Duration::from_secs(30)).await?;
let peer_read = target.client(3)?;
wait_for_replicated_bytes(&peer_read, &target_bucket, key, &body, Duration::from_secs(15)).await?;
let replica_head = peer_read.head_object().bucket(&target_bucket).key(key).send().await?;
assert_eq!(
replica_head
.metadata()
.and_then(|metadata| metadata.get("origin"))
.map(String::as_str),
Some("four-node-source")
);
assert_eq!(replica_head.replication_status().map(|status| status.as_str()), Some("REPLICA"));
let replica_tags = peer_read.get_object_tagging().bucket(&target_bucket).key(key).send().await?;
let tags: std::collections::BTreeMap<_, _> = replica_tags.tag_set().iter().map(|tag| (tag.key(), tag.value())).collect();
assert_eq!(tags.get("suite"), Some(&"distributed"));
assert_eq!(tags.get("shape"), Some(&"metadata"));
target.cluster.stop();
let outage_key = "replicated/queued-during-target-outage.bin";
let outage_body = b"retry-after-target-restart".to_vec();
put_object(&source_client, &source_bucket, outage_key, outage_body.clone()).await?;
wait_for_replication_status(
&source_client,
&source_bucket,
outage_key,
&["PENDING", "FAILED"],
Duration::from_secs(30),
)
.await?;
target.cluster.start().await?;
wait_for_ready(&target.cluster).await?;
wait_for_replicated_bytes(&target.client(2)?, &target_bucket, outage_key, &outage_body, Duration::from_secs(90)).await?;
wait_for_replication_status(&source_client, &source_bucket, outage_key, &["COMPLETED"], Duration::from_secs(45)).await?;
Ok(())
}
#[tokio::test]
async fn four_node_four_drive_hard_quota_rejects_over_limit_put() -> TestResult {
init_logging();
let dist = DistCluster::start_with_env(DistLayout::FourByFour, FAST_DATA_USAGE_SCANNER_ENV).await?;
let bucket = unique_bucket("quota");
dist.create_bucket(&bucket).await?;
set_bucket_quota(&dist.cluster, &bucket, 8 * 1024).await?;
let client = dist.client(1)?;
retrying_put(&client, &bucket, "small.bin", vec![0u8; 1024], Duration::from_secs(30)).await?;
wait_until(
Duration::from_secs(30),
|| async {
let (status, body) = super::harness::cluster_admin(
&dist.cluster,
Method::GET,
&format!("/rustfs/admin/v3/quota-stats/{bucket}"),
None,
)
.await?;
if !status.is_success() {
return Ok(false);
}
let stats: serde_json::Value =
serde_json::from_str(&body).map_err(|error| format!("quota stats returned invalid JSON: {error}: {body}"))?;
let usage = stats
.get("current_usage")
.and_then(serde_json::Value::as_u64)
.ok_or_else(|| format!("quota stats omitted current_usage: {stats}"))?;
Ok(usage >= 1024)
},
"quota stats observe small object",
)
.await?;
let oversized_key = "too-big.bin";
let error = client
.put_object()
.bucket(&bucket)
.key(oversized_key)
.body(vec![0u8; 16 * 1024].into())
.send()
.await
.expect_err("hard quota must reject the oversized PUT");
let service_error = error
.as_service_error()
.ok_or("quota rejection was not an S3 service error")?;
assert_eq!(
error.raw_response().map(|response| response.status().as_u16()),
Some(400),
"quota rejection must be HTTP 400: {error:?}"
);
assert_eq!(service_error.code(), Some("InvalidRequest"), "unexpected quota error: {error:?}");
assert!(
service_error
.message()
.is_some_and(|message| message.starts_with("Bucket quota exceeded")),
"PUT must fail specifically at quota admission: {error:?}"
);
let missing = client
.head_object()
.bucket(&bucket)
.key(oversized_key)
.send()
.await
.expect_err("an object rejected by quota must not become visible");
assert_eq!(
missing.raw_response().map(|response| response.status().as_u16()),
Some(404),
"quota-rejected object returned an unexpected HEAD result: {missing:?}"
);
let listed = client.list_objects_v2().bucket(&bucket).send().await?;
assert!(
listed.contents().iter().all(|object| object.key() != Some(oversized_key)),
"quota-rejected key leaked into ListObjectsV2"
);
Ok(())
}
@@ -1,258 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, get_object_bytes, put_object, unique_bucket};
use crate::common::{init_logging, local_http_client};
use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::presigning::PresigningConfig;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{Delete, MetadataDirective, ObjectIdentifier};
use std::time::Duration;
#[tokio::test]
async fn four_node_four_drive_s3_put_get_head_list_copy_rename_delete_and_presign() -> TestResult {
init_logging();
let dist = DistCluster::start(DistLayout::FourByFour).await?;
let bucket = unique_bucket("s3basic");
dist.create_bucket(&bucket).await?;
let writer = dist.client(0)?;
let reader = dist.client(3)?;
let key = "dir/object.bin";
let body = vec![0xA5u8; 256 * 1024];
put_object(&writer, &bucket, key, body.clone()).await?;
let head = reader.head_object().bucket(&bucket).key(key).send().await?;
assert_eq!(head.content_length(), Some(body.len() as i64));
assert_object_bytes(&reader, &bucket, key, &body).await?;
let ranged = reader
.get_object()
.bucket(&bucket)
.key(key)
.range("bytes=0-15")
.send()
.await?;
let ranged_body = ranged.body.collect().await?.into_bytes();
assert_eq!(ranged_body.as_ref(), &body[..16]);
let listed = reader.list_objects_v2().bucket(&bucket).prefix("dir/").send().await?;
let keys: Vec<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
assert_eq!(keys, vec![key]);
let copy_key = "dir/object-copy.bin";
reader
.copy_object()
.bucket(&bucket)
.key(copy_key)
.copy_source(format!("{bucket}/{key}"))
.metadata_directive(MetadataDirective::Copy)
.send()
.await?;
assert_object_bytes(&writer, &bucket, copy_key, &body).await?;
let moved_key = "dir/object-moved.bin";
writer
.copy_object()
.bucket(&bucket)
.key(moved_key)
.copy_source(format!("{bucket}/{copy_key}"))
.send()
.await?;
writer.delete_object().bucket(&bucket).key(copy_key).send().await?;
match writer.head_object().bucket(&bucket).key(copy_key).send().await {
Ok(_) => return Err("copied source still present after rename delete".into()),
Err(error) if error.as_service_error().is_some_and(|err| err.is_not_found()) => {}
Err(error) => return Err(error.into()),
}
assert_object_bytes(&reader, &bucket, moved_key, &body).await?;
let presigned = writer
.get_object()
.bucket(&bucket)
.key(key)
.presigned(PresigningConfig::expires_in(Duration::from_secs(120))?)
.await?;
let response = local_http_client().get(presigned.uri().to_string()).send().await?;
assert!(response.status().is_success(), "presigned GET failed: {}", response.status());
let presigned_body = response.bytes().await?;
assert_eq!(presigned_body.as_ref(), body.as_slice());
let empty_key = "empty";
put_object(&writer, &bucket, empty_key, Vec::new()).await?;
let empty = get_object_bytes(&reader, &bucket, empty_key).await?;
assert!(empty.is_empty());
let deleted = writer
.delete_objects()
.bucket(&bucket)
.delete(
Delete::builder()
.objects(ObjectIdentifier::builder().key(key).build()?)
.objects(ObjectIdentifier::builder().key(moved_key).build()?)
.objects(ObjectIdentifier::builder().key(empty_key).build()?)
.build()?,
)
.send()
.await?;
assert!(deleted.errors().is_empty(), "DeleteObjects reported failures: {deleted:?}");
assert_eq!(deleted.deleted().len(), 3, "DeleteObjects did not acknowledge every key");
let remaining = reader.list_objects_v2().bucket(&bucket).send().await?;
assert!(remaining.contents().is_empty(), "bucket still has objects after delete");
Ok(())
}
#[tokio::test]
async fn four_node_s3_metadata_tags_special_keys_pagination_and_multipart_abort() -> TestResult {
init_logging();
let dist = DistCluster::start(DistLayout::FourByFour).await?;
let bucket = unique_bucket("s3matrix");
dist.create_bucket(&bucket).await?;
let writer = dist.client(0)?;
let reader = dist.client(3)?;
let special_key = "unicode/测试 space+percent%25.txt";
let special_body = b"metadata and tagging survive distributed routing".to_vec();
let put = writer
.put_object()
.bucket(&bucket)
.key(special_key)
.metadata("test-meta", "distributed")
.tagging("purpose=compatibility&scope=four-by-four")
.body(ByteStream::from(special_body.clone()))
.send()
.await?;
let etag = put.e_tag().ok_or("PutObject omitted ETag")?.to_string();
let head = reader.head_object().bucket(&bucket).key(special_key).send().await?;
assert_eq!(
head.metadata()
.and_then(|metadata| metadata.get("test-meta"))
.map(String::as_str),
Some("distributed")
);
assert_eq!(head.e_tag(), Some(etag.as_str()));
let tags = reader.get_object_tagging().bucket(&bucket).key(special_key).send().await?;
let actual_tags: std::collections::BTreeMap<_, _> = tags
.tag_set()
.iter()
.map(|tag| (tag.key().to_string(), tag.value().to_string()))
.collect();
assert_eq!(actual_tags.get("purpose").map(String::as_str), Some("compatibility"));
assert_eq!(actual_tags.get("scope").map(String::as_str), Some("four-by-four"));
let conditional = reader
.get_object()
.bucket(&bucket)
.key(special_key)
.if_match(&etag)
.send()
.await?;
assert_eq!(conditional.body.collect().await?.into_bytes().as_ref(), special_body.as_slice());
let invalid_range = reader
.get_object()
.bucket(&bucket)
.key(special_key)
.range("bytes=999999-1000000")
.send()
.await
.expect_err("an unsatisfiable range must fail");
assert_eq!(
invalid_range.as_service_error().and_then(ProvideErrorMetadata::code),
Some("InvalidRange"),
"unexpected invalid-range error: {invalid_range:?}"
);
let upload_key = "multipart/aborted.bin";
let upload = writer
.create_multipart_upload()
.bucket(&bucket)
.key(upload_key)
.send()
.await?;
let upload_id = upload.upload_id().ok_or("CreateMultipartUpload omitted upload ID")?;
writer
.upload_part()
.bucket(&bucket)
.key(upload_key)
.upload_id(upload_id)
.part_number(1)
.body(ByteStream::from(vec![0x5Au8; 5 * 1024 * 1024]))
.send()
.await?;
let pending = reader
.list_multipart_uploads()
.bucket(&bucket)
.prefix("multipart/")
.send()
.await?;
assert!(pending.uploads().iter().any(|entry| entry.upload_id() == Some(upload_id)));
writer
.abort_multipart_upload()
.bucket(&bucket)
.key(upload_key)
.upload_id(upload_id)
.send()
.await?;
let after_abort = reader
.list_multipart_uploads()
.bucket(&bucket)
.prefix("multipart/")
.send()
.await?;
assert!(after_abort.uploads().iter().all(|entry| entry.upload_id() != Some(upload_id)));
let aborted_head = reader
.head_object()
.bucket(&bucket)
.key(upload_key)
.send()
.await
.expect_err("aborted multipart upload must not create an object");
assert_eq!(
aborted_head.raw_response().map(|response| response.status().as_u16()),
Some(404),
"aborted multipart object returned an unexpected HEAD result: {aborted_head:?}"
);
for index in 0..113 {
let key = format!("page/{index:04}.txt");
put_object(&writer, &bucket, &key, format!("page-{index}").into_bytes()).await?;
}
let mut token = None;
let mut paged_keys = Vec::new();
loop {
let page = reader
.list_objects_v2()
.bucket(&bucket)
.prefix("page/")
.max_keys(37)
.set_continuation_token(token.take())
.send()
.await?;
paged_keys.extend(page.contents().iter().filter_map(|object| object.key().map(str::to_string)));
if page.is_truncated() != Some(true) {
break;
}
token = Some(
page.next_continuation_token()
.ok_or("truncated ListObjectsV2 page omitted next continuation token")?
.to_string(),
);
}
assert_eq!(paged_keys.len(), 113);
let expected: Vec<_> = (0..113).map(|index| format!("page/{index:04}.txt")).collect();
assert_eq!(paged_keys, expected, "pagination lost, duplicated, or reordered keys");
Ok(())
}
@@ -1,94 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::harness::{
DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, decommission_running_with_progress,
decommission_status_json, put_inventory_retrying, rebalance_running_with_progress, rebalance_status_json,
retrying_get_equals, retrying_put, start_decommission, start_rebalance, unique_bucket, wait_for_decommission_complete,
wait_for_decommission_running_with_progress, wait_for_rebalance_complete, wait_for_rebalance_running_with_progress,
};
use crate::common::init_logging;
use std::time::Duration;
#[tokio::test]
async fn s3_put_get_list_succeed_during_decommission_and_rebalance() -> TestResult {
init_logging();
let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
let bucket = unique_bucket("s3move");
dist.create_bucket(&bucket).await?;
let client = dist.client(0)?;
let inventory = put_inventory_retrying(&client, &bucket, 96, 256 * 1024, Duration::from_secs(30)).await?;
dist.expand_to_four_pools().await?;
start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
wait_for_decommission_running_with_progress(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
let live = dist.client(2)?;
retrying_put(
&live,
&bucket,
"during-decommission.bin",
b"written-while-decommissioning".to_vec(),
Duration::from_secs(30),
)
.await?;
retrying_get_equals(
&live,
&bucket,
"during-decommission.bin",
b"written-while-decommissioning",
Duration::from_secs(30),
)
.await?;
let listed = live.list_objects_v2().bucket(&bucket).send().await?;
assert!(
listed
.contents()
.iter()
.any(|object| object.key() == Some("during-decommission.bin")),
"list during decommission missed the newly written key"
);
let status = decommission_status_json(&dist.cluster).await?;
if !decommission_running_with_progress(&status, DECOMMISSION_POOL_ID)? {
return Err(format!("decommission did not remain active across the S3 operations: {status}").into());
}
wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
assert_inventory(&live, &bucket, &inventory).await?;
let rebalance_id = start_rebalance(&dist.cluster).await?;
wait_for_rebalance_running_with_progress(&dist.cluster, &rebalance_id, Duration::from_secs(30)).await?;
retrying_put(
&live,
&bucket,
"during-rebalance.bin",
b"written-while-rebalancing".to_vec(),
Duration::from_secs(30),
)
.await?;
retrying_get_equals(
&live,
&bucket,
"during-rebalance.bin",
b"written-while-rebalancing",
Duration::from_secs(30),
)
.await?;
let status = rebalance_status_json(&dist.cluster).await?;
if !rebalance_running_with_progress(&status, &rebalance_id)? {
return Err(format!("rebalance did not remain active across the S3 operations: {status}").into());
}
wait_for_rebalance_complete(&dist.cluster, &rebalance_id, Duration::from_secs(180)).await?;
assert_inventory(&dist.client(1)?, &bucket, &inventory).await?;
Ok(())
}
@@ -1,128 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::harness::{
DistCluster, TestResult, cluster_admin_ok, enable_versioning, put_object, unique_bucket, wait_for_replicated_bytes,
wait_until,
};
use crate::common::{init_logging, signed_request};
use http::{Method, StatusCode};
use rustfs_madmin::{PeerSite, ReplicateAddStatus, SiteReplicationInfo, SyncStatus};
use std::time::Duration;
async fn site_replication_add(
cluster: &crate::common::RustFSTestClusterEnvironment,
sites: &[PeerSite],
) -> TestResult<ReplicateAddStatus> {
let url = format!("{}/rustfs/admin/v3/site-replication/add?replicateILMExpiry=false", cluster.nodes[0].url);
let response = signed_request(
Method::PUT,
&url,
&cluster.access_key,
&cluster.secret_key,
Some(serde_json::to_vec(sites)?),
Some("application/json"),
)
.await?;
if response.status() != StatusCode::OK {
let status = response.status();
let body = response.text().await.unwrap_or_default();
return Err(format!("site replication add failed: {status} {body}").into());
}
Ok(serde_json::from_slice(&response.bytes().await?)?)
}
async fn site_replication_info(cluster: &crate::common::RustFSTestClusterEnvironment) -> TestResult<SiteReplicationInfo> {
let body = cluster_admin_ok(cluster, Method::GET, "/rustfs/admin/v3/site-replication/info", None).await?;
Ok(serde_json::from_str(&body)?)
}
async fn wait_for_site_replication_enabled(cluster: &crate::common::RustFSTestClusterEnvironment) -> TestResult {
wait_until(
Duration::from_secs(30),
|| async {
let info = site_replication_info(cluster).await?;
Ok(info.enabled && info.sites.len() == 2 && info.sites.iter().all(|site| site.sync_state == SyncStatus::Enable))
},
"site replication enabled with two synchronized sites",
)
.await
}
#[tokio::test]
async fn four_node_site_replication_replicates_object_to_peer_site() -> TestResult {
init_logging();
let (site_a, site_b) = DistCluster::start_replication_pair().await?;
let bucket = unique_bucket("siterepl");
site_a.create_bucket(&bucket).await?;
site_b.create_bucket(&bucket).await?;
let client_a = site_a.client(0)?;
let client_b = site_b.client(0)?;
enable_versioning(&client_a, &bucket).await?;
enable_versioning(&client_b, &bucket).await?;
let sites = vec![
PeerSite {
name: "site-a".to_string(),
endpoint: site_a.cluster.nodes[0].url.clone(),
access_key: site_a.cluster.access_key.clone(),
secret_key: site_a.cluster.secret_key.clone(),
..Default::default()
},
PeerSite {
name: "site-b".to_string(),
endpoint: site_b.cluster.nodes[0].url.clone(),
access_key: site_b.cluster.access_key.clone(),
secret_key: site_b.cluster.secret_key.clone(),
..Default::default()
},
];
let add_status = site_replication_add(&site_a.cluster, &sites).await?;
assert!(
add_status.success && add_status.err_detail.is_empty() && add_status.initial_sync_error_message.is_empty(),
"site replication add reported failure: {add_status:?}"
);
wait_for_site_replication_enabled(&site_a.cluster).await?;
wait_for_site_replication_enabled(&site_b.cluster).await?;
let info_a = site_replication_info(&site_a.cluster).await?;
let remote = info_a
.sites
.iter()
.find(|site| site.name == "site-b")
.ok_or_else(|| format!("site A info omitted the configured site-b peer: {info_a:?}"))?;
assert_eq!(remote.endpoint, site_b.cluster.nodes[0].url);
let deployment_ids: std::collections::BTreeSet<_> = info_a.sites.iter().map(|site| site.deployment_id.as_str()).collect();
assert!(
deployment_ids.iter().all(|deployment_id| !deployment_id.is_empty()) && deployment_ids.len() == 2,
"site peers must have two distinct non-empty deployment IDs: {info_a:?}"
);
assert!(info_a.retry_stats.is_none(), "site A has pending replication retries: {info_a:?}");
assert!(info_a.pending_operation.is_none(), "site A has a pending operation: {info_a:?}");
let key = "site-object.bin";
let body = b"four-node-site-replication".to_vec();
put_object(&client_a, &bucket, key, body.clone()).await?;
wait_for_replicated_bytes(&client_b, &bucket, key, &body, Duration::from_secs(60)).await?;
let peer_b = site_b.client(3)?;
wait_for_replicated_bytes(&peer_b, &bucket, key, &body, Duration::from_secs(20)).await?;
let reverse_key = "reverse/site-object.bin";
let reverse_body = b"site-b-to-site-a".to_vec();
put_object(&site_b.client(2)?, &bucket, reverse_key, reverse_body.clone()).await?;
wait_for_replicated_bytes(&site_a.client(3)?, &bucket, reverse_key, &reverse_body, Duration::from_secs(60)).await?;
Ok(())
}
@@ -1,345 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! 4-node upgrade coverage for historical objects and IAM AK/SK.
//!
//! Complements `upgrade_compatibility_test` (single-node SSE/multipart and
//! mixed-version listing). This module pins the distributed contract the
//! hardware upgrade chain is meant to catch: after a 4-node upgrade, objects
//! written on the previous release still read back, and IAM user credentials
//! created before the upgrade still authenticate.
//!
//! Requires `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous
//! release. The `e2e-distributed` workflow downloads that binary; a local run
//! without it fails closed rather than skipping.
use super::harness::{
DistCluster, DistLayout, TestResult, assert_object_bytes, cluster_admin_ok, enable_versioning, get_object_bytes, put_object,
unique_bucket, wait_until,
};
use crate::common::{
AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, init_logging,
};
use aws_sdk_s3::Client;
use aws_sdk_s3::error::ProvideErrorMetadata;
use std::path::{Path, PathBuf};
use std::time::Duration;
use uuid::Uuid;
const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
const IAM_SECRET: &str = "UpgradeTestSecretKey1";
const WRONG_SECRET: &str = "WrongSecretKey000000";
const CREDENTIAL_TIMEOUT: Duration = Duration::from_secs(30);
struct UpgradeSeed {
history_bucket: String,
history_key: &'static str,
history_body: Vec<u8>,
versioned_bucket: String,
versioned_key: &'static str,
version1: String,
version1_body: Vec<u8>,
version2: String,
version2_body: Vec<u8>,
iam_bucket: String,
iam_key: &'static str,
iam_body: Vec<u8>,
iam_user: String,
iam_secret: &'static str,
}
fn source_binary() -> TestResult<PathBuf> {
let path = std::env::var_os(SOURCE_BINARY_ENV).map(PathBuf::from).ok_or_else(|| {
format!(
"{SOURCE_BINARY_ENV} must point to the pinned previous release binary (the e2e-distributed workflow downloads it)"
)
})?;
if !path.is_file() {
return Err(format!("upgrade source binary does not exist: {}", path.display()).into());
}
Ok(path)
}
fn capture_upgrade_logs(cluster: &mut DistCluster, label: &str) -> TestResult {
let Some(log_dir) = std::env::var_os("RUSTFS_E2E_LOG_DIR") else {
return Ok(());
};
std::fs::create_dir_all(&log_dir)?;
for node_idx in 0..cluster.cluster.nodes.len() {
let path = Path::new(&log_dir).join(format!("{label}-node-{node_idx}.log"));
cluster
.cluster
.set_node_capture_log_path(node_idx, path.to_string_lossy().into_owned())?;
}
Ok(())
}
fn iam_rw_policy(bucket: &str) -> String {
serde_json::json!({
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": ["s3:*"],
"Resource": [
format!("arn:aws:s3:::{bucket}"),
format!("arn:aws:s3:::{bucket}/*")
]
}]
})
.to_string()
}
async fn create_iam_user(dist: &DistCluster, user: &str, secret: &str, policy_name: &str, bucket: &str) -> TestResult {
let url = &dist.cluster.nodes[0].url;
let access = &dist.cluster.access_key;
let admin_secret = &dist.cluster.secret_key;
admin_create_user_via(AdminTransport::Signed, url, access, admin_secret, user, secret).await?;
admin_add_canned_policy_via(AdminTransport::Signed, url, access, admin_secret, policy_name, &iam_rw_policy(bucket)).await?;
admin_attach_user_policy_via(AdminTransport::Signed, url, access, admin_secret, policy_name, user).await?;
Ok(())
}
async fn wait_for_put(client: &Client, bucket: &str, key: &str, body: Vec<u8>, label: &str) -> TestResult {
wait_until(
CREDENTIAL_TIMEOUT,
|| {
let client = client.clone();
let bucket = bucket.to_string();
let key = key.to_string();
let body = body.clone();
async move {
put_object(&client, &bucket, &key, body).await?;
Ok(true)
}
},
label,
)
.await
}
async fn wait_for_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8], label: &str) -> TestResult {
wait_until(
CREDENTIAL_TIMEOUT,
|| {
let client = client.clone();
let bucket = bucket.to_string();
let key = key.to_string();
let expected = expected.to_vec();
async move {
let got = get_object_bytes(&client, &bucket, &key).await?;
Ok(got == expected)
}
},
label,
)
.await
}
async fn seed_history_and_iam(dist: &DistCluster) -> TestResult<UpgradeSeed> {
let history_bucket = unique_bucket("upg-hist");
let versioned_bucket = unique_bucket("upg-ver");
let iam_bucket = unique_bucket("upg-iam");
dist.create_bucket(&history_bucket).await?;
dist.create_bucket(&versioned_bucket).await?;
dist.create_bucket(&iam_bucket).await?;
let root = dist.client(0)?;
enable_versioning(&root, &versioned_bucket).await?;
let history_key = "plain-history.bin";
let history_body = b"written by the previous 4-node release".to_vec();
put_object(&root, &history_bucket, history_key, history_body.clone()).await?;
let versioned_key = "versioned-history.txt";
let version1_body = b"version-one-before-upgrade".to_vec();
let version1 = root
.put_object()
.bucket(&versioned_bucket)
.key(versioned_key)
.body(aws_sdk_s3::primitives::ByteStream::from(version1_body.clone()))
.send()
.await?
.version_id()
.ok_or("first versioned PUT omitted version ID")?
.to_string();
let version2_body = b"version-two-before-upgrade".to_vec();
let version2 = root
.put_object()
.bucket(&versioned_bucket)
.key(versioned_key)
.body(aws_sdk_s3::primitives::ByteStream::from(version2_body.clone()))
.send()
.await?
.version_id()
.ok_or("second versioned PUT omitted version ID")?
.to_string();
let iam_user = format!("upg{}", &Uuid::new_v4().simple().to_string()[..8]);
let policy_name = format!("upgpol{}", &Uuid::new_v4().simple().to_string()[..8]);
create_iam_user(dist, &iam_user, IAM_SECRET, &policy_name, &iam_bucket).await?;
let iam_key = "iam-history.bin";
let iam_body = b"written with pre-upgrade IAM AK/SK".to_vec();
let iam_client = dist.client_with_credentials(1, &iam_user, IAM_SECRET)?;
wait_for_put(&iam_client, &iam_bucket, iam_key, iam_body.clone(), "IAM user PUT before upgrade").await?;
Ok(UpgradeSeed {
history_bucket,
history_key,
history_body,
versioned_bucket,
versioned_key,
version1,
version1_body,
version2,
version2_body,
iam_bucket,
iam_key,
iam_body,
iam_user,
iam_secret: IAM_SECRET,
})
}
async fn assert_history_and_iam(dist: &DistCluster, seed: &UpgradeSeed, context: &str) -> TestResult {
let root_a = dist.client(0)?;
let root_b = dist.client(3)?;
wait_for_bytes(
&root_b,
&seed.history_bucket,
seed.history_key,
&seed.history_body,
&format!("{context}: root GET historical object"),
)
.await?;
assert_object_bytes(&root_a, &seed.history_bucket, seed.history_key, &seed.history_body).await?;
let v1 = root_b
.get_object()
.bucket(&seed.versioned_bucket)
.key(seed.versioned_key)
.version_id(&seed.version1)
.send()
.await?;
let v1_body = v1.body.collect().await?.into_bytes();
if v1_body.as_ref() != seed.version1_body.as_slice() {
return Err(format!("{context}: version 1 bytes changed after upgrade").into());
}
let v2 = root_a
.get_object()
.bucket(&seed.versioned_bucket)
.key(seed.versioned_key)
.version_id(&seed.version2)
.send()
.await?;
let v2_body = v2.body.collect().await?.into_bytes();
if v2_body.as_ref() != seed.version2_body.as_slice() {
return Err(format!("{context}: version 2 bytes changed after upgrade").into());
}
let users = cluster_admin_ok(&dist.cluster, http::Method::GET, "/rustfs/admin/v3/list-users", None).await?;
if !users.contains(&seed.iam_user) {
return Err(format!("{context}: list-users lost IAM user {}: {users}", seed.iam_user).into());
}
let iam_on_upgraded = dist.client_with_credentials(0, &seed.iam_user, seed.iam_secret)?;
let iam_on_peer = dist.client_with_credentials(3, &seed.iam_user, seed.iam_secret)?;
wait_for_bytes(
&iam_on_upgraded,
&seed.iam_bucket,
seed.iam_key,
&seed.iam_body,
&format!("{context}: IAM GET historical object on node 0"),
)
.await?;
wait_for_bytes(
&iam_on_peer,
&seed.iam_bucket,
seed.iam_key,
&seed.iam_body,
&format!("{context}: IAM GET historical object on node 3"),
)
.await?;
let post_key = format!("after-upgrade-{context}.txt");
let post_body = format!("{context}: written with the same IAM AK/SK after upgrade").into_bytes();
wait_for_put(
&iam_on_peer,
&seed.iam_bucket,
&post_key,
post_body.clone(),
&format!("{context}: IAM PUT after upgrade"),
)
.await?;
assert_object_bytes(&iam_on_upgraded, &seed.iam_bucket, &post_key, &post_body).await?;
let bad = dist.client_with_credentials(1, &seed.iam_user, WRONG_SECRET)?;
match bad.get_object().bucket(&seed.iam_bucket).key(seed.iam_key).send().await {
Ok(_) => return Err(format!("{context}: wrong secret must not read the IAM object").into()),
Err(error) => {
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
let rejected = code == Some("SignatureDoesNotMatch")
|| code == Some("InvalidAccessKeyId")
|| code == Some("AccessDenied")
|| code == Some("InvalidArgument")
|| error.raw_response().is_some_and(|response| response.status().as_u16() == 403);
if !rejected {
return Err(format!("{context}: wrong secret failed with unexpected error {error:?}").into());
}
}
}
let post_root_key = format!("root-after-{context}.bin");
let post_root_body = format!("{context}: root write after upgrade").into_bytes();
put_object(&root_a, &seed.history_bucket, &post_root_key, post_root_body.clone()).await?;
assert_object_bytes(&root_b, &seed.history_bucket, &post_root_key, &post_root_body).await?;
Ok(())
}
#[tokio::test]
async fn four_node_direct_upgrade_preserves_history_and_iam_credentials() -> TestResult {
init_logging();
let previous = source_binary()?;
let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
capture_upgrade_logs(&mut dist, "direct-upgrade")?;
dist.start_from_binary(&previous).await?;
let seed = seed_history_and_iam(&dist).await?;
dist.restart_with_current_binary().await?;
assert_history_and_iam(&dist, &seed, "direct").await?;
Ok(())
}
#[tokio::test]
async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> TestResult {
init_logging();
let previous = source_binary()?;
let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
capture_upgrade_logs(&mut dist, "rolling-upgrade")?;
dist.start_from_binary(&previous).await?;
let seed = seed_history_and_iam(&dist).await?;
dist.replace_node_with_current_binary(0).await?;
assert_history_and_iam(&dist, &seed, "one-current-node").await?;
for node_idx in [1, 2] {
dist.replace_node_with_current_binary(node_idx).await?;
}
assert_history_and_iam(&dist, &seed, "one-previous-node").await?;
dist.replace_node_with_current_binary(3).await?;
assert_history_and_iam(&dist, &seed, "homogeneous-current").await?;
Ok(())
}
@@ -1,188 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::harness::{DistCluster, DistLayout, TestResult, enable_versioning, get_object_bytes, put_object, unique_bucket};
use crate::common::init_logging;
use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
#[tokio::test]
async fn four_node_four_drive_versioning_put_list_get_delete_marker() -> TestResult {
init_logging();
let dist = DistCluster::start(DistLayout::FourByFour).await?;
let bucket = unique_bucket("version");
dist.create_bucket(&bucket).await?;
let writer = dist.client(0)?;
let reader = dist.client(3)?;
enable_versioning(&writer, &bucket).await?;
let key = "versioned.txt";
let v1_id = writer
.put_object()
.bucket(&bucket)
.key(key)
.body(b"v1".to_vec().into())
.send()
.await?
.version_id()
.ok_or("v1 PUT omitted version ID")?
.to_string();
let v2_id = writer
.put_object()
.bucket(&bucket)
.key(key)
.body(b"v2".to_vec().into())
.send()
.await?
.version_id()
.ok_or("v2 PUT omitted version ID")?
.to_string();
let versions = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?;
let matching_versions: Vec<_> = versions
.versions()
.iter()
.filter(|version| version.key() == Some(key))
.collect();
assert_eq!(matching_versions.len(), 2, "fresh key must have exactly two versions: {versions:?}");
assert!(versions.delete_markers().is_empty(), "fresh key unexpectedly has a delete marker");
assert!(
matching_versions
.iter()
.any(|version| version.version_id() == Some(v1_id.as_str()) && version.is_latest() != Some(true)),
"v1 was not the historical version: {versions:?}"
);
assert!(
matching_versions
.iter()
.any(|version| version.version_id() == Some(v2_id.as_str()) && version.is_latest() == Some(true)),
"v2 was not the latest version: {versions:?}"
);
let latest = get_object_bytes(&reader, &bucket, key).await?;
assert_eq!(latest, b"v2");
let older = reader.get_object().bucket(&bucket).key(key).version_id(&v1_id).send().await?;
let older_body = older.body.collect().await?.into_bytes();
assert_eq!(older_body.as_ref(), b"v1");
let deleted = writer.delete_object().bucket(&bucket).key(key).send().await?;
assert_eq!(deleted.delete_marker(), Some(true));
let marker_id = deleted.version_id().ok_or("DeleteObject omitted delete-marker version ID")?;
let after_delete = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?;
let matching_markers: Vec<_> = after_delete
.delete_markers()
.iter()
.filter(|marker| marker.key() == Some(key))
.collect();
assert_eq!(
matching_markers.len(),
1,
"delete marker missing or duplicated after current-version delete: {after_delete:?}"
);
assert!(
matching_markers[0].version_id() == Some(marker_id) && matching_markers[0].is_latest() == Some(true),
"DeleteObject response and ListObjectVersions disagree about the marker: {after_delete:?}"
);
let latest_after_delete = reader.get_object().bucket(&bucket).key(key).send().await;
match latest_after_delete {
Ok(_) => return Err("current version should be a delete marker".into()),
Err(error)
if error
.as_service_error()
.and_then(ProvideErrorMetadata::code)
.is_some_and(|code| code == "NoSuchKey" || code == "NotFound") => {}
Err(error) => return Err(error.into()),
}
let restored = reader.get_object().bucket(&bucket).key(key).version_id(&v1_id).send().await?;
let restored_body = restored.body.collect().await?.into_bytes();
assert_eq!(restored_body.as_ref(), b"v1");
writer
.delete_object()
.bucket(&bucket)
.key(key)
.version_id(marker_id)
.send()
.await?;
assert_eq!(get_object_bytes(&reader, &bucket, key).await?, b"v2");
Ok(())
}
#[tokio::test]
async fn four_node_versioning_suspension_keeps_one_null_version_and_history() -> TestResult {
init_logging();
let dist = DistCluster::start(DistLayout::FourByFour).await?;
let bucket = unique_bucket("suspend");
dist.create_bucket(&bucket).await?;
let writer = dist.client(0)?;
let reader = dist.client(3)?;
enable_versioning(&writer, &bucket).await?;
let key = "suspended.txt";
let original = writer
.put_object()
.bucket(&bucket)
.key(key)
.body(b"enabled-history".to_vec().into())
.send()
.await?
.version_id()
.ok_or("enabled PUT omitted version ID")?
.to_string();
writer
.put_bucket_versioning()
.bucket(&bucket)
.versioning_configuration(
VersioningConfiguration::builder()
.status(BucketVersioningStatus::Suspended)
.build(),
)
.send()
.await?;
put_object(&writer, &bucket, key, b"null-one".to_vec()).await?;
put_object(&writer, &bucket, key, b"null-two".to_vec()).await?;
assert_eq!(get_object_bytes(&reader, &bucket, key).await?, b"null-two");
let versions = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?;
let matching: Vec<_> = versions
.versions()
.iter()
.filter(|version| version.key() == Some(key))
.collect();
assert!(matching.iter().any(|version| version.version_id() == Some(original.as_str())));
let null_version_count = matching
.iter()
.filter(|version| {
matches!(
version.version_id(),
None | Some("") | Some("null") | Some("00000000-0000-0000-0000-000000000000")
)
})
.count();
assert_eq!(null_version_count, 1, "suspended overwrites must keep one null version: {versions:?}");
let historical = reader
.get_object()
.bucket(&bucket)
.key(key)
.version_id(&original)
.send()
.await?;
assert_eq!(historical.body.collect().await?.into_bytes().as_ref(), b"enabled-history");
Ok(())
}
-5
View File
@@ -378,11 +378,6 @@ mod bucket_stats_regression_test;
#[cfg(test)]
mod distributed_startup_regression_test;
// 4-node / 4-disk distributed Actions suite (S3, lock, versioning, replication,
// quota, observability, expand/decommission/rebalance, site replication, chaos).
#[cfg(test)]
mod distributed;
// P1 regression: tier/ILM transition (rustfs#5218, #5130, #5011, #4826, #5024)
#[cfg(test)]
mod tier_transition_regression_test;
+3 -5
View File
@@ -17,15 +17,13 @@ Use the canonical CI-equivalent protocol command in the parent
For targeted debugging of the core suite only:
```bash
RUSTFS_BUILD_FEATURES=ftps,webdav,sftp cargo test --package e2e_test test_protocol_core_suite -- --test-threads=1 --nocapture
python3 scripts/e2e_binary.py build --features ftps,webdav,sftp
python3 scripts/e2e_binary.py run --features ftps,webdav,sftp -- cargo test --package e2e_test test_protocol_core_suite -- --test-threads=1 --nocapture
```
This targeted command does not cover the full `e2e-protocols` profile.
`RUSTFS_BUILD_FEATURES` controls which features the test rustfs binary is
built with. When this variable is set, the protocol test runner schedules
only entries whose protocol is present in the requested feature list. Leave
it unset to run every protocol entry.
`e2e_binary.py` supplies `RUSTFS_BUILD_FEATURES` from the verified server's resolved Cargo features. The protocol runner schedules only entries present in that feature list; helpers check that their required features are available without rebuilding the server.
`--test-threads=1` is required because every entry spawns a rustfs server
on fixed bind ports.
-1
View File
@@ -215,7 +215,6 @@ serde_urlencoded.workspace = true
google-cloud-storage = { workspace = true }
google-cloud-auth = { workspace = true }
faster-hex = { workspace = true }
quick-xml = { workspace = true }
ratelimit = { workspace = true }
aws-smithy-http-client = { workspace = true, default-features = false, features = ["rustls-aws-lc"] }
+10 -18
View File
@@ -153,13 +153,12 @@ pub mod bucket {
LastSourceError, LatencyBucketSnapshot, NEGATIVE_CACHE_MAX_ENTRIES, NegativeCache, OdmBucketSnapshot, OdmLookup,
OdmOp, OdmOutcome, OdmStateError, OdmStats, OdmStatsSnapshot, OnDemandMigrationSys, PullError, PullFailureReason,
PullFollower, PullLeader, PullOutcome, PullPath, PullResult, PullSlot, SOURCE_LATENCY_BUCKET_BOUNDS_MS,
SourceLatencySnapshot, source_backend_spec, source_client_spec,
SourceLatencySnapshot, source_client_spec,
};
pub use crate::bucket::on_demand_migration::{
AzureSourceConfig, ConfigPublishHook, FilterConfig, GcsSourceConfig, HeadPolicy, ON_DEMAND_MIGRATION_CONFIG_HOOK,
ON_DEMAND_MIGRATION_CONFIG_VERSION, OnDemandMigrationConfig, OnDemandMigrationConfigError, PathStyle, PolicyConfig,
Provider, RangeGetPolicy, SourceConfig, SourceCredentials, SourceErrorPolicy, SourceTimeout, TlsConfig,
ValidationContext,
ConfigPublishHook, FilterConfig, HeadPolicy, ON_DEMAND_MIGRATION_CONFIG_HOOK, ON_DEMAND_MIGRATION_CONFIG_VERSION,
OnDemandMigrationConfig, OnDemandMigrationConfigError, PathStyle, PolicyConfig, Provider, RangeGetPolicy,
SourceConfig, SourceCredentials, SourceErrorPolicy, SourceTimeout, TlsConfig, ValidationContext,
};
pub use crate::bucket::on_demand_migration::{
EnqueueOutcome, LocalObject, MAX_MULTIPART_PARTS, OdmWriteBack, PULL_MAX_RETRIES, PULL_RETRY_BASE_DELAYS,
@@ -168,10 +167,9 @@ pub mod bucket {
idle_guarded_body,
};
pub use crate::bucket::on_demand_migration::{
FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger,
ListThroughToken, ListThroughTokenError, MAX_LIST_FETCHES_PER_SIDE, MAX_LIST_NO_PROGRESS_PAGES, MergeOutcome,
MergePick, MergeSide, SOURCE_LIST_MAX_RATE_WAIT, SOURCE_LIST_RATE_PER_SEC, SourceListPlan, SourceListRateLimiter,
decode_continuation_token, source_list_plan,
FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListThroughCursor, ListThroughMerger, ListThroughToken,
ListThroughTokenError, MAX_LIST_FETCHES_PER_SIDE, MergeOutcome, MergePick, MergeSide, SOURCE_LIST_MAX_RATE_WAIT,
SOURCE_LIST_RATE_PER_SEC, SourceListPlan, SourceListRateLimiter, decode_continuation_token, source_list_plan,
};
pub mod backfill {
pub use crate::bucket::on_demand_migration::backfill::{
@@ -186,9 +184,9 @@ pub mod bucket {
}
pub mod source_client {
pub use crate::bucket::on_demand_migration::source_client::{
AzureAuth, AzureSourceSpec, GcsSourceSpec, SourceBackendSpec, SourceClient, SourceClientSpec, SourceError,
SourceGet, SourceHead, SourceListRequest, SourceObject, SourcePage, SourceProbe, SourceProvider, SourceSse,
SourceTimeouts, USER_AGENT_SUFFIX, is_multipart_etag, range_header_value, resolve_path_style,
SourceClient, SourceClientSpec, SourceError, SourceGet, SourceHead, SourceListRequest, SourceObject, SourcePage,
SourceProbe, SourceProvider, SourceSse, SourceTimeouts, USER_AGENT_SUFFIX, is_multipart_etag, range_header_value,
resolve_path_style,
};
}
}
@@ -563,12 +561,6 @@ pub mod set_disk {
pub mod test_util {
pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test;
pub use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause};
/// Keep a namespace commit pending until the returned owner is dropped.
#[must_use]
pub fn hold_namespace_commit(store: &crate::store::ECStore) -> impl Send + Sync {
store.ctx.begin_namespace_commit()
}
}
}
File diff suppressed because it is too large Load Diff
@@ -1,172 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! One contract every [`SourceBackend`] implementation must satisfy.
//!
//! The migration pipeline talks to a source only through the trait, so a new
//! provider is correct exactly when it answers the same questions the same way:
//! the same head fields, the same range semantics, the same page shape, the
//! same error classes. Each backend supplies a fixture that answers this fixed
//! corpus in its own dialect and then runs [`assert_backend_contract`], so a
//! provider-specific mapping bug shows up as a contract failure rather than as
//! a surprise in the pull pipeline.
//!
//! Backends differ in two documented ways, declared through
//! [`BackendCapabilities`]: whether the provider's ETag is a content digest,
//! and whether the provider can resume a listing from a key.
use super::source_client::{SourceBackend, SourceError, SourceListRequest};
use crate::storage_api_contracts::range::HTTPRangeSpec;
use std::collections::HashMap;
/// The single object every fixture serves.
pub(super) const OBJECT_KEY: &str = "dir/a.txt";
pub(super) const OBJECT_BODY: &[u8] = b"hello";
/// MD5 of [`OBJECT_BODY`]; the ETag of the object on a digest provider.
pub(super) const OBJECT_MD5: &str = "5d41402abc4b2a76b9719d911017c592";
/// The second key the fixture's listing returns, on its second page.
pub(super) const SECOND_KEY: &str = "dir/b.txt";
pub(super) const COMMON_PREFIX: &str = "dir/sub/";
pub(super) const LIST_CURSOR: &str = "cursor-1";
/// A key the fixture answers with the provider's "no such object".
pub(super) const MISSING_KEY: &str = "missing";
/// A key the fixture answers with the provider's "not authorized".
pub(super) const FORBIDDEN_KEY: &str = "secret";
/// Where backends are allowed to differ.
#[derive(Clone, Copy, Debug)]
pub(super) struct BackendCapabilities {
/// The provider's ETag is an opaque token, not a digest of the bytes.
pub(super) etag_is_opaque: bool,
/// The provider can resume a listing from a key rather than only from an
/// opaque cursor.
pub(super) supports_start_after: bool,
/// The provider has an object-tagging concept at all. GCS does not, and
/// answers with an empty map instead of failing a pull.
pub(super) supports_tagging: bool,
}
/// Drives `backend` through the shared corpus. Fixtures are scripted in
/// request order, so the call order here is part of the contract.
pub(super) async fn assert_backend_contract(backend: &dyn SourceBackend, caps: BackendCapabilities) {
// 1. HEAD maps the object's shared fields.
let head = backend.head(OBJECT_KEY).await.expect("HEAD of the fixture object");
assert_eq!(head.size, OBJECT_BODY.len() as u64, "HEAD reports the object size");
assert_eq!(head.content_type.as_deref(), Some("text/plain"));
assert_eq!(
head.user_metadata,
HashMap::from([("owner".to_string(), "alice".to_string())]),
"user metadata is keyed without the provider prefix"
);
assert!(head.storage_class.is_some(), "the provider's tier is recorded");
assert!(head.last_modified.is_some(), "the provider's timestamp is parsed");
assert!(head.sse.is_none(), "the fixture object is not server-side encrypted");
assert!(!head.is_multipart_etag);
assert_eq!(head.etag_is_opaque, caps.etag_is_opaque);
match caps.etag_is_opaque {
false => assert_eq!(head.etag.as_deref(), Some(OBJECT_MD5), "a digest ETag is mapped verbatim"),
true => assert!(head.etag.is_some(), "an opaque ETag is still recorded"),
}
// 2. An unranged GET streams the whole object and reports no range.
let got = backend.get(OBJECT_KEY, None).await.expect("unranged GET");
assert_eq!(got.head.size, OBJECT_BODY.len() as u64);
assert!(got.content_range.is_none(), "an unranged GET has no content-range");
assert_eq!(got.head.etag_is_opaque, caps.etag_is_opaque, "GET and HEAD agree about the ETag");
let body = got.body.collect().await.expect("body streams").into_bytes();
assert_eq!(body.as_ref(), OBJECT_BODY);
// 3. A ranged GET returns exactly the requested interval, and `size` is
// the length of the returned bytes rather than of the object.
let range = HTTPRangeSpec {
is_suffix_length: false,
start: 1,
end: 3,
};
let got = backend.get(OBJECT_KEY, Some(&range)).await.expect("ranged GET");
assert_eq!(got.head.size, 3, "a ranged GET reports the range length");
assert_eq!(got.content_range.as_deref(), Some("bytes 1-3/5"));
let body = got.body.collect().await.expect("body streams").into_bytes();
assert_eq!(body.as_ref(), &OBJECT_BODY[1..=3]);
// 4. A delimiter listing rolls prefixes up and hands back a cursor.
let page = backend
.list(&SourceListRequest {
prefix: Some("dir/"),
delimiter: Some("/"),
max_keys: 2,
..Default::default()
})
.await
.expect("first listing page");
assert_eq!(page.objects.len(), 1, "the first page holds one object");
assert_eq!(page.objects[0].key, OBJECT_KEY, "listing keys are in the source namespace");
assert_eq!(page.objects[0].size, OBJECT_BODY.len() as u64);
assert!(page.objects[0].last_modified.is_some());
assert_eq!(page.common_prefixes, vec![COMMON_PREFIX.to_string()]);
assert!(page.is_truncated);
assert_eq!(page.next_continuation_token.as_deref(), Some(LIST_CURSOR));
// 5. The cursor is passed back verbatim and the last page ends the walk.
let page = backend
.list(&SourceListRequest {
prefix: Some("dir/"),
delimiter: Some("/"),
continuation_token: Some(LIST_CURSOR),
max_keys: 2,
..Default::default()
})
.await
.expect("second listing page");
assert_eq!(page.objects.len(), 1);
assert_eq!(page.objects[0].key, SECOND_KEY);
assert!(!page.is_truncated);
assert!(page.next_continuation_token.is_none(), "a complete listing carries no cursor");
// 6. Tags come back as a flat map, empty on a provider without tags.
let tags = backend.tagging(OBJECT_KEY).await.expect("object tags");
match caps.supports_tagging {
true => assert_eq!(tags, HashMap::from([("env".to_string(), "prod".to_string())])),
false => assert!(tags.is_empty(), "a provider without tags reports none: {tags:?}"),
}
// 7. The probe confirms the bucket or container answers.
backend.probe().await.expect("probe of the fixture bucket");
// 8. A missing object is `NotFound`, and never retried.
let err = backend.head(MISSING_KEY).await.expect_err("a missing object must fail");
assert!(matches!(err, SourceError::NotFound), "{err:?}");
assert_eq!(err.class_label(), "not_found");
assert!(!err.is_retryable());
// 9. A denied object is `AccessDenied`, and never retried.
let err = backend.head(FORBIDDEN_KEY).await.expect_err("a denied object must fail");
assert!(matches!(err, SourceError::AccessDenied), "{err:?}");
assert_eq!(err.class_label(), "access_denied");
assert!(!err.is_retryable());
// 10. A provider without a key cursor must refuse one instead of listing
// from the wrong position. This issues no request either way.
if !caps.supports_start_after {
let err = backend
.list(&SourceListRequest {
start_after: Some(OBJECT_KEY),
max_keys: 1,
..Default::default()
})
.await
.expect_err("a backend without a key cursor must refuse start_after");
assert!(matches!(err, SourceError::Unsupported(_)), "{err:?}");
}
}
@@ -25,8 +25,8 @@
//! [`BACKFILL_SAVE_INTERVAL`], and at every page end, with an `If-Match`
//! compare-and-set so a concurrent cancel or takeover is never overwritten.
//! - The `continuation_token` only advances once every pull queued from the
//! page before it has succeeded. After a failure it stays at that page,
//! so crash recovery cannot skip failed pulls (existing keys are skipped).
//! page before it has reported back, so a crash re-lists at most one page
//! (already-present keys are then skipped, never re-pulled).
//! - The owner holds a lease of [`BACKFILL_LEASE`] renewed by every save. The
//! recovery loop ([`run_backfill_recovery_loop`]) scans the buckets this
//! node has an ODM state for every [`BACKFILL_RECOVERY_INTERVAL`] and takes
@@ -367,8 +367,9 @@ pub struct LocalBackfillObject {
pub source_etag: Option<String>,
}
/// Shared report of a new or coalesced pull; absent only when not admitted.
pub type PullReport = Option<super::pull::QueuedPullReport>;
/// Receiver of one queued pull's report; `None` when the pull was coalesced
/// into one already running.
pub type PullReport = Option<oneshot::Receiver<QueuedPullOutcome>>;
/// Everything the job needs from its bucket, so the loop can run against a
/// mock in unit tests. Production: [`BucketBackfillContext`].
@@ -1190,11 +1191,9 @@ impl Job {
}
async fn main_loop(&mut self) -> Result<(), Stop> {
let mut cursor = self.checkpoint.continuation_token.clone();
let failed_at_resume = self.checkpoint.failed;
loop {
self.check_cancel()?;
let page = self.list_page(cursor.as_deref()).await?;
let page = self.list_page().await?;
for object in &page.objects {
self.check_cancel()?;
self.checkpoint.listed += 1;
@@ -1206,13 +1205,10 @@ impl Job {
self.drain_ready();
self.tick(false).await?;
}
// A persisted cursor certifies successful work, not just listing
// progress. Keep it at the first failed page for crash recovery.
// Only advance the cursor once every pull of this page reported
// back, so a takeover re-lists at most this page.
self.drain_all().await?;
cursor = page.next_continuation_token;
if self.checkpoint.failed == failed_at_resume {
self.checkpoint.continuation_token = cursor.clone();
}
self.checkpoint.continuation_token = page.next_continuation_token.clone();
self.tick(true).await?;
if !page.is_truncated {
return Ok(());
@@ -1227,7 +1223,7 @@ impl Job {
}
}
async fn list_page(&mut self, cursor: Option<&str>) -> Result<SourcePage, Stop> {
async fn list_page(&mut self) -> Result<SourcePage, Stop> {
let mut attempt = 0;
loop {
while !self.context.source_available() {
@@ -1235,7 +1231,7 @@ impl Job {
self.tick(false).await?;
}
let prefix = self.checkpoint.prefix.clone();
let token = cursor.map(str::to_string);
let token = self.checkpoint.continuation_token.clone();
match self
.context
.list_page(prefix.as_deref(), token.as_deref(), BACKFILL_LIST_PAGE_SIZE)
@@ -1309,10 +1305,9 @@ impl Job {
}
loop {
match self.context.enqueue(key) {
(EnqueueOutcome::Enqueued | EnqueueOutcome::Coalesced, report) => {
(EnqueueOutcome::Enqueued, report) => {
self.checkpoint.enqueued += 1;
let rx = report.ok_or(Stop::Unavailable)?;
{
if let Some(rx) = report {
let key = key.to_string();
self.outstanding.push(Box::pin(async move { (key, rx.await) }));
}
@@ -1327,6 +1322,11 @@ impl Job {
);
return Ok(());
}
(EnqueueOutcome::Coalesced, _) => {
// Someone else pulls it; its result is not ours to count.
self.checkpoint.enqueued += 1;
return Ok(());
}
(EnqueueOutcome::QueueFull, _) => {
// Wait, never drop: one completion frees a slot.
if self.outstanding.is_empty() {
@@ -1640,7 +1640,6 @@ mod tests {
queue_capacity: usize,
pending: Mutex<Vec<(String, oneshot::Sender<QueuedPullOutcome>)>>,
fail_keys: HashSet<String>,
coalesced: bool,
auto_complete: AtomicBool,
cancel: CancellationToken,
config_updated_at: Mutex<Option<OffsetDateTime>>,
@@ -1668,7 +1667,6 @@ mod tests {
queue_capacity: usize::MAX,
pending: Mutex::new(Vec::new()),
fail_keys: HashSet::new(),
coalesced: false,
auto_complete: AtomicBool::new(true),
cancel: CancellationToken::new(),
config_updated_at: Mutex::new(Some(ts(1_700_000_000))),
@@ -1748,12 +1746,7 @@ mod tests {
} else {
self.pending.lock().push((key.to_string(), tx));
}
let outcome = if self.coalesced {
EnqueueOutcome::Coalesced
} else {
EnqueueOutcome::Enqueued
};
(outcome, Some(futures::FutureExt::shared(rx)))
(EnqueueOutcome::Enqueued, Some(rx))
}
fn cancel_token(&self) -> CancellationToken {
@@ -1919,7 +1912,7 @@ mod tests {
#[tokio::test]
async fn failed_pulls_are_counted_hashed_and_finish_with_failures() {
let bucket = "backfill-failed";
let mut context = MockContext::new(5, 2);
let mut context = MockContext::new(5, 1000);
Arc::get_mut(&mut context)
.expect("unshared")
.fail_keys
@@ -1934,52 +1927,12 @@ mod tests {
.checkpoint;
assert_eq!(cp.state, BackfillState::CompletedWithFailures);
assert_eq!((cp.pulled, cp.failed), (4, 1));
assert_eq!(cp.continuation_token.as_deref(), Some("2"), "retain the first failed page for recovery");
assert_eq!(cp.failed_keys, vec![key_hash("k/00002")]);
let last = cp.last_error.expect("last error");
assert_eq!(last.class, "local_write");
assert_eq!(last.key_hash.as_deref(), Some(key_hash("k/00002").as_str()));
}
#[tokio::test]
async fn coalesced_pulls_block_the_checkpoint_and_report_failures() {
let bucket = "backfill-coalesced";
let mut context = MockContext::new(1, 1);
{
let ctx = Arc::get_mut(&mut context).expect("unshared");
ctx.coalesced = true;
ctx.auto_complete = AtomicBool::new(false);
ctx.fail_keys.insert("k/00000".to_string());
}
let (_dirs, store, runner) = runner_with("node-a", bucket, Arc::clone(&context)).await;
runner.start(bucket, BackfillRequest::default()).await.expect("start");
tokio::time::timeout(Duration::from_secs(10), async {
while context.pending.lock().is_empty() {
tokio::task::yield_now().await;
}
})
.await
.expect("job enqueued");
assert!(runner.is_running_locally(bucket), "coalescing is not completion");
let cp = read_checkpoint(&store, bucket)
.await
.expect("read")
.expect("checkpoint")
.checkpoint;
assert!(cp.state.is_active());
assert!(cp.continuation_token.is_none());
context.complete_pending();
runner.wait_until_idle(bucket).await;
let cp = read_checkpoint(&store, bucket)
.await
.expect("read")
.expect("checkpoint")
.checkpoint;
assert_eq!(cp.state, BackfillState::CompletedWithFailures);
assert_eq!((cp.enqueued, cp.pulled, cp.failed), (1, 0, 1));
assert_eq!(cp.failed_keys, vec![key_hash("k/00000")]);
}
#[tokio::test]
async fn listing_failure_marks_the_job_failed_with_the_error_class() {
let bucket = "backfill-list-error";
@@ -2192,68 +2145,6 @@ mod tests {
assert_eq!(runner.recover_once().await.taken_over, 0, "a finished job is not recovered");
}
#[tokio::test]
async fn recovery_advances_past_historical_failures_but_pins_new_failures() {
let bucket = "backfill-takeover-failed";
let mut context = MockContext::new(8, 2);
{
let ctx = Arc::get_mut(&mut context).expect("unshared");
ctx.auto_complete = AtomicBool::new(false);
ctx.fail_keys.insert("k/00004".to_string());
}
let (_dirs, store, runner) = runner_with("node-b", bucket, Arc::clone(&context)).await;
let crashed_at = OffsetDateTime::now_utc() - Duration::from_secs(300);
let mut crashed = BackfillCheckpoint::new(&BackfillRequest::default(), ts(1_700_000_000), "node-a", crashed_at);
crashed.continuation_token = Some("2".to_string());
crashed.failed = 1;
crashed.record_failure("local_write", Some("k/00002"), crashed_at);
write_checkpoint(&store, bucket, &crashed, None)
.await
.expect("seed failed page with an expired lease");
assert_eq!(runner.recover_once().await.taken_over, 1);
for (page_start, durable_token, failures) in [(2, "2", 1), (4, "4", 1), (6, "4", 2)] {
tokio::time::timeout(Duration::from_secs(10), async {
loop {
if context.pending.lock().len() == 2 {
break;
}
tokio::task::yield_now().await;
}
})
.await
.expect("resumed page enqueued before its reports complete");
assert_eq!(
context.pending.lock().iter().map(|(key, _)| key.clone()).collect::<Vec<_>>(),
vec![format!("k/{page_start:05}"), format!("k/{:05}", page_start + 1)]
);
let cp = read_checkpoint(&store, bucket)
.await
.expect("read persisted page boundary")
.expect("checkpoint")
.checkpoint;
assert_eq!(cp.job_id, crashed.job_id);
assert_eq!(cp.owner.as_ref().map(|owner| owner.node.as_str()), Some("node-b"));
assert_eq!(cp.continuation_token.as_deref(), Some(durable_token));
assert_eq!(cp.failed, failures);
context.complete_pending();
}
runner.wait_until_idle(bucket).await;
let cp = read_checkpoint(&store, bucket)
.await
.expect("read completed checkpoint")
.expect("checkpoint")
.checkpoint;
assert_eq!(cp.state, BackfillState::CompletedWithFailures);
assert_eq!((cp.pulled, cp.failed), (5, 2));
assert_eq!(cp.continuation_token.as_deref(), Some("4"));
assert_eq!(cp.failed_keys, vec![key_hash("k/00002"), key_hash("k/00004")]);
assert_eq!(
context.list_requests.lock().as_slice(),
&[Some("2".to_string()), Some("4".to_string()), Some("6".to_string())]
);
}
#[tokio::test]
async fn recovery_cancels_a_job_whose_config_changed_and_reclaims_own_node_jobs() {
let bucket = "backfill-recovery-config";
@@ -30,10 +30,6 @@ pub const ON_DEMAND_MIGRATION_CONFIG_VERSION: u32 = 1;
const REDACTED: &str = "REDACTED";
const AUTO_REGION: &str = "auto";
const AUTO_REGION_FALLBACK: &str = "us-east-1";
/// Public Azure Blob host suffix; the account name is the first label.
pub const AZURE_BLOB_SUFFIX: &str = "blob.core.windows.net";
/// Public Google Cloud Storage endpoint for the native provider.
pub const GCS_DEFAULT_ENDPOINT: &str = "https://storage.googleapis.com";
const KIB: u64 = 1024;
const MIB: u64 = 1024 * KIB;
@@ -79,25 +75,14 @@ pub struct SourceConfig {
pub bucket: String,
#[serde(default)]
pub path_style: PathStyle,
/// `None` means anonymous access to a public source bucket. Only the
/// SigV4 providers read it; `azure` and `gcs_native` carry their own
/// credentials in `azure` / `gcs`.
/// `None` means anonymous access to a public source bucket.
#[serde(default)]
pub credentials: Option<SourceCredentials>,
#[serde(default)]
pub tls: TlsConfig,
/// Required for [`Provider::Azure`] and rejected for every other
/// provider.
#[serde(default)]
pub azure: Option<AzureSourceConfig>,
/// Required for [`Provider::GcsNative`] and rejected for every other
/// provider. [`Provider::Gcs`] keeps using `credentials` because it
/// speaks the S3 interoperability API.
#[serde(default)]
pub gcs: Option<GcsSourceConfig>,
}
/// Source vendor family.
/// Source vendor family. `azure` is deliberately absent from this version.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum Provider {
@@ -109,12 +94,6 @@ pub enum Provider {
R2,
/// GCS XML interoperability API with HMAC keys.
Gcs,
/// Native Azure Blob service; parameters in `source.azure`.
Azure,
/// Native GCS JSON API with a service-account key; parameters in
/// `source.gcs`.
#[serde(rename = "gcs_native")]
GcsNative,
}
impl Provider {
@@ -126,22 +105,13 @@ impl Provider {
Provider::Rustfs => "rustfs",
Provider::R2 => "r2",
Provider::Gcs => "gcs",
Provider::Azure => "azure",
Provider::GcsNative => "gcs_native",
}
}
/// Providers that do not speak S3 and therefore ignore `region`,
/// `path_style` and `credentials`.
pub fn is_native(&self) -> bool {
matches!(self, Provider::Azure | Provider::GcsNative)
}
/// Providers whose SDKs accept `region = "auto"`; RustFS maps it to
/// `us-east-1` for signing. The native providers never sign with a
/// region, so they accept it as well.
/// `us-east-1` for signing.
fn accepts_auto_region(&self) -> bool {
matches!(self, Provider::R2 | Provider::Minio | Provider::Rustfs) || self.is_native()
matches!(self, Provider::R2 | Provider::Minio | Provider::Rustfs)
}
}
@@ -194,73 +164,6 @@ impl fmt::Debug for SourceCredentials {
}
}
/// Native Azure Blob source parameters. The container is `source.bucket`,
/// so a config never carries two names for the same container. Exactly one
/// of `account_key` and `sas_token` must be set: the account key signs with
/// Shared Key, the SAS token is appended to every request URL.
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct AzureSourceConfig {
/// Storage account name; also derives the default `blob.core.windows.net`
/// endpoint when `source.endpoint` is absent.
pub account: String,
/// Base64 shared key of the storage account.
#[serde(default)]
pub account_key: Option<String>,
/// SAS query string without the leading `?`.
#[serde(default)]
pub sas_token: Option<String>,
}
impl AzureSourceConfig {
/// A copy safe to return to admin clients or log: both secrets are
/// replaced by `REDACTED`, and whether each is set stays visible.
pub fn redacted(&self) -> Self {
Self {
account: self.account.clone(),
account_key: self.account_key.as_ref().map(|_| REDACTED.to_string()),
sas_token: self.sas_token.as_ref().map(|_| REDACTED.to_string()),
}
}
}
impl fmt::Debug for AzureSourceConfig {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("AzureSourceConfig")
.field("account", &self.account)
.field("account_key", &self.account_key.as_ref().map(|_| REDACTED))
.field("sas_token", &self.sas_token.as_ref().map(|_| REDACTED))
.finish()
}
}
/// Native Google Cloud Storage source parameters. The bucket is
/// `source.bucket`; only the service-account key lives here.
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct GcsSourceConfig {
/// Service-account key JSON, verbatim as downloaded from Google Cloud.
pub service_account_json: String,
}
impl GcsSourceConfig {
/// A copy safe to return to admin clients or log: the whole key JSON is
/// a secret (it embeds the private key), so it is replaced wholesale.
pub fn redacted(&self) -> Self {
Self {
service_account_json: REDACTED.to_string(),
}
}
}
impl fmt::Debug for GcsSourceConfig {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("GcsSourceConfig")
.field("service_account_json", &REDACTED)
.finish()
}
}
#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct TlsConfig {
@@ -451,14 +354,6 @@ pub enum OnDemandMigrationConfigError {
InvalidBucket(&'static str),
#[error("source credentials field {0} must not be empty")]
EmptyCredential(&'static str),
#[error("source.{0} is required for provider {1}")]
MissingProviderBlock(&'static str, Provider),
#[error("source.{0} is not valid for provider {1}")]
UnexpectedProviderBlock(&'static str, Provider),
/// Carries only the reason: the block holds account keys, SAS tokens and
/// service-account JSON, so no value of it is ever echoed.
#[error("source.{0} is invalid: {1}")]
InvalidProviderBlock(&'static str, &'static str),
#[error("source tls.ca_cert_pem is not a PEM certificate")]
InvalidCaCert,
#[error("filter.{0} must be null or a non-empty string")]
@@ -493,8 +388,6 @@ impl OnDemandMigrationConfig {
pub fn redacted(&self) -> Self {
let mut copy = self.clone();
copy.source.credentials = self.source.credentials.as_ref().map(SourceCredentials::redacted);
copy.source.azure = self.source.azure.as_ref().map(AzureSourceConfig::redacted);
copy.source.gcs = self.source.gcs.as_ref().map(GcsSourceConfig::redacted);
copy
}
@@ -540,12 +433,6 @@ impl SourceConfig {
match (&self.endpoint, self.provider) {
(Some(endpoint), _) => endpoint.clone(),
(None, Provider::Aws) => format!("https://s3.{}.amazonaws.com", self.region),
(None, Provider::Azure) => self
.azure
.as_ref()
.map(|azure| format!("https://{}.{AZURE_BLOB_SUFFIX}", azure.account))
.unwrap_or_default(),
(None, Provider::GcsNative) => GCS_DEFAULT_ENDPOINT.to_string(),
(None, _) => String::new(),
}
}
@@ -561,8 +448,6 @@ impl SourceConfig {
}
fn validate(&self) -> Result<(), OnDemandMigrationConfigError> {
self.validate_provider_block()?;
if self.region.is_empty() {
return Err(OnDemandMigrationConfigError::EmptyRegion);
}
@@ -581,9 +466,6 @@ impl SourceConfig {
));
}
}
// Both native providers derive a fixed endpoint; Azure's is built
// from the account name, already checked by `validate_provider_block`.
None if self.provider.is_native() => {}
None => return Err(OnDemandMigrationConfigError::MissingEndpoint(self.provider)),
}
@@ -614,84 +496,6 @@ impl SourceConfig {
Ok(())
}
/// The provider-specific block must be present for exactly its own
/// provider: a stray `azure` block on an `s3` source would otherwise be
/// accepted, stored, and silently ignored by the client builder.
fn validate_provider_block(&self) -> Result<(), OnDemandMigrationConfigError> {
let missing = OnDemandMigrationConfigError::MissingProviderBlock;
let unexpected = OnDemandMigrationConfigError::UnexpectedProviderBlock;
let invalid = OnDemandMigrationConfigError::InvalidProviderBlock;
if self.provider != Provider::Azure && self.azure.is_some() {
return Err(unexpected("azure", self.provider));
}
if self.provider != Provider::GcsNative && self.gcs.is_some() {
return Err(unexpected("gcs", self.provider));
}
match self.provider {
Provider::Azure => {
let azure = self.azure.as_ref().ok_or(missing("azure", self.provider))?;
if azure.account.is_empty() {
return Err(invalid("azure", "account must not be empty"));
}
// The account feeds a hostname when the endpoint is derived:
// keep it to label characters so it cannot rewrite the host.
if !azure.account.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-') {
return Err(invalid("azure", "account contains characters outside [A-Za-z0-9-]"));
}
match (azure.account_key.as_deref(), azure.sas_token.as_deref()) {
(Some(_), Some(_)) => return Err(invalid("azure", "account_key and sas_token are mutually exclusive")),
(None, None) => return Err(invalid("azure", "one of account_key and sas_token is required")),
(Some(key), None) => {
if key.is_empty() {
return Err(invalid("azure", "account_key must not be empty"));
}
// Decoded here so a mistyped key fails at the admin
// boundary instead of on the first source request.
if base64_simd::STANDARD.decode_to_vec(key.as_bytes()).is_err() {
return Err(invalid("azure", "account_key is not base64"));
}
}
(None, Some(sas)) => {
if sas.is_empty() {
return Err(invalid("azure", "sas_token must not be empty"));
}
if sas.starts_with('?') {
return Err(invalid("azure", "sas_token must not start with '?'"));
}
if sas.chars().any(char::is_whitespace) {
return Err(invalid("azure", "sas_token must not contain whitespace"));
}
}
}
}
Provider::GcsNative => {
let gcs = self.gcs.as_ref().ok_or(missing("gcs", self.provider))?;
let key: serde_json::Value = serde_json::from_str(&gcs.service_account_json)
.map_err(|_| invalid("gcs", "service_account_json is not valid JSON"))?;
let Some(object) = key.as_object() else {
return Err(invalid("gcs", "service_account_json is not a JSON object"));
};
if object.get("type").and_then(serde_json::Value::as_str) != Some("service_account") {
return Err(invalid("gcs", "service_account_json is not a service_account key"));
}
for field in ["client_email", "private_key"] {
if object
.get(field)
.and_then(serde_json::Value::as_str)
.is_none_or(str::is_empty)
{
return Err(invalid("gcs", "service_account_json is missing client_email or private_key"));
}
}
}
Provider::S3 | Provider::Aws | Provider::Minio | Provider::Rustfs | Provider::R2 | Provider::Gcs => {}
}
Ok(())
}
}
fn validate_endpoint(endpoint: &str) -> Result<(), OnDemandMigrationConfigError> {
@@ -895,15 +699,7 @@ mod tests {
),
(
"provider enum",
r#"{"source":{"provider":"swift","endpoint":"https://h","region":"r","bucket":"b"}}"#,
),
(
"azure block",
r#"{"source":{"provider":"azure","region":"auto","bucket":"b","azure":{"account":"acct","account_key":"a2V5","extra":1}}}"#,
),
(
"gcs block",
r#"{"source":{"provider":"gcs_native","region":"auto","bucket":"b","gcs":{"service_account_json":"{}","extra":1}}}"#,
r#"{"source":{"provider":"azure","endpoint":"https://h","region":"r","bucket":"b"}}"#,
),
] {
let err = OnDemandMigrationConfig::from_json(json.as_bytes()).expect_err(label);
@@ -1024,201 +820,9 @@ mod tests {
"{provider}"
);
}
// The native providers never sign with a region, so "auto" is the
// honest value to write for them.
for cfg in [azure_cfg(), gcs_native_cfg()] {
assert_eq!(cfg.source.region, "auto");
cfg.validate(empty_ctx())
.unwrap_or_else(|err| panic!("{}: {err}", cfg.source.provider));
}
assert_eq!(sample().source.effective_region(), "us-west-1");
}
const SERVICE_ACCOUNT_JSON: &str = r#"{"type":"service_account","project_id":"p","client_email":"a@b.iam.gserviceaccount.com","private_key":"-----BEGIN PRIVATE KEY-----\nsecret\n-----END PRIVATE KEY-----"}"#;
fn azure_cfg() -> OnDemandMigrationConfig {
let mut cfg = sample();
cfg.source.provider = Provider::Azure;
cfg.source.endpoint = None;
cfg.source.region = "auto".to_string();
cfg.source.credentials = None;
cfg.source.azure = Some(AzureSourceConfig {
account: "legacyaccount".to_string(),
account_key: Some("c2VjcmV0LWtleQ==".to_string()),
sas_token: None,
});
cfg
}
fn gcs_native_cfg() -> OnDemandMigrationConfig {
let mut cfg = sample();
cfg.source.provider = Provider::GcsNative;
cfg.source.endpoint = None;
cfg.source.region = "auto".to_string();
cfg.source.credentials = None;
cfg.source.gcs = Some(GcsSourceConfig {
service_account_json: SERVICE_ACCOUNT_JSON.to_string(),
});
cfg
}
#[test]
fn native_providers_derive_their_endpoint_and_round_trip_on_the_wire() {
let azure = azure_cfg();
assert_eq!(azure.source.effective_endpoint(), "https://legacyaccount.blob.core.windows.net");
let gcs = gcs_native_cfg();
assert_eq!(gcs.source.effective_endpoint(), "https://storage.googleapis.com");
for cfg in [azure_cfg(), gcs_native_cfg()] {
let json = cfg.to_json().expect("config must serialize");
assert_eq!(OnDemandMigrationConfig::from_json(&json).expect("config must parse"), cfg);
}
// The wire labels are part of the admin contract.
assert!(
String::from_utf8(azure_cfg().to_json().expect("json"))
.expect("utf8")
.contains(r#""provider":"azure""#)
);
assert!(
String::from_utf8(gcs_native_cfg().to_json().expect("json"))
.expect("utf8")
.contains(r#""provider":"gcs_native""#)
);
}
#[test]
fn an_explicit_endpoint_overrides_the_derived_native_one() {
// Azurite and fake-gcs-server are addressed this way.
let mut cfg = azure_cfg();
cfg.source.endpoint = Some("http://azurite.example.com:10000".to_string());
cfg.validate(empty_ctx()).expect("an explicit native endpoint is allowed");
assert_eq!(cfg.source.effective_endpoint(), "http://azurite.example.com:10000");
cfg.source.endpoint = Some("http://azurite.example.com:10000/devstoreaccount1".to_string());
assert!(
matches!(cfg.validate(empty_ctx()), Err(OnDemandMigrationConfigError::InvalidEndpoint(_))),
"a native endpoint is still an origin"
);
}
#[test]
fn a_provider_block_belongs_to_exactly_its_own_provider() {
let mut cfg = sample();
cfg.source.azure = azure_cfg().source.azure;
assert_eq!(
cfg.validate(empty_ctx()),
Err(OnDemandMigrationConfigError::UnexpectedProviderBlock("azure", Provider::S3))
);
let mut cfg = sample();
cfg.source.gcs = gcs_native_cfg().source.gcs;
assert_eq!(
cfg.validate(empty_ctx()),
Err(OnDemandMigrationConfigError::UnexpectedProviderBlock("gcs", Provider::S3))
);
let mut cfg = azure_cfg();
cfg.source.azure = None;
assert_eq!(
cfg.validate(empty_ctx()),
Err(OnDemandMigrationConfigError::MissingProviderBlock("azure", Provider::Azure))
);
let mut cfg = gcs_native_cfg();
cfg.source.gcs = None;
assert_eq!(
cfg.validate(empty_ctx()),
Err(OnDemandMigrationConfigError::MissingProviderBlock("gcs", Provider::GcsNative))
);
}
#[test]
fn azure_block_rules() {
let with = |account: &str, key: Option<&str>, sas: Option<&str>| {
let mut cfg = azure_cfg();
cfg.source.azure = Some(AzureSourceConfig {
account: account.to_string(),
account_key: key.map(str::to_string),
sas_token: sas.map(str::to_string),
});
cfg.validate(empty_ctx())
};
with("legacyaccount", None, Some("sv=2021-08-06&sig=abc%3D")).expect("a SAS token is a complete credential");
with("legacyaccount", Some("c2VjcmV0LWtleQ=="), None).expect("an account key is a complete credential");
for (label, result) in [
("empty account", with("", Some("c2VjcmV0LWtleQ=="), None)),
// The account becomes the first label of the derived hostname.
("account with a dot", with("legacy.account", Some("c2VjcmV0LWtleQ=="), None)),
("account with a slash", with("legacy/account", Some("c2VjcmV0LWtleQ=="), None)),
("no credential", with("legacyaccount", None, None)),
("both credentials", with("legacyaccount", Some("c2VjcmV0LWtleQ=="), Some("sv=1"))),
("empty key", with("legacyaccount", Some(""), None)),
("key that is not base64", with("legacyaccount", Some("not base64!"), None)),
("empty sas", with("legacyaccount", None, Some(""))),
("sas with a leading question mark", with("legacyaccount", None, Some("?sv=1"))),
("sas with whitespace", with("legacyaccount", None, Some("sv=1 &sig=a"))),
] {
assert!(
matches!(result, Err(OnDemandMigrationConfigError::InvalidProviderBlock("azure", _))),
"{label}: {result:?}"
);
}
}
#[test]
fn gcs_native_block_requires_a_usable_service_account_key() {
let with = |json: &str| {
let mut cfg = gcs_native_cfg();
cfg.source.gcs = Some(GcsSourceConfig {
service_account_json: json.to_string(),
});
cfg.validate(empty_ctx())
};
with(SERVICE_ACCOUNT_JSON).expect("a service-account key is accepted");
for (label, json) in [
("empty", ""),
("not json", "not json"),
("not an object", "[]"),
("wrong type", r#"{"type":"authorized_user","client_email":"a@b","private_key":"k"}"#),
("no private key", r#"{"type":"service_account","client_email":"a@b"}"#),
("empty client email", r#"{"type":"service_account","client_email":"","private_key":"k"}"#),
] {
let result = with(json);
assert!(
matches!(result, Err(OnDemandMigrationConfigError::InvalidProviderBlock("gcs", _))),
"{label}: {result:?}"
);
}
}
#[test]
fn native_secrets_never_survive_redaction_or_debug() {
let mut azure = azure_cfg();
azure.source.azure.as_mut().expect("block").sas_token = Some("sv=2021-08-06&sig=top-secret".to_string());
azure.source.azure.as_mut().expect("block").account_key = None;
let gcs = gcs_native_cfg();
for rendered in [
format!("{:?}", azure.redacted()),
format!("{azure:?}"),
String::from_utf8(azure.redacted().to_json().expect("json")).expect("utf8"),
] {
assert!(!rendered.contains("top-secret"), "{rendered}");
assert!(rendered.contains("legacyaccount"), "the account name is not a secret: {rendered}");
}
for rendered in [
format!("{:?}", gcs.redacted()),
format!("{gcs:?}"),
String::from_utf8(gcs.redacted().to_json().expect("json")).expect("utf8"),
] {
assert!(!rendered.contains("PRIVATE KEY-----"), "{rendered}");
assert!(!rendered.contains("gserviceaccount"), "{rendered}");
}
}
#[test]
fn bucket_rules() {
let mut cfg = sample();
@@ -1,506 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Native Google Cloud Storage source backend.
//!
//! The `gcs` provider already reaches GCS through its S3 interoperability API,
//! which needs an HMAC key pair. This backend is the other half: it authorizes
//! with a service-account key, the credential most GCS projects actually issue,
//! by minting OAuth tokens through the shared `google-cloud-auth` credential
//! machinery the tier layer already uses.
//!
//! Two GCS surfaces are involved, each for the half it describes best. The read
//! path uses the XML API (`/{bucket}/{object}`), whose responses carry
//! `x-goog-meta-*` user metadata and the `x-goog-hash` digest in one round trip.
//! Listing uses the JSON API (`objects.list`), whose `pageToken` maps directly
//! onto the shared page cursor and whose `prefixes` are the delimiter roll-up.
//! Both accept the same bearer token.
//!
//! Every call this backend makes needs only `storage.objects.get` and
//! `storage.objects.list`, the two permissions of the `objectViewer` role, so a
//! key scoped to exactly the migration's needs works.
//!
//! `x-goog-hash` carries a base64 MD5 for every non-composite object; it is
//! converted to hex and becomes the head's ETag, so a pulled object is checked
//! against the digest GCS itself computed. A composite object has no MD5, and
//! its ETag is then marked opaque rather than checked.
use super::native_http::{
NativeHeadFields, NativeHttp, base64_md5_to_hex, header, native_source_head, parse_http_timestamp, read_text, response_body,
};
use super::source_client::{
GcsSourceSpec, SourceBackend, SourceError, SourceGet, SourceHead, SourceListRequest, SourceObject, SourcePage,
SourceTimeouts, range_header_value,
};
use crate::bucket::remote_s3_client::RemoteS3ClientError;
use crate::storage_api_contracts::range::HTTPRangeSpec;
use google_cloud_auth::credentials::service_account::{AccessSpecifier, Builder as ServiceAccountBuilder};
use google_cloud_auth::credentials::{CacheableResource, Credentials};
use http::{HeaderMap, HeaderValue, Method};
use serde::Deserialize;
use std::collections::HashMap;
use url::Url;
/// Read-only object scope: this backend never writes to the source.
const READ_ONLY_SCOPE: &str = "https://www.googleapis.com/auth/devstorage.read_only";
const METADATA_PREFIX: &str = "x-goog-meta-";
/// GCS reports its error code in the response body, not a header; the shared
/// transport takes a header name, so it is given one that never matches and
/// classification falls back to the status.
const NO_ERROR_CODE_HEADER: &str = "x-goog-unused-error-code";
/// One `objects.list` page is small; refuse an unbounded document.
const MAX_JSON_BYTES: usize = 8 * 1024 * 1024;
pub struct GcsNativeSourceBackend {
http: NativeHttp,
bucket: String,
credentials: Credentials,
}
impl GcsNativeSourceBackend {
pub fn new(
endpoint: &str,
bucket: &str,
spec: &GcsSourceSpec,
timeouts: SourceTimeouts,
skip_tls_verify: bool,
ca_cert_pem: Option<&str>,
) -> Result<Self, RemoteS3ClientError> {
let key: serde_json::Value = serde_json::from_str(&spec.service_account_json)
.map_err(|_| RemoteS3ClientError::Credentials("gcs service account key is not valid JSON"))?;
let credentials = ServiceAccountBuilder::new(key)
.with_access_specifier(AccessSpecifier::from_scopes([READ_ONLY_SCOPE]))
.build()
.map_err(|_| RemoteS3ClientError::Credentials("gcs service account key is not usable"))?;
Ok(Self {
http: NativeHttp::new(endpoint, timeouts, skip_tls_verify, ca_cert_pem)?,
bucket: bucket.to_string(),
credentials,
})
}
/// Authorization headers for one request. A credential failure is reported
/// as `AccessDenied` with no message: the renderer of a credential error
/// has the key material in scope, and the class is what callers act on.
async fn auth_headers(&self) -> Result<HeaderMap, SourceError> {
match self.credentials.headers(http::Extensions::new()).await {
Ok(CacheableResource::New { data, .. }) => Ok(data),
// Only returned when the caller passes an entity tag, which this
// backend never does; an empty set is still the honest answer.
Ok(CacheableResource::NotModified) => Ok(HeaderMap::new()),
Err(_) => Err(SourceError::AccessDenied),
}
}
/// XML API URL of one object; `/` in the key stay path separators.
fn object_url(&self, key: &str) -> Result<Url, SourceError> {
self.http.url(std::iter::once(self.bucket.as_str()).chain(key.split('/')))
}
/// JSON API URL of the bucket's object collection.
fn objects_url(&self) -> Result<Url, SourceError> {
self.http.url(["storage", "v1", "b", self.bucket.as_str(), "o"])
}
async fn request(&self, method: Method, url: Url, mut headers: HeaderMap) -> Result<reqwest::Request, SourceError> {
for (name, value) in self.auth_headers().await? {
if let Some(name) = name {
headers.insert(name, value);
}
}
let mut request = reqwest::Request::new(method, url);
*request.headers_mut() = headers;
Ok(request)
}
/// Shared mapping for the XML API's HEAD and GET responses.
fn head_from_response(headers: &HeaderMap) -> Result<SourceHead, SourceError> {
if header(headers, "x-goog-encryption-key-sha256").is_some() {
return Err(SourceError::Unsupported(
"source object uses a customer-supplied encryption key; customer-key sources are not supported".to_string(),
));
}
// `x-goog-hash` lists digests as `name=base64`, comma separated, and may
// repeat across header lines. Only the MD5 describes the whole object.
let md5 = headers
.get_all("x-goog-hash")
.iter()
.filter_map(|value| value.to_str().ok())
.flat_map(|value| value.split(','))
.filter_map(|digest| digest.trim().strip_prefix("md5="))
.find_map(base64_md5_to_hex);
let (etag, etag_is_opaque) = match md5 {
Some(md5) => (Some(md5), false),
// A composite object has no MD5; its ETag describes the composition
// rather than the bytes, so it is provenance only.
None => (header(headers, "etag").map(str::to_string), true),
};
native_source_head(
headers,
METADATA_PREFIX,
NativeHeadFields {
etag,
etag_is_opaque,
version_id: header(headers, "x-goog-generation").map(str::to_string),
storage_class: header(headers, "x-goog-storage-class").map(str::to_string),
},
)
}
}
#[async_trait::async_trait]
impl SourceBackend for GcsNativeSourceBackend {
async fn head(&self, key: &str) -> Result<SourceHead, SourceError> {
let request = self.request(Method::HEAD, self.object_url(key)?, HeaderMap::new()).await?;
let response = self.http.send(request, NO_ERROR_CODE_HEADER).await?;
Self::head_from_response(response.headers())
}
async fn get(&self, key: &str, range: Option<&HTTPRangeSpec>) -> Result<SourceGet, SourceError> {
let mut headers = HeaderMap::new();
if let Some(range) = range.map(range_header_value).transpose()? {
headers.insert(
http::header::RANGE,
HeaderValue::from_str(&range).map_err(|_| SourceError::Other("invalid range header".to_string()))?,
);
}
let request = self.request(Method::GET, self.object_url(key)?, headers).await?;
let response = self.http.send(request, NO_ERROR_CODE_HEADER).await?;
let head = Self::head_from_response(response.headers())?;
let content_range = header(response.headers(), "content-range").map(str::to_string);
Ok(SourceGet {
head,
body: response_body(response),
content_range,
})
}
async fn list(&self, request: &SourceListRequest<'_>) -> Result<SourcePage, SourceError> {
// `objects.list` offers `startOffset`, which is inclusive, so it cannot
// express "resume after this key" without silently repeating it.
if request.start_after.is_some() {
return Err(SourceError::Unsupported(
"gcs sources cannot resume a listing from a key; use the continuation token".to_string(),
));
}
let mut url = self.objects_url()?;
{
let mut query = url.query_pairs_mut();
if let Some(prefix) = request.prefix.filter(|prefix| !prefix.is_empty()) {
query.append_pair("prefix", prefix);
}
if let Some(delimiter) = request.delimiter.filter(|delimiter| !delimiter.is_empty()) {
query.append_pair("delimiter", delimiter);
}
if let Some(token) = request.continuation_token.filter(|token| !token.is_empty()) {
query.append_pair("pageToken", token);
}
if request.max_keys > 0 {
query.append_pair("maxResults", &request.max_keys.to_string());
}
}
let request = self.request(Method::GET, url, HeaderMap::new()).await?;
let response = self.http.send(request, NO_ERROR_CODE_HEADER).await?;
let body = read_text(response, MAX_JSON_BYTES).await?;
parse_objects_list(&body)
}
/// GCS has no object tagging API; user metadata is already carried by the
/// head mapping. An empty map keeps `policy.copy_tags` from failing a pull
/// over a concept the provider does not have.
async fn tagging(&self, _key: &str) -> Result<HashMap<String, String>, SourceError> {
Ok(HashMap::new())
}
/// A one-object listing, not `buckets.get`: the migration pipeline only
/// ever needs `storage.objects.list` and `storage.objects.get`, and a key
/// scoped to exactly those (the `objectViewer` role) cannot read the bucket
/// resource. Probing with `buckets.get` would reject a correct key.
async fn probe(&self) -> Result<(), SourceError> {
let mut url = self.objects_url()?;
url.query_pairs_mut().append_pair("maxResults", "1");
let request = self.request(Method::GET, url, HeaderMap::new()).await?;
let response = self.http.send(request, NO_ERROR_CODE_HEADER).await?;
read_text(response, MAX_JSON_BYTES)
.await
.and_then(|body| parse_objects_list(&body))?;
Ok(())
}
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
struct ObjectsList {
#[serde(default)]
items: Vec<ListedObject>,
#[serde(default)]
prefixes: Vec<String>,
#[serde(default)]
next_page_token: Option<String>,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
struct ListedObject {
name: String,
/// GCS renders the size as a decimal string, not a JSON number.
#[serde(default)]
size: Option<String>,
#[serde(default)]
updated: Option<String>,
#[serde(default)]
md5_hash: Option<String>,
#[serde(default)]
etag: Option<String>,
#[serde(default)]
storage_class: Option<String>,
}
fn parse_objects_list(body: &str) -> Result<SourcePage, SourceError> {
let listing: ObjectsList =
serde_json::from_str(body).map_err(|err| SourceError::Other(format!("source listing is not valid JSON: {err}")))?;
let next_continuation_token = listing.next_page_token.filter(|token| !token.is_empty());
let objects = listing
.items
.into_iter()
.map(|item| {
let etag = item
.md5_hash
.as_deref()
.and_then(base64_md5_to_hex)
.or_else(|| item.etag.map(|etag| etag.trim_matches('"').to_string()))
.filter(|etag| !etag.is_empty());
SourceObject {
key: item.name,
etag,
size: item.size.and_then(|size| size.parse().ok()).unwrap_or(0),
last_modified: item.updated.as_deref().and_then(parse_http_timestamp),
storage_class: item.storage_class,
// GCS never encodes a part count in a digest or an ETag.
is_multipart_etag: false,
}
})
.collect();
Ok(SourcePage {
objects,
common_prefixes: listing.prefixes,
is_truncated: next_continuation_token.is_some(),
next_continuation_token,
})
}
#[cfg(test)]
mod tests {
use super::*;
use crate::bucket::on_demand_migration::backend_contract::{BackendCapabilities, assert_backend_contract};
use crate::bucket::on_demand_migration::test_http_fixture::{ScriptedResponse, scripted_server};
use google_cloud_auth::credentials::anonymous::Builder as AnonymousBuilder;
const LIST_PAGE_ONE: &str = r#"{
"kind": "storage#objects",
"nextPageToken": "cursor-1",
"prefixes": ["dir/sub/"],
"items": [
{
"name": "dir/a.txt",
"size": "5",
"updated": "2015-10-21T07:28:00.000Z",
"md5Hash": "XUFAKrxLKna5cZ2REBfFkg==",
"etag": "CJizy9Wq0McCEAE=",
"storageClass": "STANDARD"
}
]
}"#;
const LIST_PAGE_TWO: &str = r#"{
"kind": "storage#objects",
"items": [
{
"name": "dir/b.txt",
"size": "7",
"updated": "2015-10-21T07:28:00.000Z",
"etag": "\"CJizy9Wq0McCEAI=\""
}
]
}"#;
fn backend(endpoint: &Url) -> GcsNativeSourceBackend {
GcsNativeSourceBackend {
http: NativeHttp::for_test(endpoint.clone()),
bucket: "legacy".to_string(),
// Anonymous credentials add no headers, so the fixture sees exactly
// the request this backend builds.
credentials: AnonymousBuilder::new().build(),
}
}
fn object_headers() -> Vec<(&'static str, String)> {
vec![
("Content-Type", "text/plain".to_string()),
("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT".to_string()),
("ETag", "\"CJizy9Wq0McCEAE=\"".to_string()),
("x-goog-hash", "crc32c=AAAAAA==,md5=XUFAKrxLKna5cZ2REBfFkg==".to_string()),
("x-goog-meta-owner", "alice".to_string()),
("x-goog-storage-class", "STANDARD".to_string()),
("x-goog-generation", "1445412480000000".to_string()),
]
}
#[test]
fn objects_list_maps_items_prefixes_and_the_page_token() {
let page = parse_objects_list(LIST_PAGE_ONE).expect("page should parse");
assert_eq!(page.common_prefixes, vec!["dir/sub/"]);
assert!(page.is_truncated);
assert_eq!(page.next_continuation_token.as_deref(), Some("cursor-1"));
assert_eq!(page.objects.len(), 1);
assert_eq!(page.objects[0].key, "dir/a.txt");
assert_eq!(page.objects[0].size, 5, "the string size is parsed");
assert_eq!(
page.objects[0].etag.as_deref(),
Some("5d41402abc4b2a76b9719d911017c592"),
"the base64 md5Hash becomes a hex ETag"
);
assert_eq!(page.objects[0].storage_class.as_deref(), Some("STANDARD"));
assert!(page.objects[0].last_modified.is_some(), "RFC 3339 `updated` is parsed");
let page = parse_objects_list(LIST_PAGE_TWO).expect("page should parse");
assert!(!page.is_truncated);
assert!(page.next_continuation_token.is_none());
assert_eq!(
page.objects[0].etag.as_deref(),
Some("CJizy9Wq0McCEAI="),
"without md5Hash the raw etag is carried"
);
assert!(parse_objects_list("not json").is_err());
}
#[tokio::test]
async fn head_prefers_the_goog_hash_md5_over_the_etag() {
let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, object_headers(), String::new())]).await;
let head = backend(&endpoint).head("dir/a b.txt").await.expect("HEAD should map");
let recorded = recorded.lock().expect("recorder lock").clone();
assert_eq!(recorded[0].method, "HEAD");
assert_eq!(recorded[0].target, "/legacy/dir/a%20b.txt", "the XML API addresses the object by path");
assert_eq!(
head.etag.as_deref(),
Some("5d41402abc4b2a76b9719d911017c592"),
"the x-goog-hash md5 is the content digest"
);
assert!(!head.etag_is_opaque, "a GCS md5 may be checked against the pulled bytes");
assert_eq!(head.user_metadata, HashMap::from([("owner".to_string(), "alice".to_string())]));
assert_eq!(head.version_id.as_deref(), Some("1445412480000000"));
assert_eq!(head.storage_class.as_deref(), Some("STANDARD"));
}
#[tokio::test]
async fn a_composite_object_without_an_md5_keeps_an_opaque_etag() {
let headers = object_headers()
.into_iter()
.map(|(name, value)| {
if name == "x-goog-hash" {
(name, "crc32c=AAAAAA==".to_string())
} else {
(name, value)
}
})
.collect();
let (endpoint, _) = scripted_server(vec![ScriptedResponse::new(200, headers, String::new())]).await;
let head = backend(&endpoint).head("composed").await.expect("HEAD should map");
assert_eq!(head.etag.as_deref(), Some("CJizy9Wq0McCEAE="));
assert!(head.etag_is_opaque, "a composite ETag describes the composition, not the bytes");
}
#[tokio::test]
async fn customer_supplied_key_objects_are_refused() {
let mut headers = object_headers();
headers.push(("x-goog-encryption-key-sha256", "abc".to_string()));
let (endpoint, _) = scripted_server(vec![ScriptedResponse::new(200, headers, String::new())]).await;
let err = backend(&endpoint)
.head("a.txt")
.await
.expect_err("CSEK objects are unsupported");
assert!(matches!(err, SourceError::Unsupported(_)), "{err:?}");
}
#[tokio::test]
async fn list_and_probe_address_the_json_api() {
let (endpoint, recorded) = scripted_server(vec![
ScriptedResponse::new(200, Vec::new(), LIST_PAGE_ONE.to_string()),
ScriptedResponse::new(200, Vec::new(), "{}".to_string()),
])
.await;
let backend = backend(&endpoint);
backend
.list(&SourceListRequest {
prefix: Some("dir/"),
delimiter: Some("/"),
continuation_token: Some("cursor-0"),
max_keys: 2,
..Default::default()
})
.await
.expect("listing should succeed");
backend.probe().await.expect("probe should succeed");
let recorded = recorded.lock().expect("recorder lock").clone();
assert!(recorded[0].target.starts_with("/storage/v1/b/legacy/o?"), "{}", recorded[0].target);
assert!(recorded[0].target.contains("prefix=dir%2F"), "{}", recorded[0].target);
assert!(recorded[0].target.contains("delimiter=%2F"), "{}", recorded[0].target);
assert!(recorded[0].target.contains("pageToken=cursor-0"), "{}", recorded[0].target);
assert!(recorded[0].target.contains("maxResults=2"), "{}", recorded[0].target);
assert_eq!(
recorded[1].target, "/storage/v1/b/legacy/o?maxResults=1",
"the probe uses the listing permission the pipeline already needs"
);
}
#[tokio::test]
async fn gcs_native_backend_satisfies_the_shared_backend_contract() {
let mut ranged = object_headers();
ranged.push(("Content-Range", "bytes 1-3/5".to_string()));
// A HEAD reports the object size with no body, exactly as GCS does.
let mut head_only = object_headers();
head_only.push(("Content-Length", "5".to_string()));
let (endpoint, _) = scripted_server(vec![
ScriptedResponse::new(200, head_only, String::new()),
ScriptedResponse::new(200, object_headers(), "hello".to_string()),
ScriptedResponse::new(206, ranged, "ell".to_string()),
ScriptedResponse::new(200, Vec::new(), LIST_PAGE_ONE.to_string()),
ScriptedResponse::new(200, Vec::new(), LIST_PAGE_TWO.to_string()),
// GCS has no tagging call, so the contract's tag step issues no
// request; the probe is the next one on the wire.
ScriptedResponse::new(200, Vec::new(), "{}".to_string()),
ScriptedResponse::new(404, Vec::new(), String::new()),
ScriptedResponse::new(403, Vec::new(), String::new()),
])
.await;
assert_backend_contract(
&backend(&endpoint),
BackendCapabilities {
etag_is_opaque: false,
supports_start_after: false,
// GCS objects have no tags; the contract's tag step is skipped.
supports_tagging: false,
},
)
.await;
}
}
@@ -25,21 +25,13 @@ use parking_lot::Mutex;
use serde::{Deserialize, Serialize};
use std::time::{Duration, Instant};
/// The continuation-token version used by ordinary progressing pages.
/// The only continuation-token envelope version this build reads and writes.
pub const LIST_THROUGH_TOKEN_VERSION: u32 = 1;
const LIST_THROUGH_PROGRESS_TOKEN_VERSION: u32 = 2;
/// The sixteenth consecutive merged page without a key or new EOF fails.
/// This also bounds legitimate sparse listings; it is not a cycle detector.
pub const MAX_LIST_NO_PROGRESS_PAGES: u8 = 16;
/// Envelope marker. A bucket that is *not* merging hands out the local
/// listing's own marker, so the decoder needs a positive signal before it
/// treats an opaque token as a merged one.
const LIST_THROUGH_TOKEN_TAG: &str = "odm-list";
// Object keys cannot contain NUL (bucket::utils::is_valid_object_prefix),
// so this framing cannot collide with a local key used as an opaque marker.
const LIST_THROUGH_TOKEN_PREFIX: &str = "\0odm-list:";
/// Pages fetched per side per request: the first page, plus at most one refill
/// when the first one was mostly consumed by the previous page. Two pages of
@@ -94,7 +86,8 @@ pub struct MergePick {
}
/// The continuation-token envelope. Opaque to clients: it is serialized as
/// framed JSON and then base64-encoded by the same helper as a local marker.
/// JSON and then base64-encoded by the same helper that encodes a plain local
/// marker, so the wire shape is `base64(json)`.
///
/// A `null` cursor with `done = false` means "list that side from the start";
/// `done = true` means the side is finished and must not be listed again.
@@ -118,10 +111,6 @@ pub struct ListThroughToken {
/// common prefix compares as itself, never as its members.
#[serde(default)]
pub last_key: Option<String>,
/// Consecutive empty truncated merged pages, present only in v2 tokens.
/// Ordinary v1 tokens retain their original serialized shape.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub no_progress: Option<u8>,
}
impl ListThroughToken {
@@ -134,14 +123,13 @@ impl ListThroughToken {
source: source.token,
source_done: source.done,
last_key,
no_progress: None,
}
}
pub fn encode(&self) -> String {
// The envelope is built here from owned strings, so serialization
// cannot fail; the fallback keeps the signature infallible.
format!("{LIST_THROUGH_TOKEN_PREFIX}{}", serde_json::to_string(self).unwrap_or_default())
serde_json::to_string(self).unwrap_or_default()
}
}
@@ -165,35 +153,24 @@ pub enum ListThroughTokenError {
/// Classifies an already base64-decoded continuation token.
///
/// Only a framed JSON object is read as a merged token;
/// Only a JSON object carrying the envelope marker is read as a merged token;
/// anything else is a local marker, so a bucket that turns `list_through` off
/// keeps paginating with the tokens it handed out. A token that *is* an
/// envelope but was tampered with (unknown version, unknown field, truncated
/// JSON) is an error, never a silent fallback.
pub fn decode_continuation_token(decoded: &str) -> Result<ListThroughCursor, ListThroughTokenError> {
let Some(payload) = decoded.strip_prefix(LIST_THROUGH_TOKEN_PREFIX) else {
if !decoded.starts_with('{') {
return Ok(ListThroughCursor::Local(decoded.to_string()));
}
let Ok(value) = serde_json::from_str::<serde_json::Value>(decoded) else {
// Not JSON at all: an object key may legitimately start with '{'.
return Ok(ListThroughCursor::Local(decoded.to_string()));
};
let value = serde_json::from_str::<serde_json::Value>(payload).map_err(|_| ListThroughTokenError::Malformed)?;
if value.get("t").and_then(serde_json::Value::as_str) != Some(LIST_THROUGH_TOKEN_TAG) {
return Err(ListThroughTokenError::Malformed);
return Ok(ListThroughCursor::Local(decoded.to_string()));
}
match value.get("v").and_then(serde_json::Value::as_u64) {
Some(version) if version == u64::from(LIST_THROUGH_TOKEN_VERSION) => {
// v1 readers reject this field even when it is null or zero.
if value.get("no_progress").is_some() {
return Err(ListThroughTokenError::Malformed);
}
}
Some(version) if version == u64::from(LIST_THROUGH_PROGRESS_TOKEN_VERSION) => {
if !value
.get("no_progress")
.and_then(serde_json::Value::as_u64)
.is_some_and(|count| (1..u64::from(MAX_LIST_NO_PROGRESS_PAGES)).contains(&count))
{
return Err(ListThroughTokenError::Malformed);
}
}
Some(version) if version == u64::from(LIST_THROUGH_TOKEN_VERSION) => {}
Some(version) => return Err(ListThroughTokenError::UnsupportedVersion(version.min(u64::from(u32::MAX)) as u32)),
None => return Err(ListThroughTokenError::Malformed),
}
@@ -311,8 +288,6 @@ pub enum ListPageError {
Empty,
#[error("truncated listing repeats a continuation token")]
Repeated,
#[error("listing exhausted its consecutive no-progress page budget")]
NoProgress(MergeSide),
}
pub(crate) fn validate_list_page(is_truncated: bool, token: Option<&str>, next_token: Option<&str>) -> Result<(), ListPageError> {
@@ -377,7 +352,6 @@ pub struct MergeOutcome {
#[derive(Debug)]
pub struct ListThroughMerger {
max_keys: usize,
no_progress: Option<u8>,
last_key: Option<String>,
local: SideState,
source: SideState,
@@ -397,7 +371,6 @@ impl ListThroughMerger {
};
Self {
max_keys,
no_progress: token.and_then(|token| token.no_progress),
last_key,
local,
source,
@@ -463,18 +436,13 @@ impl ListThroughMerger {
Ok(())
}
/// `issue_progress_tokens` allows a v1 chain to start carrying a budget.
/// An existing v2 budget is always enforced, including on reader-only nodes.
/// Borrowing lets a source failure re-merge the fetched local buffers.
pub fn finish(&self, issue_progress_tokens: bool) -> Result<MergeOutcome, ListPageError> {
pub fn finish(self) -> MergeOutcome {
let Self {
max_keys,
no_progress,
last_key,
local,
source,
} = self;
let max_keys = *max_keys;
// A side with more pages behind it can only be trusted up to the last
// key it handed over: past that horizon the other side's entries could
@@ -540,44 +508,12 @@ impl ListThroughMerger {
let source_left = !source.disabled && (!source_cursor.done || consumed_source < source.entries.len());
let is_truncated = local_left || source_left;
let reached_eof = (!local.start.done && local_cursor.done) || (!source.start.done && source_cursor.done);
let next_no_progress = if !is_truncated || !picks.is_empty() || reached_eof {
None
} else if max_keys == 0 {
// A zero-sized request cannot consume entries. Preserve an existing
// budget without spending it or starting a new one.
*no_progress
} else if issue_progress_tokens || no_progress.is_some() {
let count = no_progress.unwrap_or(0).saturating_add(1);
if count >= MAX_LIST_NO_PROGRESS_PAGES {
// An empty truncated side closes the merge horizon. Local
// failure takes precedence; disabling the source cannot fix it.
let side = if local.more && local.entries.is_empty() {
MergeSide::Local
} else if !source.disabled && source.more && source.entries.is_empty() {
MergeSide::Source
} else {
MergeSide::Local
};
return Err(ListPageError::NoProgress(side));
}
Some(count)
} else {
None
};
let last_key = consumed_key.or_else(|| last_key.clone());
Ok(MergeOutcome {
let last_key = consumed_key.or(last_key);
MergeOutcome {
picks,
is_truncated,
next_token: is_truncated.then(|| {
let mut token = ListThroughToken::new(local_cursor, source_cursor, last_key);
if let Some(count) = next_no_progress {
token.v = LIST_THROUGH_PROGRESS_TOKEN_VERSION;
token.no_progress = Some(count);
}
token
}),
})
next_token: is_truncated.then(|| ListThroughToken::new(local_cursor, source_cursor, last_key)),
}
}
}
@@ -705,7 +641,7 @@ mod tests {
.push_page(fetch.side, kept, truncated, next)
.expect("reference provider pages must advance");
}
let outcome = merger.finish(false).expect("valid merge outcome");
let outcome = merger.finish();
assert_eq!(outcome.is_truncated, outcome.next_token.is_some());
if outcome.is_truncated {
assert_ne!(outcome.next_token, token, "every truncated merged page must make progress");
@@ -788,7 +724,7 @@ mod tests {
.push_page(MergeSide::Local, vec![ListEntryKey::object("a")], false, None)
.expect("local EOF is valid");
assert_eq!(merger.next_fetch(), None);
let outcome = merger.finish(false).expect("valid merge outcome");
let outcome = merger.finish();
assert_eq!(outcome.picks.len(), 1);
assert!(!outcome.is_truncated);
assert!(outcome.next_token.is_none());
@@ -804,7 +740,6 @@ mod tests {
source: Some("source-1".to_string()),
source_done: false,
last_key: Some("a".to_string()),
no_progress: None,
};
let mut merger = ListThroughMerger::new(1, Some(&resume));
merger.disable_source();
@@ -816,7 +751,7 @@ mod tests {
Some("local-2".to_string()),
)
.expect("local cursor advances");
let outcome = merger.finish(false).expect("valid merge outcome");
let outcome = merger.finish();
assert!(outcome.is_truncated);
let token = outcome.next_token.expect("truncated page carries a token");
assert_eq!(token.source.as_deref(), Some("source-1"), "the source cursor must not move");
@@ -895,7 +830,7 @@ mod tests {
.expect("opaque cursor advances regardless of sort order");
}
assert!(merger.next_fetch().is_none(), "two source fetches exhaust the request budget");
let outcome = merger.finish(false).expect("valid merge outcome");
let outcome = merger.finish();
assert!(outcome.picks.is_empty());
assert!(outcome.is_truncated);
let token = outcome.next_token.expect("empty progressing page has a cursor");
@@ -905,7 +840,7 @@ mod tests {
merger
.push_page(MergeSide::Source, vec![ListEntryKey::object("result")], false, None)
.expect("source EOF");
let outcome = merger.finish(false).expect("valid merge outcome");
let outcome = merger.finish();
assert_eq!(
outcome.picks,
vec![MergePick {
@@ -952,7 +887,7 @@ mod tests {
Err(ListPageError::Repeated)
);
merger.disable_source();
let outcome = merger.finish(false).expect("valid merge outcome");
let outcome = merger.finish();
assert_eq!(
outcome.picks,
vec![MergePick {
@@ -1044,279 +979,21 @@ mod tests {
let encoded = token.encode();
assert_eq!(decode_continuation_token(&encoded), Ok(ListThroughCursor::Merged(Box::new(token))));
let bumped = encoded.replace("\"v\":1", "\"v\":3");
assert_eq!(decode_continuation_token(&bumped), Err(ListThroughTokenError::UnsupportedVersion(3)));
let bumped = encoded.replace("\"v\":1", "\"v\":2");
assert_eq!(decode_continuation_token(&bumped), Err(ListThroughTokenError::UnsupportedVersion(2)));
let extra = encoded.replace("{", "{\"x\":1,");
assert_eq!(decode_continuation_token(&extra), Err(ListThroughTokenError::Malformed));
let truncated = &encoded[..encoded.len() - 3];
assert_eq!(decode_continuation_token(truncated), Err(ListThroughTokenError::Malformed));
assert_eq!(decode_continuation_token(truncated), Ok(ListThroughCursor::Local(truncated.to_string())));
let no_version = "\0odm-list:{\"t\":\"odm-list\"}";
let no_version = "{\"t\":\"odm-list\"}";
assert_eq!(decode_continuation_token(no_version), Err(ListThroughTokenError::Malformed));
}
fn progress_token(count: Option<u8>, local_done: bool, source_done: bool) -> ListThroughToken {
let mut token = ListThroughToken::new(
SideCursor {
token: None,
done: local_done,
},
SideCursor {
token: Some("A".into()),
done: source_done,
},
Some("last-key".into()),
);
if let Some(count) = count {
token.v = LIST_THROUGH_PROGRESS_TOKEN_VERSION;
token.no_progress = Some(count);
}
token
}
fn push_empty_pages(merger: &mut ListThroughMerger, side: MergeSide) {
for _ in 0..MAX_LIST_FETCHES_PER_SIDE {
let fetch = merger.next_fetch().expect("empty truncated side must be fetched");
assert_eq!(fetch.side, side);
let next = format!("{}:next", fetch.token.unwrap_or_default());
merger
.push_page(side, vec![], true, Some(next))
.expect("opaque cursor advances");
}
}
#[test]
fn progress_tokens_preserve_v1_bytes_and_validate_v2_counts() {
let token = progress_token(None, true, false);
assert_eq!(
token.encode(),
r#"{"t":"odm-list","v":1,"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key"}"#
);
for count in 1..MAX_LIST_NO_PROGRESS_PAGES {
let token = progress_token(Some(count), true, false);
assert_eq!(decode_continuation_token(&token.encode()), Ok(ListThroughCursor::Merged(Box::new(token))));
}
for version in [1, 2] {
for value in ["null", "0", "16", "-1", "1.5", "256", "18446744073709551616", "\"1\""] {
let encoded = format!(r#"{{"t":"odm-list","v":{version},"no_progress":{value}}}"#);
assert_eq!(decode_continuation_token(&encoded), Err(ListThroughTokenError::Malformed), "{encoded}");
}
}
for encoded in [
r#"{"t":"odm-list","v":1,"no_progress":1}"#,
r#"{"t":"odm-list","v":2}"#,
r#"{"t":"odm-list","v":2,"no_progress":1,"extra":true}"#,
] {
assert_eq!(decode_continuation_token(encoded), Err(ListThroughTokenError::Malformed), "{encoded}");
}
}
#[test]
fn reader_only_nodes_do_not_start_a_budget_but_mixed_readers_preserve_one() {
let mut token = progress_token(None, true, false);
for _ in 0..MAX_LIST_NO_PROGRESS_PAGES {
let mut merger = ListThroughMerger::new(2, Some(&token));
push_empty_pages(&mut merger, MergeSide::Source);
token = merger
.finish(false)
.expect("reader-only v1 behavior")
.next_token
.expect("truncated cursor");
assert_eq!(token.v, 1);
assert_eq!(token.no_progress, None);
}
for count in 1..=MAX_LIST_NO_PROGRESS_PAGES {
let mut merger = ListThroughMerger::new(2, Some(&token));
push_empty_pages(&mut merger, MergeSide::Source);
assert!(merger.next_fetch().is_none(), "the per-request two-fetch limit stays intact");
let outcome = merger.finish(count % 2 == 1);
if count == MAX_LIST_NO_PROGRESS_PAGES {
assert_eq!(outcome, Err(ListPageError::NoProgress(MergeSide::Source)));
break;
}
token = outcome.expect("budget not exhausted").next_token.expect("truncated cursor");
assert_eq!(token.no_progress, Some(count));
let ListThroughCursor::Merged(decoded) = decode_continuation_token(&token.encode()).expect("round-trip v2") else {
panic!("merged cursor expected");
};
token = *decoded;
}
}
#[test]
fn objects_and_common_prefixes_reset_a_budget_at_the_boundary() {
for entry in [ListEntryKey::object("result"), ListEntryKey::prefix("result/")] {
for issue_tokens in [false, true] {
let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), true, false);
let mut merger = ListThroughMerger::new(2, Some(&resume));
merger
.push_page(MergeSide::Source, vec![], true, Some("B".into()))
.expect("empty advancing page");
merger
.push_page(MergeSide::Source, vec![entry.clone()], true, Some("C".into()))
.expect("real progress");
let outcome = merger
.finish(issue_tokens)
.expect("real progress does not exhaust the budget");
assert_eq!(
outcome.picks,
vec![MergePick {
side: MergeSide::Source,
index: 0
}]
);
let next = outcome.next_token.expect("source remains truncated");
assert_eq!(next.last_key.as_deref(), Some(entry.name.as_str()));
assert_eq!(next.v, 1);
assert_eq!(next.no_progress, None);
assert!(!next.encode().contains("no_progress"));
}
}
}
#[test]
fn only_a_new_eof_transition_resets_the_empty_page_budget() {
for finished_side in [MergeSide::Local, MergeSide::Source] {
let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), false, false);
let mut merger = ListThroughMerger::new(2, Some(&resume));
if finished_side == MergeSide::Local {
merger
.push_page(MergeSide::Local, vec![], false, None)
.expect("new local EOF");
push_empty_pages(&mut merger, MergeSide::Source);
} else {
push_empty_pages(&mut merger, MergeSide::Local);
merger
.push_page(MergeSide::Source, vec![], false, None)
.expect("new source EOF");
}
let next = merger
.finish(false)
.expect("new EOF is progress")
.next_token
.expect("other side truncated");
assert_eq!(next.no_progress, None);
assert_eq!(next.v, 1);
assert_eq!(next.local_done, finished_side == MergeSide::Local);
assert_eq!(next.source_done, finished_side == MergeSide::Source);
let mut merger = ListThroughMerger::new(2, Some(&next));
let remaining = if finished_side == MergeSide::Local {
MergeSide::Source
} else {
MergeSide::Local
};
push_empty_pages(&mut merger, remaining);
let next = merger
.finish(true)
.expect("a new budget starts")
.next_token
.expect("truncated");
assert_eq!(next.no_progress, Some(1), "an already-done side cannot reset every page");
}
let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), true, false);
let mut merger = ListThroughMerger::new(2, Some(&resume));
merger.push_page(MergeSide::Source, vec![], false, None).expect("final EOF");
let outcome = merger.finish(false).expect("EOF succeeds at the budget boundary");
assert!(!outcome.is_truncated);
assert!(outcome.next_token.is_none());
}
#[test]
fn filtered_duplicates_cannot_reset_the_no_progress_budget() {
let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), true, false);
let mut merger = ListThroughMerger::new(2, Some(&resume));
for next in ["B", "C"] {
let entries = [ListEntryKey::object("last-key"), ListEntryKey::object("earlier")]
.into_iter()
.filter(|entry| merger.accepts(&entry.name))
.collect::<Vec<_>>();
assert!(entries.is_empty(), "both provider entries were already consumed");
merger
.push_page(MergeSide::Source, entries, true, Some(next.into()))
.expect("advancing cursor");
}
assert_eq!(merger.finish(false), Err(ListPageError::NoProgress(MergeSide::Source)));
}
#[test]
fn no_progress_is_attributed_to_local_when_source_cannot_unblock_it() {
for source_mode in ["disabled", "done", "empty", "data"] {
let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), false, source_mode == "done");
let mut merger = ListThroughMerger::new(2, Some(&resume));
if source_mode == "disabled" {
merger.disable_source();
}
push_empty_pages(&mut merger, MergeSide::Local);
match source_mode {
"empty" => push_empty_pages(&mut merger, MergeSide::Source),
"data" => merger
.push_page(MergeSide::Source, vec![ListEntryKey::object("source")], false, None)
.expect("source data"),
_ => {}
}
assert_eq!(merger.finish(false), Err(ListPageError::NoProgress(MergeSide::Local)), "{source_mode}");
}
}
#[test]
fn source_budget_failure_remerges_local_objects_and_prefixes_without_refetching() {
let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), false, false);
let mut merger = ListThroughMerger::new(2, Some(&resume));
merger
.push_page(MergeSide::Local, vec![ListEntryKey::object("local")], true, Some("L1".into()))
.expect("local object");
merger
.push_page(MergeSide::Local, vec![ListEntryKey::prefix("prefix/")], true, Some("L2".into()))
.expect("local prefix");
push_empty_pages(&mut merger, MergeSide::Source);
assert_eq!(merger.finish(false), Err(ListPageError::NoProgress(MergeSide::Source)));
merger.disable_source();
assert!(merger.next_fetch().is_none(), "fallback does not perform another fetch");
let outcome = merger.finish(false).expect("local data makes progress");
assert_eq!(
outcome.picks,
vec![
MergePick {
side: MergeSide::Local,
index: 0
},
MergePick {
side: MergeSide::Local,
index: 1
}
]
);
let token = outcome.next_token.expect("remaining local page");
assert_eq!(token.local.as_deref(), Some("L2"));
assert_eq!(token.source.as_deref(), Some("A"));
assert_eq!(token.last_key.as_deref(), Some("prefix/"));
assert_eq!(token.no_progress, None);
assert_eq!(token.v, 1);
}
#[test]
fn a_zero_sized_merge_preserves_an_existing_budget() {
let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), true, false);
let mut merger = ListThroughMerger::new(0, Some(&resume));
merger
.push_page(MergeSide::Source, vec![ListEntryKey::object("result")], true, Some("B".into()))
.expect("source page");
let outcome = merger.finish(false).expect("a zero-sized request cannot consume entries");
assert!(outcome.picks.is_empty());
assert_eq!(outcome.next_token.expect("unconsumed source").no_progress, resume.no_progress);
}
#[test]
fn a_plain_local_marker_stays_local() {
for marker in [
r#"{"t":"odm-list","v":1}"#,
r#"{"t":"odm-list","v":2,"local_done":true}"#,
r#"{"t":"odm-list"}"#,
] {
assert_eq!(decode_continuation_token(marker), Ok(ListThroughCursor::Local(marker.to_string())));
}
assert_eq!(
decode_continuation_token("photos/2024/01.jpg"),
Ok(ListThroughCursor::Local("photos/2024/01.jpg".to_string()))
@@ -19,42 +19,30 @@
//! client, and the per-node runtime (`sys`) that turns configs into live
//! clients guarded by a breaker, a negative cache, singleflight and a pull
//! concurrency limit (rustfs/backlog#2147).
//!
//! A source is reached through one `SourceBackend`: the S3 dialect for every
//! S3-compatible provider, and a native backend for the providers that have no
//! S3 API (`azure`, `gcs_native`).
pub mod azure;
#[cfg(test)]
mod backend_contract;
pub mod backfill;
pub mod breaker;
pub mod config;
pub mod gcs;
pub mod list_through;
mod native_http;
pub mod negative_cache;
pub mod pull;
pub mod source_client;
pub mod stats;
pub mod sys;
#[cfg(test)]
mod test_http_fixture;
pub use breaker::{
BREAKER_FAILURE_THRESHOLD, BREAKER_FAILURE_WINDOW, BREAKER_HALF_OPEN_MAX_PROBES, BREAKER_OPEN_DURATION, Breaker,
BreakerState, BreakerTransition, BreakerVerdict,
};
pub use config::{
AzureSourceConfig, ConfigPublishHook, FilterConfig, GcsSourceConfig, HeadPolicy, ON_DEMAND_MIGRATION_CONFIG_HOOK,
ON_DEMAND_MIGRATION_CONFIG_VERSION, OnDemandMigrationConfig, OnDemandMigrationConfigError, PathStyle, PolicyConfig, Provider,
RangeGetPolicy, SourceConfig, SourceCredentials, SourceErrorPolicy, SourceTimeout, TlsConfig, ValidationContext,
ConfigPublishHook, FilterConfig, HeadPolicy, ON_DEMAND_MIGRATION_CONFIG_HOOK, ON_DEMAND_MIGRATION_CONFIG_VERSION,
OnDemandMigrationConfig, OnDemandMigrationConfigError, PathStyle, PolicyConfig, Provider, RangeGetPolicy, SourceConfig,
SourceCredentials, SourceErrorPolicy, SourceTimeout, TlsConfig, ValidationContext,
};
pub use list_through::{
FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger,
ListThroughToken, ListThroughTokenError, MAX_LIST_FETCHES_PER_SIDE, MAX_LIST_NO_PROGRESS_PAGES, MergeOutcome, MergePick,
MergeSide, SOURCE_LIST_MAX_RATE_WAIT, SOURCE_LIST_RATE_PER_SEC, SourceListPlan, SourceListRateLimiter,
decode_continuation_token, source_list_plan,
FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListThroughCursor, ListThroughMerger, ListThroughToken,
ListThroughTokenError, MAX_LIST_FETCHES_PER_SIDE, MergeOutcome, MergePick, MergeSide, SOURCE_LIST_MAX_RATE_WAIT,
SOURCE_LIST_RATE_PER_SEC, SourceListPlan, SourceListRateLimiter, decode_continuation_token, source_list_plan,
};
pub use negative_cache::{NEGATIVE_CACHE_MAX_ENTRIES, NegativeCache};
pub use pull::{
@@ -68,6 +56,5 @@ pub use stats::{
};
pub use sys::{
ApplyOutcome, BucketOdmState, GLOBAL_ON_DEMAND_MIGRATION_SYS, OdmBucketSnapshot, OdmLookup, OdmStateError,
OnDemandMigrationSys, PullError, PullFollower, PullLeader, PullOutcome, PullResult, PullSlot, source_backend_spec,
source_client_spec,
OnDemandMigrationSys, PullError, PullFollower, PullLeader, PullOutcome, PullResult, PullSlot, source_client_spec,
};
@@ -1,415 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Shared HTTP transport for the on-demand migration source backends that do
//! not speak S3 (Azure Blob, native GCS).
//!
//! The S3 backend rides the AWS SDK; these providers have no SigV4 dialect, so
//! they talk plain HTTP through one `reqwest` client that carries the same
//! connect/read timeouts and TLS policy the operator configured for the source.
//! Redirects are refused: the endpoint passed the outbound policy gate once, and
//! following a source-chosen `Location` would leave that gate behind.
//!
//! Errors never render the request URL. A SAS token lives in the query string,
//! so a `reqwest` error rendered with its URL would print the credential into
//! the log line and the admin response.
use super::source_client::{SourceError, SourceHead, SourceTimeouts, USER_AGENT_SUFFIX, classify_status, is_multipart_etag};
use crate::bucket::remote_s3_client::{RemoteS3ClientError, validate_remote_endpoint, validate_target_ca_pem};
use aws_sdk_s3::primitives::ByteStream;
use aws_smithy_types::body::SdkBody;
use futures::StreamExt;
use http::HeaderMap;
use std::collections::HashMap;
use std::time::SystemTime;
use time::OffsetDateTime;
use time::format_description::well_known::{Rfc2822, Rfc3339};
use url::Url;
/// Origin the native backends are allowed to address, plus the HTTP client
/// that reaches it.
pub(super) struct NativeHttp {
client: reqwest::Client,
endpoint: Url,
}
impl NativeHttp {
/// `endpoint` must be a bare `scheme://host[:port]` origin; it is checked
/// against the outbound policy exactly like an S3 source endpoint.
pub(super) fn new(
endpoint: &str,
timeouts: SourceTimeouts,
skip_tls_verify: bool,
ca_cert_pem: Option<&str>,
) -> Result<Self, RemoteS3ClientError> {
let endpoint = Url::parse(endpoint.trim()).map_err(|err| RemoteS3ClientError::InvalidEndpoint(err.to_string()))?;
if !matches!(endpoint.scheme(), "http" | "https") {
return Err(RemoteS3ClientError::InvalidEndpoint(format!(
"unsupported scheme {}; expected http or https",
endpoint.scheme()
)));
}
if endpoint.host_str().is_none_or(str::is_empty) {
return Err(RemoteS3ClientError::InvalidEndpoint("endpoint has no host".to_string()));
}
if !endpoint.username().is_empty() || endpoint.password().is_some() {
return Err(RemoteS3ClientError::InvalidEndpoint("endpoint must not carry userinfo".to_string()));
}
if !matches!(endpoint.path(), "" | "/") || endpoint.query().is_some() || endpoint.fragment().is_some() {
return Err(RemoteS3ClientError::InvalidEndpoint(
"endpoint must be an origin without path, query or fragment".to_string(),
));
}
validate_remote_endpoint(&endpoint).map_err(RemoteS3ClientError::EndpointNotAllowed)?;
let mut builder = reqwest::Client::builder()
.connect_timeout(timeouts.connect)
.read_timeout(timeouts.read)
.redirect(reqwest::redirect::Policy::none())
.user_agent(USER_AGENT_SUFFIX);
if skip_tls_verify {
builder = builder.danger_accept_invalid_certs(true);
} else if let Some(pem) = ca_cert_pem.map(str::trim).filter(|pem| !pem.is_empty()) {
// Reject a malformed bundle the same way the S3 path does, so the
// operator sees "invalid CA PEM" instead of a TLS handshake failure.
validate_target_ca_pem(pem)?;
let certificate = reqwest::Certificate::from_pem(pem.as_bytes())
.map_err(|err| RemoteS3ClientError::InvalidCaPem(err.to_string()))?;
builder = builder.add_root_certificate(certificate);
}
let client = builder
.build()
.map_err(|err| RemoteS3ClientError::InvalidEndpoint(format!("http client cannot be built: {err}")))?;
Ok(Self { client, endpoint })
}
#[cfg(test)]
pub(super) fn for_test(endpoint: Url) -> Self {
Self {
client: reqwest::Client::builder()
.redirect(reqwest::redirect::Policy::none())
.build()
.expect("test http client should build"),
endpoint,
}
}
/// A URL under the endpoint origin. `segments` are percent-encoded as
/// path segments, so a key containing `?`, `#` or a space cannot rewrite
/// the request target.
pub(super) fn url<'a>(&self, segments: impl IntoIterator<Item = &'a str>) -> Result<Url, SourceError> {
let mut url = self.endpoint.clone();
{
let mut path = url
.path_segments_mut()
.map_err(|_| SourceError::Other("source endpoint cannot carry a path".to_string()))?;
path.clear();
path.extend(segments);
}
Ok(url)
}
/// Sends the request and returns the response only for a 2xx status.
/// Non-2xx statuses are classified from the status and the provider's own
/// error-code header; response bodies are not read, so no provider message
/// can smuggle credentials or markup into a log line.
pub(super) async fn send(
&self,
request: reqwest::Request,
error_code_header: &str,
) -> Result<reqwest::Response, SourceError> {
let response = self.client.execute(request).await.map_err(classify_transport_error)?;
let status = response.status();
if status.is_success() {
return Ok(response);
}
let code = response
.headers()
.get(error_code_header)
.and_then(|value| value.to_str().ok())
.map(str::to_string);
Err(classify_status(
status.as_u16(),
None,
match &code {
Some(code) => format!("source returned HTTP {status} ({code})"),
None => format!("source returned HTTP {status}"),
},
))
}
}
/// Renders a transport failure without the request URL: a SAS token or a
/// signed query would otherwise reach logs and admin responses.
pub(super) fn classify_transport_error(err: reqwest::Error) -> SourceError {
let is_timeout = err.is_timeout();
let is_connect = err.is_connect();
let message = err.without_url().to_string();
if is_timeout {
SourceError::Timeout
} else if is_connect {
SourceError::Connect(message)
} else {
SourceError::Other(message)
}
}
/// Streams the response body without buffering it.
pub(super) fn response_body(response: reqwest::Response) -> ByteStream {
let stream = response.bytes_stream().map(|chunk| {
chunk
.map(http_body::Frame::data)
.map_err(|err| std::io::Error::other(err.without_url().to_string()))
});
ByteStream::new(SdkBody::from_body_1_x(http_body_util::StreamBody::new(stream)))
}
/// Reads a bounded response body as UTF-8, for the XML and JSON listings.
pub(super) async fn read_text(response: reqwest::Response, max_bytes: usize) -> Result<String, SourceError> {
let mut body = Vec::new();
let mut stream = response.bytes_stream();
while let Some(chunk) = stream.next().await {
let chunk = chunk.map_err(classify_transport_error)?;
if body.len().saturating_add(chunk.len()) > max_bytes {
return Err(SourceError::Other("source listing response exceeded the size limit".to_string()));
}
body.extend_from_slice(&chunk);
}
String::from_utf8(body).map_err(|_| SourceError::Other("source listing response is not valid UTF-8".to_string()))
}
/// Base64 digest (`Content-MD5`, `md5Hash`, `x-goog-hash`) as lowercase hex.
/// `None` when the value is not a 16-byte digest, so a CRC32C never passes as
/// an MD5.
pub(super) fn base64_md5_to_hex(value: &str) -> Option<String> {
let raw = base64_simd::STANDARD.decode_to_vec(value.trim().as_bytes()).ok()?;
(raw.len() == 16).then(|| faster_hex::hex_string(&raw))
}
pub(super) fn header<'a>(headers: &'a HeaderMap, name: &str) -> Option<&'a str> {
headers.get(name).and_then(|value| value.to_str().ok()).map(str::trim)
}
fn header_string(headers: &HeaderMap, name: &str) -> Option<String> {
header(headers, name).filter(|value| !value.is_empty()).map(str::to_string)
}
/// `Last-Modified` and friends arrive as an HTTP date; the JSON dialects use
/// RFC 3339 for the same field, so both are accepted.
pub(super) fn parse_http_timestamp(value: &str) -> Option<SystemTime> {
OffsetDateTime::parse(value, &Rfc2822)
.or_else(|_| OffsetDateTime::parse(value, &Rfc3339))
.ok()
.map(SystemTime::from)
}
/// Provider-specific fields the shared header mapping cannot infer.
pub(super) struct NativeHeadFields {
pub(super) etag: Option<String>,
/// The ETag is an opaque token rather than a digest of the bytes.
pub(super) etag_is_opaque: bool,
pub(super) version_id: Option<String>,
pub(super) storage_class: Option<String>,
}
/// Maps a HEAD or GET response onto [`SourceHead`]. `metadata_prefix` is the
/// provider's user-metadata header prefix (`x-ms-meta-`, `x-goog-meta-`); the
/// stored shape drops it, matching the `x-amz-meta-` handling of the S3 path.
pub(super) fn native_source_head(
headers: &HeaderMap,
metadata_prefix: &str,
fields: NativeHeadFields,
) -> Result<SourceHead, SourceError> {
let size = header(headers, "content-length")
.and_then(|value| value.parse::<u64>().ok())
.ok_or_else(|| SourceError::Other("source response has no valid content-length".to_string()))?;
let mut user_metadata = HashMap::new();
for (name, value) in headers {
let name = name.as_str();
if let Some(key) = name.strip_prefix(metadata_prefix)
&& !key.is_empty()
&& let Ok(value) = value.to_str()
{
user_metadata.insert(key.to_string(), value.to_string());
}
}
let etag = fields
.etag
.map(|etag| etag.trim().trim_matches('"').to_string())
.filter(|etag| !etag.is_empty());
// An opaque ETag never encodes a part count, so the multipart flag stays
// false for it however the provider happens to spell the token.
let is_multipart_etag = !fields.etag_is_opaque && etag.as_deref().is_some_and(is_multipart_etag);
Ok(SourceHead {
etag,
size,
last_modified: header(headers, "last-modified").and_then(parse_http_timestamp),
content_type: header_string(headers, "content-type"),
content_encoding: header_string(headers, "content-encoding"),
content_disposition: header_string(headers, "content-disposition"),
content_language: header_string(headers, "content-language"),
cache_control: header_string(headers, "cache-control"),
expires: header_string(headers, "expires"),
user_metadata,
version_id: fields.version_id,
storage_class: fields.storage_class,
// Neither native provider hands back ciphertext: a customer-key object
// is refused by the backend before it reaches this mapping, and the
// service-managed encryption is transparent to the reader.
sse: None,
is_multipart_etag,
etag_is_opaque: fields.etag_is_opaque,
})
}
#[cfg(test)]
mod tests {
use super::*;
use http::HeaderValue;
fn headers(pairs: &[(&str, &str)]) -> HeaderMap {
let mut headers = HeaderMap::new();
for (name, value) in pairs {
headers.insert(
http::HeaderName::from_bytes(name.as_bytes()).expect("test header name"),
HeaderValue::from_str(value).expect("test header value"),
);
}
headers
}
fn fields() -> NativeHeadFields {
NativeHeadFields {
etag: None,
etag_is_opaque: false,
version_id: None,
storage_class: None,
}
}
#[test]
fn native_source_head_maps_content_headers_and_prefixed_metadata() {
let headers = headers(&[
("content-length", "1234"),
("content-type", "text/plain"),
("content-encoding", "gzip"),
("content-language", "en"),
("content-disposition", "attachment"),
("cache-control", "max-age=60"),
("expires", "Thu, 01 Jan 2026 00:00:00 GMT"),
("last-modified", "Wed, 21 Oct 2015 07:28:00 GMT"),
("x-ms-meta-owner", "alice"),
("x-goog-meta-owner", "not-mine"),
]);
let head = native_source_head(
&headers,
"x-ms-meta-",
NativeHeadFields {
etag: Some("\"0x8DCE1D2\"".to_string()),
etag_is_opaque: true,
version_id: Some("2026-01-01T00:00:00.0000000Z".to_string()),
storage_class: Some("Hot".to_string()),
},
)
.expect("head should map");
assert_eq!(head.size, 1234);
assert_eq!(head.content_type.as_deref(), Some("text/plain"));
assert_eq!(head.content_encoding.as_deref(), Some("gzip"));
assert_eq!(head.content_language.as_deref(), Some("en"));
assert_eq!(head.content_disposition.as_deref(), Some("attachment"));
assert_eq!(head.cache_control.as_deref(), Some("max-age=60"));
assert_eq!(head.expires.as_deref(), Some("Thu, 01 Jan 2026 00:00:00 GMT"));
assert_eq!(
head.last_modified,
Some(SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1_445_412_480)),
"HTTP-date Last-Modified must parse"
);
assert_eq!(
head.user_metadata,
HashMap::from([("owner".to_string(), "alice".to_string())]),
"only the provider's own metadata prefix is read"
);
assert_eq!(head.etag.as_deref(), Some("0x8DCE1D2"), "quotes are stripped, the token is kept");
assert!(head.etag_is_opaque);
assert!(!head.is_multipart_etag);
assert_eq!(head.storage_class.as_deref(), Some("Hot"));
assert!(head.sse.is_none());
}
#[test]
fn native_source_head_requires_a_content_length() {
let err = native_source_head(&headers(&[("content-type", "text/plain")]), "x-ms-meta-", fields())
.expect_err("a response without content-length is unusable");
assert!(matches!(err, SourceError::Other(_)), "{err:?}");
}
#[test]
fn opaque_etag_never_reads_as_a_multipart_etag() {
// A digest-shaped ETag keeps the S3 reading; the same string marked
// opaque must not be split into "digest-partcount".
for (opaque, expected) in [(false, true), (true, false)] {
let head = native_source_head(
&headers(&[("content-length", "1")]),
"x-ms-meta-",
NativeHeadFields {
etag: Some("d41d8cd98f00b204e9800998ecf8427e-3".to_string()),
etag_is_opaque: opaque,
..fields()
},
)
.expect("head should map");
assert_eq!(head.is_multipart_etag, expected, "opaque = {opaque}");
}
}
#[test]
fn base64_md5_converts_only_sixteen_byte_digests() {
assert_eq!(
base64_md5_to_hex("1B2M2Y8AsgTpgAmY7PhCfg==").as_deref(),
Some("d41d8cd98f00b204e9800998ecf8427e")
);
assert_eq!(base64_md5_to_hex("not base64!").as_deref(), None);
// A CRC32C digest is four bytes: it must not pass as an MD5.
assert_eq!(base64_md5_to_hex("AAAAAA==").as_deref(), None);
}
#[test]
fn native_http_rejects_endpoints_that_are_not_bare_origins() {
for bad in [
"ftp://source.example.com",
"https://user:pw@source.example.com",
"https://source.example.com/container",
"https://source.example.com/?x=1",
"not a url",
] {
assert!(
NativeHttp::new(bad, SourceTimeouts::default(), false, None).is_err(),
"{bad} must be rejected"
);
}
}
#[test]
fn native_http_percent_encodes_every_path_segment() {
let http = NativeHttp::for_test(Url::parse("https://acct.blob.core.windows.net").expect("origin"));
let url = http.url(["container", "dir", "a b?c#d.txt"]).expect("url should build");
assert_eq!(url.as_str(), "https://acct.blob.core.windows.net/container/dir/a%20b%3Fc%23d.txt");
assert_eq!(url.query(), None, "a key with '?' must not become a query");
}
}
@@ -46,10 +46,10 @@ use super::stats::{PullFailureReason, PullPath};
use super::sys::{BucketOdmState, OnDemandMigrationSys, PullError, PullOutcome, PullSlot};
use async_trait::async_trait;
use bytes::Bytes;
use futures::{FutureExt, Stream, StreamExt, future::Shared};
use futures::{Stream, StreamExt};
use parking_lot::Mutex;
use rand::RngExt;
use std::collections::HashMap;
use std::collections::{HashMap, HashSet};
use std::fmt;
use std::io;
use std::pin::Pin;
@@ -133,8 +133,6 @@ pub enum QueuedPullOutcome {
Failed(PullError),
}
pub type QueuedPullReport = Shared<oneshot::Receiver<QueuedPullOutcome>>;
/// Result of [`PullQueue::enqueue`].
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum EnqueueOutcome {
@@ -253,7 +251,6 @@ pub struct WriteBackRequest {
pub preserve_etag: bool,
/// `policy.emit_events`.
pub emit_events: bool,
pub respect_delete_marker: bool,
/// Source tags to copy (`policy.copy_tags`), `None` to skip.
pub tags: Option<HashMap<String, String>>,
}
@@ -269,7 +266,6 @@ impl WriteBackRequest {
pulled_at: OffsetDateTime::now_utc(),
preserve_etag: config.policy.preserve_etag,
emit_events: config.policy.emit_events,
respect_delete_marker: config.policy.respect_local_delete_marker,
tags,
}
}
@@ -834,7 +830,7 @@ pub struct PullQueue {
bucket: String,
tx: mpsc::Sender<PullJob>,
/// Keys queued or running; the job removes its key when it ends.
pending: Mutex<HashMap<String, QueuedPullReport>>,
pending: Mutex<HashSet<String>>,
capacity: usize,
cancel: CancellationToken,
stats: Arc<super::stats::OdmStats>,
@@ -873,7 +869,7 @@ impl PullQueue {
let queue = Arc::new(Self {
bucket: state.bucket().to_string(),
tx,
pending: Mutex::new(HashMap::new()),
pending: Mutex::new(HashSet::new()),
capacity,
cancel: state.cancel_token(),
stats: Arc::clone(state.stats()),
@@ -907,24 +903,29 @@ impl PullQueue {
self.enqueue_with_report(key, reason).0
}
/// [`Self::enqueue`] with a shared report, including for coalesced pulls.
pub fn enqueue_with_report(&self, key: &str, reason: PullReason) -> (EnqueueOutcome, Option<QueuedPullReport>) {
/// [`Self::enqueue`] that also hands back the job's report channel when
/// a new job was queued (`Coalesced` pulls report to their first
/// requester only).
pub fn enqueue_with_report(
&self,
key: &str,
reason: PullReason,
) -> (EnqueueOutcome, Option<oneshot::Receiver<QueuedPullOutcome>>) {
if self.cancel.is_cancelled() {
return (EnqueueOutcome::Unavailable, None);
}
let mut pending = self.pending.lock();
if let Some(report) = pending.get(key) {
return (EnqueueOutcome::Coalesced, Some(report.clone()));
if pending.contains(key) {
return (EnqueueOutcome::Coalesced, None);
}
let (report_tx, report_rx) = oneshot::channel();
let report_rx = report_rx.shared();
match self.tx.try_send(PullJob {
key: key.to_string(),
reason,
report: Some(report_tx),
}) {
Ok(()) => {
pending.insert(key.to_string(), report_rx.clone());
pending.insert(key.to_string());
(EnqueueOutcome::Enqueued, Some(report_rx))
}
Err(TrySendError::Full(_)) => {
@@ -1071,7 +1072,7 @@ impl BucketOdmState {
self: &Arc<Self>,
key: &str,
reason: PullReason,
) -> (EnqueueOutcome, Option<QueuedPullReport>) {
) -> (EnqueueOutcome, Option<oneshot::Receiver<QueuedPullOutcome>>) {
match self.pull_queue() {
Some(queue) => queue.enqueue_with_report(key, reason),
None => (EnqueueOutcome::Unavailable, None),
@@ -1118,8 +1119,6 @@ mod tests {
session_token: None,
}),
tls: TlsConfig::default(),
azure: None,
gcs: None,
},
filter: FilterConfig::default(),
policy: PolicyConfig::default(),
@@ -1400,21 +1399,13 @@ mod tests {
assert_eq!(queue.capacity(), 1024);
let mut outcomes = HashMap::new();
let mut shared_report = None;
for _ in 0..100 {
let (outcome, report) = queue.enqueue_with_report("a", PullReason::RangeGet);
*outcomes.entry(outcome).or_insert(0) += 1;
shared_report = report;
*outcomes.entry(queue.enqueue("a", PullReason::RangeGet)).or_insert(0) += 1;
}
assert_eq!(outcomes.get(&EnqueueOutcome::Enqueued), Some(&1));
assert_eq!(outcomes.get(&EnqueueOutcome::Coalesced), Some(&99));
assert_eq!(queue.pending_keys(), 1);
assert_eq!(
shared_report.expect("coalesced report").await,
Ok(QueuedPullOutcome::Stored { size: 1000 })
);
wait_until("first pull to finish", || queue.pending_keys() == 0).await;
assert_eq!(source.head_calls.load(Ordering::SeqCst), 1);
assert_eq!(source.get_calls.load(Ordering::SeqCst), 1);
@@ -1447,23 +1438,6 @@ mod tests {
assert_eq!(queue.enqueue("a", PullReason::RangeGet), EnqueueOutcome::Unavailable);
}
#[tokio::test]
async fn coalesced_enqueues_share_failure_reports() {
let sys = OnDemandMigrationSys::new();
let state = enabled_state(&sys, &config()).await;
let source = MockSource::with_object("missing", 1000, BodyKind::Bytes(body_bytes(1000)));
let queue = PullQueue::start(Arc::clone(&state), source, Arc::new(MockWriteBack::default()));
let (first, first_report) = queue.enqueue_with_report("absent", PullReason::RangeGet);
let (second, second_report) = queue.enqueue_with_report("absent", PullReason::Backfill);
assert_eq!(first, EnqueueOutcome::Enqueued);
assert_eq!(second, EnqueueOutcome::Coalesced);
let (first, second) = tokio::join!(first_report.expect("leader report"), second_report.expect("coalesced report"));
assert_eq!(first, second);
assert!(matches!(first, Ok(QueuedPullOutcome::Failed(_))));
sys.remove(BUCKET);
queue.wait_until_stopped().await;
}
#[tokio::test]
async fn queue_full_is_reported_and_cancel_drains_without_leaking_tasks() {
let sys = OnDemandMigrationSys::new();
@@ -1493,8 +1467,7 @@ mod tests {
wait_until("dispatcher to wait for a slot", || state.stats().queue_depth() == 1).await;
assert_eq!(queue.enqueue("c", PullReason::LargeObject), EnqueueOutcome::Enqueued);
assert_eq!(queue.enqueue("d", PullReason::LargeObject), EnqueueOutcome::QueueFull);
let (coalesced, canceled_report) = queue.enqueue_with_report("c", PullReason::LargeObject);
assert_eq!(coalesced, EnqueueOutcome::Coalesced);
assert_eq!(queue.enqueue("c", PullReason::LargeObject), EnqueueOutcome::Coalesced);
assert_eq!(queue.pending_keys(), 3);
assert_eq!(failures(&state).get("queue_full"), Some(&1));
assert!(!queue.is_stopped());
@@ -1504,12 +1477,6 @@ mod tests {
.await
.expect("dispatcher and in-flight job must exit after cancel");
assert!(queue.is_stopped());
assert!(
tokio::time::timeout(Duration::from_secs(5), canceled_report.expect("coalesced cancellation report"))
.await
.expect("cancellation closes the report")
.is_err()
);
assert_eq!(queue.pending_keys(), 0);
assert_eq!(state.inflight_keys(), 0);
assert_eq!(state.stats().inflight_pulls(), 0);
@@ -25,8 +25,6 @@
//! Client-supplied `If-*`, `Authorization`, `Host` and SSE-C headers are never
//! forwarded: v1 rejects SSE-C source objects outright.
use super::azure::AzureSourceBackend;
use super::gcs::GcsNativeSourceBackend;
use super::list_through::{ListPageError, validate_list_page};
use crate::bucket::remote_s3_client::{
PathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3EndpointSpec, RemoteS3RetryPolicy, build_remote_s3_config,
@@ -67,10 +65,6 @@ pub enum SourceProvider {
/// Generic S3-compatible service.
#[default]
S3,
/// Native Azure Blob service; not an S3 dialect.
Azure,
/// Native GCS JSON API with a service-account key; not an S3 dialect.
GcsNative,
}
impl SourceProvider {
@@ -82,8 +76,6 @@ impl SourceProvider {
"minio" => Some(Self::Minio),
"rustfs" => Some(Self::Rustfs),
"s3" => Some(Self::S3),
"azure" => Some(Self::Azure),
"gcs_native" => Some(Self::GcsNative),
_ => None,
}
}
@@ -96,8 +88,6 @@ impl SourceProvider {
Self::Minio => "minio",
Self::Rustfs => "rustfs",
Self::S3 => "s3",
Self::Azure => "azure",
Self::GcsNative => "gcs_native",
}
}
@@ -163,75 +153,12 @@ pub struct SourceClientSpec {
/// Wire requests one logical source call may cost. The pull pipeline and
/// the backfill job own the retry budget (`pull.rs` `PULL_MAX_RETRIES`,
/// `backfill.rs` `LIST_MAX_RETRIES`) and the breaker counts logical calls,
/// so ODM declares [`RemoteS3RetryPolicy::Disabled`]. An ambiguous HEAD
/// 404 additionally probes the bucket before declaring a key absent.
/// so ODM declares [`RemoteS3RetryPolicy::Disabled`] and keeps one counted
/// failure equal to one request against a struggling source.
pub retry: RemoteS3RetryPolicy,
/// Bytes per second the pull pipeline may consume from this source;
/// `None` means unlimited. Enforced by the consumer, not by this client.
pub bandwidth_limit: Option<NonZeroU64>,
/// Which [`SourceBackend`] to build. The S3 variant reads `region`,
/// `path_style` and `credentials`; the native variants ignore all three
/// and carry their own credentials.
pub backend: SourceBackendSpec,
}
/// Provider-specific half of [`SourceClientSpec`].
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub enum SourceBackendSpec {
#[default]
S3,
Azure(AzureSourceSpec),
Gcs(GcsSourceSpec),
}
/// Native Azure Blob parameters. The container is [`SourceClientSpec::bucket`].
#[derive(Clone, PartialEq, Eq)]
pub struct AzureSourceSpec {
pub account: String,
pub auth: AzureAuth,
}
impl fmt::Debug for AzureSourceSpec {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("AzureSourceSpec")
.field("account", &self.account)
.field("auth", &self.auth)
.finish()
}
}
/// How Azure requests are authorized.
#[derive(Clone, PartialEq, Eq)]
pub enum AzureAuth {
/// Base64 storage-account key, signed per request with Shared Key.
SharedKey(String),
/// SAS query string without the leading `?`, appended to every URL.
Sas(String),
}
impl fmt::Debug for AzureAuth {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
// Both variants are secrets; only the scheme may be rendered.
f.write_str(match self {
Self::SharedKey(_) => "SharedKey(REDACTED)",
Self::Sas(_) => "Sas(REDACTED)",
})
}
}
/// Native GCS parameters. The bucket is [`SourceClientSpec::bucket`].
#[derive(Clone, PartialEq, Eq)]
pub struct GcsSourceSpec {
/// Service-account key JSON.
pub service_account_json: String,
}
impl fmt::Debug for GcsSourceSpec {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("GcsSourceSpec")
.field("service_account_json", &"REDACTED")
.finish()
}
}
impl SourceClientSpec {
@@ -335,7 +262,7 @@ const THROTTLE_CODES: &[&str] = &[
"TooManyRequests",
"RequestThrottled",
];
const NOT_FOUND_CODES: &[&str] = &["NoSuchKey"];
const NOT_FOUND_CODES: &[&str] = &["NoSuchKey", "NotFound", "NoSuchBucket", "NoSuchVersion"];
const ACCESS_DENIED_CODES: &[&str] = &[
"AccessDenied",
"InvalidAccessKeyId",
@@ -345,7 +272,7 @@ const ACCESS_DENIED_CODES: &[&str] = &[
"InvalidToken",
];
pub(super) fn classify_status(status: u16, code: Option<&str>, message: String) -> SourceError {
fn classify_status(status: u16, code: Option<&str>, message: String) -> SourceError {
if let Some(code) = code {
if THROTTLE_CODES.contains(&code) {
return SourceError::Throttled;
@@ -358,6 +285,7 @@ pub(super) fn classify_status(status: u16, code: Option<&str>, message: String)
}
}
match status {
404 => SourceError::NotFound,
401 | 403 => SourceError::AccessDenied,
429 | 503 => SourceError::Throttled,
500..=599 => SourceError::ServerError(status),
@@ -417,11 +345,6 @@ pub struct SourceHead {
pub storage_class: Option<String>,
pub sse: Option<SourceSse>,
pub is_multipart_etag: bool,
/// The provider's ETag is not derived from the object bytes (Azure
/// stamps an opaque concurrency token). Such an ETag is recorded for
/// provenance but must never be read as a content digest, so the
/// write-back path refuses to use it as the expected MD5.
pub etag_is_opaque: bool,
}
/// Per-operation fields shared by HEAD and GET outputs.
@@ -443,7 +366,7 @@ struct HeadParts {
sse_customer_algorithm: Option<String>,
}
pub(super) fn normalize_etag(etag: Option<String>) -> Option<String> {
fn normalize_etag(etag: Option<String>) -> Option<String> {
etag.map(|etag| etag.trim().trim_matches('"').to_string())
.filter(|etag| !etag.is_empty())
}
@@ -492,7 +415,6 @@ fn source_head(parts: HeadParts) -> Result<SourceHead, SourceError> {
storage_class: parts.storage_class,
sse,
is_multipart_etag,
etag_is_opaque: false,
})
}
@@ -703,52 +625,14 @@ impl fmt::Debug for SourceClient {
impl SourceClient {
pub async fn new(spec: &SourceClientSpec) -> Result<Self, RemoteS3ClientError> {
match &spec.backend {
SourceBackendSpec::S3 => {
let endpoint = spec.endpoint_spec()?;
let config = build_remote_s3_config(&endpoint).await?;
Ok(Self::from_config_builder(config, endpoint.endpoint_url(), spec))
}
SourceBackendSpec::Azure(azure) => {
let backend = AzureSourceBackend::new(
&spec.endpoint,
&spec.bucket,
azure,
spec.timeouts,
spec.skip_tls_verify,
spec.ca_cert_pem.as_deref(),
)?;
Ok(Self::from_backend(Box::new(backend), spec))
}
SourceBackendSpec::Gcs(gcs) => {
let backend = GcsNativeSourceBackend::new(
&spec.endpoint,
&spec.bucket,
gcs,
spec.timeouts,
spec.skip_tls_verify,
spec.ca_cert_pem.as_deref(),
)?;
Ok(Self::from_backend(Box::new(backend), spec))
}
}
}
/// Wraps a ready backend in the prefix-mapping client. The endpoint is
/// kept only for `Debug` and admin status.
fn from_backend(backend: Box<dyn SourceBackend>, spec: &SourceClientSpec) -> Self {
Self {
backend,
endpoint: spec.endpoint.clone(),
bucket: spec.bucket.clone(),
source_prefix: spec.source_prefix.clone().filter(|prefix| !prefix.is_empty()),
timeouts: spec.timeouts,
bandwidth_limit: spec.bandwidth_limit,
}
let endpoint = spec.endpoint_spec()?;
let config = build_remote_s3_config(&endpoint).await?;
Ok(Self::from_config_builder(config, endpoint.endpoint_url(), spec))
}
/// `config` must come from [`SourceClientSpec::endpoint_spec`], which is
/// where the policy disabling SDK-level retries is declared.
/// where the retry policy that keeps one logical call equal to one wire
/// request is declared.
fn from_config_builder(config: aws_sdk_s3::config::Builder, endpoint: String, spec: &SourceClientSpec) -> Self {
let client = S3Client::from_conf(config.interceptor(SourceProxyMarkerInterceptor::new()).build());
Self {
@@ -870,16 +754,15 @@ impl SourceClient {
#[async_trait::async_trait]
impl SourceBackend for S3SourceBackend {
async fn head(&self, key: &str) -> Result<SourceHead, SourceError> {
match self.client.head_object().bucket(&self.bucket).key(key).send().await {
Ok(output) => source_head_from_head_output(output),
Err(err) if err.raw_response().is_some_and(|response| response.status().as_u16() == 404) => {
// HEAD has no error body: a missing bucket must not poison
// the per-key negative cache as though only the key was absent.
self.probe().await?;
Err(SourceError::NotFound)
}
Err(err) => Err(classify_sdk_error(err)),
}
let output = self
.client
.head_object()
.bucket(&self.bucket)
.key(key)
.send()
.await
.map_err(classify_sdk_error)?;
source_head_from_head_output(output)
}
/// Streams the object; `range` is passed through as an HTTP `Range`
@@ -926,8 +809,8 @@ impl SourceBackend for S3SourceBackend {
.contents
.unwrap_or_default()
.into_iter()
.map(s3_source_object)
.collect::<Result<Vec<_>, _>>()?;
.filter_map(s3_source_object)
.collect();
let common_prefixes = output
.common_prefixes
.unwrap_or_default()
@@ -966,20 +849,14 @@ impl SourceBackend for S3SourceBackend {
}
}
fn s3_source_object(object: SdkObject) -> Result<SourceObject, SourceError> {
let key = object
.key
.ok_or_else(|| SourceError::Other("source listing object has no key".to_string()))?;
let size = object
.size
.and_then(|size| u64::try_from(size).ok())
.ok_or_else(|| SourceError::Other("source listing object has no valid size".to_string()))?;
fn s3_source_object(object: SdkObject) -> Option<SourceObject> {
let key = object.key?;
let etag = normalize_etag(object.e_tag);
let is_multipart_etag = etag.as_deref().is_some_and(is_multipart_etag);
Ok(SourceObject {
Some(SourceObject {
key,
etag,
size,
size: object.size.and_then(|size| u64::try_from(size).ok()).unwrap_or(0),
last_modified: system_time(object.last_modified),
storage_class: object.storage_class.map(|class| class.as_str().to_string()),
is_multipart_etag,
@@ -989,7 +866,6 @@ fn s3_source_object(object: SdkObject) -> Result<SourceObject, SourceError> {
#[cfg(test)]
mod tests {
use super::*;
use crate::bucket::on_demand_migration::backend_contract::{BackendCapabilities, OBJECT_MD5, assert_backend_contract};
use aws_smithy_runtime_api::client::http::{HttpConnector, HttpConnectorFuture, SharedHttpConnector, http_client_fn};
use aws_smithy_runtime_api::client::orchestrator::HttpRequest;
use aws_smithy_runtime_api::client::result::ConnectorError;
@@ -1107,7 +983,6 @@ mod tests {
retry: RemoteS3RetryPolicy::Disabled,
timeouts: SourceTimeouts::default(),
bandwidth_limit: NonZeroU64::new(1_000_000),
backend: SourceBackendSpec::S3,
}
}
@@ -1614,10 +1489,7 @@ mod tests {
#[tokio::test]
async fn source_error_classification_covers_every_class() {
let cases: Vec<(Scripted, &str, bool)> = vec![
(status(404, ""), "other", false),
(status(404, "<Error><Code>NoSuchKey</Code></Error>"), "not_found", false),
(status(404, "<Error><Code>NoSuchBucket</Code></Error>"), "other", false),
(status(404, "<Error><Code>NoSuchVersion</Code></Error>"), "other", false),
(status(404, ""), "not_found", false),
(status(403, ACCESS_DENIED_BODY), "access_denied", false),
(status(401, ""), "access_denied", false),
(status(429, ""), "throttled", true),
@@ -1640,35 +1512,14 @@ mod tests {
}
}
let (client, requests) = scripted_client(&spec(None), vec![status(404, ""), status(200, "")]).await;
// HEAD carries no error body, so the classification must work from the
// status alone as well.
let (client, _) = scripted_client(&spec(None), vec![status(404, "")]).await;
assert!(matches!(client.head_object("missing").await, Err(SourceError::NotFound)));
assert_eq!(recorded(&requests).len(), 2, "ambiguous HEAD 404 must check the bucket");
let (client, _) = scripted_client(&spec(None), vec![status(404, ""), status(404, "")]).await;
assert!(matches!(client.head_object("missing").await, Err(SourceError::Other(_))));
let (client, _) = scripted_client(&spec(None), vec![status(404, ""), status(403, "")]).await;
assert!(matches!(client.head_object("missing").await, Err(SourceError::AccessDenied)));
let (client, _) = scripted_client(&spec(None), vec![status(403, "")]).await;
assert!(matches!(client.head_object("secret").await, Err(SourceError::AccessDenied)));
}
#[test]
fn source_listing_rejects_missing_and_negative_sizes() {
for size in [None, Some(-1)] {
let object = SdkObject::builder().key("key").set_size(size).build();
assert!(matches!(s3_source_object(object), Err(SourceError::Other(_))));
}
assert!(matches!(
s3_source_object(SdkObject::builder().size(0).build()),
Err(SourceError::Other(_))
));
assert_eq!(
s3_source_object(SdkObject::builder().key("empty").size(0).build())
.expect("empty object")
.size,
0
);
}
#[tokio::test]
async fn source_client_debug_redacts_credentials() {
let (client, _) = scripted_client(&spec(Some("data/")), Vec::new()).await;
@@ -1735,101 +1586,7 @@ mod tests {
assert_eq!(resolve_path_style(PathStyle::VirtualHost, Minio, "10.0.0.1"), PathStyle::VirtualHost);
assert_eq!(resolve_path_style(PathStyle::Path, Aws, "s3.amazonaws.com"), PathStyle::Path);
assert_eq!(SourceProvider::from_label(" AWS "), Some(Aws));
assert_eq!(SourceProvider::from_label(" Azure "), Some(Azure));
assert_eq!(SourceProvider::from_label("gcs_native"), Some(GcsNative));
assert_eq!(SourceProvider::from_label("swift"), None);
}
const CONTRACT_LIST_PAGE_ONE: &str = r#"<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Name>source-bucket</Name>
<IsTruncated>true</IsTruncated>
<NextContinuationToken>cursor-1</NextContinuationToken>
<Contents>
<Key>dir/a.txt</Key>
<LastModified>2015-10-21T07:28:00.000Z</LastModified>
<ETag>&quot;5d41402abc4b2a76b9719d911017c592&quot;</ETag>
<Size>5</Size>
<StorageClass>STANDARD</StorageClass>
</Contents>
<CommonPrefixes><Prefix>dir/sub/</Prefix></CommonPrefixes>
</ListBucketResult>"#;
const CONTRACT_LIST_PAGE_TWO: &str = r#"<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Name>source-bucket</Name>
<IsTruncated>false</IsTruncated>
<Contents>
<Key>dir/b.txt</Key>
<LastModified>2015-10-21T07:28:00.000Z</LastModified>
<ETag>&quot;7d41402abc4b2a76b9719d911017c592&quot;</ETag>
<Size>7</Size>
</Contents>
</ListBucketResult>"#;
const CONTRACT_TAGGING: &str = r#"<?xml version="1.0" encoding="UTF-8"?>
<Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><TagSet>
<Tag><Key>env</Key><Value>prod</Value></Tag>
</TagSet></Tagging>"#;
fn contract_object_headers(content_length: u64) -> Vec<(&'static str, String)> {
vec![
("etag", format!("\"{OBJECT_MD5}\"")),
("content-length", content_length.to_string()),
("content-type", "text/plain".to_string()),
("last-modified", "Wed, 21 Oct 2015 07:28:00 GMT".to_string()),
("x-amz-meta-owner", "alice".to_string()),
("x-amz-storage-class", "STANDARD".to_string()),
]
}
/// The S3 backend behind the scripted connector, without the prefix-mapping
/// client on top: the contract is a property of the backend itself.
async fn scripted_s3_backend(responses: Vec<Scripted>) -> S3SourceBackend {
let spec = spec(None);
let connector = SharedHttpConnector::new(ScriptedConnector {
requests: Arc::new(Mutex::new(Vec::new())),
responses: Arc::new(Mutex::new(responses.into_iter().collect())),
});
let http_client = http_client_fn(move |_settings, _components| connector.clone());
let endpoint = spec.endpoint_spec().expect("test spec endpoint should parse");
let config = build_remote_s3_config(&endpoint)
.await
.expect("test spec should build")
.http_client(http_client)
.interceptor(SourceProxyMarkerInterceptor::new());
S3SourceBackend {
client: S3Client::from_conf(config.build()),
bucket: spec.bucket.clone(),
}
}
#[tokio::test]
async fn s3_backend_satisfies_the_shared_backend_contract() {
let mut ranged = contract_object_headers(3);
ranged.push(("content-range", "bytes 1-3/5".to_string()));
let backend = scripted_s3_backend(vec![
ok(contract_object_headers(5), ""),
ok(contract_object_headers(5), "hello"),
ok(ranged, "ell"),
ok(Vec::new(), CONTRACT_LIST_PAGE_ONE),
ok(Vec::new(), CONTRACT_LIST_PAGE_TWO),
ok(Vec::new(), CONTRACT_TAGGING),
ok(Vec::new(), ""),
status(404, ""),
status(403, ACCESS_DENIED_BODY),
])
.await;
assert_backend_contract(
&backend,
BackendCapabilities {
etag_is_opaque: false,
supports_start_after: true,
supports_tagging: true,
},
)
.await;
assert_eq!(SourceProvider::from_label("azure"), None);
}
fn prefix_client(prefix: Option<String>) -> SourceClient {
@@ -47,10 +47,7 @@ use super::config::{
use super::list_through::{SOURCE_LIST_RATE_PER_SEC, SourceListRateLimiter};
use super::negative_cache::NegativeCache;
use super::pull::{OdmWriteBack, PullQueue};
use super::source_client::{
AzureAuth, AzureSourceSpec, GcsSourceSpec, SourceBackendSpec, SourceClient, SourceClientSpec, SourceError, SourceProvider,
SourceTimeouts,
};
use super::source_client::{SourceClient, SourceClientSpec, SourceError, SourceProvider, SourceTimeouts};
use super::stats::{GaugeGuard, OdmStats, OdmStatsSnapshot, PullFailureReason};
use crate::bucket::remote_s3_client::{
PathStyle as ClientPathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3RetryPolicy,
@@ -622,7 +619,6 @@ pub fn source_client_spec(config: &OnDemandMigrationConfig) -> SourceClientSpec
// load on a source that is already failing.
retry: RemoteS3RetryPolicy::Disabled,
bandwidth_limit: policy.bandwidth_limit_bytes_per_sec.and_then(NonZeroU64::new),
backend: source_backend_spec(source),
}
}
@@ -634,31 +630,6 @@ fn source_provider(provider: Provider) -> SourceProvider {
Provider::Rustfs => SourceProvider::Rustfs,
Provider::R2 => SourceProvider::R2,
Provider::Gcs => SourceProvider::Gcs,
Provider::Azure => SourceProvider::Azure,
Provider::GcsNative => SourceProvider::GcsNative,
}
}
/// Which backend the client builds. A native provider whose block is missing
/// falls back to the S3 spec, where the builder reports the missing
/// credentials: the config layer already refuses to store that shape, so this
/// only covers a config written by an older or hand-edited build.
pub fn source_backend_spec(source: &SourceConfig) -> SourceBackendSpec {
match (source.provider, source.azure.as_ref(), source.gcs.as_ref()) {
(Provider::Azure, Some(azure), _) => SourceBackendSpec::Azure(AzureSourceSpec {
account: azure.account.clone(),
auth: match (&azure.account_key, &azure.sas_token) {
(Some(key), _) => AzureAuth::SharedKey(key.clone()),
(None, Some(sas)) => AzureAuth::Sas(sas.clone()),
// Refused by `SourceConfig::validate`; an empty shared key
// fails closed at the builder rather than signing with none.
(None, None) => AzureAuth::SharedKey(String::new()),
},
}),
(Provider::GcsNative, _, Some(gcs)) => SourceBackendSpec::Gcs(GcsSourceSpec {
service_account_json: gcs.service_account_json.clone(),
}),
_ => SourceBackendSpec::S3,
}
}
@@ -958,8 +929,6 @@ mod tests {
session_token: None,
}),
tls: TlsConfig::default(),
azure: None,
gcs: None,
},
filter: FilterConfig {
prefix: prefix.map(str::to_string),
@@ -1,120 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Scripted HTTP server for the native source backends' tests.
//!
//! The S3 backend can be driven through the SDK's own connector; the native
//! backends talk to a real socket, so their tests need a server that answers a
//! fixed script and records what it was asked. Every response closes its
//! connection, which keeps one request on one socket and makes the script order
//! exactly the request order.
use std::sync::{Arc, Mutex};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpListener;
use url::Url;
pub(super) struct ScriptedResponse {
status: u16,
headers: Vec<(&'static str, String)>,
body: String,
}
impl ScriptedResponse {
pub(super) fn new(status: u16, headers: Vec<(&'static str, String)>, body: String) -> Self {
Self { status, headers, body }
}
}
#[derive(Clone, Debug)]
pub(super) struct RecordedRequest {
pub(super) method: String,
/// Request target as it appeared on the wire: path plus query.
pub(super) target: String,
pub(super) headers: Vec<(String, String)>,
}
impl RecordedRequest {
pub(super) fn header(&self, name: &str) -> Option<&str> {
self.headers
.iter()
.find(|(key, _)| key.eq_ignore_ascii_case(name))
.map(|(_, value)| value.as_str())
}
}
pub(super) type Recorder = Arc<Mutex<Vec<RecordedRequest>>>;
/// Binds a loopback listener that answers `responses` in order and returns its
/// origin plus the recorder. The task ends once the script is exhausted.
pub(super) async fn scripted_server(responses: Vec<ScriptedResponse>) -> (Url, Recorder) {
let listener = TcpListener::bind("127.0.0.1:0").await.expect("fixture listener should bind");
let port = listener.local_addr().expect("fixture address").port();
let recorder: Recorder = Arc::new(Mutex::new(Vec::new()));
let sink = Arc::clone(&recorder);
tokio::spawn(async move {
for response in responses {
let Ok((mut stream, _)) = listener.accept().await else {
return;
};
let mut request = Vec::new();
let mut buffer = [0_u8; 2048];
while !request.windows(4).any(|window| window == b"\r\n\r\n") {
match stream.read(&mut buffer).await {
Ok(0) | Err(_) => break,
Ok(read) => request.extend_from_slice(&buffer[..read]),
}
}
let text = String::from_utf8_lossy(&request).into_owned();
let mut lines = text.lines();
let start = lines.next().unwrap_or_default().to_string();
let mut parts = start.split_whitespace();
sink.lock().expect("recorder lock").push(RecordedRequest {
method: parts.next().unwrap_or_default().to_string(),
target: parts.next().unwrap_or_default().to_string(),
headers: lines
.take_while(|line| !line.is_empty())
.filter_map(|line| line.split_once(':'))
.map(|(name, value)| (name.trim().to_string(), value.trim().to_string()))
.collect(),
});
// A scripted HEAD declares the object size in its own headers while
// carrying no body, so an explicit `Content-Length` wins over the
// body length.
let declares_length = response
.headers
.iter()
.any(|(name, _)| name.eq_ignore_ascii_case("content-length"));
let mut rendered = match declares_length {
true => format!("HTTP/1.1 {} Scripted\r\nConnection: close\r\n", response.status),
false => format!(
"HTTP/1.1 {} Scripted\r\nContent-Length: {}\r\nConnection: close\r\n",
response.status,
response.body.len()
),
};
for (name, value) in response.headers {
rendered.push_str(&format!("{name}: {value}\r\n"));
}
rendered.push_str("\r\n");
rendered.push_str(&response.body);
let _ = stream.write_all(rendered.as_bytes()).await;
let _ = stream.flush().await;
}
});
(Url::parse(&format!("http://127.0.0.1:{port}")).expect("fixture endpoint"), recorder)
}
-3
View File
@@ -956,9 +956,6 @@ pub struct ObjectOptions {
pub preserve_etag: Option<String>,
pub metadata_chg: bool,
pub http_preconditions: Option<HTTPPreconditions>,
/// Internal create-only writes may also preserve an acknowledged deletion.
/// Evaluated with `http_preconditions` under the namespace commit lock.
pub preserve_delete_marker: bool,
pub delete_replication: Option<ReplicationState>,
pub delete_replication_config_snapshot: Option<Arc<DeleteReplicationConfigSnapshot>>,
-112
View File
@@ -78,21 +78,6 @@ pub(crate) struct ScannerPublicationLeaseEntry {
pub(crate) _operation_guard: OwnedRwLockReadGuard<()>,
}
pub(crate) struct NamespaceCommitGuard {
ctx: Arc<InstanceContext>,
counted: bool,
}
impl Drop for NamespaceCommitGuard {
fn drop(&mut self) {
if self.counted {
// Publish the new generation before a zero-pending publication probe.
self.ctx.advance_namespace_commit_generation();
self.ctx.namespace_commits.fetch_sub(1, Ordering::AcqRel);
}
}
}
/// Runtime state owned by a single `ECStore` instance.
///
/// This is intentionally minimal in the first migration slice; subsequent
@@ -224,13 +209,9 @@ pub struct InstanceContext {
/// Last storage-owned movement snapshot observed under the operation
/// gate. SetDisks cache writers fail closed until ECStore refreshes it.
scanner_publication_state: AtomicU8,
namespace_commits: AtomicU64,
namespace_commit_generation: AtomicU64,
/// Resolves object-encryption material at the application boundary.
object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
#[cfg(test)]
suppress_tier_delete_journal_recovery: bool,
transition_transaction_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
tier_delete_journal_recovery_wakeup: tokio::sync::Notify,
}
@@ -275,12 +256,8 @@ impl InstanceContext {
data_movement_generation_exhausted: AtomicBool::new(false),
data_movement_generation_notify: Arc::new(Notify::new()),
scanner_publication_state: AtomicU8::new(SCANNER_PUBLICATION_STATE_UNKNOWN),
namespace_commits: AtomicU64::new(0),
namespace_commit_generation: AtomicU64::new(0),
object_encryption_resolver: OnceLock::new(),
tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
#[cfg(test)]
suppress_tier_delete_journal_recovery: false,
transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()),
tier_delete_journal_recovery_wakeup: tokio::sync::Notify::new(),
}
@@ -408,36 +385,6 @@ impl InstanceContext {
&& self.scanner_publication_state.load(Ordering::Acquire) == SCANNER_PUBLICATION_STATE_ALLOWED
}
pub(crate) fn begin_namespace_commit(self: &Arc<Self>) -> Arc<NamespaceCommitGuard> {
let counted = self
.namespace_commits
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| count.checked_add(1))
.is_ok();
if counted {
self.advance_namespace_commit_generation();
} else {
self.namespace_commit_generation.store(u64::MAX, Ordering::Release);
}
Arc::new(NamespaceCommitGuard {
ctx: Arc::clone(self),
counted,
})
}
fn advance_namespace_commit_generation(&self) {
let _ = self
.namespace_commit_generation
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |generation| Some(generation.saturating_add(1)));
}
pub(crate) fn namespace_commit_generation(&self) -> u64 {
self.namespace_commit_generation.load(Ordering::Acquire)
}
pub(crate) fn namespace_commits_pending(&self) -> bool {
self.namespace_commits.load(Ordering::Acquire) != 0 || self.namespace_commit_generation() == u64::MAX
}
pub(crate) fn set_scanner_publication_state(&self, blocked: bool) {
self.scanner_publication_state.store(
if blocked {
@@ -693,21 +640,12 @@ impl InstanceContext {
}
pub(crate) fn mark_tier_delete_journal_recovery_started(&self, store_id: Uuid) -> bool {
#[cfg(test)]
if self.suppress_tier_delete_journal_recovery {
return false;
}
self.tier_delete_journal_recovery_stores
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.insert(store_id)
}
#[cfg(test)]
pub(crate) fn suppress_tier_delete_journal_recovery_for_test(&mut self) {
self.suppress_tier_delete_journal_recovery = true;
}
pub(crate) fn mark_transition_transaction_recovery_started(&self, store_id: Uuid) -> bool {
self.transition_transaction_recovery_stores
.lock()
@@ -818,50 +756,6 @@ pub fn bootstrap_ctx() -> Arc<InstanceContext> {
mod tests {
use super::*;
#[test]
fn namespace_commit_guards_are_instance_local_and_count_until_last_owner() {
let first = Arc::new(InstanceContext::new());
let other = Arc::new(InstanceContext::new());
first.set_scanner_publication_state(false);
other.set_scanner_publication_state(false);
assert!(first.scanner_publication_state_allowed());
let one = first.begin_namespace_commit();
let shared_owner = Arc::clone(&one);
let two = first.begin_namespace_commit();
assert!(first.namespace_commits_pending());
assert!(first.scanner_publication_state_allowed(), "pending writes must not block scan admission");
assert_eq!(first.namespace_commit_generation(), 2);
assert!(!other.namespace_commits_pending());
assert_eq!(other.namespace_commit_generation(), 0);
assert!(other.scanner_publication_state_allowed());
drop(one);
assert_eq!(first.namespace_commit_generation(), 2);
drop(shared_owner);
assert!(first.namespace_commits_pending());
assert_eq!(first.namespace_commit_generation(), 3);
drop(two);
assert!(!first.namespace_commits_pending());
assert_eq!(first.namespace_commit_generation(), 4);
assert!(first.scanner_publication_state_allowed());
}
#[test]
fn namespace_commit_counter_exhaustion_keeps_publication_blocked() {
for (count, generation) in [(0, u64::MAX - 1), (u64::MAX, 0)] {
let ctx = Arc::new(InstanceContext::new());
ctx.set_scanner_publication_state(false);
ctx.namespace_commits.store(count, Ordering::Release);
ctx.namespace_commit_generation.store(generation, Ordering::Release);
let guard = ctx.begin_namespace_commit();
assert!(ctx.namespace_commits_pending());
assert_eq!(ctx.namespace_commit_generation(), u64::MAX);
drop(guard);
assert!(ctx.namespace_commits_pending());
assert_eq!(ctx.namespace_commit_generation(), u64::MAX);
assert_eq!(ctx.namespace_commits.load(Ordering::Acquire), count);
}
}
// The SetupType inputs must derive the exact (is_erasure,
// is_dist_erasure, is_erasure_sd) triples that the original three
// process-global erasure bools produced via update_erasure_type().
@@ -1179,12 +1073,6 @@ mod tests {
assert!(!ctx_a.mark_tier_delete_journal_recovery_started(store_a));
assert!(ctx_a.mark_tier_delete_journal_recovery_started(store_b));
assert!(ctx_b.mark_tier_delete_journal_recovery_started(store_a));
let mut manual_ctx = InstanceContext::new();
manual_ctx.suppress_tier_delete_journal_recovery_for_test();
assert!(!manual_ctx.mark_tier_delete_journal_recovery_started(store_a));
assert!(!manual_ctx.mark_tier_delete_journal_recovery_started(store_b));
assert!(ctx_b.mark_tier_delete_journal_recovery_started(store_b));
}
#[test]
+19 -168
View File
@@ -3541,7 +3541,7 @@ impl TierConfigMgr {
// Get tier configuration and create new driver
let tier_config = self.tiers.get(tier_name).ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?;
let driver = construct_warm_backend(tier_config).await?;
let driver = new_warm_backend(tier_config, false).await?;
self.replace_driver(tier_name, driver)?;
Ok(self
@@ -4486,11 +4486,6 @@ impl TierConfigMgr {
let committed_coordinator_intent =
committed_tier_mutation_intent(coordinator_intent.as_ref(), &committed_config_etag)
.map_err(TierConfigUpdateError::Save)?;
// Persist Committed before notifying refresh; a Prepared disk record
// would restore the prepared block and invalidate our publish allowance.
let coordinator_commit =
commit_coordinator_tier_mutation_intent(api.clone(), coordinator_intent.as_ref(), &committed_config_etag)
.await;
if let Some(intent) = committed_coordinator_intent.as_ref() {
TierConfigMgr::apply_committed_mutation_intent_block(&handle, intent)
.await
@@ -4501,9 +4496,9 @@ impl TierConfigMgr {
.map_err(TierConfigUpdateError::Publish)?,
);
}
// Config is already saved: retain the committed fence and wake recovery
// even when the coordinator commit failed or its outcome is unknown.
coordinator_commit.map_err(TierConfigUpdateError::Save)?;
commit_coordinator_tier_mutation_intent(api.clone(), coordinator_intent.as_ref(), &committed_config_etag)
.await
.map_err(TierConfigUpdateError::Save)?;
if coordinated_config_update {
drop(update.take());
drop(config_lock.take());
@@ -10608,11 +10603,6 @@ mod tests {
.expect_err("coordinator committed-state CAS failure must be observable");
assert!(matches!(err, TierConfigUpdateError::Save(_)));
assert!(manager.read().await.tiers.contains_key("COLD-A"));
assert!(TierConfigMgr::has_committed_mutation_block(&manager).await);
let refresh = TierConfigMgr::mutation_refresh_notifier(&manager).await;
tokio::time::timeout(Duration::from_secs(1), refresh.notified())
.await
.expect("failed coordinator commit must notify recovery after saving config");
let blocked = match TierConfigMgr::acquire_operation_lease(&manager, "COLD-A").await {
Ok(_) => panic!("failed coordinator commit CAS must retain the local committed fence"),
Err(err) => err,
@@ -14339,12 +14329,6 @@ mod tests {
after_commit: bool,
}
#[derive(Debug, Default)]
struct CasCoordinatorCommitBarrier {
arrived: tokio::sync::Notify,
release: tokio::sync::Notify,
}
#[derive(Debug)]
struct CasConfigStore {
objects: tokio::sync::Mutex<HashMap<String, (Vec<u8>, String)>>,
@@ -14357,7 +14341,6 @@ mod tests {
fail_delete_prefix: tokio::sync::Mutex<Option<(String, usize)>>,
delete_log: tokio::sync::Mutex<Vec<String>>,
list_barrier: tokio::sync::Mutex<Option<Arc<CasListBarrier>>>,
coordinator_commit_barrier: tokio::sync::Mutex<Option<Arc<CasCoordinatorCommitBarrier>>>,
intent_list_calls: AtomicUsize,
fail_reference_walk: AtomicBool,
reference_walk_send_count: AtomicUsize,
@@ -14380,7 +14363,6 @@ mod tests {
fail_delete_prefix: tokio::sync::Mutex::new(None),
delete_log: tokio::sync::Mutex::new(Vec::new()),
list_barrier: tokio::sync::Mutex::new(None),
coordinator_commit_barrier: tokio::sync::Mutex::new(None),
intent_list_calls: AtomicUsize::new(0),
fail_reference_walk: AtomicBool::new(false),
reference_walk_send_count: AtomicUsize::new(0),
@@ -14572,19 +14554,6 @@ mod tests {
}
let mut payload = Vec::new();
tokio::io::AsyncReadExt::read_to_end(&mut data.stream, &mut payload).await?;
if object.starts_with(crate::services::tier::tier_mutation_intent::TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX)
&& opts
.http_preconditions
.as_ref()
.and_then(HTTPPreconditions::if_match_value)
.is_some()
{
let barrier = self.coordinator_commit_barrier.lock().await.take();
if let Some(barrier) = barrier {
barrier.arrived.notify_one();
barrier.release.notified().await;
}
}
let race_rewrite = if opts
.http_preconditions
.as_ref()
@@ -15682,7 +15651,14 @@ mod tests {
);
}
async fn assert_lifecycle_only_reference_obeys_force(clear: bool, force: bool) {
#[tokio::test]
async fn force_remove_and_save_bypasses_lifecycle_only_reference() {
// rustfs/rustfs#6832: reproduces the admin RemoveTier path (not just the lower-level
// reference-proof function) for a tier with zero transitioned objects but a lifecycle
// rule still pointing at it — the exact shape of
// `test_manual_transition_async_tier_failure_reports_terminal_partial` in e2e_test,
// which force-removes a tier a lifecycle rule still references to simulate a
// decommissioned backend.
let store = Arc::new(CasConfigStore::default());
let tier = build_rustfs_tier("COLD-A");
let mut persisted = empty_mgr();
@@ -15723,55 +15699,22 @@ mod tests {
let manager = TierConfigMgr::new();
manager.write().await.tiers.insert("COLD-A".to_string(), tier);
let mutation = if clear {
TierCandidateMutation::Clear(force)
} else {
TierCandidateMutation::Remove("COLD-A".to_string(), force)
};
let result = TIER_DRIVER_TEST_FACTORY
.scope(
healthy_driver_factory(),
TierConfigMgr::update_candidate_with_config_lock(&manager, store.clone(), mutation),
)
.await;
if force {
result.expect("force mutation must bypass a lifecycle-config-only reference");
} else {
let err = result.expect_err("non-force mutation must reject a lifecycle-only reference");
let TierConfigUpdateError::Publish(err) = err else {
panic!("non-force mutation must fail during reference proof: {err:?}");
};
assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
assert!(err.message.contains("move-current"), "{err}");
}
TierConfigMgr::remove_and_save_with(&manager, store.clone(), "COLD-A", true)
.await
.expect("force remove must bypass a lifecycle-config-only reference");
assert_eq!(manager.read().await.tiers.contains_key("COLD-A"), !force);
assert_eq!(
load_tier_config_for_update(store)
assert!(!manager.read().await.tiers.contains_key("COLD-A"));
assert!(
!load_tier_config_for_update(store)
.await
.expect("config should still reload")
.0
.tiers
.contains_key("COLD-A"),
!force,
"persisted state must match the force mutation result"
"force removal must persist the empty candidate"
);
}
#[tokio::test]
async fn remove_with_config_lock_obeys_force_for_lifecycle_only_reference() {
for force in [false, true] {
assert_lifecycle_only_reference_obeys_force(false, force).await;
}
}
#[tokio::test]
async fn clear_with_config_lock_obeys_force_for_lifecycle_only_reference() {
for force in [false, true] {
assert_lifecycle_only_reference_obeys_force(true, force).await;
}
}
#[tokio::test]
async fn zero_reference_proof_blocks_clear_before_config_save() {
let store = Arc::new(CasConfigStore::default());
@@ -17312,98 +17255,6 @@ mod tests {
assert_ne!(manager_a.read().await.empty(), manager_b.read().await.empty());
}
async fn assert_coordinator_commit_refresh_succeeds(mutation: TierCandidateMutation) {
let adding = matches!(mutation, TierCandidateMutation::Add(..));
let manager = TierConfigMgr::new();
let store = Arc::new(CasConfigStore::default());
if !adding {
let mut persisted = empty_mgr();
persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
persisted
.save_tiering_config_if_current(store.clone(), None)
.await
.expect("existing tier fixture should persist");
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
}
let barrier = Arc::new(CasCoordinatorCommitBarrier::default());
*store.coordinator_commit_barrier.lock().await = Some(barrier.clone());
let update_manager = manager.clone();
let update_store = store.clone();
let update = tokio::spawn(async move {
TIER_DRIVER_TEST_FACTORY
.scope(
healthy_driver_factory(),
TIER_MUTATION_TEST_PEERS.scope(
Vec::new(),
TierConfigMgr::update_candidate_with_config_lock(&update_manager, update_store, mutation),
),
)
.await
});
tokio::time::timeout(Duration::from_secs(5), barrier.arrived.notified())
.await
.expect("mutation should reach coordinator commit after saving config");
assert_eq!(
load_tier_config_for_update(store.clone())
.await
.expect("saved config should be readable before coordinator commit")
.0
.tiers
.contains_key("COLD-A"),
adding
);
assert_eq!(
TierConfigMgr::load_coordinator_mutation_intents(store.clone())
.await
.expect("coordinator intent should remain readable")[0]
.state,
TierMutationIntentState::Prepared
);
let lock_requests = lock_unpoisoned(&store.lock_requests).len();
// Also exercise an independently scheduled refresh while the durable
// coordinator record is still Prepared, before its commit notification.
TierConfigMgr::request_committed_mutation_refresh(&manager).await;
TIER_MUTATION_TEST_PEERS
.scope(Vec::new(), async {
let worker = TierConfigMgr::refresh_tier_config_handle_with(manager.clone(), store.clone());
tokio::pin!(worker);
tokio::time::timeout(Duration::from_secs(5), async {
while lock_unpoisoned(&store.lock_requests).len() == lock_requests {
tokio::select! {
_ = &mut worker => panic!("refresh worker must remain available"),
_ = tokio::task::yield_now() => {}
}
}
})
.await
.expect("refresh should reconcile the Prepared record before waiting for the config lock");
barrier.release.notify_one();
let result = tokio::time::timeout(Duration::from_secs(5), async {
tokio::select! {
_ = &mut worker => panic!("refresh worker must remain available"),
result = update => result.expect("tier mutation task should join"),
}
})
.await
.expect("tier mutation should finish with refresh running");
result.expect("saved tier mutation must publish successfully on the first attempt");
})
.await;
assert_eq!(manager.read().await.tiers.contains_key("COLD-A"), adding);
}
#[tokio::test]
async fn tier_add_succeeds_with_refresh_during_coordinator_commit() {
assert_coordinator_commit_refresh_succeeds(TierCandidateMutation::Add(build_rustfs_tier("COLD-A"), true)).await;
}
#[tokio::test]
async fn tier_remove_succeeds_with_refresh_during_coordinator_commit() {
assert_coordinator_commit_refresh_succeeds(TierCandidateMutation::Remove("COLD-A".to_string(), true)).await;
}
async fn committed_refresh_fixture(fail_cleanup: bool) -> (Arc<RwLock<TierConfigMgr>>, Arc<CasConfigStore>, uuid::Uuid) {
let manager = TierConfigMgr::new();
{
@@ -37,7 +37,7 @@ use crate::services::tier::{
use bytes::Bytes;
use http::StatusCode;
use rustfs_s3_client::credentials::{Credentials, SignatureType, Static, Value};
use rustfs_s3_client::transition_api::{BucketLookupType, Options, TransitionClient, TransitionClientTimeouts, TransitionCore};
use rustfs_s3_client::transition_api::{BucketLookupType, Options, TransitionClient, TransitionCore};
use rustfs_s3_client::{
admin_handler_utils::AdminError,
api_error_response::to_error_response,
@@ -320,27 +320,6 @@ pub(crate) fn endpoint_authority(url: &url::Url) -> Result<String, std::io::Erro
}
}
fn transition_timeout_from_env(env_key: &str, default_secs: u64) -> Duration {
Duration::from_secs(rustfs_utils::get_env_u64(env_key, default_secs))
}
pub(crate) fn transition_client_timeouts_from_env() -> TransitionClientTimeouts {
TransitionClientTimeouts::new(
transition_timeout_from_env(
rustfs_config::ENV_TIER_REMOTE_CONNECT_TIMEOUT_SECS,
rustfs_config::DEFAULT_TIER_REMOTE_CONNECT_TIMEOUT_SECS,
),
transition_timeout_from_env(
rustfs_config::ENV_TIER_REMOTE_REQUEST_TIMEOUT_SECS,
rustfs_config::DEFAULT_TIER_REMOTE_REQUEST_TIMEOUT_SECS,
),
transition_timeout_from_env(
rustfs_config::ENV_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS,
rustfs_config::DEFAULT_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS,
),
)
}
/// Build the [`WarmBackendS3`] shared by the S3-compatible warm backend providers.
///
/// Credential, bucket, and endpoint validation run in this order because the
@@ -371,7 +350,6 @@ pub(crate) async fn new_s3_compatible_warm_backend(
signer_type: SignatureType::SignatureV4,
..Default::default()
}));
let timeouts = transition_client_timeouts_from_env();
let opts = Options {
creds,
secure: u.scheme() == "https",
@@ -384,7 +362,7 @@ pub(crate) async fn new_s3_compatible_warm_backend(
// Run the SSRF guard after the host-presence check so a host-less endpoint
// keeps this constructor's stable error text.
(params.validate_endpoint)(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
let client = TransitionClient::new_with_timeouts(&endpoint, opts, params.provider_tag, timeouts).await?;
let client = TransitionClient::new(&endpoint, opts, params.provider_tag).await?;
let client = Arc::new(client);
let core = TransitionCore(Arc::clone(&client));
@@ -26,7 +26,7 @@ use crate::services::tier::{
tier_config::TierS3,
warm_backend::{
TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts,
build_transition_put_options, endpoint_authority, transition_client_timeouts_from_env,
build_transition_put_options, endpoint_authority,
},
};
use http::HeaderMap;
@@ -139,7 +139,6 @@ impl WarmBackendS3 {
} else {
return Err(std::io::Error::other("insufficient parameters for S3 backend authentication"));
}
let timeouts = transition_client_timeouts_from_env();
let opts = Options {
creds,
secure: u.scheme() == "https",
@@ -148,7 +147,7 @@ impl WarmBackendS3 {
..Default::default()
};
let endpoint = endpoint_authority(&u)?;
let client = TransitionClient::new_with_timeouts(&endpoint, opts, tier_type, timeouts).await?;
let client = TransitionClient::new(&endpoint, opts, tier_type).await?;
let client = Arc::new(client);
let core = TransitionCore(Arc::clone(&client));
+71 -243
View File
@@ -3558,11 +3558,6 @@ impl RenameRollbackReceipt {
}
}
struct RenameRollbackOwnership {
receipt: Option<RenameRollbackReceipt>,
namespace_commit_guard: Option<Arc<crate::runtime::instance::NamespaceCommitGuard>>,
}
async fn inspect_incomplete_rename_rollback(
disks: &[Option<DiskStore>],
bucket: &str,
@@ -3609,12 +3604,8 @@ async fn rollback_failed_rename(
dispatch_states: &[RenameDispatchState],
rollback_dirs: &[Option<Uuid>],
dst: (&str, &str),
ownership: RenameRollbackOwnership,
receipt: Option<RenameRollbackReceipt>,
) {
let RenameRollbackOwnership {
receipt,
namespace_commit_guard,
} = ownership;
let owned_disks = disks.to_vec();
let owned_errs = errs.to_vec();
let owned_dispatch_states = dispatch_states.to_vec();
@@ -3660,9 +3651,7 @@ async fn rollback_failed_rename(
let fi = std::mem::take(&mut file_infos[disk_index]);
let bucket = bucket.to_string();
let object = object.to_string();
let disk_namespace_commit_guard = namespace_commit_guard.clone();
let task = tokio::spawn(async move {
let _namespace_commit_guard = disk_namespace_commit_guard;
#[allow(clippy::let_unit_value)]
let _task_guard = SetDisks::rename_fanout_task_guard(&object);
SetDisks::rename_fanout_barrier(&object, disk_index, rename_fanout_barrier_phase::ROLLBACK).await;
@@ -3683,9 +3672,6 @@ async fn rollback_failed_rename(
});
tasks.push(async move { (disk_index, task.await) });
}
#[cfg(test)]
rollback_fault_injection::after_undo_dispatch(object);
let _namespace_commit_guard = namespace_commit_guard;
for (disk_index, result) in join_all(tasks).await {
outcomes[disk_index].outcome = rename_rollback_task_outcome(result);
}
@@ -3792,7 +3778,6 @@ pub(in crate::set_disk) struct RenameDataFenceOptions<'a> {
write_quorum: usize,
scanner_publication_lease_tokens: Option<&'a HashMap<String, Uuid>>,
scanner_publication_commit_scope: Option<crate::object_api::ScannerPublicationCommitScope>,
namespace_commit_guard: Option<Arc<crate::runtime::instance::NamespaceCommitGuard>>,
rollback_receipt: Option<RenameRollbackReceipt>,
}
@@ -3805,7 +3790,6 @@ impl<'a> RenameDataFenceOptions<'a> {
write_quorum,
scanner_publication_lease_tokens,
scanner_publication_commit_scope: None,
namespace_commit_guard: None,
rollback_receipt: None,
}
}
@@ -3822,14 +3806,6 @@ impl<'a> RenameDataFenceOptions<'a> {
self.scanner_publication_commit_scope = scanner_publication_commit_scope;
self
}
pub(in crate::set_disk) fn with_namespace_commit_guard(
mut self,
namespace_commit_guard: Option<Arc<crate::runtime::instance::NamespaceCommitGuard>>,
) -> Self {
self.namespace_commit_guard = namespace_commit_guard;
self
}
}
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
@@ -4188,7 +4164,6 @@ impl SetDisks {
write_quorum,
scanner_publication_lease_tokens,
scanner_publication_commit_scope: _scanner_publication_commit_scope,
namespace_commit_guard,
rollback_receipt,
} = fence_options;
if let Some(file_info) = disks
@@ -4235,9 +4210,7 @@ impl SetDisks {
let dst_object = fanout_dst_object.clone();
let file_info = file_info.clone();
let successful_rename_completion_rank = successful_rename_completion_rank.clone();
let namespace_commit_guard = namespace_commit_guard.clone();
tasks.spawn(async move {
let _namespace_commit_guard = namespace_commit_guard;
let mut dispatch_state = RenameDispatchState::NotDispatched;
let result = std::panic::AssertUnwindSafe(async {
#[allow(clippy::let_unit_value)]
@@ -4399,10 +4372,7 @@ impl SetDisks {
&dispatch_states,
&data_dirs,
(&fanout_dst_bucket, &fanout_dst_object),
RenameRollbackOwnership {
receipt: rollback_receipt,
namespace_commit_guard,
},
rollback_receipt,
)
.await;
if let Some(commit_tx) = commit_tx.take() {
@@ -4558,7 +4528,6 @@ impl SetDisks {
write_quorum,
scanner_publication_lease_tokens,
scanner_publication_commit_scope,
namespace_commit_guard,
rollback_receipt,
} = fence_options;
if let Some(file_info) = disks
@@ -4592,7 +4561,6 @@ impl SetDisks {
let fanout_dst_bucket = dst_bucket.clone();
let fanout_dst_object = dst_object.clone();
let fanout_publication_scope = scanner_publication_commit_scope.clone();
let fanout_namespace_commit_guard = namespace_commit_guard.clone();
// Keep one coordinator task so a cancelled caller cannot drop partially
// completed disk mutations. Per-disk futures stay ordered in `join_all`,
// preserving slot-indexed quorum and convergence accounting without a
@@ -4601,7 +4569,6 @@ impl SetDisks {
// Keep the storage-owned movement permit attached to the actual
// fan-out owner, even if the caller future is cancelled.
let _fanout_publication_scope = fanout_publication_scope;
let _namespace_commit_guard = fanout_namespace_commit_guard;
let successful_rename_completion_rank =
rustfs_io_metrics::put_stage_metrics_enabled().then(|| Arc::new(AtomicUsize::new(0)));
let futures = fanout_disks
@@ -4823,10 +4790,7 @@ impl SetDisks {
&dispatch_states,
&data_dirs,
(&dst_bucket, &dst_object),
RenameRollbackOwnership {
receipt: rollback_receipt,
namespace_commit_guard,
},
rollback_receipt,
)
.await;
return Err(ret_err);
@@ -6539,9 +6503,9 @@ impl SetDisks {
match oi {
Ok(oi) => {
// Ordinary writes may proceed past a top-level delete marker;
// data movement and guarded internal writes must preserve it.
// data movement must not replace an acknowledged deletion.
if oi.delete_marker {
return (opts.data_movement || opts.preserve_delete_marker).then_some(StorageError::PreconditionFailed);
return opts.data_movement.then_some(StorageError::PreconditionFailed);
}
let if_none_match = http_preconditions.if_none_match_value().map(str::to_owned);
let if_match = http_preconditions.if_match_value().map(str::to_owned);
@@ -6790,7 +6754,6 @@ pub(in crate::set_disk) mod rollback_fault_injection {
VolumeNotFoundAfterRename,
PanicAfterRename,
CoordinatorPanic,
RollbackCoordinatorPanic,
}
fn registry() -> &'static Mutex<HashMap<String, (usize, Fault)>> {
@@ -6853,17 +6816,6 @@ pub(in crate::set_disk) mod rollback_fault_injection {
panic!("injected rename coordinator panic");
}
}
pub(super) fn after_undo_dispatch(object: &str) {
let fault = registry()
.lock()
.expect("rollback registry should not poison")
.get(object)
.copied();
if matches!(fault, Some((_, Fault::RollbackCoordinatorPanic))) {
panic!("injected rollback coordinator panic");
}
}
}
/// Test-only per-disk call counters for the metadata fan-out (backlog#1325,
@@ -7025,7 +6977,7 @@ pub(crate) mod rename_fanout_barrier {
use tokio::sync::Notify;
pub use super::rename_fanout_barrier_phase::{
CLEANUP as PHASE_CLEANUP, READ_VERSION as PHASE_READ_VERSION, RENAME as PHASE_RENAME, ROLLBACK as PHASE_ROLLBACK,
CLEANUP as PHASE_CLEANUP, READ_VERSION as PHASE_READ_VERSION, RENAME as PHASE_RENAME,
};
/// One armed barrier: the fan-out task matching `(disk_index, phase)` pauses.
@@ -10862,177 +10814,79 @@ mod tests {
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn rename_rollback_incomplete_receipt_waits_for_undo_barrier() {
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
for (allow_early_ack, cancel_caller, object) in [
(false, false, "rollback-barrier-object"),
(false, true, "rollback-barrier-cancelled"),
(true, false, "rollback-barrier-early-object"),
(true, true, "rollback-barrier-early-cancelled"),
] {
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
let bucket = "rename-rollback-barrier";
let (dirs, disks) = call_counter_local_disks(bucket, 4).await;
prepare_rename_source_dirs(&dirs, &disks, "source").await;
let mut old = metadata_test_fileinfo(object);
old.mod_time = Some(OffsetDateTime::now_utc());
old.data = Some(Bytes::from_static(b"old-inline-body"));
old.set_inline_data();
old.metadata.insert("etag".to_string(), "old-etag".to_string());
for disk in disks.iter().flatten() {
disk.write_metadata(bucket, bucket, object, old.clone())
.await
.expect("old metadata should be staged");
}
let _rename_fault = rename_fault_injection::fail_rename_on(object, &[2, 3]);
let _undo_fault = rollback_fault_injection::arm(object, 0, rollback_fault_injection::Fault::Io);
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier_phase::ROLLBACK);
let receipt = RenameRollbackReceipt::default();
let mut rename = Box::pin(SetDisks::rename_data_owned_with_fence(
&disks,
(RUSTFS_META_TMP_BUCKET, "source"),
rename_commit_fileinfos(object, 4, "new-etag"),
(bucket, object),
allow_early_ack,
RenameDataFenceOptions::new(3, None)
.with_rollback_receipt(receipt.clone())
.with_namespace_commit_guard(Some(ctx.begin_namespace_commit())),
));
tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
tokio::select! {
() = barrier.wait_until_paused() => {}
_ = rename.as_mut() => panic!("rename returned before the armed rollback barrier"),
}
})
.await
.expect("undo must reach its disk barrier");
assert!(receipt.0.get().is_none(), "pending undo must not be recorded as success");
assert!(ctx.namespace_commits_pending());
assert_eq!(ctx.namespace_commit_generation(), 1);
if cancel_caller {
drop(rename);
assert!(ctx.namespace_commits_pending(), "caller cancellation must not retire pending undo work");
assert_eq!(ctx.namespace_commit_generation(), 1);
barrier.release();
tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
while receipt.0.get().is_none() || ctx.namespace_commits_pending() {
tokio::task::yield_now().await;
}
})
for cancel_caller in [false, true] {
let bucket = "rename-rollback-barrier";
let object = if cancel_caller {
"rollback-barrier-cancelled"
} else {
"rollback-barrier-object"
};
let (dirs, disks) = call_counter_local_disks(bucket, 4).await;
prepare_rename_source_dirs(&dirs, &disks, "source").await;
let mut old = metadata_test_fileinfo(object);
old.mod_time = Some(OffsetDateTime::now_utc());
old.data = Some(Bytes::from_static(b"old-inline-body"));
old.set_inline_data();
old.metadata.insert("etag".to_string(), "old-etag".to_string());
for disk in disks.iter().flatten() {
disk.write_metadata(bucket, bucket, object, old.clone())
.await
.expect("cancelled caller must not cancel rollback accounting");
} else {
barrier.release();
assert!(rename.await.is_err());
}
assert!(
!ctx.namespace_commits_pending(),
"the completed rollback must release its namespace ownership"
);
assert_eq!(ctx.namespace_commit_generation(), 2);
assert!(receipt.is_incomplete(), "drained undo failure must survive in the receipt");
for dir in dirs.iter().skip(1) {
let reopened = reopen_local_disk(dir).await;
let restored = reopened
.read_version(
"",
bucket,
object,
"",
&ReadOptions {
read_data: true,
..Default::default()
},
)
.await
.expect("old version must remain readable after caller cancellation");
assert_eq!(restored.data.as_deref(), Some(b"old-inline-body".as_slice()));
}
.expect("old metadata should be staged");
}
})
.await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn rename_rollback_children_keep_namespace_ownership_after_coordinator_panic() {
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
for (allow_early_ack, object) in [
(false, "rollback-coordinator-panic"),
(true, "rollback-coordinator-panic-early"),
] {
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
let bucket = "rename-rollback-coordinator-panic";
let (dirs, disks) = call_counter_local_disks(bucket, 4).await;
prepare_rename_source_dirs(&dirs, &disks, "source").await;
let mut old = metadata_test_fileinfo(object);
old.mod_time = Some(OffsetDateTime::now_utc());
old.data = Some(Bytes::from_static(b"old-inline-body"));
old.set_inline_data();
old.metadata.insert("etag".to_string(), "old-etag".to_string());
for disk in disks.iter().flatten() {
disk.write_metadata(bucket, bucket, object, old.clone())
.await
.expect("old metadata should be staged");
let _rename_fault = rename_fault_injection::fail_rename_on(object, &[2, 3]);
let _undo_fault = rollback_fault_injection::arm(object, 0, rollback_fault_injection::Fault::Io);
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier_phase::ROLLBACK);
let receipt = RenameRollbackReceipt::default();
let mut rename = Box::pin(SetDisks::rename_data_owned_with_fence(
&disks,
(RUSTFS_META_TMP_BUCKET, "source"),
rename_commit_fileinfos(object, 4, "new-etag"),
(bucket, object),
false,
RenameDataFenceOptions::new(3, None).with_rollback_receipt(receipt.clone()),
));
tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
tokio::select! {
() = barrier.wait_until_paused() => {}
_ = rename.as_mut() => panic!("rename returned before the armed rollback barrier"),
}
let _rename_fault = rename_fault_injection::fail_rename_on(object, &[2, 3]);
let _rollback_fault =
rollback_fault_injection::arm(object, 0, rollback_fault_injection::Fault::RollbackCoordinatorPanic);
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier_phase::ROLLBACK);
let receipt = RenameRollbackReceipt::default();
let result = tokio::time::timeout(
BARRIER_PAUSE_GUARD,
SetDisks::rename_data_owned_with_fence(
&disks,
(RUSTFS_META_TMP_BUCKET, "source"),
rename_commit_fileinfos(object, 4, "new-etag"),
(bucket, object),
allow_early_ack,
RenameDataFenceOptions::new(3, None)
.with_rollback_receipt(receipt.clone())
.with_namespace_commit_guard(Some(ctx.begin_namespace_commit())),
),
)
.await
.expect("coordinator failure must return without waiting for detached undo tasks");
assert!(result.is_err());
tokio::time::timeout(BARRIER_PAUSE_GUARD, barrier.wait_until_paused())
.await
.expect("detached undo must reach its disk barrier");
assert!(
receipt.is_incomplete(),
"coordinator failure must preserve indeterminate recovery evidence"
);
assert!(ctx.namespace_commits_pending(), "the paused child must retain namespace ownership");
assert_eq!(ctx.namespace_commit_generation(), 1);
})
.await
.expect("undo must reach its disk barrier");
assert!(receipt.0.get().is_none(), "pending undo must not be recorded as success");
if cancel_caller {
drop(rename);
barrier.release();
tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
while ctx.namespace_commits_pending() {
while receipt.0.get().is_none() {
tokio::task::yield_now().await;
}
})
.await
.expect("completed undo children must release their namespace ownership");
assert_eq!(ctx.namespace_commit_generation(), 2);
for dir in &dirs {
let reopened = reopen_local_disk(dir).await;
let restored = reopened
.read_version(
"",
bucket,
object,
"",
&ReadOptions {
read_data: true,
..Default::default()
},
)
.await
.expect("old version must remain readable after rollback coordinator failure");
assert_eq!(restored.data.as_deref(), Some(b"old-inline-body".as_slice()));
}
.expect("cancelled caller must not cancel rollback accounting");
} else {
barrier.release();
assert!(rename.await.is_err());
}
})
.await;
assert!(receipt.is_incomplete(), "drained undo failure must survive in the receipt");
for dir in dirs.iter().skip(1) {
let reopened = reopen_local_disk(dir).await;
let restored = reopened
.read_version(
"",
bucket,
object,
"",
&ReadOptions {
read_data: true,
..Default::default()
},
)
.await
.expect("old version must remain readable after caller cancellation");
assert_eq!(restored.data.as_deref(), Some(b"old-inline-body".as_slice()));
}
}
}
#[tokio::test]
@@ -11147,35 +11001,9 @@ mod tests {
let mut file_infos = rename_commit_fileinfos(object, DISKS, "fresh-rollback-etag");
file_infos[3] = FileInfo::default();
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
ctx.set_scanner_publication_state(false);
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_ROLLBACK);
let rename = SetDisks::rename_data_owned_with_fence(
&disks,
(RUSTFS_META_TMP_BUCKET, "source"),
file_infos,
(bucket, object),
false,
RenameDataFenceOptions::new(4, None).with_namespace_commit_guard(Some(ctx.begin_namespace_commit())),
);
let control = async {
barrier.wait_until_paused().await;
assert!(ctx.namespace_commits_pending(), "rollback must retain namespace publication ownership");
assert!(ctx.scanner_publication_state_allowed(), "rollback must not disable namespace walks");
assert_eq!(ctx.namespace_commit_generation(), 1);
barrier.release();
};
let (result, ()) = tokio::time::timeout(BARRIER_PAUSE_GUARD, async { tokio::join!(rename, control) })
SetDisks::rename_data(&disks, RUSTFS_META_TMP_BUCKET, "source", &file_infos, bucket, object, 4)
.await
.expect("rename rollback must reach its barrier and finish after release");
assert_eq!(
result.err(),
Some(DiskError::ErasureWriteQuorum),
"three successful disks must fail a strict write quorum of four"
);
assert!(!ctx.namespace_commits_pending());
assert!(ctx.scanner_publication_state_allowed());
assert_eq!(ctx.namespace_commit_generation(), 2);
.expect_err("three successful disks must fail a strict write quorum of four");
for (idx, dir) in dirs.iter().enumerate() {
let reopened = reopen_local_disk(dir).await;
+3 -363
View File
@@ -2490,9 +2490,9 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks {
return Ok((result, err.map(|e| e.into())));
}
// The inner heal and missing-object report read the registry again;
// release this snapshot guard before a topology writer can queue between reads.
let disks = self.get_disks_internal().await;
let disks = self.disks.read().await;
let disks = disks.clone();
let (_, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, false, false, false)
.await
.map_err(|e| to_object_err(e.into(), vec![bucket, object]))?;
@@ -3419,366 +3419,6 @@ mod heal_result_report_tests {
assert_eq!(unformatted, DiskError::UnformattedDisk);
}
#[derive(Clone, Copy)]
enum InventoryWriterHealCase {
Existing,
Missing,
MissingVersion,
}
async fn assert_heal_object_inventory_writer(case: InventoryWriterHealCase) {
use crate::set_disk::core::io_primitives::disk_call_counters;
use std::time::Duration;
use tokio::io::AsyncReadExt;
let (_temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
let bucket = "heal-inventory-writer-bucket";
let object = match case {
InventoryWriterHealCase::Existing => "heal-inventory-writer-existing",
InventoryWriterHealCase::Missing => "heal-inventory-writer-missing",
InventoryWriterHealCase::MissingVersion => "heal-inventory-writer-missing-version",
};
set.make_bucket(
bucket,
&MakeBucketOptions {
versioning_enabled: true,
..Default::default()
},
)
.await
.expect("heal fixture bucket should be created");
let body = vec![0x67; 64 * 1024];
let stored_version = Uuid::new_v4();
let stored_version_string = stored_version.to_string();
let published = if matches!(case, InventoryWriterHealCase::Missing) {
None
} else {
let mut reader = PutObjReader::from_vec(body.clone());
let info = set
.put_object(
bucket,
object,
&mut reader,
&ObjectOptions {
no_lock: true,
versioned: true,
version_id: Some(stored_version_string.clone()),
..Default::default()
},
)
.await
.expect("full-fanout PUT should seed the heal fixture");
for disk in &disks {
let metadata = disk
.read_version("", bucket, object, &stored_version_string, &ReadOptions::default())
.await
.expect("the seeded version must be present on every disk");
assert_eq!(metadata.version_id, Some(stored_version));
assert_eq!(metadata.size, i64::try_from(body.len()).expect("fixture size should fit i64"));
}
Some(info)
};
let requested_version = match case {
InventoryWriterHealCase::Existing => stored_version_string.clone(),
InventoryWriterHealCase::Missing => String::new(),
InventoryWriterHealCase::MissingVersion => Uuid::new_v4().to_string(),
};
let opts = HealOpts {
no_lock: true,
..Default::default()
};
let calls = disk_call_counters::observe(object);
let read_gate = set.disks.read().await;
// UFCS selects the trait's outer precheck, not the same-named inherent heal.
let heal = <SetDisks as crate::storage_api_contracts::heal::HealOperations>::heal_object(
set.as_ref(),
bucket,
object,
&requested_version,
&opts,
);
tokio::pin!(heal);
assert!(matches!(
futures::poll!(tokio::task::unconstrained(heal.as_mut())),
std::task::Poll::Pending
));
// These tests use the current-thread runtime: full-wait metadata tasks
// have been spawned, but cannot run during the single unconstrained poll.
assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), 0);
let writer = set.disks.write();
tokio::pin!(writer);
assert!(matches!(
futures::poll!(tokio::task::unconstrained(writer.as_mut())),
std::task::Poll::Pending
));
assert!(set.disks.try_read().is_err(), "the writer must already block new inventory readers");
tokio::time::timeout(Duration::from_secs(5), async {
while calls.total(disk_call_counters::KIND_READ_VERSION) < 4 {
tokio::task::yield_now().await;
}
})
.await
.expect("the suspended trait heal must have started the real metadata fanout");
for disk_index in 0..4 {
assert_eq!(calls.for_disk(disk_call_counters::KIND_READ_VERSION, disk_index), 1);
}
drop(read_gate);
let (_, outcome) =
tokio::time::timeout(Duration::from_secs(5), async { tokio::join!(async { drop(writer.await) }, heal) })
.await
.expect("trait heal must not deadlock its nested inventory read with the queued writer");
let (result, error) = outcome.expect("heal should report the object's outcome");
match case {
InventoryWriterHealCase::Existing => assert!(error.is_none(), "existing object heal failed: {error:?}"),
InventoryWriterHealCase::Missing => assert!(matches!(error, Some(Error::FileNotFound))),
InventoryWriterHealCase::MissingVersion => assert!(matches!(error, Some(Error::FileVersionNotFound))),
}
assert_eq!(result.bucket, bucket);
assert_eq!(result.object, object);
assert_eq!(result.version_id, requested_version);
assert_eq!(result.disk_count, 4);
assert_eq!(result.before.drives.len(), 4);
assert_eq!(result.after.drives.len(), 4);
for disk_index in 0..4 {
let endpoint = set.set_endpoints[disk_index].to_string();
assert_eq!(result.before.drives[disk_index].endpoint, endpoint);
assert_eq!(result.after.drives[disk_index].endpoint, endpoint);
}
if let Some(published) = published {
tokio::time::timeout(Duration::from_secs(10), async {
let mut reader = set
.get_object_reader(
bucket,
object,
None,
Default::default(),
&ObjectOptions {
versioned: true,
version_id: Some(stored_version_string),
..Default::default()
},
)
.await
.expect("the stored version must remain readable after heal");
assert_eq!(reader.object_info.etag, published.etag);
assert_eq!(reader.object_info.version_id, Some(stored_version));
let mut observed_body = Vec::new();
reader
.stream
.read_to_end(&mut observed_body)
.await
.expect("stored body should stream");
assert_eq!(observed_body, body);
})
.await
.expect("GET must finish after the inventory writer and heal");
}
}
#[tokio::test]
async fn heal_object_inventory_writer_existing() {
assert_heal_object_inventory_writer(InventoryWriterHealCase::Existing).await;
}
#[tokio::test]
async fn heal_object_inventory_writer_missing() {
assert_heal_object_inventory_writer(InventoryWriterHealCase::Missing).await;
}
#[tokio::test]
async fn heal_object_inventory_writer_missing_version() {
assert_heal_object_inventory_writer(InventoryWriterHealCase::MissingVersion).await;
}
#[tokio::test]
#[serial_test::serial]
async fn heal_object_with_queued_disk_renewal() {
use crate::layout::endpoints::SetupType;
use crate::runtime::instance::InstanceContext;
use crate::set_disk::core::io_primitives::disk_call_counters;
use std::collections::HashMap;
use std::future::Future;
use std::task::Poll;
use std::time::Duration;
use tokio::io::AsyncReadExt;
// renew_disk still registers local disks on the ambient context. Match
// the default serial group used by its other setup/registry fixtures,
// and restore only this temporary endpoint, including on a failed join.
struct RenewDiskTestState {
ctx: Arc<InstanceContext>,
was_dist_erasure: bool,
map: Arc<RwLock<HashMap<String, Option<DiskStore>>>>,
endpoint: String,
previous_disk: Option<Option<DiskStore>>,
}
impl Drop for RenewDiskTestState {
fn drop(&mut self) {
let ctx = self.ctx.clone();
let was_dist_erasure = self.was_dist_erasure;
let map = self.map.clone();
let endpoint = self.endpoint.clone();
let previous_disk = self.previous_disk.take();
let handle = tokio::runtime::Handle::current();
std::thread::spawn(move || {
handle.block_on(async move {
let mut map = map.write().await;
match previous_disk {
Some(disk) => {
map.insert(endpoint, disk);
}
None => {
map.remove(&endpoint);
}
}
drop(map);
if was_dist_erasure {
ctx.update_erasure_type(SetupType::DistErasure).await;
}
});
})
.join()
.expect("renew fixture state restoration should finish");
}
}
let (_temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
let endpoint = set.set_endpoints[0].clone();
let ctx = crate::runtime::global::current_ctx();
let map = ctx.local_disk_map();
let restore = RenewDiskTestState {
ctx: ctx.clone(),
was_dist_erasure: ctx.is_dist_erasure().await,
map: map.clone(),
endpoint: endpoint.to_string(),
previous_disk: map.read().await.get(&endpoint.to_string()).cloned(),
};
// Only distributed erasure needs an override to avoid the ambient slot array.
if restore.was_dist_erasure {
ctx.update_erasure_type(SetupType::Erasure).await;
}
let bucket = "heal-disk-renewal-bucket";
let object = "heal-disk-renewal-object";
set.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("renew fixture bucket should be created");
let body = vec![0x73; 64 * 1024];
let mut reader = PutObjReader::from_vec(body.clone());
let published = set
.put_object(
bucket,
object,
&mut reader,
&ObjectOptions {
no_lock: true,
..Default::default()
},
)
.await
.expect("full-fanout PUT should seed the renewal fixture");
for disk in &disks {
let metadata = disk
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.expect("the seeded object must be present on every disk");
assert_eq!(metadata.size, i64::try_from(body.len()).expect("fixture size should fit i64"));
}
let opts = HealOpts {
no_lock: true,
..Default::default()
};
let calls = disk_call_counters::observe(object);
let read_gate = set.disks.read().await;
let heal = <SetDisks as crate::storage_api_contracts::heal::HealOperations>::heal_object(
set.as_ref(),
bucket,
object,
"",
&opts,
);
tokio::pin!(heal);
assert!(matches!(futures::poll!(tokio::task::unconstrained(heal.as_mut())), Poll::Pending));
assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), 0);
let renew = set.renew_disk(&endpoint);
tokio::pin!(renew);
tokio::time::timeout(
Duration::from_secs(5),
futures::future::poll_fn(|cx| {
assert!(
std::pin::pin!(tokio::task::unconstrained(renew.as_mut()))
.poll(cx)
.is_pending(),
"renewal must reach its inventory write before returning"
);
if set.disks.try_read().is_err() {
Poll::Ready(())
} else {
Poll::Pending
}
}),
)
.await
.expect("real renewal must queue its topology writer behind the read gate");
let registered = map
.read()
.await
.get(&endpoint.to_string())
.cloned()
.flatten()
.expect("renewal must register the connected disk before its inventory write");
assert!(!Arc::ptr_eq(&registered, &disks[0]), "renewal must construct a new disk handle");
tokio::time::timeout(Duration::from_secs(5), async {
while calls.total(disk_call_counters::KIND_READ_VERSION) < 4 {
tokio::task::yield_now().await;
}
})
.await
.expect("the suspended trait heal must have started the real metadata fanout");
for disk_index in 0..4 {
assert_eq!(calls.for_disk(disk_call_counters::KIND_READ_VERSION, disk_index), 1);
}
drop(read_gate);
let (_, outcome) = tokio::time::timeout(Duration::from_secs(5), async { tokio::join!(renew, heal) })
.await
.expect("trait heal and real disk renewal must finish without a nested inventory read deadlock");
let (report, error) = outcome.expect("heal should report the existing object");
assert!(error.is_none(), "existing object heal failed after renewal: {error:?}");
assert_eq!(report.bucket, bucket);
assert_eq!(report.object, object);
assert_eq!(report.disk_count, 4);
let renewed = set.get_disks_internal().await[0]
.clone()
.expect("the renewed slot must remain online");
assert!(Arc::ptr_eq(&renewed, &registered), "the set must publish the newly connected handle");
assert_eq!(renewed.endpoint(), endpoint);
let format = load_format_erasure(&renewed, false)
.await
.expect("renewed disk format should remain readable");
assert_eq!(format.erasure.this, set.format.erasure.sets[0][0]);
tokio::time::timeout(Duration::from_secs(10), async {
let mut reader = set
.get_object_reader(bucket, object, None, Default::default(), &ObjectOptions::default())
.await
.expect("the object must remain readable after renewal and heal");
assert_eq!(reader.object_info.etag, published.etag);
let mut observed_body = Vec::new();
reader
.stream
.read_to_end(&mut observed_body)
.await
.expect("stored body should stream");
assert_eq!(observed_body, body);
})
.await
.expect("GET must finish after renewal and heal");
}
// Regression for #955: an offline disk must contribute exactly one drive
// record. Before the fix the offline branch fell through and pushed a second
// (Corrupt) record for the same disk, so `before/after.drives` grew to
+23 -119
View File
@@ -2452,9 +2452,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
let write_quorum = fi.write_quorum(self.default_write_quorum());
let read_quorum = fi.read_quorum(self.default_read_quorum());
// Release the registry guard before recovery and cleanup read it again:
// a queued topology writer would otherwise deadlock those nested reads.
let disks = self.get_disks_internal().await;
let disks = self.disks.read().await;
let disks = disks.clone();
// let disks = Self::shuffle_disks(&disks, &fi.erasure.distribution);
let part_path = format!("{}/{}/", upload_id_path, fi.data_dir.unwrap_or(Uuid::nil()));
self.recover_part_transactions(&part_path, read_quorum, write_quorum)
@@ -4050,7 +4051,6 @@ mod tests {
let _ = drain_global_dirty_scopes();
let rename_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let rename_tasks = rename_fanout_barrier::observe_tasks(object);
let complete_store = Arc::clone(&set_disks);
let mut complete = tokio::spawn(async move {
let mut opts = ObjectOptions::default();
@@ -4062,6 +4062,16 @@ mod tests {
tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused())
.await
.expect("multipart completion should pause one tail disk during rename");
assert!(
tokio::time::timeout(Duration::from_millis(100), &mut complete).await.is_err(),
"multipart completion must not publish success while a tail rename is still paused"
);
let initial = drain_global_dirty_scopes().into_iter().collect::<HashSet<_>>();
assert!(
initial.is_empty(),
"capacity must not be marked as committed before the full multipart rename finishes"
);
let abort_store = Arc::clone(&set_disks);
let abort = tokio::spawn(async move {
@@ -4070,46 +4080,21 @@ mod tests {
.await
});
signaling.wait_for_attempts(2).await;
assert!(!abort.is_finished(), "the in-flight completion must retain the multipart upload guard");
// A paused rename does not establish that the other disks reached quorum.
let retained_staging = tokio::time::timeout(Duration::from_secs(30), async {
loop {
let mut retained = 0;
for result in futures::future::join_all(
disk_stores
.iter()
.map(|disk| disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &staged_part)),
)
.await
{
match result {
Ok(_) => retained += 1,
Err(DiskError::FileNotFound) => {}
Err(error) => panic!("staged rename source lookup failed: {error}"),
}
}
if retained <= 1 && rename_tasks.running() == 1 {
break retained;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
let retained_staging = futures::future::join_all(
disk_stores
.iter()
.map(|disk| disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &staged_part)),
)
.await
.expect("unpaused multipart renames should finish before the tail is released");
.into_iter()
.filter(|result| result.is_ok())
.count();
assert_eq!(
retained_staging, 1,
"only the paused tail disk should still retain the multipart rename source"
);
assert!(
tokio::time::timeout(Duration::from_millis(100), &mut complete).await.is_err(),
"multipart completion must not publish success while a tail rename is still paused"
);
let initial = drain_global_dirty_scopes().into_iter().collect::<HashSet<_>>();
assert!(
initial.is_empty(),
"capacity must not be marked as committed before the full multipart rename finishes"
);
assert!(!abort.is_finished(), "the in-flight completion must retain the multipart upload guard");
signaling.set_target(rustfs_lock::ObjectKey::new(bucket, object));
let object_attempt = signaling.attempts.load(Ordering::Acquire) + 1;
@@ -6758,87 +6743,6 @@ mod tests {
.await;
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn complete_multipart_releases_disk_snapshot_before_cleanup() {
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "multipart-topology-lock-bucket";
let object = "object";
let body = vec![0x65; 4096];
make_bucket_on_all(&disk_stores, bucket).await;
let (upload_id, parts) =
stage_upload_with_create_opts(&set_disks, bucket, object, &body, &ObjectOptions::default()).await;
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
for dir in &temp_dirs {
assert!(
dir.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&upload_id_path).exists(),
"the test must create real upload staging on every disk"
);
}
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterObjectPublication);
let complete_store = set_disks.clone();
let complete_upload_id = upload_id.clone();
let complete = tokio::spawn(async move {
complete_store
.complete_multipart_upload(bucket, object, &complete_upload_id, parts, &ObjectOptions::default())
.await
});
barrier.wait_until_paused().await;
// Hold a separate read gate so the real writer queues even when completion
// correctly releases its snapshot guard. Polling Pending proves admission
// to Tokio's write-preferring queue before the cleanup attempts another read.
let read_gate = set_disks.disks.read().await;
let writer = set_disks.disks.write();
tokio::pin!(writer);
assert!(matches!(
futures::poll!(tokio::task::unconstrained(writer.as_mut())),
std::task::Poll::Pending
));
assert!(
set_disks.disks.try_read().is_err(),
"the pending writer must already block new readers before the cleanup resumes"
);
drop(read_gate);
barrier.release();
let writer_guard = tokio::time::timeout(Duration::from_secs(5), writer)
.await
.expect("a queued topology writer must not deadlock with multipart cleanup's disk snapshot");
// A reconnect can publish the same handles; this test isolates admission
// order without changing the disks that contain the committed object.
drop(writer_guard);
tokio::time::timeout(Duration::from_secs(10), complete)
.await
.expect("multipart cleanup must finish after the topology writer releases")
.expect("completion task should not panic")
.expect("completion should preserve the successful object commit");
let mut reader = tokio::time::timeout(
Duration::from_secs(10),
set_disks.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()),
)
.await
.expect("GET should finish after completion")
.expect("the completed object should remain readable");
let mut observed_body = Vec::new();
tokio::time::timeout(Duration::from_secs(10), reader.stream.read_to_end(&mut observed_body))
.await
.expect("the completed object body should finish streaming")
.expect("the completed object body should be readable");
assert_eq!(observed_body, body);
assert!(matches!(
set_disks.check_upload_id_exists(bucket, object, &upload_id, false).await,
Err(StorageError::InvalidUploadID(..))
));
for dir in &temp_dirs {
assert!(
!dir.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&upload_id_path).exists(),
"successful completion must remove its upload staging from every disk"
);
}
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn complete_releases_object_lock_before_cleanup_and_keeps_upload_lock() {
+1 -4
View File
@@ -4459,10 +4459,7 @@ impl SetDisks {
commit_scanner_publication_lease_tokens.as_ref(),
)
.with_publication_scope(commit_scanner_publication_scope.clone())
.with_rollback_receipt(commit_rollback_receipt.clone())
.with_namespace_commit_guard(
(!is_meta_bucketname(&commit_bucket)).then(|| commit_set.ctx.begin_namespace_commit()),
),
.with_rollback_receipt(commit_rollback_receipt.clone()),
)
.await;
if let Some(scope) = commit_scanner_publication_scope.as_ref() {
+2 -12
View File
@@ -1059,7 +1059,6 @@ mod tests {
use crate::storage_api_contracts::{
bucket::{BucketOperations as _, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp},
list::ListOperations as _,
namespace::NamespaceLocking as _,
object::{ObjectIO as _, ObjectOperations as _},
};
use crate::store::{ECStore, init_local_disks_with_instance_ctx};
@@ -1487,19 +1486,10 @@ mod tests {
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
.await
.expect("object should be written");
let lock = ecstore.pools[0].disk_set[0]
.new_ns_lock(bucket, object)
.await
.expect("fixture namespace lock should be created");
drop(
lock.get_write_lock(Duration::from_secs(30))
.await
.expect("fixture rename tail should finish before checking its generation"),
);
assert_eq!(
ecstore.scanner_namespace_mutation_generation(),
generation_before_put.saturating_add(3),
"successful object creation must observe the logical mutation and both fanout boundaries"
generation_before_put.saturating_add(1),
"successful object creation should advance scanner namespace activity"
);
ecstore
.get_object_info(bucket, object, &ObjectOptions::default())
+25 -517
View File
@@ -787,12 +787,6 @@ impl ECStore {
pub fn single_pool(&self) -> bool {
self.pools.len() == 1
}
/// The set-local create-only check is atomic only when every object
/// mutation uses that same, enabled namespace lock domain.
pub fn supports_atomic_create_only_write_back(&self) -> bool {
!self.ctx.lock_manager().is_disabled() && self.pools.len() == 1 && self.pools[0].disk_set.len() == 1
}
}
#[cfg(test)]
@@ -2133,7 +2127,7 @@ mod tests {
.iter()
.map(|&drives_per_set| (1, drives_per_set))
.collect::<Vec<_>>();
build_isolated_test_store_with_layout(temp_dir, cmd_line, &pool_layouts, shutdown, None).await
build_isolated_test_store_with_layout(temp_dir, cmd_line, &pool_layouts, shutdown).await
}
async fn build_isolated_test_store_with_layout(
@@ -2141,7 +2135,6 @@ mod tests {
cmd_line: &str,
pool_layouts: &[(usize, usize)],
shutdown: CancellationToken,
instance_ctx: Option<Arc<crate::runtime::instance::InstanceContext>>,
) -> (
Arc<crate::runtime::instance::InstanceContext>,
Arc<crate::store::ECStore>,
@@ -2174,7 +2167,7 @@ mod tests {
let endpoint_pools = EndpointServerPools(pools);
crate::services::notification_sys::install_cross_pool_fence_fleet_proof_for_test();
let instance_ctx = instance_ctx.unwrap_or_else(|| Arc::new(crate::runtime::instance::InstanceContext::new()));
let instance_ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
crate::store::init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
.await
.expect("register local disks into the fresh context");
@@ -2542,348 +2535,6 @@ mod tests {
shutdown.cancel();
}
#[cfg(feature = "test-util")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial_test::serial(storage_class_env)]
async fn early_ack_put_tails_block_scanner_publication_until_all_renames_finish() {
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
let temp_dir = tempfile::tempdir().expect("create scanner PUT tail store dir");
let (ctx, store, shutdown) =
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "scanner-put-tails", &[4])).await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
let bucket = format!("scanner-put-tails-{}", Uuid::new_v4());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create scanner PUT tail bucket");
let set = &store.pools[0].disk_set[0];
let objects = [("scanner-tail-a", vec![0xA1; 273]), ("scanner-tail-b", vec![0xB2; 379])];
temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
let (active, blocked, movement_generation) = store.scanner_data_movement_activity().await;
assert!(!active && !blocked);
assert!(ctx.scanner_publication_state_allowed(), "the set admission cache should start allowed");
let (old_lease, _) = store
.acquire_scanner_publication_lease(movement_generation, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
.await
.expect("publication lease should be admitted before either PUT starts");
let barriers: Vec<_> = objects
.iter()
.map(|(object, _)| {
crate::set_disk::rename_fanout_barrier::arm(object, 0, crate::set_disk::rename_fanout_barrier::PHASE_RENAME)
})
.collect();
let trackers: Vec<_> = objects
.iter()
.map(|(object, _)| crate::set_disk::rename_fanout_barrier::observe_tasks(object))
.collect();
let puts: Vec<_> = objects
.iter()
.map(|(object, body)| {
let put_store = Arc::clone(&store);
let put_bucket = bucket.clone();
let object = *object;
let body = body.clone();
tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(body);
put_store
.put_object(&put_bucket, object, &mut reader, &ObjectOptions::default())
.await
})
})
.collect();
let committed = tokio::time::timeout(Duration::from_secs(30), async {
for barrier in &barriers {
barrier.wait_until_paused().await;
}
let mut committed = Vec::with_capacity(puts.len());
for put in puts {
committed.push(
put.await
.expect("early-ACK PUT task should join while its tail is paused")
.expect("root PUT should return after quorum without waiting for its tail"),
);
}
committed
})
.await
.expect("both root PUTs must quorum-ACK while their tail disks remain paused");
assert!(trackers.iter().all(|tracker| tracker.running() >= 1));
assert!(ctx.namespace_commits_pending());
assert!(
ctx.scanner_publication_state_allowed(),
"pending PUT tails must not disable scanner namespace walks"
);
let (active, blocked, observed_movement_generation) = store.scanner_data_movement_activity().await;
assert!(!active, "ordinary PUT tails are not decommission or rebalance work");
assert!(!blocked, "ordinary PUT tails must not block the movement-only scan baseline");
assert_eq!(observed_movement_generation, movement_generation);
assert!(store.scanner_data_usage_publication_blocked().await);
assert!(store.scanner_data_usage_publication_admission_guard().await.is_some());
assert!(set.scanner_data_usage_publication_admission_guard().await.is_some());
for error in [
store
.acquire_scanner_publication_lease(
movement_generation,
crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL,
)
.await
.expect_err("a new remote publication lease must reject pending PUT tails"),
store
.validate_scanner_publication_lease(old_lease, movement_generation)
.await
.expect_err("an existing remote lease must not bypass pending PUT tails"),
store
.acquire_scanner_publication_lease_guard(old_lease)
.await
.expect_err("target-side publication admission must reject pending PUT tails"),
] {
assert!(
error.to_string().contains("blocked"),
"publication must fail because of active tails: {error}"
);
}
store.release_scanner_publication_lease(old_lease).await;
for (index, barrier) in barriers.iter().enumerate() {
let commit_generation = ctx.namespace_commit_generation();
let namespace_generation = store.scanner_namespace_mutation_generation();
barrier.release();
tokio::time::timeout(Duration::from_secs(30), async {
while trackers[index].running() != 0 || ctx.namespace_commit_generation() <= commit_generation {
tokio::task::yield_now().await;
}
if index + 1 == barriers.len() {
while ctx.namespace_commits_pending() {
tokio::task::yield_now().await;
}
}
})
.await
.expect("released tail must drain and publish its terminal namespace generation");
assert!(store.scanner_namespace_mutation_generation() > namespace_generation);
let pending = index + 1 < barriers.len();
assert_eq!(ctx.namespace_commits_pending(), pending);
assert_eq!(store.scanner_data_usage_publication_blocked().await, pending);
assert!(!store.scanner_data_movement_activity().await.1);
assert!(store.scanner_data_usage_publication_admission_guard().await.is_some());
assert!(set.scanner_data_usage_publication_admission_guard().await.is_some());
}
let (lease, generation) = store
.acquire_scanner_publication_lease(movement_generation, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
.await
.expect("remote publication lease should resume after both tails drain");
store
.validate_scanner_publication_lease(lease, generation)
.await
.expect("a resumed remote publication lease should validate");
drop(
store
.acquire_scanner_publication_lease_guard(lease)
.await
.expect("target-side publication admission should resume after both tails drain"),
);
assert!(store.release_scanner_publication_lease(lease).await);
let disks = set.disk_inventory().await;
assert_eq!(disks.len(), 4);
for ((object, body), committed) in objects.iter().zip(&committed) {
let logical_size = i64::try_from(body.len()).expect("fixture payload size should fit i64");
let etag = committed.etag.as_ref().expect("root PUT should return a committed ETag");
for (disk_index, disk) in disks.iter().enumerate() {
let file_info = disk
.as_ref()
.expect("every fixture disk should remain online")
.read_version(
"",
&bucket,
object,
"",
&crate::disk::ReadOptions {
read_data: true,
..Default::default()
},
)
.await
.unwrap_or_else(|err| panic!("disk {disk_index} should publish {object} after its tail finishes: {err}"));
assert_eq!(file_info.size, logical_size);
assert_eq!(file_info.metadata.get(http::header::ETAG.as_str()), Some(etag));
assert!(
file_info.inline_data(),
"small fixture payloads should have an inline shard on every disk"
);
let inline_data = file_info.data.as_ref().expect("every disk should retain its inline shard");
let erasure = crate::erasure::coding::Erasure::try_new_with_options(
file_info.erasure.data_blocks,
file_info.erasure.parity_blocks,
file_info.erasure.block_size,
file_info.uses_legacy_checksum,
)
.expect("persisted erasure geometry should be valid");
let shard_size =
usize::try_from(erasure.shard_file_size(logical_size)).expect("fixture shard size should fit usize");
crate::erasure::coding::bitrot_verify(
Cursor::new(inline_data.clone()),
inline_data.len(),
shard_size,
rustfs_utils::HashAlgorithm::HighwayHash256S,
erasure.shard_size(),
)
.await
.unwrap_or_else(|err| panic!("disk {disk_index} should retain a complete valid shard for {object}: {err}"));
}
let mut reader = store
.get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("fully drained PUT should be readable");
let mut actual = Vec::new();
reader.stream.read_to_end(&mut actual).await.expect("PUT body should drain");
assert_eq!(&actual, body);
}
let generation_before_internal_put = ctx.namespace_commit_generation();
let internal_object = "scanner-tail-regression/internal-metadata";
let internal_body = b"scanner metadata must not invalidate its own publication";
let mut internal_reader = PutObjReader::from_vec(internal_body.to_vec());
store
.put_object(RUSTFS_META_BUCKET, internal_object, &mut internal_reader, &ObjectOptions::default())
.await
.expect("internal metadata PUT should commit without scanner self-invalidation");
let internal_lock = set
.new_ns_lock(RUSTFS_META_BUCKET, internal_object)
.await
.expect("internal metadata tail lock should be available");
drop(
internal_lock
.get_write_lock(Duration::from_secs(30))
.await
.expect("internal metadata tail should drain"),
);
assert_eq!(ctx.namespace_commit_generation(), generation_before_internal_put);
assert!(!ctx.namespace_commits_pending());
assert!(store.scanner_data_usage_publication_admission_guard().await.is_some());
assert!(set.scanner_data_usage_publication_admission_guard().await.is_some());
let mut internal_reader = store
.get_object_reader(RUSTFS_META_BUCKET, internal_object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("internal metadata should remain readable");
let mut actual = Vec::new();
internal_reader
.stream
.read_to_end(&mut actual)
.await
.expect("internal metadata body should drain");
assert_eq!(actual, internal_body);
})
.await;
shutdown.cancel();
}
#[cfg(feature = "test-util")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial_test::serial(storage_class_env)]
async fn cancelled_early_ack_put_keeps_scanner_publication_blocked_until_tail_finishes() {
let temp_dir = tempfile::tempdir().expect("create cancelled scanner PUT tail store dir");
let (ctx, store, shutdown) =
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "scanner-cancelled-put-tail", &[4])).await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
let bucket = format!("scanner-cancelled-put-tail-{}", Uuid::new_v4());
let object = "scanner-cancelled-tail";
let body = vec![0xC3; 273];
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create cancelled scanner PUT tail bucket");
temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
let tracker = crate::set_disk::rename_fanout_barrier::observe_tasks(object);
let tail =
crate::set_disk::rename_fanout_barrier::arm(object, 0, crate::set_disk::rename_fanout_barrier::PHASE_RENAME);
let quorum = crate::set_disk::PutObjectCommitBarrier::install(
&bucket,
object,
crate::set_disk::PutObjectCommitPause::AfterRenameQuorum,
);
let handoff = crate::set_disk::PutObjectCommitBarrier::install(
&bucket,
object,
crate::set_disk::PutObjectCommitPause::AfterRenameHandoff,
);
let put_store = Arc::clone(&store);
let put_bucket = bucket.clone();
let put_body = body.clone();
let put = tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(put_body);
put_store
.put_object(&put_bucket, object, &mut reader, &ObjectOptions::default())
.await
});
tokio::time::timeout(Duration::from_secs(30), tail.wait_until_paused())
.await
.expect("cancelled PUT should pause one disk before rename");
quorum.wait_until_paused().await;
put.abort();
assert!(
put.await
.expect_err("caller should be cancelled after rename quorum")
.is_cancelled()
);
quorum.release();
handoff.wait_until_paused().await;
assert!(tracker.running() >= 1);
assert!(ctx.namespace_commits_pending());
assert!(!store.scanner_data_movement_activity().await.1);
assert!(store.scanner_data_usage_publication_blocked().await);
assert!(store.scanner_data_usage_publication_admission_guard().await.is_some());
assert!(
store.pools[0].disk_set[0]
.scanner_data_usage_publication_admission_guard()
.await
.is_some()
);
let generation = store.scanner_namespace_mutation_generation();
handoff.release();
tail.release();
tokio::time::timeout(Duration::from_secs(30), async {
while tracker.running() != 0 || ctx.namespace_commits_pending() {
tokio::task::yield_now().await;
}
})
.await
.expect("cancelled request's detached fanout must release scanner admission after finishing");
assert!(store.scanner_namespace_mutation_generation() > generation);
assert!(!store.scanner_data_usage_publication_blocked().await);
assert!(store.scanner_data_usage_publication_admission_guard().await.is_some());
for (disk_index, disk) in store.pools[0].disk_set[0].disk_inventory().await.iter().enumerate() {
let file_info = disk
.as_ref()
.expect("cancelled PUT fixture disk should remain online")
.read_version("", &bucket, object, "", &crate::disk::ReadOptions::default())
.await
.unwrap_or_else(|err| panic!("cancelled PUT must still publish on disk {disk_index}: {err}"));
assert_eq!(file_info.size, i64::try_from(body.len()).expect("fixture body size should fit i64"));
}
let mut reader = store
.get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("a cancelled caller must not discard its quorum-committed object");
let mut actual = Vec::new();
reader
.stream
.read_to_end(&mut actual)
.await
.expect("cancelled PUT body should drain");
assert_eq!(actual, body);
})
.await;
shutdown.cancel();
}
#[cfg(feature = "test-util")]
#[test]
#[serial_test::serial(storage_class_env)]
@@ -3335,9 +2986,8 @@ mod tests {
) -> crate::core::pools::DecommissionTestFaultDecision {
let target_bucket = bucket.to_string();
let target_object = object.to_string();
Arc::new(move |stage, bucket, object, attempt, succeeded| {
Arc::new(move |stage, bucket, object, _attempt, succeeded| {
if !succeeded
|| attempt >= crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS
|| stage != DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT
|| bucket != target_bucket
|| object != target_object
@@ -3347,7 +2997,6 @@ mod tests {
// Entry retries reset the local attempt; real copy errors can skip
// successful attempts. Only injected faults spend this global budget.
// A real failure may consume an attempt, so preserve the final chance.
faults
.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |faults| {
(faults < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS.saturating_sub(1))
@@ -5369,7 +5018,6 @@ mod tests {
"decommission-delete-fence",
&[(2, 4), (1, 4)],
CancellationToken::new(),
None,
))
.await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
@@ -5501,15 +5149,7 @@ mod tests {
#[test]
fn decommission_retry_fault_budget_counts_successes_across_attempt_changes() {
let cases: &[&[(usize, bool, bool)]] = &[
&[(1, true, true), (2, true, true), (3, true, false)],
&[(1, true, true), (1, true, true), (2, true, false)],
&[(1, true, true), (3, true, false), (3, true, false)],
&[(1, true, true), (2, false, false), (1, true, true), (2, true, false)],
&[(1, true, true), (2, false, false), (3, true, false)],
&[(3, true, false), (4, true, false)],
];
for case in cases {
for attempts in [[1, 2, 3], [1, 1, 2], [1, 3, 3]] {
let faults = Arc::new(AtomicUsize::new(0));
let hook = decommission_retry_fault_hook("bucket", "object", Arc::clone(&faults));
@@ -5523,16 +5163,14 @@ mod tests {
}
assert_eq!(faults.load(Ordering::SeqCst), 0, "unrelated or failed copies must not consume faults");
let mut expected_faults = 0;
for &(attempt, succeeded, expected) in *case {
for (index, attempt) in attempts.into_iter().enumerate() {
assert_eq!(
hook(DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "object", attempt, succeeded),
expected,
"fault plan {case:?} at attempt {attempt}"
hook(DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "object", attempt, true),
index < 2,
"attempts={attempts:?}, index={index}"
);
expected_faults += usize::from(expected);
assert_eq!(faults.load(Ordering::SeqCst), expected_faults);
}
assert_eq!(faults.load(Ordering::SeqCst), 2, "attempts={attempts:?}");
}
}
@@ -5668,15 +5306,6 @@ mod tests {
changed_result.expect("SourceChanged entry retry must converge");
other_result.expect("other bucket entry must continue through ordinary copy retries");
assert_eq!(
store.pool_meta.read().await.pools[0]
.decommission
.as_ref()
.expect("decommission progress should remain available")
.items_decommission_failed,
0,
"entry completion must not hide an exhausted copy failure"
);
assert!(!rx.is_cancelled(), "entry-level SourceChanged must not cancel the shared worker token");
assert_eq!(mutation_calls.load(Ordering::SeqCst), 2, "entry must be re-listed after SourceChanged");
assert_eq!(ordinary_faults.load(Ordering::SeqCst), 2, "ordinary copy must consume the retry budget");
@@ -6305,7 +5934,6 @@ mod tests {
"reverse-decommission-fixed-target",
&[(1, 4), (1, 4)],
CancellationToken::new(),
None,
))
.await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
@@ -6727,7 +6355,6 @@ mod tests {
"multi-set-decommission-source-cleanup",
&[(2, 4)],
CancellationToken::new(),
None,
))
.await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
@@ -9243,17 +8870,18 @@ mod tests {
const MANIFEST_COUNT: usize = 10;
let temp_dir = tempfile::tempdir().expect("create fast manifest pass recovery store dir");
let mut instance_ctx = crate::runtime::instance::InstanceContext::new();
instance_ctx.suppress_tier_delete_journal_recovery_for_test();
let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
temp_dir.path(),
"tier-delete-fast-manifest-pass",
&[(1, 4)],
CancellationToken::new(),
Some(Arc::new(instance_ctx)),
))
.await;
let (ctx, store, _shutdown) =
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-delete-fast-manifest-pass", &[4])).await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = "tier-delete-fast-manifest-pass-bucket";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("fast manifest pass bucket should be created");
let incarnation = store
.bucket_incarnation_id(bucket)
.await
.expect("fast manifest pass bucket incarnation should resolve");
let tier_name = "FAST-MANIFEST-PASS";
let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
@@ -9261,19 +8889,9 @@ mod tests {
.expect("fast manifest pass tier lease should resolve")
.backend_identity();
for index in 0..MANIFEST_COUNT {
// Pagination must not depend on same-bucket lock wait deadlines.
let bucket = format!("tier-delete-fast-manifest-pass-{index}");
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("fast manifest pass bucket should be created");
let incarnation = store
.bucket_incarnation_id(&bucket)
.await
.expect("fast manifest pass bucket incarnation should resolve");
install_aborting_dispatch_fixture(
store.clone(),
&bucket,
bucket,
incarnation,
&format!("manifest-page-{index:06}/"),
tier_name,
@@ -9304,78 +8922,12 @@ mod tests {
"one production pass must cross the default eight-manifest page limit"
);
assert_eq!(stats.manifests.scanned, MANIFEST_COUNT);
assert_eq!(stats.manifests.deleted, MANIFEST_COUNT, "full recovery result: {stats:?}");
assert_eq!(stats.manifests.failed, 0, "full recovery result: {stats:?}");
assert_eq!(stats.manifests.deleted, MANIFEST_COUNT);
assert_eq!(stats.manifests.failed, 0);
assert_eq!(manifest_marker, None);
assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
assert_eq!(tier_delete_journal_count(store).await, 0);
assert_eq!(backend.remove_count().await, 0, "rollback recovery must not call the remote tier");
shutdown.cancel();
}
#[cfg(feature = "test-util")]
#[tokio::test]
#[serial_test::serial(storage_class_env)]
async fn tier_delete_manual_pass_retains_manifest_owned_by_startup_recovery() {
let temp_dir = tempfile::tempdir().expect("create automatic recovery ownership store dir");
let (ctx, store, shutdown) =
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-delete-auto-owner", &[4])).await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = "tier-delete-auto-owner-bucket";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("automatic recovery bucket should be created");
let incarnation = store.bucket_incarnation_id(bucket).await.expect("bucket incarnation");
let tier_name = "AUTO-OWNER";
let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
.await
.expect("automatic recovery tier lease")
.backend_identity();
// The automatic worker must not observe a partially installed fixture.
let lifecycle_guard = store
.acquire_bucket_lifecycle_write_lock(bucket)
.await
.expect("fixture lifecycle lock");
let (manifest_name, entries) =
install_aborting_dispatch_fixture(store.clone(), bucket, incarnation, "auto-owner/", tier_name, identity, 1).await;
let journal_name = tier_delete_journal_object_name(&entries[0]);
let hook = TierDeleteDispatchRollbackTestHook::install_slow_delete(&journal_name, &journal_name);
drop(lifecycle_guard);
ctx.wake_tier_delete_journal_recovery();
tokio::time::timeout(Duration::from_secs(30), hook.wait_until_delete_paused())
.await
.expect("startup recovery should own the manifest before a manual pass");
assert!(tier_delete_dispatch_manifest_recovery_inflight_for_test(&store, &manifest_name));
let stats = recover_tier_delete_dispatch_manifests(store.clone(), 8, None)
.await
.expect("manual recovery scan");
assert_eq!(stats.scanned, 1, "{stats:?}");
assert_eq!(stats.retained, 1, "{stats:?}");
assert_eq!(stats.deleted, 0, "{stats:?}");
assert_eq!(stats.failed, 0, "{stats:?}");
assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 1);
assert_eq!(tier_delete_journal_count(store.clone()).await, 1);
hook.release_delete();
tokio::time::timeout(Duration::from_secs(30), async {
loop {
let manifest_gone = matches!(com::read_config(store.clone(), &manifest_name).await, Err(Error::ConfigNotFound));
if manifest_gone && !tier_delete_dispatch_manifest_recovery_inflight_for_test(&store, &manifest_name) {
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("automatic recovery should converge without a manual retry");
assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
assert_eq!(tier_delete_journal_count(store).await, 0);
assert_eq!(backend.remove_count().await, 0, "rollback must not delete from the remote tier");
shutdown.cancel();
}
#[cfg(feature = "test-util")]
@@ -10750,17 +10302,8 @@ mod tests {
const JOURNAL_COUNT: usize = 40;
let temp_dir = tempfile::tempdir().expect("create rollback retry store dir");
// Manual retries must own progress between fault removal and the next attempt.
let mut instance_ctx = crate::runtime::instance::InstanceContext::new();
instance_ctx.suppress_tier_delete_journal_recovery_for_test();
let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
temp_dir.path(),
"dispatch-rollback-retry",
&[(1, 4)],
CancellationToken::new(),
Some(Arc::new(instance_ctx)),
))
.await;
let (ctx, store, _shutdown) =
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-rollback-retry", &[4])).await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = "dispatch-rollback-retry-bucket";
store
@@ -10834,7 +10377,6 @@ mod tests {
assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
assert_eq!(backend.remove_count().await, 0, "rollback retries must never call the remote tier");
shutdown.cancel();
}
#[cfg(feature = "test-util")]
@@ -13662,7 +13204,6 @@ mod tests {
"partial-set-prefix-delete",
&[(2, 4)],
CancellationToken::new(),
None,
))
.await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
@@ -17035,7 +16576,6 @@ mod tests {
"prepared-directory-recovery",
&[(2, 4)],
shutdown,
None,
))
.await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
@@ -17688,38 +17228,6 @@ mod tests {
.expect("test thread should complete");
}
#[cfg(feature = "test-util")]
#[tokio::test]
#[serial_test::serial(storage_class_env)]
async fn odm_write_back_requires_one_set_and_enabled_namespace_locking() {
for (layout, locking, supported) in [
(&[(1, 4)][..], true, true),
(&[(1, 4), (1, 4)][..], true, false),
(&[(2, 4)][..], true, false),
(&[(1, 4)][..], false, false),
] {
temp_env::async_with_vars([("RUSTFS_LOCK_ENABLED", Some(if locking { "true" } else { "false" }))], async {
let dir = tempfile::tempdir().expect("isolated topology");
let shutdown = CancellationToken::new();
let (_ctx, store, _) = without_storage_class_env(build_isolated_test_store_with_layout(
dir.path(),
"odm-topology",
layout,
shutdown.clone(),
None,
))
.await;
assert_eq!(
store.supports_atomic_create_only_write_back(),
supported,
"layout={layout:?}, locking={locking}"
);
shutdown.cancel();
})
.await;
}
}
#[cfg(feature = "test-util")]
#[tokio::test]
#[serial_test::serial(storage_class_env)]
+7 -8
View File
@@ -848,7 +848,7 @@ impl ECStore {
}
pub fn scanner_namespace_mutation_generation(&self) -> u64 {
list_objects::scanner_namespace_mutation_generation().saturating_add(self.ctx.namespace_commit_generation())
list_objects::scanner_namespace_mutation_generation()
}
pub async fn scanner_data_movement_active(&self) -> bool {
@@ -857,7 +857,7 @@ impl ECStore {
}
/// Return the storage-owned movement state and generation as one
/// authenticated activity snapshot. The read lock is acquired before
/// authenticated activity snapshot. The read lock is acquired before
/// the state locks (cancelers, pool metadata, then rebalance metadata),
/// matching the transition writer order and preventing a terminal state
/// from being reported with the preceding generation.
@@ -886,12 +886,11 @@ impl ECStore {
/// Returns whether scanner metadata may still be hidden by a local
/// data-movement state. Terminal failed/canceled decommission entries
/// remain suspended until an operator clears or retries them, so they are
/// a publication barrier even after the worker has stopped. Active PUT
/// rename fanouts also defer publication, including post-ACK tails.
/// a publication barrier even after the worker has stopped.
pub async fn scanner_data_usage_publication_blocked(&self) -> bool {
let operation_gate = self.ctx.data_movement_operation_gate();
let _operation_guard = operation_gate.read_owned().await;
self.scanner_data_usage_publication_snapshot_blocked().await || self.ctx.namespace_commits_pending()
self.scanner_data_usage_publication_snapshot_blocked().await
}
pub async fn scanner_data_movement_pause_status(&self) -> ScannerDataMovementPauseStatus {
@@ -1071,7 +1070,7 @@ impl ECStore {
{
return Err(Error::other("scanner publication lease generation is stale"));
}
if self.scanner_data_movement_snapshot_locked().await.1 || self.ctx.namespace_commits_pending() {
if self.scanner_data_movement_snapshot_locked().await.1 {
return Err(Error::other("scanner publication lease is blocked by data movement"));
}
@@ -1110,7 +1109,7 @@ impl ECStore {
{
return Err(Error::other("scanner publication lease generation is stale"));
}
if self.scanner_data_movement_snapshot_locked().await.1 || self.ctx.namespace_commits_pending() {
if self.scanner_data_movement_snapshot_locked().await.1 {
return Err(Error::other("scanner publication lease is blocked by data movement"));
}
if !self.ctx.scanner_publication_lease_is_active(token).await {
@@ -1130,7 +1129,7 @@ impl ECStore {
if self.ctx.data_movement_generation_exhausted() || self.ctx.data_movement_operation_epoch_exhausted() {
return Err(Error::other("scanner publication lease generation is exhausted"));
}
if self.scanner_data_movement_snapshot_locked().await.1 || self.ctx.namespace_commits_pending() {
if self.scanner_data_movement_snapshot_locked().await.1 {
return Err(Error::other("scanner publication lease is blocked by data movement"));
}
let Some(lease_generation) = self.ctx.scanner_publication_lease_generation(token).await else {
-4
View File
@@ -45,10 +45,6 @@ use uuid::Uuid;
use crate::heal::task::{HealOptions, HealPriority, HealRequest, HealType};
/// Read-only inspection of committed MRF checkpoints. The legacy consumer
/// remains unchanged until ownership-aware replay is deployed.
pub mod snapshot;
/// Journal location inside the metadata bucket, following the resume-state
/// layout.
pub(crate) const MRF_JOURNAL_PATH: &str = "buckets/.heal/mrf/journal.bin";
-681
View File
@@ -1,681 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Reader-first support for owner-local MRF checkpoints.
//!
//! Each of two slots has a payload and a commit manifest. The manifest binds
//! the writer identity, persistent sequence, length and whole-payload digest.
//! Replacing the inactive slot must leave the previous committed slot intact.
//! Production publication and reclamation are deliberately not enabled here.
//! An unreadable commit path cannot prove that only legacy data exists. This
//! explicit inspection API fails closed and never mutates recovery anchors.
//! It is not wired into the legacy consumer: that transition requires the
//! ownership-aware replay and producer handoff before writer activation.
//! One surviving committed replica supports process restart recovery only;
//! this reader does not establish a replication quorum or a power-loss policy.
use super::{MRF_JOURNAL_PATH, MRF_SCOPED_JOURNAL_PATH, decode_journal};
use crate::heal::RUSTFS_META_BUCKET;
use crate::heal::storage_api::owner::{EcstoreDiskAPI, EcstoreDiskError, EcstoreDiskStore};
use sha2::{Digest, Sha256};
use std::collections::HashMap;
use tokio::io::AsyncReadExt;
use uuid::Uuid;
// Root-level control files avoid requiring a new directory before the first
// atomic commit. They remain inside the storage owner's metadata volume.
const PAYLOAD_PATHS: [&str; 2] = [".heal-mrf-snapshot.0.bin", ".heal-mrf-snapshot.1.bin"];
const MANIFEST_PATHS: [&str; 2] = [".heal-mrf-commit.0.bin", ".heal-mrf-commit.1.bin"];
const MAGIC: &[u8; 8] = b"RFMRFC01";
const MANIFEST_LEN: usize = 8 + 1 + 16 + 8 + 8 + 32 + 32;
const VERSION: u8 = 1;
#[derive(Debug, thiserror::Error)]
pub enum SnapshotError {
#[error("MRF checkpoint has an invalid or incomplete commit record")]
Corrupt,
#[error("MRF checkpoint format is unsupported")]
Unsupported,
#[error("MRF checkpoint exceeds the configured byte limit")]
TooLarge,
#[error("MRF checkpoint replicas disagree at the same sequence")]
Conflict,
#[error("MRF checkpoint storage is unavailable")]
Disk(#[source] EcstoreDiskError),
#[error("MRF checkpoint body could not be read")]
Read(#[source] std::io::Error),
}
#[derive(Debug, PartialEq, Eq)]
struct Manifest {
owner: Uuid,
sequence: u64,
payload_len: usize,
payload_digest: [u8; 32],
}
impl Manifest {
fn decode(bytes: &[u8], limit: usize) -> Result<Self, SnapshotError> {
if bytes.len() != MANIFEST_LEN || &bytes[..8] != MAGIC {
return Err(SnapshotError::Corrupt);
}
if bytes[8] != VERSION {
return Err(SnapshotError::Unsupported);
}
let signed = MANIFEST_LEN - 32;
let checksum: [u8; 32] = Sha256::digest(&bytes[..signed]).into();
if checksum != bytes[signed..] {
return Err(SnapshotError::Corrupt);
}
let owner = Uuid::from_slice(&bytes[9..25]).map_err(|_| SnapshotError::Corrupt)?;
let sequence = u64::from_le_bytes(bytes[25..33].try_into().map_err(|_| SnapshotError::Corrupt)?);
let payload_len = u64::from_le_bytes(bytes[33..41].try_into().map_err(|_| SnapshotError::Corrupt)?);
let payload_len = usize::try_from(payload_len).map_err(|_| SnapshotError::TooLarge)?;
if owner.is_nil() || sequence == 0 || sequence == u64::MAX {
return Err(SnapshotError::Corrupt);
}
if payload_len > limit {
return Err(SnapshotError::TooLarge);
}
Ok(Self {
owner,
sequence,
payload_len,
payload_digest: bytes[41..73].try_into().map_err(|_| SnapshotError::Corrupt)?,
})
}
}
#[derive(Debug)]
pub struct CommittedSnapshot {
manifest: Manifest,
payload: Vec<u8>,
}
impl CommittedSnapshot {
/// Persistent single-writer sequence, not a process UUID ordering.
pub fn sequence(&self) -> u64 {
self.manifest.sequence
}
/// Identity recorded by the committed checkpoint's writer.
pub fn owner(&self) -> Uuid {
self.manifest.owner
}
/// Complete, checksum-validated record bytes. Inspection does not consume
/// these records or acknowledge completion to any producer.
pub fn payload(&self) -> &[u8] {
&self.payload
}
fn decode(manifest: &[u8], payload: Vec<u8>, limit: usize) -> Result<Self, SnapshotError> {
let manifest = Manifest::decode(manifest, limit)?;
let checksum: [u8; 32] = Sha256::digest(&payload).into();
if payload.len() != manifest.payload_len || checksum != manifest.payload_digest {
return Err(SnapshotError::Corrupt);
}
if decode_journal(&payload).1 != 0 {
return Err(SnapshotError::Corrupt);
}
Ok(Self { manifest, payload })
}
}
#[derive(Debug)]
pub enum RecoverySnapshot {
/// An intact legacy snapshot, without a comparable commit sequence.
Legacy(Vec<u8>),
/// A committed checkpoint requiring ownership-aware replay before use.
Committed(CommittedSnapshot),
}
async fn read_bounded(disk: &EcstoreDiskStore, path: &str, limit: usize) -> Result<Option<Vec<u8>>, SnapshotError> {
let reader = match EcstoreDiskAPI::read_file(disk.as_ref(), RUSTFS_META_BUCKET, path).await {
Ok(reader) => reader,
Err(EcstoreDiskError::FileNotFound | EcstoreDiskError::VolumeNotFound) => return Ok(None),
Err(error) => return Err(SnapshotError::Disk(error)),
};
let maximum = limit.checked_add(1).ok_or(SnapshotError::TooLarge)?;
let maximum = u64::try_from(maximum).map_err(|_| SnapshotError::TooLarge)?;
let mut bytes = Vec::new();
reader
.take(maximum)
.read_to_end(&mut bytes)
.await
.map_err(SnapshotError::Read)?;
if bytes.len() > limit {
return Err(SnapshotError::TooLarge);
}
Ok(Some(bytes))
}
fn select_snapshot(selected: &mut Option<CommittedSnapshot>, candidate: CommittedSnapshot) -> Result<(), SnapshotError> {
if let Some(current) = selected {
if current.manifest.sequence == candidate.manifest.sequence
&& (current.manifest != candidate.manifest || current.payload != candidate.payload)
{
return Err(SnapshotError::Conflict);
}
if current.manifest.sequence >= candidate.manifest.sequence {
return Ok(());
}
}
*selected = Some(candidate);
Ok(())
}
async fn read_committed(disks: &[EcstoreDiskStore], limit: usize) -> Result<Option<CommittedSnapshot>, SnapshotError> {
let mut selected = None;
let mut damaged = None;
let mut identities = HashMap::new();
for disk in disks {
for (manifest_path, payload_path) in MANIFEST_PATHS.into_iter().zip(PAYLOAD_PATHS) {
let candidate = async {
let Some(manifest) = read_bounded(disk, manifest_path, MANIFEST_LEN).await? else {
return Ok(None);
};
let header = Manifest::decode(&manifest, limit)?;
let payload = read_bounded(disk, payload_path, header.payload_len)
.await?
.ok_or(SnapshotError::Corrupt)?;
CommittedSnapshot::decode(&manifest, payload, limit).map(Some)
}
.await;
match candidate {
Ok(Some(candidate)) => {
let identity = (
candidate.manifest.owner,
candidate.manifest.payload_len,
candidate.manifest.payload_digest,
);
if identities
.insert(candidate.manifest.sequence, identity)
.is_some_and(|previous| previous != identity)
{
return Err(SnapshotError::Conflict);
}
select_snapshot(&mut selected, candidate)?;
}
Ok(None) => {}
// A future committed format may supersede all readable slots.
Err(SnapshotError::Unsupported) => return Err(SnapshotError::Unsupported),
Err(error) => damaged = Some(error),
}
}
}
match (selected, damaged) {
(Some(snapshot), _) => Ok(Some(snapshot)),
(None, Some(error)) => Err(error),
(None, None) => Ok(None),
}
}
async fn read_legacy(disks: &[EcstoreDiskStore], path: &str, limit: usize) -> Result<Option<Vec<u8>>, SnapshotError> {
let mut selected = None;
let mut incomplete: Option<Vec<u8>> = None;
for disk in disks {
match read_bounded(disk, path, limit).await {
Ok(Some(payload)) if decode_journal(&payload).1 == 0 => {
if selected.as_ref().is_some_and(|current| *current != payload) {
// Legacy snapshots have no sequence. There is no evidence
// that the first, longest or nonempty replica is newest.
return Err(SnapshotError::Conflict);
}
selected = Some(payload);
}
Ok(Some(payload)) => {
if let Some(previous) = &incomplete {
if previous.starts_with(&payload) {
continue;
}
if !payload.starts_with(previous) {
return Err(SnapshotError::Corrupt);
}
}
incomplete = Some(payload);
}
Ok(None) => {}
Err(error) => return Err(error),
}
}
if let Some(prefix) = incomplete
&& !selected.as_ref().is_some_and(|payload| payload.starts_with(&prefix))
{
// In particular, an empty O_TRUNC replica cannot supersede another
// replica containing intact records followed by a torn tail.
return Err(SnapshotError::Corrupt);
}
Ok(selected)
}
/// Inspect local MRF checkpoints without replaying, acknowledging or deleting.
///
/// `max_bytes` bounds each payload read. Every local replica is examined and
/// ambiguous identities, unavailable proof or unsupported formats return a
/// typed error. This API must not authorize a writer without the separate
/// ownership and mixed-version activation checks.
pub async fn inspect_local_recovery_snapshot(max_bytes: usize) -> Result<Option<RecoverySnapshot>, SnapshotError> {
read_recovery_snapshot(&super::journal_disks().await, max_bytes).await
}
async fn read_recovery_snapshot(disks: &[EcstoreDiskStore], limit: usize) -> Result<Option<RecoverySnapshot>, SnapshotError> {
if let Some(snapshot) = read_committed(disks, limit).await? {
return Ok(Some(RecoverySnapshot::Committed(snapshot)));
}
// RUSTFS_COMPAT_TODO(backlog-2263): inspect retained legacy MRF journals. Remove after all supported upgrade and rollback readers understand committed snapshots and retained journals have migrated.
if let Some(payload) = read_legacy(disks, MRF_SCOPED_JOURNAL_PATH, limit).await? {
return Ok(Some(RecoverySnapshot::Legacy(payload)));
}
Ok(read_legacy(disks, MRF_JOURNAL_PATH, limit)
.await?
.map(RecoverySnapshot::Legacy))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::heal::mrf_queue::encode_intent;
use crate::heal::storage_api::owner::{EcstoreConditionalFileUpdate, EcstoreDiskBytes};
use crate::heal::{DiskOption, Endpoint, new_disk};
use rustfs_common::mrf_channel::{MrfIntent, MrfKind, MrfScope};
use std::sync::Arc;
use tempfile::TempDir;
fn payload(object: &str) -> Vec<u8> {
let intent = MrfIntent {
bucket: Arc::from("bucket"),
object: Arc::from(object),
version_id: None,
kind: MrfKind::PartialWrite,
scope: None,
lease: None,
enqueued_at_ms: 1234,
attempts: 0,
};
let mut bytes = Vec::new();
assert!(encode_intent(&intent, &mut bytes), "fixture must encode a full record");
bytes
}
fn manifest(owner: Uuid, sequence: u64, payload: &[u8]) -> Vec<u8> {
let mut bytes = Vec::with_capacity(MANIFEST_LEN);
bytes.extend_from_slice(MAGIC);
bytes.push(VERSION);
bytes.extend_from_slice(owner.as_bytes());
bytes.extend_from_slice(&sequence.to_le_bytes());
bytes.extend_from_slice(&u64::try_from(payload.len()).expect("fixture length fits").to_le_bytes());
bytes.extend_from_slice(&Sha256::digest(payload));
bytes.extend_from_slice(&Sha256::digest(&bytes));
bytes
}
async fn disk(root: &TempDir, name: &str) -> EcstoreDiskStore {
let path = root.path().join(name);
std::fs::create_dir_all(&path).expect("create disk directory");
let endpoint = Endpoint::try_from(path.to_string_lossy().as_ref()).expect("valid disk endpoint");
let disk = new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("open disk");
let result = EcstoreDiskAPI::make_volume(disk.as_ref(), RUSTFS_META_BUCKET).await;
assert!(
matches!(result, Ok(()) | Err(EcstoreDiskError::VolumeExists)),
"metadata volume: {result:?}"
);
disk
}
// Exercise the existing storage owner's atomic CAS primitive. No production
// caller publishes this format until ownership-aware replay is available.
async fn install(disk: &EcstoreDiskStore, path: &str, bytes: &[u8]) {
let expected = EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, path).await.ok();
let result = EcstoreDiskAPI::compare_and_update_file(
disk.as_ref(),
RUSTFS_META_BUCKET,
path,
expected,
Some(EcstoreDiskBytes::copy_from_slice(bytes)),
)
.await
.expect("atomic snapshot slot write");
assert_eq!(result, EcstoreConditionalFileUpdate::Updated);
}
async fn commit(disk: &EcstoreDiskStore, slot: usize, owner: Uuid, sequence: u64, bytes: &[u8]) {
install(disk, PAYLOAD_PATHS[slot], bytes).await;
install(disk, MANIFEST_PATHS[slot], &manifest(owner, sequence, bytes)).await;
}
#[test]
fn manifest_validates_identity_sequence_length_and_digest() {
let bytes = payload("object");
let owner = Uuid::new_v4();
assert!(CommittedSnapshot::decode(&manifest(owner, 1, &bytes), bytes.clone(), bytes.len()).is_ok());
for (owner, sequence) in [(Uuid::nil(), 1), (owner, 0), (owner, u64::MAX)] {
assert!(matches!(
Manifest::decode(&manifest(owner, sequence, &bytes), bytes.len()),
Err(SnapshotError::Corrupt)
));
}
assert!(matches!(
Manifest::decode(&manifest(owner, 1, &bytes), bytes.len() - 1),
Err(SnapshotError::TooLarge)
));
let mut corrupt = manifest(owner, 1, &bytes);
corrupt[25] ^= 1;
assert!(matches!(Manifest::decode(&corrupt, bytes.len()), Err(SnapshotError::Corrupt)));
let mut unsupported = manifest(owner, 1, &bytes);
unsupported[8] = 2;
assert!(matches!(Manifest::decode(&unsupported, bytes.len()), Err(SnapshotError::Unsupported)));
}
#[test]
fn whole_payload_integrity_is_required_even_with_a_valid_manifest() {
let bytes = payload("object");
let owner = Uuid::new_v4();
let header = manifest(owner, 1, &bytes);
assert!(matches!(
CommittedSnapshot::decode(&header, bytes[..bytes.len() - 1].to_vec(), bytes.len()),
Err(SnapshotError::Corrupt)
));
let invalid = b"not an MRF record".to_vec();
assert!(matches!(
CommittedSnapshot::decode(&manifest(owner, 2, &invalid), invalid, bytes.len()),
Err(SnapshotError::Corrupt)
));
}
#[tokio::test]
async fn newest_complete_replica_wins_in_both_disk_orders() {
let root = TempDir::new().expect("test directory");
let first = disk(&root, "first").await;
let second = disk(&root, "second").await;
let owner = Uuid::new_v4();
commit(&first, 0, owner, 1, &payload("old")).await;
commit(&second, 1, owner, 2, &payload("new")).await;
for disks in [vec![first.clone(), second.clone()], vec![second.clone(), first.clone()]] {
let recovered = read_committed(&disks, 4096)
.await
.expect("read replicas")
.expect("committed snapshot");
assert_eq!(recovered.manifest.sequence, 2);
assert_eq!(recovered.payload, payload("new"));
}
}
#[tokio::test]
async fn divergent_commits_at_same_sequence_fail_closed() {
let root = TempDir::new().expect("test directory");
let first = disk(&root, "first").await;
let second = disk(&root, "second").await;
let owner = Uuid::new_v4();
commit(&first, 0, owner, 7, &payload("a")).await;
commit(&second, 1, owner, 7, &payload("b")).await;
assert!(matches!(read_committed(&[first, second], 4096).await, Err(SnapshotError::Conflict)));
}
#[tokio::test]
async fn newer_slot_does_not_hide_a_conflicting_commit_history() {
let root = TempDir::new().expect("test directory");
let first = disk(&root, "first").await;
let second = disk(&root, "second").await;
let owner = Uuid::new_v4();
commit(&first, 0, owner, 8, &payload("newest")).await;
commit(&first, 1, owner, 7, &payload("a")).await;
commit(&second, 1, owner, 7, &payload("b")).await;
assert!(matches!(read_committed(&[first, second], 4096).await, Err(SnapshotError::Conflict)));
}
#[tokio::test]
async fn uncommitted_or_torn_successor_preserves_previous_slot() {
let root = TempDir::new().expect("test directory");
let disk = disk(&root, "disk").await;
let owner = Uuid::new_v4();
let old = payload("old");
let next = payload("next");
commit(&disk, 0, owner, 1, &old).await;
install(&disk, PAYLOAD_PATHS[1], &next).await;
let recovered = read_committed(std::slice::from_ref(&disk), 4096)
.await
.expect("staged payload is not a commit")
.expect("old snapshot");
assert_eq!(recovered.payload, old);
install(&disk, MANIFEST_PATHS[1], &manifest(owner, 2, &next)[..20]).await;
let recovered = read_committed(std::slice::from_ref(&disk), 4096)
.await
.expect("torn manifest preserves old slot")
.expect("old snapshot");
assert_eq!(recovered.manifest.sequence, 1);
install(&disk, MANIFEST_PATHS[1], &manifest(owner, 2, &next)).await;
install(&disk, PAYLOAD_PATHS[1], b"torn").await;
let recovered = read_committed(&[disk], 4096)
.await
.expect("torn payload preserves old slot")
.expect("old snapshot");
assert_eq!(recovered.manifest.sequence, 1);
}
#[tokio::test]
async fn stale_manifest_cas_cannot_replace_committed_anchor() {
let root = TempDir::new().expect("test directory");
let disk = disk(&root, "disk").await;
let owner = Uuid::new_v4();
let bytes = payload("object");
commit(&disk, 0, owner, 1, &bytes).await;
let result = EcstoreDiskAPI::compare_and_update_file(
disk.as_ref(),
RUSTFS_META_BUCKET,
MANIFEST_PATHS[0],
None,
Some(manifest(owner, 2, &bytes).into()),
)
.await
.expect("CAS call");
assert_eq!(result, EcstoreConditionalFileUpdate::Mismatch);
let recovered = read_committed(&[disk], 4096)
.await
.expect("read old anchor")
.expect("snapshot");
assert_eq!(recovered.manifest.sequence, 1);
}
#[tokio::test]
async fn legacy_import_requires_complete_consistent_replicas() {
let root = TempDir::new().expect("test directory");
let first = disk(&root, "first").await;
let second = disk(&root, "second").await;
let bytes = payload("object");
for (disk, data) in [(&first, &bytes[..bytes.len() - 1]), (&second, bytes.as_slice())] {
EcstoreDiskAPI::write_all(
disk.as_ref(),
RUSTFS_META_BUCKET,
MRF_SCOPED_JOURNAL_PATH,
EcstoreDiskBytes::copy_from_slice(data),
)
.await
.expect("legacy fixture");
}
let disks = [first.clone(), second];
assert!(
matches!(read_recovery_snapshot(&disks, 4096).await.expect("intact legacy replica"), Some(RecoverySnapshot::Legacy(data)) if data == bytes)
);
EcstoreDiskAPI::write_all(first.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH, payload("different").into())
.await
.expect("divergent fixture");
assert!(matches!(read_recovery_snapshot(&disks, 4096).await, Err(SnapshotError::Conflict)));
}
#[tokio::test]
async fn committed_inspection_leaves_payload_and_manifest_unchanged() {
let root = TempDir::new().expect("test directory");
let disk = disk(&root, "disk").await;
let owner = Uuid::new_v4();
let bytes = payload("object");
commit(&disk, 0, owner, 3, &bytes).await;
assert!(matches!(
read_recovery_snapshot(std::slice::from_ref(&disk), 4096)
.await
.expect("new snapshot"),
Some(RecoverySnapshot::Committed(_))
));
assert_eq!(
EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, MANIFEST_PATHS[0])
.await
.expect("manifest retained")
.as_ref(),
manifest(owner, 3, &bytes)
);
assert_eq!(
EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, PAYLOAD_PATHS[0])
.await
.expect("payload retained")
.as_ref(),
bytes
);
}
#[tokio::test]
async fn legacy_inspection_rejects_complete_subsets_and_scope_ambiguity() {
let scoped = |set_index| {
let intent = MrfIntent {
bucket: Arc::from("bucket"),
object: Arc::from("a"),
version_id: None,
kind: MrfKind::PartialWrite,
scope: Some(MrfScope {
pool_index: 0,
set_index,
}),
lease: None,
enqueued_at_ms: 1234,
attempts: 0,
};
let mut bytes = Vec::new();
assert!(encode_intent(&intent, &mut bytes), "scoped fixture must encode");
bytes
};
let mut superset = payload("a");
superset.extend_from_slice(&payload("b"));
for (case, first_bytes, second_bytes) in [
("complete-subset", payload("a"), superset),
("different-set", scoped(1), scoped(2)),
("unknown-scope", payload("a"), scoped(1)),
] {
let root = TempDir::new().expect("test directory");
let first = disk(&root, "first").await;
let second = disk(&root, "second").await;
for (disk, bytes) in [(&first, &first_bytes), (&second, &second_bytes)] {
assert_eq!(decode_journal(bytes).1, 0, "{case}: complete fixture");
EcstoreDiskAPI::write_all(
disk.as_ref(),
RUSTFS_META_BUCKET,
MRF_SCOPED_JOURNAL_PATH,
EcstoreDiskBytes::copy_from_slice(bytes),
)
.await
.expect("write legacy replica");
}
for disks in [vec![first.clone(), second.clone()], vec![second.clone(), first.clone()]] {
assert!(
matches!(read_recovery_snapshot(&disks, 4096).await, Err(SnapshotError::Conflict)),
"{case}: neither replica order proves a latest snapshot"
);
}
for (disk, bytes) in [(&first, &first_bytes), (&second, &second_bytes)] {
assert_eq!(
EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH)
.await
.expect("legacy evidence retained")
.as_ref(),
bytes.as_slice(),
"{case}: inspection must preserve both source replicas"
);
}
}
}
#[tokio::test]
async fn oversized_or_corrupt_scoped_snapshot_never_falls_back_to_legacy() {
let root = TempDir::new().expect("test directory");
let disk = disk(&root, "disk").await;
EcstoreDiskAPI::write_all(disk.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH, vec![0; 1025].into())
.await
.expect("oversized fixture");
EcstoreDiskAPI::write_all(disk.as_ref(), RUSTFS_META_BUCKET, MRF_JOURNAL_PATH, payload("old").into())
.await
.expect("legacy fixture");
assert!(matches!(
read_recovery_snapshot(std::slice::from_ref(&disk), 1024).await,
Err(SnapshotError::TooLarge)
));
assert!(matches!(read_recovery_snapshot(&[disk], 2048).await, Err(SnapshotError::Corrupt)));
}
#[tokio::test]
async fn empty_legacy_replica_cannot_erase_records_in_a_torn_replica() {
let root = TempDir::new().expect("test directory");
let first = disk(&root, "first").await;
let second = disk(&root, "second").await;
let mut incomplete = payload("durable-object");
incomplete.extend_from_slice(b"torn");
EcstoreDiskAPI::write_all(first.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH, Vec::new().into())
.await
.expect("empty truncated replica");
EcstoreDiskAPI::write_all(second.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH, incomplete.clone().into())
.await
.expect("records and torn tail");
for disks in [vec![first.clone(), second.clone()], vec![second.clone(), first.clone()]] {
assert!(matches!(read_recovery_snapshot(&disks, 4096).await, Err(SnapshotError::Corrupt)));
}
assert_eq!(
EcstoreDiskAPI::read_all(second.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH)
.await
.expect("recovery anchor preserved")
.as_ref(),
incomplete
);
}
#[tokio::test]
async fn unreadable_commit_record_never_implies_legacy_only() {
let root = TempDir::new().expect("test directory");
let disk = disk(&root, "disk").await;
let legacy = payload("old");
EcstoreDiskAPI::write_all(disk.as_ref(), RUSTFS_META_BUCKET, MRF_JOURNAL_PATH, legacy.clone().into())
.await
.expect("legacy fixture");
// Opening a directory as a record either fails at open or at read,
// depending on the platform. Neither outcome proves absence.
std::fs::create_dir(root.path().join("disk").join(RUSTFS_META_BUCKET).join(MANIFEST_PATHS[0]))
.expect("unreadable manifest fixture");
let recovered = read_recovery_snapshot(std::slice::from_ref(&disk), 4096).await;
assert!(
matches!(recovered, Err(SnapshotError::Disk(_) | SnapshotError::Read(_))),
"must preserve unavailable proof: {recovered:?}"
);
assert_eq!(
EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, MRF_JOURNAL_PATH)
.await
.expect("legacy remains")
.as_ref(),
legacy
);
}
}
@@ -44,9 +44,7 @@ use walkdir::WalkDir;
mod storage_api;
use storage_api::integration::{
BucketOperations, ECStore, MakeBucketOptions, NamespaceLocking as _, ObjectIO as _, ObjectOperations as _,
};
use storage_api::integration::{BucketOperations, ECStore, MakeBucketOptions, ObjectIO as _, ObjectOperations as _};
/// 256 KiB + change: large enough to be stored as non-inline erasure shards
/// (so each data version materializes as an on-disk `part.*` file we can assert
@@ -108,7 +106,6 @@ async fn put_versioned(ecstore: &Arc<ECStore>, bucket: &str, object: &str, data:
.put_object(bucket, object, &mut reader, &opts)
.await
.expect("versioned put_object failed");
wait_for_put_tail(ecstore, bucket, object).await;
info.version_id
.map(|u| u.to_string())
.expect("versioned put must return a version id")
@@ -120,7 +117,6 @@ async fn put_unversioned(ecstore: &Arc<ECStore>, bucket: &str, object: &str, dat
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
.await
.expect("unversioned put_object failed");
wait_for_put_tail(ecstore, bucket, object).await;
}
/// Create a delete-marker as the latest version (versioned:true, no version_id)
@@ -164,16 +160,20 @@ fn xl_meta_path(obj_dir: &Path) -> PathBuf {
obj_dir.join("xl.meta")
}
async fn wait_for_put_tail(ecstore: &Arc<ECStore>, bucket: &str, object: &str) {
// Shards and xl.meta can exist before the detached PUT owner finishes.
let lock = ecstore
.new_ns_lock(bucket, object)
.await
.expect("fixture namespace lock should be created");
let _settled = lock
.get_write_lock(Duration::from_secs(30))
.await
.expect("PUT rename tail must finish before inspecting or wiping the fixture");
async fn wait_for_two_version_copies(disks: &[PathBuf], bucket: &str, object: &str) {
tokio::time::timeout(Duration::from_secs(5), async {
loop {
if disks.iter().all(|disk| {
let object_dir = object_dir(disk, bucket, object);
xl_meta_path(&object_dir).exists() && count_part_files(&object_dir) >= 2
}) {
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("PUT rename tails must converge before wiping the versioned fixture");
}
fn recreate_heal_opts() -> HealOpts {
@@ -305,13 +305,7 @@ mod serial_tests {
let data_v2 = versioned_test_data(20);
let v1 = put_versioned(&ecstore, bucket, object, &data_v1).await; // OLD, non-latest
let v2 = put_versioned(&ecstore, bucket, object, &data_v2).await; // latest
assert!(
disk_paths.iter().all(|disk| {
let dir = object_dir(disk, bucket, object);
xl_meta_path(&dir).exists() && count_part_files(&dir) >= 2
}),
"both versions must exist on every disk before wiping the fixture"
);
wait_for_two_version_copies(&disk_paths, bucket, object).await;
// ── Pre-wipe: prove the fixture actually has 2 versions on disk[0] ──
let obj_dir0 = object_dir(&disk_paths[0], bucket, object);
-1
View File
@@ -23,7 +23,6 @@ pub(crate) mod integration {
pub(crate) use rustfs_ecstore::api::storage::ECStore;
pub(crate) use rustfs_storage_api::BucketOperations;
pub(crate) use rustfs_storage_api::MakeBucketOptions;
pub(crate) use rustfs_storage_api::NamespaceLocking;
pub(crate) use rustfs_storage_api::ObjectIO;
pub(crate) use rustfs_storage_api::ObjectOperations;
}
+44 -460
View File
@@ -43,10 +43,6 @@ const ERR_LIFECYCLE_BUCKET_LOCKED: &str =
"ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an object locked bucket";
const ERR_LIFECYCLE_TOO_MANY_RULES: &str = "Lifecycle configuration should have at most 1000 rules";
const ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS: &str = "'Days' for Expiration action must be a positive integer";
const ERR_LIFECYCLE_EXPIRATION_DAYS_DATE_CONFLICT: &str = "Expiration cannot specify both Days and Date";
const ERR_LIFECYCLE_MULTIPLE_TRANSITIONS: &str = "Only one Transition action per lifecycle rule is supported";
const ERR_LIFECYCLE_MULTIPLE_NONCURRENT_TRANSITIONS: &str =
"Only one NoncurrentVersionTransition action per lifecycle rule is supported";
const ERR_LIFECYCLE_INVALID_NONCURRENT_EXPIRATION_DAYS: &str =
"'NoncurrentDays' for NoncurrentVersionExpiration action must be a positive integer";
const ERR_LIFECYCLE_INVALID_ABORT_INCOMPLETE_MPU_DAYS: &str =
@@ -365,12 +361,6 @@ impl Lifecycle for BucketLifecycleConfiguration {
{
return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_EXPIRED_OBJECT_ALL_VERSIONS));
}
if expiration.days.is_some() && expiration.date.is_some() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
ERR_LIFECYCLE_EXPIRATION_DAYS_DATE_CONFLICT,
));
}
if let Some(expiration_date) = &expiration.date {
let date = OffsetDateTime::from(expiration_date.clone());
if date.hour() != 0 || date.minute() != 0 || date.second() != 0 || date.nanosecond() != 0 {
@@ -404,20 +394,11 @@ impl Lifecycle for BucketLifecycleConfiguration {
}
}
if let Some(transitions) = &r.transitions {
if transitions.len() > 1 {
return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, ERR_LIFECYCLE_MULTIPLE_TRANSITIONS));
}
for transition in transitions {
TransitionOps::validate(transition)?;
}
}
if let Some(noncurrent_transitions) = &r.noncurrent_version_transitions {
if noncurrent_transitions.len() > 1 {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
ERR_LIFECYCLE_MULTIPLE_NONCURRENT_TRANSITIONS,
));
}
for transition in noncurrent_transitions {
NoncurrentVersionTransitionOps::validate(transition)?;
}
@@ -492,8 +473,6 @@ impl Lifecycle for BucketLifecycleConfiguration {
}
async fn eval(&self, obj: &ObjectOpts) -> Event {
// A single-object lookup cannot prove how many newer historical versions
// survive. Count-dependent actions wait for the complete-group evaluator.
self.eval_inner(obj, OffsetDateTime::now_utc(), 0).await
}
@@ -557,8 +536,23 @@ impl Lifecycle for BucketLifecycleConfiguration {
return Event::default();
};
if let Some(event) = obj.restored_copy_expiry(now) {
events.push(event);
if let Some(restore_expires) = obj.restore_expires
&& restore_expires.unix_timestamp() != 0
&& now.unix_timestamp() > restore_expires.unix_timestamp()
{
let mut action = IlmAction::DeleteRestoredAction;
if !obj.is_latest {
action = IlmAction::DeleteRestoredVersionAction;
}
events.push(Event {
action,
due: Some(now),
rule_id: "".into(),
noncurrent_days: 0,
newer_noncurrent_versions: 0,
storage_class: "".into(),
});
}
if let Some(ref lc_rules) = self.filter_rules(obj).await {
@@ -617,12 +611,17 @@ impl Lifecycle for BucketLifecycleConfiguration {
continue;
}
if !obj.is_latest
&& let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration
&& let Some(retain_newer_noncurrent_versions) = noncurrent_version_expiration.newer_noncurrent_versions
&& newer_noncurrent_versions < usize::try_from(retain_newer_noncurrent_versions).unwrap_or(usize::MAX)
{
continue;
}
if !obj.is_latest
&& let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration
&& let Some(noncurrent_days) = noncurrent_version_expiration.noncurrent_days
&& noncurrent_version_expiration
.newer_noncurrent_versions
.is_none_or(|retain| usize::try_from(retain).is_ok_and(|retain| newer_noncurrent_versions >= retain))
{
if let Some(successor_mod_time) = obj.successor_mod_time {
let expected_expiry = expected_expiry_time(successor_mod_time, noncurrent_days);
@@ -652,11 +651,7 @@ impl Lifecycle for BucketLifecycleConfiguration {
&& let Some(noncurrent_version_transition) = rule
.noncurrent_version_transitions
.as_ref()
.filter(|transitions| transitions.len() == 1)
.and_then(|transitions| transitions.first())
&& noncurrent_version_transition
.newer_noncurrent_versions
.is_none_or(|retain| usize::try_from(retain).is_ok_and(|retain| newer_noncurrent_versions >= retain))
&& let Some(storage_class) = noncurrent_version_transition.storage_class.as_ref()
&& !storage_class.as_str().is_empty()
&& !obj.delete_marker
@@ -740,11 +735,7 @@ impl Lifecycle for BucketLifecycleConfiguration {
}
if obj.transition_status != TRANSITION_COMPLETE
&& let Some(transition) = rule
.transitions
.as_ref()
.filter(|transitions| transitions.len() == 1)
.and_then(|transitions| transitions.first())
&& let Some(transition) = rule.transitions.as_ref().and_then(|transitions| transitions.first())
&& let Some(storage_class) = transition.storage_class.as_ref()
&& !storage_class.as_str().is_empty()
{
@@ -767,15 +758,18 @@ impl Lifecycle for BucketLifecycleConfiguration {
}
if !events.is_empty() {
// Eligible expiration takes precedence over transition, even when a
// failed transition has an earlier deadline. Within each action class,
// prefer the earliest deadline using a deterministic total order.
// Select the winning event using a strict total order (MinIO semantics):
// the earliest `due` wins, and ties break toward delete-type actions. A
// missing `due` is treated as UNIX_EPOCH. This replaces a hand-written
// `sort_by` comparator that was not a strict weak ordering (it could return
// `Ordering::Less` for both `(a, b)` and `(b, a)`), which panics on the
// repository toolchain and did not deterministically pick the earliest event.
let event = events
.iter()
.min_by_key(|event| {
(
ilm_action_priority_rank(&event.action),
event.due.unwrap_or(OffsetDateTime::UNIX_EPOCH).unix_timestamp(),
ilm_action_priority_rank(&event.action),
)
})
.cloned()
@@ -1048,27 +1042,6 @@ impl ObjectOpts {
pub fn expired_object_deletemarker(&self) -> bool {
self.delete_marker && self.is_latest && self.num_versions == 1
}
pub(crate) fn restored_copy_expiry(&self, now: OffsetDateTime) -> Option<Event> {
let restore_expires = self.restore_expires?;
// Restore metadata alone does not prove that a durable remote copy exists.
if self.transition_status != TRANSITION_COMPLETE
|| restore_expires.unix_timestamp() == 0
|| now.unix_timestamp() <= restore_expires.unix_timestamp()
{
return None;
}
let action = if self.is_latest {
IlmAction::DeleteRestoredAction
} else {
IlmAction::DeleteRestoredVersionAction
};
expiration_action_has_valid_target(action, self.version_id, self.is_latest, self.delete_marker).then(|| Event {
action,
due: Some(now),
..Default::default()
})
}
}
/// Returns whether an expiry action has enough identity to target the object
@@ -1091,8 +1064,11 @@ pub fn expiration_action_has_valid_target(
}
}
/// Eligible logical expiration takes precedence over transition and restore-copy
/// cleanup. Deadlines break ties within an action class.
/// Total-order rank for lifecycle actions used to break `due` ties.
///
/// Delete-type actions rank before every other action so that, when two events
/// share the same `due`, a delete wins (MinIO semantics). The concrete numeric
/// values only matter relative to each other.
fn ilm_action_priority_rank(action: &IlmAction) -> u8 {
match action {
IlmAction::DeleteAllVersionsAction
@@ -4183,392 +4159,6 @@ mod tests {
assert_eq!(event.action, IlmAction::NoneAction);
}
mod adversarial_regressions {
use super::*;
use s3s::dto::NoncurrentVersionExpiration;
fn run(test: impl std::future::Future<Output = ()>) {
with_default_ilm_process_time(|| {
tokio::runtime::Builder::new_current_thread()
.build()
.expect("lifecycle regression runtime should build")
.block_on(test);
});
}
fn noncurrent_object() -> ObjectOpts {
ObjectOpts {
name: "logs/object".to_string(),
mod_time: Some(datetime!(2020-01-01 00:00:00 UTC)),
successor_mod_time: Some(datetime!(2020-01-02 00:00:00 UTC)),
version_id: Some(Uuid::from_u128(1)),
size: 1024 * 1024,
..Default::default()
}
}
#[test]
#[serial]
fn noncurrent_transition_retains_the_requested_newer_versions() {
run(async {
let mut rule = enabled_rule(None, None, Some("retain-two-hot-versions"));
rule.filter = Some(LifecycleRuleFilter::default());
rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition {
noncurrent_days: Some(1),
newer_noncurrent_versions: Some(2),
storage_class: Some(TransitionStorageClass::from_static("WARM")),
}]);
let lc = Arc::new(BucketLifecycleConfiguration {
rules: vec![rule],
expiry_updated_at: None,
});
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("valid noncurrent transition policy");
let objects = (0..4)
.map(|index| ObjectOpts {
mod_time: Some(datetime!(2020-01-05 00:00:00 UTC) - Duration::days(index)),
successor_mod_time: (index > 0).then_some(datetime!(2020-01-06 00:00:00 UTC) - Duration::days(index)),
version_id: Some(Uuid::from_u128(u128::try_from(index + 1).expect("small version index"))),
is_latest: index == 0,
num_versions: 4,
..noncurrent_object()
})
.collect::<Vec<_>>();
let actions = crate::Evaluator::new(lc)
.eval(&objects)
.await
.expect("complete version chain should evaluate")
.into_iter()
.map(|event| event.action)
.collect::<Vec<_>>();
assert_eq!(
actions,
[
IlmAction::NoneAction,
IlmAction::NoneAction,
IlmAction::NoneAction,
IlmAction::TransitionVersionAction
],
"the two newest noncurrent versions must remain in their current storage class"
);
});
}
#[test]
#[serial]
fn noncurrent_transition_checks_count_age_and_single_object_context() {
run(async {
let mut rule = enabled_rule(None, None, Some("retain-two"));
rule.filter = Some(LifecycleRuleFilter::default());
rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition {
noncurrent_days: Some(3),
newer_noncurrent_versions: Some(2),
storage_class: Some(TransitionStorageClass::from_static("WARM")),
}]);
let mut lc = BucketLifecycleConfiguration {
rules: vec![rule],
expiry_updated_at: None,
};
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("valid counted transition");
let object = noncurrent_object();
let now = datetime!(2020-01-10 00:00:00 UTC);
for (newer, expected) in [
(0, IlmAction::NoneAction),
(1, IlmAction::NoneAction),
(2, IlmAction::TransitionVersionAction),
(3, IlmAction::TransitionVersionAction),
] {
assert_eq!(lc.eval_inner(&object, now, newer).await.action, expected, "newer count: {newer}");
}
assert_eq!(
lc.eval_inner(&object, datetime!(2020-01-04 00:00:00 UTC), 2).await.action,
IlmAction::NoneAction,
"the retention count does not replace the age condition"
);
assert_eq!(
lc.eval(&object).await.action,
IlmAction::NoneAction,
"a single-object lookup must not assume a complete version history"
);
for retain in [None, Some(0), Some(-1), Some(i32::MAX)] {
lc.rules[0]
.noncurrent_version_transitions
.as_mut()
.expect("transition exists")[0]
.newer_noncurrent_versions = retain;
let expected = if matches!(retain, None | Some(0)) {
IlmAction::TransitionVersionAction
} else {
IlmAction::NoneAction
};
assert_eq!(lc.eval_inner(&object, now, 2).await.action, expected, "retention: {retain:?}");
}
});
}
#[test]
#[serial]
fn noncurrent_expiration_and_transition_have_independent_retention_counts() {
run(async {
let mut rule = enabled_rule(None, None, Some("independent-counts"));
rule.filter = Some(LifecycleRuleFilter::default());
rule.noncurrent_version_expiration = Some(NoncurrentVersionExpiration {
noncurrent_days: Some(90),
newer_noncurrent_versions: Some(4),
});
rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition {
noncurrent_days: Some(30),
newer_noncurrent_versions: Some(2),
storage_class: Some(TransitionStorageClass::from_static("WARM")),
}]);
let lc = BucketLifecycleConfiguration {
rules: vec![rule],
expiry_updated_at: None,
};
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("valid independent retention limits");
let object = noncurrent_object();
let now = datetime!(2020-05-01 00:00:00 UTC);
for (newer, expected) in [
(1, IlmAction::NoneAction),
(2, IlmAction::TransitionVersionAction),
(3, IlmAction::TransitionVersionAction),
(4, IlmAction::DeleteVersionAction),
] {
assert_eq!(lc.eval_inner(&object, now, newer).await.action, expected, "newer count: {newer}");
}
});
}
#[test]
#[serial]
fn expiration_retention_does_not_skip_an_independent_transition() {
run(async {
let mut rule = enabled_rule(None, None, Some("transition-then-expire"));
rule.filter = Some(LifecycleRuleFilter::default());
rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition {
noncurrent_days: Some(1),
newer_noncurrent_versions: None,
storage_class: Some(TransitionStorageClass::from_static("WARM")),
}]);
let mut lc = BucketLifecycleConfiguration {
rules: vec![rule],
expiry_updated_at: None,
};
let object = noncurrent_object();
let now = datetime!(2020-01-10 00:00:00 UTC);
let transition_only = lc.eval_inner(&object, now, 0).await;
assert_eq!(transition_only.action, IlmAction::TransitionVersionAction);
lc.rules[0].noncurrent_version_expiration = Some(NoncurrentVersionExpiration {
noncurrent_days: Some(90),
newer_noncurrent_versions: Some(2),
});
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("valid combined policy");
let combined = lc.eval_inner(&object, now, 0).await;
assert_eq!(combined.action, transition_only.action, "retention limits expiration, not transition");
assert_eq!(combined.storage_class, transition_only.storage_class);
});
}
#[test]
#[serial]
fn current_transition_rejects_multiple_stages_in_any_order() {
run(async {
let mut rule = enabled_rule(None, None, Some("two-current-transitions"));
rule.transitions = Some(vec![
Transition {
date: Some(datetime!(2020-03-01 00:00:00 UTC).into()),
days: None,
storage_class: Some(TransitionStorageClass::from_static("COLD")),
},
Transition {
date: Some(datetime!(2020-01-03 00:00:00 UTC).into()),
days: None,
storage_class: Some(TransitionStorageClass::from_static("WARM")),
},
]);
let mut lc = BucketLifecycleConfiguration {
rules: vec![rule],
expiry_updated_at: None,
};
let object = ObjectOpts {
is_latest: true,
..noncurrent_object()
};
let now = datetime!(2020-01-10 00:00:00 UTC);
for status in [ExpirationStatus::ENABLED, ExpirationStatus::DISABLED] {
lc.rules[0].status = ExpirationStatus::from_static(status);
for _ in 0..2 {
let err = lc
.validate(&ObjectLockConfiguration::default())
.await
.expect_err("multiple transition stages must be rejected");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
assert_eq!(err.to_string(), ERR_LIFECYCLE_MULTIPLE_TRANSITIONS);
assert_eq!(
lc.eval_inner(&object, now, 0).await.action,
IlmAction::NoneAction,
"legacy multi-stage configurations must not silently execute their first stage"
);
lc.rules[0]
.transitions
.as_mut()
.expect("transition array is present")
.reverse();
}
}
lc.rules[0]
.transitions
.as_mut()
.expect("transition array is present")
.remove(0);
lc.rules[0].status = ExpirationStatus::from_static(ExpirationStatus::ENABLED);
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("one stage is supported");
let event = lc.eval_inner(&object, now, 0).await;
assert_eq!(event.action, IlmAction::TransitionAction);
assert_eq!(event.storage_class, "WARM");
});
}
#[test]
#[serial]
fn noncurrent_transition_rejects_multiple_stages_in_any_order() {
run(async {
let mut rule = enabled_rule(None, None, Some("two-noncurrent-transitions"));
rule.noncurrent_version_transitions = Some(vec![
NoncurrentVersionTransition {
noncurrent_days: Some(30),
newer_noncurrent_versions: None,
storage_class: Some(TransitionStorageClass::from_static("COLD")),
},
NoncurrentVersionTransition {
noncurrent_days: Some(1),
newer_noncurrent_versions: None,
storage_class: Some(TransitionStorageClass::from_static("WARM")),
},
]);
let mut lc = BucketLifecycleConfiguration {
rules: vec![rule],
expiry_updated_at: None,
};
let object = noncurrent_object();
let now = datetime!(2020-01-10 00:00:00 UTC);
for status in [ExpirationStatus::ENABLED, ExpirationStatus::DISABLED] {
lc.rules[0].status = ExpirationStatus::from_static(status);
for _ in 0..2 {
let err = lc
.validate(&ObjectLockConfiguration::default())
.await
.expect_err("multiple noncurrent transition stages must be rejected");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
assert_eq!(err.to_string(), ERR_LIFECYCLE_MULTIPLE_NONCURRENT_TRANSITIONS);
assert_eq!(
lc.eval_inner(&object, now, 0).await.action,
IlmAction::NoneAction,
"legacy multi-stage configurations must not silently execute their first stage"
);
lc.rules[0]
.noncurrent_version_transitions
.as_mut()
.expect("transition array is present")
.reverse();
}
}
lc.rules[0]
.noncurrent_version_transitions
.as_mut()
.expect("transition array is present")
.remove(0);
lc.rules[0].status = ExpirationStatus::from_static(ExpirationStatus::ENABLED);
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("one stage is supported");
let event = lc.eval_inner(&object, now, 0).await;
assert_eq!(event.action, IlmAction::TransitionVersionAction);
assert_eq!(event.storage_class, "WARM");
});
}
#[test]
#[serial]
fn expiration_rejects_simultaneous_days_and_date() {
run(async {
let mut lc = BucketLifecycleConfiguration {
rules: vec![enabled_rule(
Some(LifecycleExpiration {
days: Some(1),
..Default::default()
}),
None,
Some("ambiguous-expiry"),
)],
expiry_updated_at: None,
};
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("a single Days expiration is valid");
lc.rules[0].expiration.as_mut().expect("expiration is present").date =
Some(datetime!(2099-01-01 00:00:00 UTC).into());
let err = lc
.validate(&ObjectLockConfiguration::default())
.await
.expect_err("Days and Date are mutually exclusive; accepting both silently overrides Days");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
assert_eq!(err.to_string(), ERR_LIFECYCLE_EXPIRATION_DAYS_DATE_CONFLICT);
});
}
#[test]
#[serial]
fn overdue_transition_does_not_starve_permanent_expiration() {
run(async {
let mut rule = enabled_rule(
Some(LifecycleExpiration {
days: Some(90),
..Default::default()
}),
None,
Some("archive-then-delete"),
);
rule.transitions = Some(vec![Transition {
days: Some(30),
date: None,
storage_class: Some(TransitionStorageClass::from_static("WARM")),
}]);
let lc = BucketLifecycleConfiguration {
rules: vec![rule],
expiry_updated_at: None,
};
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("valid transition and expiration policy");
let object = ObjectOpts {
is_latest: true,
version_id: None,
transition_status: TRANSITION_PENDING.to_string(),
..noncurrent_object()
};
let before_expiration = lc.eval_inner(&object, datetime!(2020-02-15 00:00:00 UTC), 0).await;
assert_eq!(before_expiration.action, IlmAction::TransitionAction);
let overdue = lc.eval_inner(&object, datetime!(2020-05-01 00:00:00 UTC), 0).await;
assert_eq!(
overdue.action,
IlmAction::DeleteAction,
"an unavailable tier must not prevent permanent expiration indefinitely"
);
});
}
}
/// Property-based tests for the rule evaluator (backlog#1148 ilm-14,
/// follow-up to backlog#1030 / rustfs#4455).
///
@@ -4579,7 +4169,7 @@ mod tests {
///
/// * `eval_inner` never panics and is deterministic for a fixed input;
/// * the winning event matches an independently recomputed candidate set:
/// eligible expiration wins over transition, then earliest `due` wins (the
/// earliest `due` wins, ties break toward delete-class actions (the
/// `min_by_key` selection that replaced the rustfs#4455 comparator);
/// * `expected_expiry_time` is monotonically non-decreasing in `days` and
/// always lands on the processing boundary, both at production defaults
@@ -4868,8 +4458,8 @@ mod tests {
/// consider for a live current version under `selection`-shaped rules
/// (expiration and first-transition only, no filters): expiration
/// fires when `now >= due`, transition when `now > due` and the object
/// has not already transitioned. Eligible expiration wins over transition;
/// the earliest deadline wins within the selected action class.
/// has not already transitioned. Selection semantics under test:
/// earliest due wins, ties prefer delete-class.
fn oracle_candidates(lc: &BucketLifecycleConfiguration, obj: &ObjectOpts, now: OffsetDateTime) -> Vec<Candidate> {
let mod_time = obj.mod_time.expect("selection strategy always sets mod_time");
let mut candidates = Vec::new();
@@ -4958,8 +4548,8 @@ mod tests {
/// Differential test of winner selection (the rustfs#4455 fix):
/// for a live current version under randomized expiration and
/// transition rules, `eval_inner`'s winner must carry the
/// earliest expiration from the independently recomputed candidate
/// set, or the earliest transition when no expiration is eligible,
/// minimum `(due, rank)` of the independently recomputed
/// candidate set — earliest due wins, ties prefer delete-class —
/// and must be `NoneAction` exactly when that set is empty.
#[test]
#[serial]
@@ -4988,13 +4578,7 @@ mod tests {
// Oracle and evaluator must observe the same (pinned) time env.
let (event, expected) = with_production_time_env(|| {
let candidates = oracle_candidates(&lc, &obj, now);
let expected = candidates
.iter()
.filter(|(_, rank)| *rank == 0)
.min()
.copied()
.or_else(|| candidates.into_iter().min());
let expected = oracle_candidates(&lc, &obj, now).into_iter().min();
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
+7 -93
View File
@@ -116,10 +116,13 @@ impl Evaluator {
break 'top_loop;
}
}
// Restore expiry removes only the temporary local copy; the
// retained logical version and its remote data remain intact.
IlmAction::DeleteAction | IlmAction::DeleteVersionAction if self.is_object_locked(obj) => {
event = obj.restored_copy_expiry(now).unwrap_or_default();
IlmAction::DeleteAction
| IlmAction::DeleteRestoredAction
| IlmAction::DeleteVersionAction
| IlmAction::DeleteRestoredVersionAction
if self.is_object_locked(obj) =>
{
event = Event::default();
}
_ => {}
}
@@ -203,95 +206,6 @@ mod tests {
use super::*;
use rustfs_replication::{ReplicationStatusType, VersionPurgeStatusType};
#[tokio::test]
async fn adversarial_restore_expiry_survives_legal_hold() {
let mut policy = (*latest_expiration_lifecycle()).clone();
policy.rules[0].status = ExpirationStatus::from_static(ExpirationStatus::DISABLED);
let policy = Arc::new(policy);
policy
.validate(&lock_enabled_without_default_retention())
.await
.expect("valid disabled lifecycle rule");
let mut objects = [true, false].map(|is_latest| ObjectOpts {
is_latest,
num_versions: 2,
mod_time: Some(
OffsetDateTime::from_unix_timestamp(if is_latest { 1_200_000 } else { 1_000_000 })
.expect("fixed version timestamp"),
),
successor_mod_time: (!is_latest)
.then(|| OffsetDateTime::from_unix_timestamp(1_200_000).expect("fixed successor timestamp")),
transition_status: crate::TRANSITION_COMPLETE.to_string(),
restore_expires: Some(OffsetDateTime::from_unix_timestamp(2_000_000).expect("fixed expired restore timestamp")),
..current_object_opts(ReplicationStatusType::Completed)
});
let evaluator = Evaluator::new(policy).with_lock_retention(Some(lock_enabled_without_default_retention()));
let expected = [IlmAction::DeleteRestoredAction, IlmAction::DeleteRestoredVersionAction];
let unlocked = evaluator
.eval(&objects)
.await
.expect("unlocked restored versions should evaluate");
assert_eq!(unlocked.iter().map(|event| event.action).collect::<Vec<_>>(), expected);
for object in &mut objects {
object
.user_defined
.insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_string(), "ON".to_string());
}
let locked = evaluator
.eval(&objects)
.await
.expect("locked restored versions should evaluate");
assert_eq!(
locked.iter().map(|event| event.action).collect::<Vec<_>>(),
expected,
"expiring a restored local copy preserves the retained logical version and remote object"
);
let mut expiring_policy = (*latest_expiration_lifecycle()).clone();
expiring_policy.rules[0].noncurrent_version_expiration = Some(NoncurrentVersionExpiration {
noncurrent_days: Some(1),
newer_noncurrent_versions: None,
});
let expiring_evaluator =
Evaluator::new(Arc::new(expiring_policy)).with_lock_retention(Some(lock_enabled_without_default_retention()));
let locked = expiring_evaluator
.eval(&objects)
.await
.expect("locked expired versions should evaluate");
assert_eq!(
locked.iter().map(|event| event.action).collect::<Vec<_>>(),
expected,
"blocked logical expiration must still allow an eligible restore-copy cleanup"
);
for status in [ReplicationStatusType::Pending, ReplicationStatusType::Failed] {
for object in &mut objects {
object.replication_status = status.clone();
}
for evaluator in [&evaluator, &expiring_evaluator] {
let events = evaluator.eval(&objects).await.expect("pending replication should evaluate");
assert!(events.iter().all(|event| event.action == IlmAction::NoneAction));
}
}
for object in &mut objects {
object.replication_status = ReplicationStatusType::Completed;
}
for transition_status in ["", crate::TRANSITION_PENDING, "unknown"] {
for object in &mut objects {
object.transition_status = transition_status.to_string();
}
for evaluator in [&evaluator, &expiring_evaluator] {
let events = evaluator.eval(&objects).await.expect("incomplete transition should evaluate");
assert!(
events.iter().all(|event| event.action == IlmAction::NoneAction),
"restore metadata cannot authorize cleanup without a completed transition"
);
}
}
}
fn expired_marker_lifecycle() -> Arc<BucketLifecycleConfiguration> {
Arc::new(BucketLifecycleConfiguration {
expiry_updated_at: None,
@@ -1 +1 @@
{"bucket":"photos","config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":null,"gcs":null},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z"}
{"bucket":"photos","config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null}},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z"}
@@ -1 +1 @@
{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"sourceSecretKey123","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":null,"gcs":null},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}}
{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"sourceSecretKey123","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null}},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}}
@@ -1 +1 @@
{"bucket":"photos","dry_run":false,"config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":null,"gcs":null},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z","probe":{"reachable":true,"listable":true,"sample_key":"photos/2024/01.jpg"}}
{"bucket":"photos","dry_run":false,"config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null}},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z","probe":{"reachable":true,"listable":true,"sample_key":"photos/2024/01.jpg"}}
-88
View File
@@ -78,18 +78,10 @@ pub struct OnDemandMigrationSource {
#[serde(default)]
pub path_style: OnDemandMigrationPathStyle,
/// `None` means anonymous access to a public source bucket.
/// `None` means anonymous access to a public source bucket. The native
/// providers carry their credentials in `azure` / `gcs` instead.
#[serde(default)]
pub credentials: Option<OnDemandMigrationCredentials>,
#[serde(default)]
pub tls: OnDemandMigrationTls,
/// Required for `azure` and rejected for every other provider.
#[serde(default)]
pub azure: Option<OnDemandMigrationAzure>,
/// Required for `gcs_native` and rejected for every other provider.
#[serde(default)]
pub gcs: Option<OnDemandMigrationGcs>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
@@ -100,49 +92,7 @@ pub enum OnDemandMigrationProvider {
Minio,
Rustfs,
R2,
/// GCS XML interoperability API with HMAC keys.
Gcs,
/// Native Azure Blob service.
Azure,
/// Native GCS JSON API with a service-account key.
#[serde(rename = "gcs_native")]
GcsNative,
}
/// Native Azure Blob parameters. The container is `source.bucket`; exactly one
/// of `account_key` and `sas_token` is set. Responses carry both as `REDACTED`.
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OnDemandMigrationAzure {
pub account: String,
#[serde(default)]
pub account_key: Option<String>,
#[serde(default)]
pub sas_token: Option<String>,
}
impl fmt::Debug for OnDemandMigrationAzure {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("OnDemandMigrationAzure")
.field("account", &self.account)
.field("account_key", &self.account_key.as_ref().map(|_| "REDACTED"))
.field("sas_token", &self.sas_token.as_ref().map(|_| "REDACTED"))
.finish()
}
}
/// Native GCS parameters. The bucket is `source.bucket`; the key JSON embeds a
/// private key, so responses carry it as `REDACTED`.
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OnDemandMigrationGcs {
pub service_account_json: String,
}
impl fmt::Debug for OnDemandMigrationGcs {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("OnDemandMigrationGcs")
.field("service_account_json", &"REDACTED")
.finish()
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
@@ -856,8 +806,6 @@ mod tests {
session_token: None,
}),
tls: OnDemandMigrationTls::default(),
azure: None,
gcs: None,
});
let mut expected: OnDemandMigrationConfig = serde_json::from_str(SET_REQUEST_FIXTURE.trim()).expect("fixture");
expected.filter.source_prefix = None;
@@ -873,42 +821,6 @@ mod tests {
assert!(minimal.source.credentials.is_none());
}
#[test]
fn native_provider_documents_round_trip_and_hide_their_secrets() {
for (label, json) in [
(
"azure",
r#"{"provider":"azure","endpoint":null,"region":"auto","bucket":"legacy-photos","path_style":"auto","credentials":null,"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":{"account":"legacyaccount","account_key":null,"sas_token":"sv=2021-08-06&sig=topsecret"},"gcs":null}"#,
),
(
"gcs_native",
r#"{"provider":"gcs_native","endpoint":null,"region":"auto","bucket":"legacy-photos","path_style":"auto","credentials":null,"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":null,"gcs":{"service_account_json":"{\"type\":\"service_account\"}"}}"#,
),
] {
let source: OnDemandMigrationSource = serde_json::from_str(json).unwrap_or_else(|err| panic!("{label}: {err}"));
assert_eq!(
serde_json::to_string(&source).expect("re-encodes"),
json,
"{label} must reproduce the server wire shape byte for byte"
);
}
let azure = OnDemandMigrationAzure {
account: "legacyaccount".to_string(),
account_key: Some("c2VjcmV0".to_string()),
sas_token: Some("sig=topsecret".to_string()),
};
let rendered = format!("{azure:?}");
assert!(rendered.contains("legacyaccount"));
assert!(!rendered.contains("c2VjcmV0"), "{rendered}");
assert!(!rendered.contains("topsecret"), "{rendered}");
let gcs = OnDemandMigrationGcs {
service_account_json: r#"{"private_key":"-----BEGIN PRIVATE KEY-----"}"#.to_string(),
};
assert!(!format!("{gcs:?}").contains("PRIVATE KEY"), "{gcs:?}");
}
#[test]
fn credentials_debug_never_prints_secrets() {
let credentials = OnDemandMigrationCredentials {
+48 -204
View File
@@ -120,10 +120,18 @@ impl TransitionClient {
let h = resp.headers().clone();
let mut body = resp.into_body();
let body_vec = if let Some(limit) = max_response_bytes {
self.collect_response_body(resp.into_body(), limit).await?
collect_response_body(body, limit).await?
} else {
self.collect_response_body_unbounded(resp.into_body()).await?
let mut body_vec = Vec::new();
while let Some(frame) = body.frame().await {
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
if let Some(data) = frame.data_ref() {
body_vec.extend_from_slice(data);
}
}
body_vec
};
Ok((object_stat, h, BufReader::new(Cursor::new(body_vec))))
}
@@ -135,7 +143,7 @@ mod bounded_response_tests {
use crate::{
api_get_options::GetObjectOptions,
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{BucketLookupType, Options, TransitionClient, TransitionClientTimeouts, collect_response_body},
transition_api::{BucketLookupType, Options, TransitionClient, collect_response_body},
};
use http_body_util::Full;
use hyper::body::Bytes;
@@ -167,31 +175,7 @@ mod bounded_response_tests {
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
}
fn test_options() -> Options {
Options {
creds: Credentials::new(Static(Value {
access_key_id: "access-key".to_string(),
secret_access_key: "secret-key".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
})),
region: "us-east-1".to_string(),
bucket_lookup: BucketLookupType::BucketLookupPath,
max_retries: 1,
..Default::default()
}
}
async fn client_for_endpoint(endpoint: &str, timeouts: TransitionClientTimeouts) -> TransitionClient {
TransitionClient::new_with_timeouts(endpoint, test_options(), "", timeouts)
.await
.expect("fixture client should build")
}
async fn bounded_get_fixture_with_timeouts(
body: &'static [u8],
timeouts: TransitionClientTimeouts,
) -> Option<(TransitionClient, tokio::task::JoinHandle<String>)> {
async fn bounded_get_fixture(body: &'static [u8]) -> Option<(TransitionClient, tokio::task::JoinHandle<String>)> {
let listener = match TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None,
@@ -225,14 +209,27 @@ mod bounded_response_tests {
stream.write_all(body).await.expect("fixture should write response body");
request
});
let client = client_for_endpoint(&endpoint, timeouts).await;
let client = TransitionClient::new(
&endpoint,
Options {
creds: Credentials::new(Static(Value {
access_key_id: "access-key".to_string(),
secret_access_key: "secret-key".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
})),
region: "us-east-1".to_string(),
bucket_lookup: BucketLookupType::BucketLookupPath,
max_retries: 1,
..Default::default()
},
"",
)
.await
.expect("fixture client should build");
Some((client, request))
}
async fn bounded_get_fixture(body: &'static [u8]) -> Option<(TransitionClient, tokio::task::JoinHandle<String>)> {
bounded_get_fixture_with_timeouts(body, TransitionClientTimeouts::default()).await
}
#[tokio::test]
async fn real_transport_accepts_the_exact_closed_range_length() {
let Some((client, request)) = bounded_get_fixture(b"RustFS!").await else {
@@ -295,7 +292,24 @@ mod bounded_response_tests {
.local_addr()
.expect("listener local address should be available")
.to_string();
let client = client_for_endpoint(&endpoint, TransitionClientTimeouts::default()).await;
let client = TransitionClient::new(
&endpoint,
Options {
creds: Credentials::new(Static(Value {
access_key_id: "access-key".to_string(),
secret_access_key: "secret-key".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
})),
region: "us-east-1".to_string(),
bucket_lookup: BucketLookupType::BucketLookupPath,
max_retries: 1,
..Default::default()
},
"",
)
.await
.expect("fixture client should build");
let mut opts = GetObjectOptions::default();
opts.headers
.insert("range".to_string(), "bytes=0-18446744073709551615".to_string());
@@ -312,176 +326,6 @@ mod bounded_response_tests {
.is_err()
);
}
#[tokio::test]
async fn connection_refused_returns_without_waiting_for_the_request_timeout() {
let listener = match TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
Err(err) => panic!("test listener should bind: {err}"),
};
let endpoint = listener
.local_addr()
.expect("listener local address should be available")
.to_string();
drop(listener);
let client = client_for_endpoint(
&endpoint,
TransitionClientTimeouts::new(Duration::from_secs(1), Duration::from_secs(5), Duration::from_secs(1)),
)
.await;
let mut opts = GetObjectOptions::default();
opts.set_range(0, 6).expect("the probe range should be valid");
let result = tokio::time::timeout(Duration::from_secs(2), client.get_object_inner("bucket", "probe", &opts))
.await
.expect("connection refused should return before the broader request timeout");
assert!(result.is_err(), "connection refused must fail instead of hanging");
}
#[tokio::test]
async fn response_header_stall_returns_timed_out() {
let listener = match TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
Err(err) => panic!("test listener should bind: {err}"),
};
let endpoint = listener
.local_addr()
.expect("listener local address should be available")
.to_string();
let fixture = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await.expect("fixture should accept one GET");
let mut request = Vec::new();
let mut buffer = [0; 1024];
loop {
let read = stream.read(&mut buffer).await.expect("fixture should read request headers");
assert_ne!(read, 0, "connection closed before request headers were received");
request.extend_from_slice(&buffer[..read]);
if request.windows(4).any(|window| window == b"\r\n\r\n") {
break;
}
}
tokio::time::sleep(Duration::from_millis(200)).await;
});
let client = client_for_endpoint(
&endpoint,
TransitionClientTimeouts::new(Duration::from_secs(1), Duration::from_millis(50), Duration::from_secs(1)),
)
.await;
let mut opts = GetObjectOptions::default();
opts.set_range(0, 6).expect("the probe range should be valid");
let err = client
.get_object_inner("bucket", "probe", &opts)
.await
.expect_err("response header stalls must be bounded");
assert_eq!(err.kind(), std::io::ErrorKind::TimedOut);
fixture.await.expect("fixture should join");
}
#[tokio::test]
async fn response_body_idle_stall_returns_timed_out() {
let listener = match TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
Err(err) => panic!("test listener should bind: {err}"),
};
let endpoint = listener
.local_addr()
.expect("listener local address should be available")
.to_string();
let fixture = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await.expect("fixture should accept one GET");
let mut request = Vec::new();
let mut buffer = [0; 1024];
loop {
let read = stream.read(&mut buffer).await.expect("fixture should read request headers");
assert_ne!(read, 0, "connection closed before request headers were received");
request.extend_from_slice(&buffer[..read]);
if request.windows(4).any(|window| window == b"\r\n\r\n") {
break;
}
}
stream
.write_all(b"HTTP/1.1 206 Partial Content\r\nContent-Length: 7\r\nConnection: close\r\n\r\nRu")
.await
.expect("fixture should write the first body chunk");
tokio::time::sleep(Duration::from_millis(200)).await;
});
let client = client_for_endpoint(
&endpoint,
TransitionClientTimeouts::new(Duration::from_secs(1), Duration::from_secs(1), Duration::from_millis(50)),
)
.await;
let mut opts = GetObjectOptions::default();
opts.set_range(0, 6).expect("the probe range should be valid");
let err = client
.get_object_inner("bucket", "probe", &opts)
.await
.expect_err("body stalls after partial progress must be bounded");
assert_eq!(err.kind(), std::io::ErrorKind::TimedOut);
fixture.await.expect("fixture should join");
}
#[tokio::test]
async fn response_body_idle_timer_resets_on_progress() {
let listener = match TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
Err(err) => panic!("test listener should bind: {err}"),
};
let endpoint = listener
.local_addr()
.expect("listener local address should be available")
.to_string();
let fixture = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await.expect("fixture should accept one GET");
let mut request = Vec::new();
let mut buffer = [0; 1024];
loop {
let read = stream.read(&mut buffer).await.expect("fixture should read request headers");
assert_ne!(read, 0, "connection closed before request headers were received");
request.extend_from_slice(&buffer[..read]);
if request.windows(4).any(|window| window == b"\r\n\r\n") {
break;
}
}
stream
.write_all(b"HTTP/1.1 206 Partial Content\r\nContent-Length: 7\r\nConnection: close\r\n\r\n")
.await
.expect("fixture should write response headers");
for byte in b"RustFS!" {
stream.write_all(&[*byte]).await.expect("fixture should write body progress");
tokio::time::sleep(Duration::from_millis(20)).await;
}
});
let client = client_for_endpoint(
&endpoint,
TransitionClientTimeouts::new(Duration::from_millis(10), Duration::from_secs(1), Duration::from_millis(100)),
)
.await;
let mut opts = GetObjectOptions::default();
opts.set_range(0, 6).expect("the probe range should be valid");
let (_, _, mut reader) = client
.get_object_inner("bucket", "probe", &opts)
.await
.expect("continuous body progress must not be killed by the idle timer");
let mut body = Vec::new();
reader
.read_to_end(&mut body)
.await
.expect("bounded response should be readable");
assert_eq!(body, b"RustFS!");
fixture.await.expect("fixture should join");
}
}
#[derive(Default)]
+10 -82
View File
@@ -27,6 +27,7 @@ use crate::{
transition_api::{ReaderImpl, RequestMetadata, TransitionClient, collect_response_body},
};
use http::{HeaderMap, StatusCode};
use http_body_util::BodyExt;
use hyper::body::Body;
use hyper::body::Bytes;
use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE;
@@ -123,9 +124,14 @@ impl TransitionClient {
}
//let mut list_bucket_result = ListBucketV2Result::default();
let body_vec = self
.collect_response_body(resp.into_body(), MAX_S3_CLIENT_RESPONSE_SIZE)
.await?;
let mut body_vec = Vec::new();
let mut body = resp.into_body();
while let Some(frame) = body.frame().await {
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
if let Some(data) = frame.data_ref() {
body_vec.extend_from_slice(data);
}
}
let mut list_bucket_result = match quick_xml::de::from_str::<ListBucketV2Result>(&String::from_utf8_lossy(&body_vec)) {
Ok(result) => result,
Err(err) => {
@@ -208,9 +214,7 @@ impl TransitionClient {
let resp_status = resp.status();
let headers = resp.headers().clone();
let body = self
.collect_response_body(resp.into_body(), MAX_S3_CLIENT_RESPONSE_SIZE)
.await?;
let body = collect_response_body(resp.into_body(), MAX_S3_CLIENT_RESPONSE_SIZE).await?;
if resp_status != StatusCode::OK {
return Err(std::io::Error::other(http_resp_to_error_response(
resp_status,
@@ -424,30 +428,6 @@ fn decode_s3_name(name: &str, encoding_type: &str) -> Result<String, std::io::Er
#[cfg(test)]
mod tests {
use super::*;
use crate::{
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{BucketLookupType, Options, TransitionClientTimeouts},
};
use std::time::Duration;
use tokio::{
io::{AsyncReadExt, AsyncWriteExt},
net::TcpListener,
};
fn timeout_test_options() -> Options {
Options {
creds: Credentials::new(Static(Value {
access_key_id: "access-key".to_string(),
secret_access_key: "secret-key".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
})),
region: "us-east-1".to_string(),
bucket_lookup: BucketLookupType::BucketLookupPath,
max_retries: 1,
..Default::default()
}
}
#[test]
fn list_versions_xml_preserves_versions_and_delete_markers() {
@@ -545,56 +525,4 @@ mod tests {
assert_eq!(parsed.common_prefixes.len(), 1);
assert_eq!(parsed.common_prefixes[0].prefix, "subdir/");
}
#[tokio::test]
async fn list_objects_v2_body_stall_returns_timed_out() {
let listener = match TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
Err(err) => panic!("test listener should bind: {err}"),
};
let endpoint = listener
.local_addr()
.expect("listener local address should be available")
.to_string();
let fixture = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await.expect("fixture should accept one list request");
let mut request = Vec::new();
let mut buffer = [0; 1024];
loop {
let read = stream.read(&mut buffer).await.expect("fixture should read request headers");
assert_ne!(read, 0, "connection closed before request headers were received");
request.extend_from_slice(&buffer[..read]);
if request.windows(4).any(|window| window == b"\r\n\r\n") {
break;
}
}
stream
.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 512\r\nConnection: close\r\n\r\n<ListBucketResult><Name>warm")
.await
.expect("fixture should write a partial list response");
tokio::time::sleep(Duration::from_millis(200)).await;
});
let client = TransitionClient::new_with_timeouts(
&endpoint,
timeout_test_options(),
"",
TransitionClientTimeouts::new(Duration::from_secs(1), Duration::from_secs(1), Duration::from_millis(50)),
)
.await
.expect("fixture client should build");
client
.bucket_loc_cache
.lock()
.expect("location cache should lock")
.set("bucket", "us-east-1");
let err = client
.list_objects_v2_query("bucket", "", "", false, false, "", "", 1, HeaderMap::new())
.await
.expect_err("a stalled ListObjectsV2 body must be bounded");
assert_eq!(err.kind(), std::io::ErrorKind::TimedOut);
fixture.await.expect("fixture should join");
}
}
@@ -18,6 +18,7 @@
#![allow(clippy::all)]
use http::{HeaderMap, HeaderName, StatusCode};
use http_body_util::BodyExt;
use hyper::body::Bytes;
use s3s::S3ErrorCode;
use std::collections::HashMap;
@@ -246,9 +247,14 @@ impl TransitionClient {
// Parse the CreateMultipartUpload response for the UploadId. Returning a
// default (empty) result here made every multipart transition fail at the
// first UploadPart with "UploadID cannot be empty" (rustfs/rustfs#4811).
let body_vec = self
.collect_response_body(resp.into_body(), rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE)
.await?;
let mut body_vec = Vec::new();
let mut body = resp.into_body();
while let Some(frame) = body.frame().await {
let frame = frame.map_err(|e| std::io::Error::other(e.to_string()))?;
if let Some(data) = frame.data_ref() {
body_vec.extend_from_slice(data);
}
}
let initiate_multipart_upload_result =
quick_xml::de::from_str::<InitiateMultipartUploadResult>(&String::from_utf8_lossy(&body_vec))
.map_err(|e| std::io::Error::other(format!("failed to parse CreateMultipartUpload response: {e}")))?;
+9 -3
View File
@@ -19,6 +19,7 @@
#![allow(clippy::all)]
use http::{HeaderMap, HeaderValue, Method, StatusCode};
use http_body_util::BodyExt;
use hyper::body::Body;
use hyper::body::Bytes;
use rustfs_utils::HashAlgorithm;
@@ -350,9 +351,14 @@ impl TransitionClient {
)
.await?;
let body_vec = self
.collect_response_body(resp.into_body(), rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE)
.await?;
let mut body_vec = Vec::new();
let mut body = resp.into_body();
while let Some(frame) = body.frame().await {
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
if let Some(data) = frame.data_ref() {
body_vec.extend_from_slice(data);
}
}
process_remove_multi_objects_response(
ReaderImpl::Body(Bytes::from(body_vec)),
bucket_name,
+11 -72
View File
@@ -19,6 +19,7 @@
#![allow(clippy::all)]
use http::{HeaderMap, HeaderValue, StatusCode};
use http_body_util::BodyExt;
use hyper::body::Body;
use hyper::body::Bytes;
use rustfs_utils::EMPTY_STRING_SHA256_HASH;
@@ -118,9 +119,14 @@ impl TransitionClient {
let resp_status = resp.status();
let h = resp.headers().clone();
let body_vec = self
.collect_response_body(resp.into_body(), rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE)
.await?;
let mut body_vec = Vec::new();
let mut body = resp.into_body();
while let Some(frame) = body.frame().await {
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
if let Some(data) = frame.data_ref() {
body_vec.extend_from_slice(data);
}
}
let resperr = http_resp_to_error_response(resp_status, &h, body_vec, bucket_name, "");
warn!("bucket exists, resperr: {:?}", resperr);
@@ -164,13 +170,11 @@ impl TransitionClient {
let resp_status = resp.status();
let h = resp.headers().clone();
let body_vec = self
.collect_response_body(resp.into_body(), rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE)
.await?;
let body_vec = collect_response_body(resp.into_body(), rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE).await?;
parse_bucket_versioning_response(resp_status, &h, body_vec, bucket_name)
}
Err(err) => Err(err),
Err(err) => Err(std::io::Error::other(err)),
}
}
@@ -270,14 +274,8 @@ impl TransitionClient {
#[cfg(test)]
mod tests {
use super::parse_bucket_versioning_response;
use crate::{
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{BucketLookupType, Options, TransitionClient, TransitionClientTimeouts},
};
use http::{HeaderMap, StatusCode};
use s3s::dto::BucketVersioningStatus;
use std::time::Duration;
use tokio::{io::AsyncReadExt, net::TcpListener};
#[test]
fn parses_bucket_versioning_statuses_mfa_delete_and_unversioned_state() {
@@ -340,63 +338,4 @@ mod tests {
assert_eq!(strict_err.kind(), std::io::ErrorKind::InvalidData);
}
}
#[tokio::test]
async fn get_bucket_versioning_preserves_request_timeout_kind() {
let listener = match TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
Err(err) => panic!("test listener should bind: {err}"),
};
let endpoint = listener
.local_addr()
.expect("listener local address should be available")
.to_string();
let fixture = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await.expect("fixture should accept one versioning request");
let mut request = Vec::new();
let mut buffer = [0; 1024];
loop {
let read = stream.read(&mut buffer).await.expect("fixture should read request headers");
assert_ne!(read, 0, "connection closed before request headers were received");
request.extend_from_slice(&buffer[..read]);
if request.windows(4).any(|window| window == b"\r\n\r\n") {
break;
}
}
tokio::time::sleep(Duration::from_millis(200)).await;
});
let client = TransitionClient::new_with_timeouts(
&endpoint,
Options {
creds: Credentials::new(Static(Value {
access_key_id: "access-key".to_string(),
secret_access_key: "secret-key".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
})),
region: "us-east-1".to_string(),
bucket_lookup: BucketLookupType::BucketLookupPath,
max_retries: 1,
..Default::default()
},
"",
TransitionClientTimeouts::new(Duration::from_secs(1), Duration::from_millis(50), Duration::from_secs(1)),
)
.await
.expect("fixture client should build");
client
.bucket_loc_cache
.lock()
.expect("location cache should lock")
.set("bucket", "us-east-1");
let err = client
.get_bucket_versioning("bucket")
.await
.expect_err("a stalled versioning request must time out");
assert_eq!(err.kind(), std::io::ErrorKind::TimedOut);
fixture.await.expect("fixture should join");
}
}
+10 -5
View File
@@ -26,6 +26,7 @@ use crate::{
transition_api::{CreateBucketConfiguration, LocationConstraint, TransitionClient},
};
use http::Request;
use http_body_util::BodyExt;
use hyper::StatusCode;
use hyper::body::Body;
use hyper::body::Bytes;
@@ -85,7 +86,7 @@ impl TransitionClient {
let req = self.get_bucket_location_request(bucket_name)?;
let mut resp = self.doit(req).await?;
location = process_bucket_location_response(self, resp, bucket_name, &self.tier_type).await?;
location = process_bucket_location_response(resp, bucket_name, &self.tier_type).await?;
{
if let Ok(mut bucket_loc_cache) = self.bucket_loc_cache.lock() {
bucket_loc_cache.set(bucket_name, &location);
@@ -197,7 +198,6 @@ impl TransitionClient {
}
async fn process_bucket_location_response(
client: &TransitionClient,
mut resp: http::Response<Incoming>,
bucket_name: &str,
tier_type: &str,
@@ -237,9 +237,14 @@ async fn process_bucket_location_response(
}
//}
let body_vec = client
.collect_response_body(resp.into_body(), MAX_S3_CLIENT_RESPONSE_SIZE)
.await?;
let mut body_vec = Vec::new();
let mut body = resp.into_body();
while let Some(frame) = body.frame().await {
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
if let Some(data) = frame.data_ref() {
body_vec.extend_from_slice(data);
}
}
let mut location = "".to_string();
if tier_type == "huaweicloud" {
if let Ok(body_str) = String::from_utf8(body_vec) {
+41 -328
View File
@@ -41,7 +41,7 @@ use http::{
request::{Builder, Request},
};
use http_body::Body;
use http_body_util::BodyExt;
use http_body_util::{BodyExt, LengthLimitError, Limited};
use hyper::body::Bytes;
use hyper::body::Incoming;
use hyper_rustls::{ConfigBuilderExt, HttpsConnector};
@@ -67,12 +67,10 @@ use s3s::dto::Owner;
use s3s::dto::ReplicationStatus;
use serde::{Deserialize, Serialize};
use sha2::Sha256;
use std::error::Error as StdError;
use std::io::Cursor;
use std::pin::Pin;
use std::sync::atomic::{AtomicI32, Ordering};
use std::task::{Context, Poll};
use std::time::Duration as StdDuration;
use std::{
collections::HashMap,
sync::{Arc, Mutex},
@@ -81,108 +79,28 @@ use time::Duration;
use time::OffsetDateTime;
use tokio::io::BufReader;
use tokio::io::{AsyncRead, AsyncReadExt};
use tracing::{debug, error, trace, warn};
use tracing::{debug, error, warn};
use url::{Url, form_urlencoded};
use uuid::Uuid;
const C_USER_AGENT: &str = "RustFS (linux; x86)";
pub const MAX_S3_ERROR_RESPONSE_SIZE: usize = 64 * 1024;
const EVENT_TIER_REMOTE_TRANSPORT: &str = "tier_remote_transport";
const LOG_COMPONENT_S3_CLIENT: &str = "s3_client";
const LOG_SUBSYSTEM_TIER: &str = "tier";
const SUCCESS_STATUS: [StatusCode; 3] = [StatusCode::OK, StatusCode::NO_CONTENT, StatusCode::PARTIAL_CONTENT];
fn response_body_exceeds_limit_error() -> std::io::Error {
std::io::Error::new(std::io::ErrorKind::InvalidData, "remote tier response body exceeds limit")
}
fn remote_tier_timeout_error(message: &'static str) -> std::io::Error {
std::io::Error::new(std::io::ErrorKind::TimedOut, message)
}
fn source_chain_has_io_kind(error: &(dyn StdError + 'static), kind: std::io::ErrorKind) -> bool {
let mut current = Some(error);
while let Some(error) = current {
if error
.downcast_ref::<std::io::Error>()
.is_some_and(|io_error| io_error.kind() == kind)
{
return true;
}
current = error.source();
}
false
}
fn transition_transport_error(err: hyper_util::client::legacy::Error) -> std::io::Error {
if source_chain_has_io_kind(&err, std::io::ErrorKind::TimedOut) {
return remote_tier_timeout_error("remote tier connection timed out");
}
std::io::Error::other(err)
}
async fn next_response_body_data<B>(
mut body: Pin<&mut B>,
idle_timeout: Option<StdDuration>,
) -> Result<Option<Bytes>, std::io::Error>
where
B: Body<Data = Bytes>,
B::Error: Into<Box<dyn StdError + Send + Sync>>,
{
let next_nonempty_data = async {
loop {
let Some(frame) = std::future::poll_fn(|cx| body.as_mut().poll_frame(cx)).await else {
return Ok(None);
};
let frame = frame.map_err(std::io::Error::other)?;
let Ok(data) = frame.into_data() else {
continue;
};
if !data.is_empty() {
return Ok(Some(data));
}
}
};
if let Some(idle_timeout) = idle_timeout {
tokio::time::timeout(idle_timeout, next_nonempty_data)
.await
.map_err(|_| remote_tier_timeout_error("remote tier response body stalled"))?
} else {
next_nonempty_data.await
}
}
async fn collect_response_body_inner<B>(
body: B,
limit: Option<usize>,
idle_timeout: Option<StdDuration>,
) -> Result<Vec<u8>, std::io::Error>
where
B: Body<Data = Bytes>,
B::Error: Into<Box<dyn StdError + Send + Sync>>,
{
let mut body_vec = Vec::new();
let mut body = std::pin::pin!(body);
while let Some(data) = next_response_body_data(body.as_mut(), idle_timeout).await? {
let Some(new_len) = body_vec.len().checked_add(data.len()) else {
return Err(response_body_exceeds_limit_error());
};
if limit.is_some_and(|limit| new_len > limit) {
return Err(response_body_exceeds_limit_error());
}
body_vec.extend_from_slice(&data);
}
Ok(body_vec)
}
pub async fn collect_response_body<B>(body: B, limit: usize) -> Result<Vec<u8>, std::io::Error>
where
B: Body<Data = Bytes>,
B::Error: Into<Box<dyn StdError + Send + Sync>>,
B::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
{
collect_response_body_inner(body, Some(limit), None).await
let body = Limited::new(body, limit).collect().await.map_err(|err| {
if err.is::<LengthLimitError>() {
std::io::Error::new(std::io::ErrorKind::InvalidData, "remote tier response body exceeds limit")
} else {
std::io::Error::other(err)
}
})?;
Ok(body.to_bytes().to_vec())
}
const C_UNKNOWN: i32 = -1;
@@ -278,62 +196,6 @@ pub struct TransitionClient {
pub trailing_header_support: bool,
pub max_retries: i64,
pub tier_type: String,
pub timeouts: TransitionClientTimeouts,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct TransitionClientTimeouts {
pub connect_timeout: StdDuration,
pub request_timeout: StdDuration,
pub response_body_idle_timeout: StdDuration,
}
impl TransitionClientTimeouts {
pub const fn new(
connect_timeout: StdDuration,
request_timeout: StdDuration,
response_body_idle_timeout: StdDuration,
) -> Self {
Self {
connect_timeout,
request_timeout,
response_body_idle_timeout,
}
}
fn validate(self) -> Result<Self, std::io::Error> {
if self.connect_timeout.is_zero() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"remote tier connect timeout must be greater than zero",
));
}
if self.request_timeout.is_zero() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"remote tier request timeout must be greater than zero",
));
}
if self.response_body_idle_timeout.is_zero() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"remote tier response body idle timeout must be greater than zero",
));
}
Ok(self)
}
}
impl Default for TransitionClientTimeouts {
fn default() -> Self {
Self {
connect_timeout: StdDuration::from_secs(rustfs_config::DEFAULT_TIER_REMOTE_CONNECT_TIMEOUT_SECS),
request_timeout: StdDuration::from_secs(rustfs_config::DEFAULT_TIER_REMOTE_REQUEST_TIMEOUT_SECS),
response_body_idle_timeout: StdDuration::from_secs(
rustfs_config::DEFAULT_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS,
),
}
}
}
#[derive(Debug, Default)]
@@ -426,28 +288,12 @@ async fn build_tls_config() -> Result<rustls::ClientConfig, std::io::Error> {
impl TransitionClient {
pub async fn new(endpoint: &str, opts: Options, tier_type: &str) -> Result<TransitionClient, std::io::Error> {
Self::private_new(endpoint, opts, tier_type, TransitionClientTimeouts::default()).await
let client = Self::private_new(endpoint, opts, tier_type).await?;
Ok(client)
}
/// Builds a transition client with explicit transport timeout budgets.
///
/// [`Self::new`] keeps the historical constructor surface and uses the
/// production defaults from [`TransitionClientTimeouts::default`].
pub async fn new_with_timeouts(
endpoint: &str,
opts: Options,
tier_type: &str,
timeouts: TransitionClientTimeouts,
) -> Result<TransitionClient, std::io::Error> {
Self::private_new(endpoint, opts, tier_type, timeouts).await
}
async fn private_new(
endpoint: &str,
opts: Options,
tier_type: &str,
timeouts: TransitionClientTimeouts,
) -> Result<TransitionClient, std::io::Error> {
async fn private_new(endpoint: &str, opts: Options, tier_type: &str) -> Result<TransitionClient, std::io::Error> {
if rustls::crypto::CryptoProvider::get_default().is_none() {
// No default provider is set yet; try to install aws-lc-rs.
// `install_default` can only fail if another thread races us and installs a provider
@@ -460,19 +306,15 @@ impl TransitionClient {
}
let endpoint_url = get_endpoint_url(endpoint, opts.secure)?;
let timeouts = timeouts.validate()?;
let tls = build_tls_config().await?;
let mut http = HttpConnector::new();
http.enforce_http(false);
http.set_connect_timeout(Some(timeouts.connect_timeout));
let https = hyper_rustls::HttpsConnectorBuilder::new()
.with_tls_config(tls)
.https_or_http()
.enable_http1()
.enable_http2()
.wrap_connector(http);
.build();
let http_client = Client::builder(TokioExecutor::new()).build(https);
let mut client = TransitionClient {
@@ -495,7 +337,6 @@ impl TransitionClient {
trailing_header_support: opts.trailing_headers,
max_retries: opts.max_retries,
tier_type: tier_type.to_string(),
timeouts,
};
{
@@ -660,43 +501,29 @@ impl TransitionClient {
}
pub async fn doit(&self, req: Request<s3s::Body>) -> Result<Response<Incoming>, std::io::Error> {
let req_method;
let req_uri;
let resp;
let http_client = self.http_client.clone();
let req_method = req.method().clone();
let resp = tokio::time::timeout(self.timeouts.request_timeout, http_client.request(req)).await;
{
req_method = req.method().clone();
req_uri = req.uri().clone();
debug!("endpoint_url: {}", self.endpoint_url.as_str().to_string());
resp = http_client.request(req);
}
let resp = resp.await;
debug!("http_client url: {} {}", req_method, req_uri);
if let Err(err) = resp {
error!("http_client call error: {:?}", err);
return Err(std::io::Error::other(err));
}
let resp = match resp {
Ok(Ok(resp)) => resp,
Ok(Err(err)) => {
let err = transition_transport_error(err);
error!(
event = EVENT_TIER_REMOTE_TRANSPORT,
component = LOG_COMPONENT_S3_CLIENT,
subsystem = LOG_SUBSYSTEM_TIER,
method = %req_method,
error_kind = ?err.kind(),
"remote tier request failed"
);
return Err(err);
}
Err(_) => {
warn!(
event = EVENT_TIER_REMOTE_TRANSPORT,
component = LOG_COMPONENT_S3_CLIENT,
subsystem = LOG_SUBSYSTEM_TIER,
method = %req_method,
timeout_ms = self.timeouts.request_timeout.as_millis(),
"remote tier request timed out before response headers"
);
return Err(remote_tier_timeout_error("remote tier request timed out before response headers"));
}
Ok(r) => r,
Err(_) => return Err(std::io::Error::other("Unexpected error in response")),
};
trace!(
event = EVENT_TIER_REMOTE_TRANSPORT,
component = LOG_COMPONENT_S3_CLIENT,
subsystem = LOG_SUBSYSTEM_TIER,
method = %req_method,
status = %resp.status(),
"remote tier response received"
);
debug!(status = %resp.status(), "remote tier response received");
//let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
//debug!("http_resp_body: {}", String::from_utf8(b).unwrap());
@@ -710,15 +537,7 @@ impl TransitionClient {
.and_then(|value| value.to_str().ok())
.unwrap_or_default()
.to_string();
warn!(
event = EVENT_TIER_REMOTE_TRANSPORT,
component = LOG_COMPONENT_S3_CLIENT,
subsystem = LOG_SUBSYSTEM_TIER,
method = %req_method,
status = %status,
request_id,
"remote tier request rejected"
);
warn!(status = %status, request_id, "remote tier request rejected");
}
Ok(resp)
}
@@ -762,9 +581,7 @@ impl TransitionClient {
let resp_status = resp.status();
let h = resp.headers().clone();
let body_vec = self
.collect_response_body(resp.into_body(), MAX_S3_ERROR_RESPONSE_SIZE)
.await?;
let body_vec = collect_response_body(resp.into_body(), MAX_S3_ERROR_RESPONSE_SIZE).await?;
let parsed_error =
http_resp_to_error_response(resp_status, &h, body_vec, &metadata.bucket_name, &metadata.object_name);
let routing_region = parsed_error.region;
@@ -818,22 +635,6 @@ impl TransitionClient {
Err(std::io::Error::other("remote tier request did not produce a response"))
}
pub async fn collect_response_body<B>(&self, body: B, limit: usize) -> Result<Vec<u8>, std::io::Error>
where
B: Body<Data = Bytes>,
B::Error: Into<Box<dyn StdError + Send + Sync>>,
{
collect_response_body_inner(body, Some(limit), Some(self.timeouts.response_body_idle_timeout)).await
}
pub async fn collect_response_body_unbounded<B>(&self, body: B) -> Result<Vec<u8>, std::io::Error>
where
B: Body<Data = Bytes>,
B::Error: Into<Box<dyn StdError + Send + Sync>>,
{
collect_response_body_inner(body, None, Some(self.timeouts.response_body_idle_timeout)).await
}
async fn new_request(
&self,
method: &http::Method,
@@ -1703,17 +1504,12 @@ pub struct CreateBucketConfiguration {
mod tests {
use super::{
MAX_S3_CLIENT_RESPONSE_SIZE, MAX_S3_ERROR_RESPONSE_SIZE, SignatureType, build_tls_config, collect_response_body,
collect_response_body_inner, signer_error_to_io_error, to_object_info_for_provider, validate_header_values,
with_rustls_init_guard,
signer_error_to_io_error, to_object_info_for_provider, validate_header_values, with_rustls_init_guard,
};
use crate::provider_versions::{BucketVersioningState, ProviderVersionCapabilities, RemoteVersion};
use futures::stream;
use http::{HeaderMap, HeaderValue, Request};
use http_body::Frame;
use http_body_util::{Full, StreamBody};
use http::{HeaderMap, HeaderValue};
use http_body_util::Full;
use hyper::body::Bytes;
use std::time::Duration as StdDuration;
use tokio::net::TcpListener;
use uuid::Uuid;
#[tokio::test]
@@ -1744,77 +1540,6 @@ mod tests {
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
}
#[tokio::test]
async fn empty_data_frames_do_not_reset_the_body_idle_timeout() {
let frames = stream::unfold((), |_| async {
tokio::time::sleep(StdDuration::from_millis(10)).await;
Some((Ok::<_, std::io::Error>(Frame::data(Bytes::new())), ()))
});
let body = StreamBody::new(Box::pin(frames));
let err = tokio::time::timeout(
StdDuration::from_millis(200),
collect_response_body_inner(body, Some(1), Some(StdDuration::from_millis(50))),
)
.await
.expect("the collector should enforce its own body idle timeout")
.expect_err("empty frames must not count as body progress");
assert_eq!(err.kind(), std::io::ErrorKind::TimedOut);
}
#[tokio::test]
async fn public_body_collector_accepts_non_unpin_bodies() {
let body = StreamBody::new(stream::once(async { Ok::<_, std::io::Error>(Frame::data(Bytes::from_static(b"ok"))) }));
let collected = collect_response_body(body, 2)
.await
.expect("the public collector should pin non-Unpin bodies internally");
assert_eq!(collected, b"ok");
}
#[tokio::test]
async fn https_endpoints_reach_the_transport_connector() {
let listener = match TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
Err(err) => panic!("test listener should bind: {err}"),
};
let endpoint = listener
.local_addr()
.expect("listener local address should be available")
.to_string();
let accepted = tokio::spawn(async move {
let (stream, _) = tokio::time::timeout(StdDuration::from_secs(1), listener.accept())
.await
.expect("HTTPS connector should reach the TCP listener")
.expect("fixture should accept the HTTPS connection");
drop(stream);
});
let client = super::TransitionClient::new_with_timeouts(
&endpoint,
super::Options {
secure: true,
..Default::default()
},
"",
super::TransitionClientTimeouts::new(StdDuration::from_secs(1), StdDuration::from_secs(1), StdDuration::from_secs(1)),
)
.await
.expect("fixture client should build");
let request = Request::builder()
.uri(format!("https://{endpoint}/"))
.body(s3s::Body::empty())
.expect("fixture request should build");
client
.doit(request)
.await
.expect_err("the fixture closes before completing the TLS handshake");
accepted.await.expect("fixture should join");
}
#[test]
fn rustls_guard_converts_panics_to_io_errors() {
let err = with_rustls_init_guard(|| -> Result<(), std::io::Error> { panic!("missing provider") })
@@ -1848,18 +1573,6 @@ mod tests {
assert!(outcome.is_ok(), "provider install guard must not panic when a provider is already set");
}
#[test]
fn transition_timeouts_reject_zero_budgets() {
for timeouts in [
super::TransitionClientTimeouts::new(StdDuration::ZERO, StdDuration::from_secs(1), StdDuration::from_secs(1)),
super::TransitionClientTimeouts::new(StdDuration::from_secs(1), StdDuration::ZERO, StdDuration::from_secs(1)),
super::TransitionClientTimeouts::new(StdDuration::from_secs(1), StdDuration::from_secs(1), StdDuration::ZERO),
] {
let err = timeouts.validate().expect_err("zero timeout budgets must fail closed");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
}
}
#[test]
fn validate_header_values_returns_header_name_for_non_utf8_values() {
let mut headers = HeaderMap::new();
+2 -10
View File
@@ -196,7 +196,7 @@ pub(crate) async fn read_config_revision<S: ScannerObjectIO>(store: Arc<S>, path
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
#[derive(Clone, Debug)]
pub(crate) struct DataUsageCacheRevisions {
main: DataUsageCacheRevision,
backup: Option<DataUsageCacheRevision>,
@@ -503,10 +503,6 @@ pub struct DataUsageCacheInfo {
pub lkg_leader_epoch: Option<u64>,
#[serde(default)]
pub lkg_scan_plan_digest: Option<DataUsageScanPlanDigest>,
/// Activity-sensitive identity for same-cycle set snapshot reuse. The
/// structural plan remains reusable across ordinary bucket writes.
#[serde(default)]
pub scan_execution_digest: Option<DataUsageScanPlanDigest>,
}
impl Serialize for DataUsageCacheInfo {
@@ -523,8 +519,7 @@ impl Serialize for DataUsageCacheInfo {
+ usize::from(self.lkg_next_cycle.is_some())
+ usize::from(self.lkg_last_update.is_some())
+ usize::from(self.lkg_leader_epoch.is_some())
+ usize::from(self.lkg_scan_plan_digest.is_some())
+ usize::from(self.scan_execution_digest.is_some());
+ usize::from(self.lkg_scan_plan_digest.is_some());
let mut state = serializer.serialize_map(Some(field_count))?;
state.serialize_entry("name", &self.name)?;
state.serialize_entry("next_cycle", &self.next_cycle)?;
@@ -563,9 +558,6 @@ impl Serialize for DataUsageCacheInfo {
if let Some(scan_plan_digest) = self.lkg_scan_plan_digest {
state.serialize_entry("lkg_scan_plan_digest", &scan_plan_digest)?;
}
if let Some(scan_execution_digest) = self.scan_execution_digest {
state.serialize_entry("scan_execution_digest", &scan_execution_digest)?;
}
state.end()
}
}
@@ -1067,7 +1067,6 @@ fn test_data_usage_cache_info_deserialize_defaults_scan_resume_after() {
assert!(decoded.source.is_none());
assert!(!decoded.snapshot_complete);
assert!(decoded.scan_plan_digest.is_none());
assert!(decoded.scan_execution_digest.is_none());
assert_eq!(decoded.cache_key_format, 0);
}
@@ -1110,7 +1109,6 @@ fn test_data_usage_cache_info_unmarshal_old_msgpack_defaults_scan_resume_after()
assert!(decoded.source.is_none());
assert!(!decoded.snapshot_complete);
assert!(decoded.scan_plan_digest.is_none());
assert!(decoded.scan_execution_digest.is_none());
assert_eq!(decoded.cache_key_format, 0);
}
@@ -1147,7 +1145,6 @@ fn test_new_data_usage_cache_msgpack_round_trips_and_supports_old_reader() {
source: Some(DataUsageCacheSource::new(1, 2)),
snapshot_complete: true,
scan_plan_digest: Some(TEST_PLAN_DIGEST),
scan_execution_digest: Some(DataUsageScanPlanDigest([42; 32])),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
@@ -1167,7 +1164,6 @@ fn test_new_data_usage_cache_msgpack_round_trips_and_supports_old_reader() {
assert_eq!(current.info.source, Some(DataUsageCacheSource::new(1, 2)));
assert!(current.info.snapshot_complete);
assert_eq!(current.info.scan_plan_digest, Some(TEST_PLAN_DIGEST));
assert_eq!(current.info.scan_execution_digest, Some(DataUsageScanPlanDigest([42; 32])));
assert_eq!(current.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
assert_eq!(current.find("bucket").map(|entry| entry.objects), Some(3));
+5 -31
View File
@@ -1616,7 +1616,7 @@ where
// Refresh the storage-owned movement snapshot before reading background
// heal state. A missing heal object yields an in-memory default; do not
// let that default influence a cycle while publication is blocked.
if storeapi.scanner_data_movement_pause_status().await.paused {
if storeapi.scanner_data_usage_publication_blocked().await {
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
}
@@ -1816,19 +1816,6 @@ where
let publication_defer_reason = publication_defer_reason
.or(remote_lease_defer_reason)
.or(remote_lease_fence_defer_reason);
// A PUT tail can finish between the walk and lease acquisition without
// changing the movement epoch accepted by those leases. Re-prove the
// namespace baseline only after every peer has granted publication.
let post_lease_activity_defer_reason = if publication_defer_reason.is_none()
&& remote_publication_leases.is_some()
&& let Ok(result) = &scan_result
&& result.status == ScannerCycleStatus::Complete
{
scanner_post_lease_activity_defer_reason(result.activity_digest(), probe_scanner_activity(storeapi.as_ref(), true).await)
} else {
None
};
let publication_defer_reason = publication_defer_reason.or(post_lease_activity_defer_reason);
// Include reasons discovered while acquiring or validating remote leases.
let publication_deferred = publication_defer_reason.is_some();
let budget_elapsed = cycle_budget.budget_elapsed() && !ctx.is_cancelled();
@@ -3253,21 +3240,6 @@ where
}
}
fn scanner_post_lease_activity_defer_reason(
expected_digest: Option<[u8; 32]>,
activity: Result<ScannerActivitySnapshot, String>,
) -> Option<ScannerCycleDeferReason> {
match activity {
Ok(snapshot)
if scanner_activity_allows_usage_publication(&snapshot)
&& expected_digest == Some(scanner_activity_snapshot_digest(&snapshot)) =>
{
None
}
Ok(_) | Err(_) => Some(ScannerCycleDeferReason::ActivityBaselineUnavailable),
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum ScannerCyclePreCommitOutcome {
RecoverCacheCycle(u64),
@@ -3456,11 +3428,13 @@ use cycle_state::*;
use leadership::*;
use usage_store::*;
#[cfg(test)]
pub(crate) use activity::scanner_activity_snapshot_digest;
pub use activity::scanner_topology_digest;
pub(crate) use activity::{
ScannerActivitySnapshot, ScannerDirtyUsageAcknowledgement, probe_scanner_activity, scanner_activity_allows_usage_publication,
scanner_activity_dirty_usage_state_for_host, scanner_activity_publication_lease_targets, scanner_activity_snapshot_digest,
scanner_activity_structural_digest, scanner_dirty_usage_acknowledgements,
scanner_activity_dirty_usage_state_for_host, scanner_activity_publication_lease_targets, scanner_activity_structural_digest,
scanner_dirty_usage_acknowledgements,
};
pub(crate) use activity::{ScannerCycleOutcome, scanner_cycle_outcome_with_pending_maintenance};
pub use backlog::{
+1
View File
@@ -902,6 +902,7 @@ where
observation
}
#[cfg(test)]
pub(crate) fn scanner_activity_snapshot_digest(snapshot: &ScannerActivitySnapshot) -> [u8; 32] {
let mut hasher = Sha256::new();
hasher.update(u64::try_from(snapshot.len()).unwrap_or(u64::MAX).to_be_bytes());
+1 -172
View File
@@ -15,8 +15,7 @@
use super::heal_info::{classify_background_heal_read_error, decode_background_heal_info};
use super::*;
use crate::EcstoreResult;
use crate::storage_api::owner::ecstore_hold_namespace_commit;
use crate::storage_api::scan::{BucketOperations as _, ObjectIO as _};
use crate::storage_api::scan::BucketOperations as _;
use crate::{
DATA_USAGE_BLOOM_RECOVERY_PATH, DATA_USAGE_CACHE_KEY_FORMAT, DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT,
DataUsageCachePrepareOutcome, DataUsageCacheSource, DataUsageEntry, DataUsageScanPlanDigest, Endpoint, EndpointServerPools,
@@ -1166,116 +1165,6 @@ async fn run_data_scanner_cycle_publishes_activity_for_owner_lifetime() {
global_metrics().set_cycle(None).await;
}
#[tokio::test]
#[serial]
async fn coordinator_walks_during_pending_put_without_persisting_or_acknowledging_usage() {
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
let (_temp_dir, store) = setup_scanner_cycle_store().await;
let bucket = format!("scanner-coordinator-pending-{}", Uuid::new_v4().simple());
store
.make_bucket(&bucket, &crate::storage_api::scan::MakeBucketOptions::default())
.await
.expect("fixture bucket should be created");
let mut reader = PutObjReader::from_vec(b"first".to_vec());
store.pools[0].disk_set[0]
.put_object(
&bucket,
"object",
&mut reader,
&ObjectOptions {
no_lock: true,
..Default::default()
},
)
.await
.expect("fixture object should finish its rename fanout");
crate::scanner_io::record_dirty_usage_bucket(&bucket);
let dirty_before = crate::scanner_io::dirty_usage_buckets_for_tests();
let baseline = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("fixture usage baseline should be readable");
let pending = ecstore_hold_namespace_commit(store.as_ref());
let ctx = CancellationToken::new();
let budget = ScannerCycleBudget::new_with_progress_tracking(&ctx, ScannerCycleBudgetConfig::default());
let mut cycle_info = CurrentCycle {
next: 1,
..Default::default()
};
let mut revision = DataUsageCacheRevision::Missing;
let outcome = tokio::time::timeout(
Duration::from_secs(30),
run_data_scanner_cycle_with_budget(&ctx, &store, &mut cycle_info, &mut revision, 1, Arc::clone(&budget)),
)
.await
.expect("the coordinator must finish its namespace walk while a PUT is pending");
assert_eq!(budget.progress().0, 1, "the coordinator must reach actual object traversal");
assert_eq!(outcome, ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
assert_eq!(cycle_info.next, 1, "a rejected publication must not advance the cycle");
assert_eq!(revision, DataUsageCacheRevision::Missing);
assert_eq!(crate::scanner_io::dirty_usage_buckets_for_tests(), dirty_before);
assert_eq!(
read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("the prior authoritative usage must remain readable"),
baseline,
"the pending candidate must not replace the authoritative baseline"
);
let committed_body = b"committed-after-walk";
let mut reader = PutObjReader::from_vec(committed_body.to_vec());
store.pools[0].disk_set[0]
.put_object(
&bucket,
"object",
&mut reader,
&ObjectOptions {
no_lock: true,
..Default::default()
},
)
.await
.expect("the pending tail must change the physical object before it drains");
assert_eq!(crate::scanner_io::dirty_usage_buckets_for_tests(), dirty_before);
drop(pending);
let retry_budget = ScannerCycleBudget::new_with_progress_tracking(&ctx, ScannerCycleBudgetConfig::default());
let outcome = tokio::time::timeout(
Duration::from_secs(30),
run_data_scanner_cycle_with_budget(&ctx, &store, &mut cycle_info, &mut revision, 1, Arc::clone(&retry_budget)),
)
.await
.expect("the same cycle must converge after the pending PUT drains");
assert_eq!(
retry_budget.progress().0,
1,
"the same-cycle retry must not reuse the pre-tail bucket cache"
);
assert!(matches!(
outcome,
ScannerCycleOutcome::Completed | ScannerCycleOutcome::CompletedWithPendingMaintenance
));
assert_eq!(cycle_info.next, 2);
assert!(!crate::scanner_io::dirty_usage_buckets_for_tests().contains_key(&bucket));
let usage = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("the converged usage should be persisted");
let usage: DataUsageInfo = serde_json::from_slice(&usage).expect("the persisted usage should decode");
assert_eq!(usage.usage_snapshot_converged, Some(true));
assert_eq!(usage.scanner_cycle, Some(1));
assert_eq!(usage.objects_total_count, 1);
assert_eq!(
usage.objects_total_size,
u64::try_from(committed_body.len()).expect("fixture body length")
);
let bucket_usage = usage
.buckets_usage
.get(&bucket)
.expect("the scanned bucket should be published");
assert_eq!(bucket_usage.objects_count, 1);
assert_eq!(bucket_usage.size, u64::try_from(committed_body.len()).expect("fixture body length"));
global_metrics().set_cycle(None).await;
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
}
#[tokio::test]
#[serial]
async fn test_finalize_partial_scan_cycle_advances_and_persists_counter() {
@@ -8596,66 +8485,6 @@ fn scanner_node_activity(epoch: &str, namespace_generation: u64, maintenance_gen
}
}
#[test]
fn post_lease_activity_proof_rejects_a_put_tail_that_finished_before_lease_acquisition() {
let before = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]);
let expected_digest = Some(scanner_activity_snapshot_digest(&before));
assert_eq!(scanner_post_lease_activity_defer_reason(expected_digest, Ok(before.clone())), None);
let mut after = before.clone();
after
.get_mut("node-2")
.expect("writer should be present")
.namespace_generation += 1;
assert_eq!(
before["node-2"].movement_generation, after["node-2"].movement_generation,
"the existing movement-only lease remains valid after a PUT tail drains"
);
assert!(scanner_activity_allows_usage_publication(&after));
let reason = scanner_post_lease_activity_defer_reason(expected_digest, Ok(after));
assert_eq!(reason, Some(ScannerCycleDeferReason::ActivityBaselineUnavailable));
let result = ScannerCycleResult::new(ScannerCycleStatus::Complete, None).with_remote_dirty_usage_acknowledgements(vec![
ScannerDirtyUsageAcknowledgement {
host: "node-2".to_string(),
instance_id: "epoch-a".to_string(),
generation: 5,
},
]);
let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(
result,
DataUsagePersistOutcome::Deferred(reason.expect("changed namespace should defer publication")),
);
assert_eq!(
outcome,
ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
);
assert!(
acknowledgements.is_empty(),
"a rejected publication must not acknowledge the peer's dirty usage"
);
}
#[test]
fn post_lease_activity_proof_requires_a_complete_matching_baseline() {
let before = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]);
let digest = scanner_activity_snapshot_digest(&before);
let mut blocked = before.clone();
blocked.get_mut("node-2").expect("peer should be present").publication_blocked = true;
let blocked_digest = scanner_activity_snapshot_digest(&blocked);
for (expected, observed) in [
(None, Ok(before)),
(Some(digest), Err("peer is unavailable".to_string())),
(Some(digest), Ok(BTreeMap::new())),
(Some(blocked_digest), Ok(blocked)),
] {
assert_eq!(
scanner_post_lease_activity_defer_reason(expected, observed),
Some(ScannerCycleDeferReason::ActivityBaselineUnavailable)
);
}
}
#[test]
fn scanner_activity_snapshot_digest_fences_storage_topology() {
let first = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]);
+2 -18
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::data_usage_define::{DATA_USAGE_CACHE_KEY_FORMAT, DataUsageCacheRevisions};
use crate::data_usage_define::DATA_USAGE_CACHE_KEY_FORMAT;
use crate::scanner_budget::ScannerCycleBudget;
use crate::scanner_folder::{ScannerItem, scan_data_folder};
use crate::sleeper::SCANNER_SLEEPER;
@@ -271,8 +271,6 @@ pub struct ScannerBucketScanPlan {
all_buckets: Arc<Vec<BucketInfo>>,
scope: ScannerBucketScanScope,
digest: DataUsageScanPlanDigest,
// Cache work must invalidate on namespace completion even when its scoped baseline remains reusable.
execution_digest: DataUsageScanPlanDigest,
leader_epoch: u64,
tier_registry_generation: u64,
/// Epoch captured once for the whole scanner cycle. `None` is retained
@@ -458,12 +456,9 @@ async fn scanner_cycle_activity_status<S>(
where
S: ScannerStorage,
{
// Read the pending-commit barrier before sampling its completion generation.
// A tail that drains during this await must invalidate the earlier baseline.
let publication_blocked = store.scanner_data_usage_publication_blocked().await;
match crate::scanner::probe_scanner_activity(store, distributed).await {
Ok(after) => {
let status = if !publication_blocked && after == *before {
let status = if after == *before {
ScannerCycleActivityStatus::Unchanged
} else {
ScannerCycleActivityStatus::Changed
@@ -765,7 +760,6 @@ fn scanner_activity_preflight(
pub(crate) struct ScannerCycleResult {
pub(crate) status: ScannerCycleStatus,
publication_epoch: Option<u64>,
activity_digest: Option<[u8; 32]>,
observational_snapshot_published: bool,
dirty_usage_clear: Option<DirtyUsageBuckets>,
remote_dirty_usage_acknowledgements: Vec<crate::scanner::ScannerDirtyUsageAcknowledgement>,
@@ -780,7 +774,6 @@ impl ScannerCycleResult {
Self {
status,
publication_epoch: None,
activity_digest: None,
observational_snapshot_published: false,
dirty_usage_clear,
remote_dirty_usage_acknowledgements: Vec::new(),
@@ -800,15 +793,6 @@ impl ScannerCycleResult {
self.publication_epoch
}
fn with_activity_digest(mut self, activity_digest: [u8; 32]) -> Self {
self.activity_digest = Some(activity_digest);
self
}
pub(crate) fn activity_digest(&self) -> Option<[u8; 32]> {
self.activity_digest
}
pub(crate) fn with_observational_snapshot_published(mut self, published: bool) -> Self {
self.observational_snapshot_published = published;
self
+12 -30
View File
@@ -604,12 +604,10 @@ pub(super) async fn persist_and_publish_cache_snapshot(
store: Arc<SetDisks>,
updates: &mpsc::Sender<DataUsageCache>,
mut cache_snapshot: DataUsageCache,
initial_revisions: Option<&DataUsageCacheRevisions>,
cache_cycle_floor: &AtomicU64,
expected_publication_epoch: u64,
) -> Option<SystemTime> {
let source = cache_snapshot.info.source?;
let execution_digest = cache_snapshot.info.scan_execution_digest?;
let guard = match acquire_scanner_cache_locks(store.as_ref(), DATA_USAGE_CACHE_NAME, source).await {
Ok(guard) => guard,
Err(err) => {
@@ -674,36 +672,20 @@ pub(super) async fn persist_and_publish_cache_snapshot(
);
return None;
}
if persisted.info.scan_execution_digest == Some(execution_digest)
&& matches!(
current_cache_root_entry_with_generation(
&persisted,
DATA_USAGE_ROOT,
source,
cache_snapshot.info.next_cycle,
cache_snapshot.info.leader_epoch,
scan_plan_digest,
cache_snapshot.info.tier_registry_generation,
),
Ok(Some(_))
)
{
if matches!(
current_cache_root_entry_with_generation(
&persisted,
DATA_USAGE_ROOT,
source,
cache_snapshot.info.next_cycle,
cache_snapshot.info.leader_epoch,
scan_plan_digest,
cache_snapshot.info.tier_registry_generation,
),
Ok(Some(_))
) {
cache_snapshot = persisted;
} else {
// A later execution may have completed while this scan was walking.
// Only replace the cache revision from which this scan started.
if initial_revisions != Some(&revisions) {
warn!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
state = "scan_baseline_revision_changed",
cache_name = DATA_USAGE_CACHE_NAME,
"Scanner skipped set snapshot without an unchanged baseline revision"
);
return None;
}
if guard.is_lock_lost() {
error!(
target: "rustfs::scanner::io",
+15 -24
View File
@@ -118,7 +118,6 @@ impl ScannerIOCache for SetDisks {
all_buckets,
scope,
digest: scan_plan_digest,
execution_digest,
leader_epoch,
tier_registry_generation,
publication_epoch,
@@ -138,24 +137,20 @@ impl ScannerIOCache for SetDisks {
.ok_or_else(|| StorageError::other("scanner cache publication is blocked by data movement"))?,
};
let mut old_cache = DataUsageCache::default();
let initial_revisions = match old_cache.load_with_revisions(self.clone(), DATA_USAGE_CACHE_NAME).await {
Ok(revisions) => Some(revisions),
Err(e) => {
warn!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
pool = self.pool_index,
set = self.set_index,
cache_name = DATA_USAGE_CACHE_NAME,
state = "old_cache_load_failed",
error = %e,
"Scanner old data usage cache load failed; rebuilding from bucket caches"
);
None
}
};
if let Err(e) = old_cache.load(self.clone(), DATA_USAGE_CACHE_NAME).await {
warn!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
pool = self.pool_index,
set = self.set_index,
cache_name = DATA_USAGE_CACHE_NAME,
state = "old_cache_load_failed",
error = %e,
"Scanner old data usage cache load failed; rebuilding from bucket caches"
);
}
let scoped_scan = prepare_scoped_set_scan(
&old_cache,
&buckets,
@@ -200,7 +195,6 @@ impl ScannerIOCache for SetDisks {
};
cache.info.last_update = Some(now);
cache.info.snapshot_complete = true;
cache.info.scan_execution_digest = Some(execution_digest);
cache.info.lkg_snapshot_complete = false;
cache.info.lkg_next_cycle = None;
cache.info.lkg_last_update = None;
@@ -214,7 +208,6 @@ impl ScannerIOCache for SetDisks {
self,
&updates,
cache,
initial_revisions.as_ref(),
cache_cycle_floor.as_ref(),
expected_publication_epoch,
)
@@ -644,7 +637,7 @@ impl ScannerIOCache for SetDisks {
let cache_name = path_join_buf(&[&bucket.name, DATA_USAGE_CACHE_NAME]);
let bucket_scan_plan_digest =
scanner_bucket_cache_digest(execution_digest, dirty_usage_buckets_clone.get(&bucket.name).copied());
scanner_bucket_cache_digest(scan_plan_digest, dirty_usage_buckets_clone.get(&bucket.name).copied());
if let Some(server_epoch) = remote_server_epoch {
let request_sequence = remote_session_sequence;
@@ -1367,7 +1360,6 @@ impl ScannerIOCache for SetDisks {
cache.info.next_cycle = want_cycle;
cache.info.last_update.get_or_insert_with(SystemTime::now);
cache.info.snapshot_complete = true;
cache.info.scan_execution_digest = Some(execution_digest);
cache.info.lkg_snapshot_complete = false;
cache.info.lkg_next_cycle = None;
cache.info.lkg_last_update = None;
@@ -1379,7 +1371,6 @@ impl ScannerIOCache for SetDisks {
self.clone(),
&updates,
cache_snapshot,
initial_revisions.as_ref(),
cache_cycle_floor.as_ref(),
expected_publication_epoch,
)
+1 -9
View File
@@ -180,7 +180,7 @@ where
// canceled decommission remains suspended after its worker exits, so
// starting a scan in that state could build a snapshot that cannot be
// routed to the authoritative metadata object.
if store.scanner_data_movement_pause_status().await.paused {
if store.scanner_data_usage_publication_blocked().await {
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
@@ -260,13 +260,8 @@ where
}
}
bucket_plan_complete &= buckets_by_source.keys().copied().collect::<HashSet<_>>() == *expected_sources;
let activity_digest = crate::scanner::scanner_activity_snapshot_digest(&activity_before);
let scan_plan_digest =
scanner_bucket_plan_digest(&all_buckets, crate::scanner::scanner_activity_structural_digest(&activity_before));
let mut execution_hasher = Sha256::new();
execution_hasher.update(scan_plan_digest.0);
execution_hasher.update(activity_digest);
let execution_digest = DataUsageScanPlanDigest(execution_hasher.finalize().into());
let dirty_usage_snapshot = Arc::new(snapshot_dirty_usage_buckets(&all_buckets, dirty_generation_before_bucket_list));
let scan_scope = resolve_scanner_bucket_scan_scope(
store,
@@ -331,7 +326,6 @@ where
};
return Ok(ScannerCycleResult::new(status, dirty_usage_clear)
.with_publication_epoch(publication_epoch)
.with_activity_digest(activity_digest)
.with_observational_snapshot_published(observational_snapshot_published)
.with_remote_publication_lease_targets(remote_publication_lease_targets)
.with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements));
@@ -416,7 +410,6 @@ where
all_buckets: Arc::clone(&all_buckets),
scope: scan_scope.clone(),
digest: scan_plan_digest,
execution_digest,
leader_epoch,
tier_registry_generation,
publication_epoch,
@@ -605,7 +598,6 @@ where
};
Ok(ScannerCycleResult::new(cycle_status, dirty_usage_clear)
.with_publication_epoch(publication_epoch)
.with_activity_digest(activity_digest)
.with_observational_snapshot_published(observational_snapshot_published)
.with_remote_publication_lease_targets(remote_publication_lease_targets)
.with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements)

Some files were not shown because too many files have changed in this diff Show More