Compare commits

..

17 Commits

Author SHA1 Message Date
overtrue 657835f12c fix(ecstore): satisfy delete fence lint checks 2026-08-22 17:05:47 +08:00
overtrue ca8d4c2ea7 test(ecstore): align decommission fence barriers 2026-08-22 17:05:47 +08:00
overtrue ca999e42f5 fix(ecstore): match decommission lock backend domain 2026-08-22 17:05:46 +08:00
overtrue 65a0eabfbf fix(ecstore): preserve distributed decommission set locks 2026-08-22 17:05:46 +08:00
overtrue 84f6b10186 fix(ecstore): unblock decommission delete fences 2026-08-22 17:05:46 +08:00
overtrue 7202c9937e test(ecstore): fix decommission fence fixtures 2026-08-22 17:05:46 +08:00
overtrue 5677201fc6 fix(ecstore): annotate batch delete fallback 2026-08-22 17:05:46 +08:00
overtrue 77604a5908 fix(ecstore): fence decommission commit loss 2026-08-22 17:05:46 +08:00
overtrue f1bf990588 fix(ecstore): reuse fixed fence for reverse decommission 2026-08-22 17:05:46 +08:00
overtrue 4a07a0a917 test(ecstore): finish decommission delete fence scenario 2026-08-22 17:05:46 +08:00
overtrue 906d982ced test(ecstore): exercise decommission delete fences 2026-08-22 17:05:46 +08:00
overtrue 971b84c4b5 fix(ecstore): retain source-set lock during cleanup 2026-08-22 17:05:46 +08:00
overtrue 631a30d6e2 fix(ecstore): preserve batch delete pool errors 2026-08-22 17:05:46 +08:00
overtrue cd06111dcb fix(ecstore): route batch delete markers to active pools 2026-08-22 17:05:46 +08:00
overtrue d139c8a8c3 fix(ecstore): preserve delete markers during source cleanup 2026-08-22 17:05:46 +08:00
overtrue 3793552885 fix(ecstore): preserve decommission target write locks 2026-08-22 17:05:46 +08:00
overtrue fdd193ff36 fix(ecstore): fence deletes against decommission commits 2026-08-22 17:05:46 +08:00
272 changed files with 4379 additions and 43175 deletions
-20
View File
@@ -1,20 +0,0 @@
# Report-only calibration baseline from https://github.com/rustfs/rustfs/actions/runs/29394996173.
# Update counts only with a linked coverage run and a reviewed explanation.
phase = "report-only"
allowed_drop_percentage_points = 1.0
[crates."crates/iam"]
covered = 5149
count = 8131
[crates."crates/kms"]
covered = 2950
count = 4200
[crates."crates/policy"]
covered = 4636
count = 5464
[crates."crates/crypto"]
covered = 469
count = 494
+2 -2
View File
@@ -1,2 +1,2 @@
sha256-darwin=ae412c27e2e43f3fda3e0c21ef616b48a6403354faab8db8d4154202e3c4eec5
sha256-linux=ef3be856bd3257c2c369428f66a48ad073dad8187229fe95840a600a81edf22b
sha256-darwin=b4ae71aa894e5c7795ae3eb8116f1777a7601d0f5db3898be2e48faf3329bd9b
sha256-linux=433debd9d9defa832986269abdf0f1d131597b2d7a417ce930e17c1fd47d85ba
-2
View File
@@ -36,8 +36,6 @@ script-tests: ## Run shell script tests
./scripts/test_manual_transition_runbooks.sh
./scripts/check_embedded_secrets.sh --self-test
python3 ./scripts/check_test_wiring.py --self-test
python3 ./scripts/check_security_coverage.py --self-test
python3 ./scripts/check_scheduled_validation_freshness.py --self-test
python3 ./scripts/s3-tests/test_report_compat.py
bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
python3 ./scripts/check_object_data_cache_follower_samples.py --self-test
+4 -14
View File
@@ -78,12 +78,6 @@ test-group = 'embedded-test-ports'
filter = 'package(rustfs-ecstore) & test(manual_transition_page_checkpoint_persists_durable_job_progress)'
test-group = 'ecstore-serial-flaky'
# The durable ILM decommission regressions build isolated multi-pool stores and
# deliberately take source or target disks offline while checking fencing.
[[profile.default.overrides]]
filter = 'package(rustfs-ecstore) & (test(decommission_migrates_and_verifies_registered_durable_ilm_records) | test(decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success) | test(decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup) | test(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page) | test(decommission_durable_ilm_recovery_keeps_multiple_active_sources))'
test-group = 'ecstore-serial-flaky'
# Serialize the bucket-incarnation / lifecycle-fence tests. They drive
# init_bucket_metadata_sys and bucket_metadata_sys_of, i.e. process-global
# OnceLock state that serial_test's #[serial] cannot protect across nextest's
@@ -196,10 +190,6 @@ test-group = 'embedded-test-ports'
filter = 'package(rustfs-ecstore) & test(manual_transition_page_checkpoint_persists_durable_job_progress)'
test-group = 'ecstore-serial-flaky'
[[profile.ci.overrides]]
filter = 'package(rustfs-ecstore) & (test(decommission_migrates_and_verifies_registered_durable_ilm_records) | test(decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success) | test(decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup) | test(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page) | test(decommission_durable_ilm_recovery_keeps_multiple_active_sources))'
test-group = 'ecstore-serial-flaky'
# Serialize the bucket-incarnation / lifecycle-fence tests under the ci profile
# too (see the matching default-profile override near the top). No retries.
[[profile.ci.overrides]]
@@ -315,7 +305,7 @@ slow-timeout = { period = "60s", terminate-after = 2, grace-period = "10s" }
# the target incl. multipart and the resync path, SSE-C and
# target-without-KMS stay fail-closed), and one guards event/history
# observers.
# * 13 `_real_dual_node` site-replication tests — each spawns TWO full rustfs
# * 12 `_real_dual_node` site-replication tests — each spawns TWO full rustfs
# servers and drives the cross-process site-replication control plane.
# * 1 `_real_three_node` site-replication test.
# * 1 `_real_single_node` service-account round-trip test.
@@ -335,8 +325,8 @@ slow-timeout = { period = "60s", terminate-after = 2, grace-period = "10s" }
#
# Wired by .github/workflows/e2e-replication-nightly.yml (schedule +
# workflow_dispatch), which builds the rustfs binary once, installs awscurl so
# the STS dual-node test actually exercises its path (the test fails when
# awscurl is absent), and routes scheduled failures
# the STS dual-node test actually exercises its path (it skips gracefully with
# a visible log line when awscurl is absent), and routes scheduled failures
# through .github/actions/schedule-failure-issue (ci-8). Explicit division of
# labor with e2e-full: these tests run only in the consolidated nightly
# workflow, not in the merge/main lane.
@@ -406,7 +396,7 @@ path = "junit.xml"
# object_lambda) — too heavy for the merge budget; they run in the
# e2e-nightly serial cluster-fault lane.
# * replication_extension_test — repl-1 already splits it into the PR
# `e2e-smoke` (20 fast) and `e2e-repl-nightly` (56 slow) lanes and reserves
# `e2e-smoke` (20 fast) and `e2e-repl-nightly` (55 slow) lanes and reserves
# it for those, so e2e-full does not double-run it.
# * #[ignore]d tests — nextest skips them by default (no --run-ignored); the
# manual-localhost:9000 reliant/policy tests are ci-13's migration.
@@ -14,10 +14,9 @@
name: "Schedule Failure Issue"
description: >-
Open (or update) a tracking issue when a scheduled workflow run fails or
does not complete normally.
Open (or update) a tracking issue when a scheduled workflow run fails.
Dedupes by workflow name: if an open issue titled
"[scheduled-failure] <workflow name>" already exists, the result is
"[scheduled-failure] <workflow name>" already exists, the failure is
appended as a comment; otherwise a new issue is created. This is the
single alerting mechanism for all scheduled pipelines (backlog#1149 ci-8).
@@ -39,30 +38,6 @@ inputs:
Set to an empty string to skip labeling.
required: false
default: "infrastructure"
source-run-id:
description: "Run ID to report. Defaults to the current workflow run."
required: false
default: ${{ github.run_id }}
source-run-attempt:
description: "Run attempt to report. Defaults to the current attempt."
required: false
default: ${{ github.run_attempt }}
source-event:
description: "Trigger event of the run being reported."
required: false
default: ${{ github.event_name }}
source-ref-name:
description: "Ref name of the run being reported."
required: false
default: ${{ github.ref_name }}
source-sha:
description: "Commit SHA of the run being reported."
required: false
default: ${{ github.sha }}
details-file:
description: "Optional Markdown file appended to the issue body."
required: false
default: ""
runs:
using: "composite"
@@ -73,22 +48,17 @@ runs:
GH_TOKEN: ${{ inputs.github-token }}
WORKFLOW_NAME: ${{ inputs.workflow-name }}
ISSUE_LABEL: ${{ inputs.label }}
SOURCE_RUN_ID: ${{ inputs.source-run-id }}
SOURCE_RUN_ATTEMPT: ${{ inputs.source-run-attempt }}
SOURCE_EVENT: ${{ inputs.source-event }}
SOURCE_REF_NAME: ${{ inputs.source-ref-name }}
SOURCE_SHA: ${{ inputs.source-sha }}
DETAILS_FILE: ${{ inputs.details-file }}
run: |
set -euo pipefail
title="[scheduled-failure] ${WORKFLOW_NAME}"
run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}"
run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
# Inspect the reported run attempt. It can be the current in-workflow
# failure or a completed run observed by the external watchdog.
# Failed job names for this run attempt. The alert job runs while the
# run as a whole is still in progress, so inspect the jobs that have
# already completed with a non-success conclusion.
failed_jobs="$(gh api \
"repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}/attempts/${SOURCE_RUN_ATTEMPT}/jobs" \
"repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}/jobs" \
--paginate \
--jq '.jobs[]
| select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "cancelled")
@@ -97,26 +67,15 @@ runs:
failed_jobs="- (failed job not recorded yet — see the run page)"
fi
details=""
if [ -n "${DETAILS_FILE}" ]; then
if [ -f "${DETAILS_FILE}" ]; then
details="$(cat "${DETAILS_FILE}")"
else
details="Details file was not available: \`${DETAILS_FILE}\`"
fi
fi
body="$(cat <<EOF
Run of **${WORKFLOW_NAME}** did not complete successfully.
Scheduled run of **${WORKFLOW_NAME}** failed.
- Run: ${run_url} (attempt ${SOURCE_RUN_ATTEMPT})
- Event: \`${SOURCE_EVENT}\`
- Ref: \`${SOURCE_REF_NAME}\` @ \`${SOURCE_SHA}\`
- Run: ${run_url} (attempt ${GITHUB_RUN_ATTEMPT})
- Event: \`${GITHUB_EVENT_NAME}\`
- Ref: \`${GITHUB_REF_NAME}\` @ \`${GITHUB_SHA}\`
Non-success jobs:
Failed jobs:
${failed_jobs}
${details}
EOF
)"
-14
View File
@@ -1,14 +0,0 @@
[
{ "workflow": ".github/workflows/audit.yml", "max_age_hours": 36 },
{ "workflow": ".github/workflows/build.yml", "max_age_hours": 192 },
{ "workflow": ".github/workflows/ci.yml", "max_age_hours": 192 },
{ "workflow": ".github/workflows/coverage.yml", "max_age_hours": 192 },
{ "workflow": ".github/workflows/e2e-replication-nightly.yml", "max_age_hours": 36 },
{ "workflow": ".github/workflows/e2e-s3tests.yml", "max_age_hours": 192 },
{ "workflow": ".github/workflows/fuzz.yml", "max_age_hours": 36 },
{ "workflow": ".github/workflows/mint.yml", "max_age_hours": 192 },
{ "workflow": ".github/workflows/minio-interop.yml", "max_age_hours": 36 },
{ "workflow": ".github/workflows/nightly-gnu.yml", "max_age_hours": 36 },
{ "workflow": ".github/workflows/performance-ab.yml", "max_age_hours": 36 },
{ "workflow": ".github/workflows/runner-hygiene.yml", "max_age_hours": 792 }
]
+1 -1
View File
@@ -46,7 +46,7 @@ on:
# advisory could sit unnoticed for seven days. The check list is unchanged —
# splitting it into a light daily advisories-only run and a weekly full run
# would create runs where sources/bans/licenses go unverified.
- cron: '23 3 * * *' # Daily 03:23 UTC
- cron: '0 3 * * *' # Daily 03:00 UTC (staggered after the midnight ci/build crons)
workflow_dispatch:
permissions:
+1 -21
View File
@@ -52,7 +52,7 @@ on:
- ".dockerignore"
- "flake.lock"
schedule:
- cron: "13 1 * * 0" # Weekly on Sunday 01:13 UTC
- cron: "0 1 * * 0" # Weekly on Sunday 01:00 UTC (staggered after the ci.yml midnight cron)
workflow_dispatch:
inputs:
build_docker:
@@ -1032,23 +1032,3 @@ jobs:
echo "🎉 Released $TAG successfully!"
echo "📄 Release URL: ${{ needs.create-release.outputs.release_url }}"
alert-on-failure:
name: Alert on scheduled failure
needs: [build-check, prepare-platform-matrix, build-rustfs, build-summary]
if: >-
always() && github.event_name == 'schedule' &&
(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
issues: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Open or update failure-tracking issue
uses: ./.github/actions/schedule-failure-issue
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
+3 -6
View File
@@ -94,9 +94,6 @@ concurrency:
env:
CARGO_TERM_COLOR: always
# Swatinem/rust-cache hashes every RUST* variable. Keep this aligned with
# ci.yml or the writer and readers use disjoint cache keys.
RUST_BACKTRACE: 1
jobs:
# Readers: test-and-lint, test-ilm-integration-serial, build-rustfs-debug-binary,
@@ -104,7 +101,7 @@ jobs:
warm-ci-dev:
name: Warm ci-dev
runs-on: sm-standard-4
timeout-minutes: 120
timeout-minutes: 90
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
steps:
@@ -194,7 +191,7 @@ jobs:
warm-ci-feat-rio:
name: Warm ci-feat-rio
runs-on: sm-standard-4
timeout-minutes: 120
timeout-minutes: 90
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
steps:
@@ -222,7 +219,7 @@ jobs:
warm-ci-feat-proto:
name: Warm ci-feat-proto
runs-on: sm-standard-4
timeout-minutes: 120
timeout-minutes: 90
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
steps:
+1 -4
View File
@@ -126,10 +126,7 @@ jobs:
run: ./scripts/check_embedded_secrets.sh
- name: Check test wiring
run: |
python3 ./scripts/check_test_wiring.py --self-test
python3 ./scripts/check_scheduled_validation_freshness.py --self-test
python3 ./scripts/check_test_wiring.py
run: python3 ./scripts/check_test_wiring.py
- name: Check no planning docs committed
run: ./scripts/check_no_planning_docs.sh
+2 -66
View File
@@ -59,7 +59,7 @@ on:
merge_group:
types: [ checks_requested ]
schedule:
- cron: "11 0 * * 0" # Weekly on Sunday 00:11 UTC
- cron: "0 0 * * 0" # Weekly on Sunday at midnight UTC
workflow_dispatch:
permissions:
@@ -161,10 +161,7 @@ jobs:
run: ./scripts/check_embedded_secrets.sh
- name: Check test wiring
run: |
python3 ./scripts/check_test_wiring.py --self-test
python3 ./scripts/check_scheduled_validation_freshness.py --self-test
python3 ./scripts/check_test_wiring.py
run: python3 ./scripts/check_test_wiring.py
- name: Check no planning docs committed
run: ./scripts/check_no_planning_docs.sh
@@ -681,19 +678,6 @@ jobs:
cache-save-if: 'false'
install-build-packaging-tools: 'false'
- name: Set up Python
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: "3.12"
- name: Install awscurl
run: |
python3 -m pip install --user --upgrade pip "awscurl==0.44"
echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"
- name: Verify awscurl
run: test -x "$AWSCURL_PATH"
# Download after the cache restore so the freshly built binary from the
# build job always wins over anything restored into target/debug.
- name: Download debug binary
@@ -816,20 +800,6 @@ jobs:
- name: Verify awscurl
run: test -x "$AWSCURL_PATH"
- name: Install mc
env:
MC_VERSION: RELEASE.2025-08-13T08-35-41Z
MC_SHA256: 01f866e9c5f9b87c2b09116fa5d7c06695b106242d829a8bb32990c00312e891
run: |
MC_BINARY="mc.linux-amd64.${MC_VERSION}"
curl -fsSLo "$RUNNER_TEMP/mc" "https://github.com/minio/mc/releases/download/${MC_VERSION}/${MC_BINARY}"
echo "${MC_SHA256} $RUNNER_TEMP/mc" | sha256sum --check --status
chmod +x "$RUNNER_TEMP/mc"
echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
- name: Verify mc
run: mc --version
- name: Install Vault
run: |
VAULT_VERSION="1.17.6"
@@ -1062,37 +1032,3 @@ jobs:
path: artifacts/s3tests-single/**
if-no-files-found: ignore
retention-days: 3
alert-on-failure:
name: Alert on scheduled failure
needs:
- typos
- quick-checks
- test-and-lint
- test-ilm-integration-serial
- test-and-lint-rio-v2
- test-and-lint-protocols
- build-rustfs-debug-binary
- build-rustfs-debug-binary-rio-v2
- uring-integration
- e2e-tests
- e2e-full
- e2e-tests-rio-v2
- s3-implemented-tests
- s3-lifecycle-behavior-tests
if: >-
always() && github.event_name == 'schedule' &&
(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
issues: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Open or update failure-tracking issue
uses: ./.github/actions/schedule-failure-issue
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
+13 -30
View File
@@ -12,12 +12,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Workspace line-coverage baseline and security-crate calibration
# (backlog#1153 infra-5/infra-6).
# Weekly workspace line-coverage baseline (backlog#1153 infra-5).
#
# NON-BLOCKING by design: the weekly job gives coverage a visible baseline and
# trend, while relevant pull requests run a report-only security-crate
# comparison. Neither job is a required check during calibration.
# NON-BLOCKING by design: this workflow only runs on schedule and manual
# dispatch, so it never attaches a status to a PR and must never be made a
# required check. It exists to give coverage a visible baseline and trend
# (per-crate table in the job summary, lcov artifact kept 90 days) — the
# per-crate ratchet for the security-critical crates builds on it later
# (backlog#1153 infra-6, report-only first per the ci-11 ladder).
#
# Measurement scope matches the PR test gate (ci.yml "Run tests"):
# `--workspace --exclude e2e_test` with the `ci` nextest profile. Doctests are
@@ -29,28 +31,13 @@
name: coverage
on:
pull_request:
branches: [main]
paths:
- "crates/iam/**"
- "crates/kms/**"
- "crates/policy/**"
- "crates/crypto/**"
- ".config/coverage-baselines.toml"
- "scripts/coverage_per_crate.py"
- "scripts/check_security_coverage.py"
- ".github/workflows/coverage.yml"
workflow_dispatch:
schedule:
# 07:00 UTC Sunday — staggered clear of the other Sunday crons: ci (00:00),
# build (01:00), e2e-s3tests (02:00), audit (03:00), nix-flake-update
# (05:00), mint (06:00), and the daily fuzz (02:00), minio-interop (03:17),
# e2e-replication-nightly (04:00) and performance-ab (06:00) lanes.
- cron: "43 7 * * 0"
concurrency:
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name != 'schedule' }}
- cron: "0 7 * * 0"
# Only alert-on-failure needs more than read access; it declares its own
# job-level `issues: write`.
@@ -59,14 +46,12 @@ permissions:
jobs:
coverage:
name: Workspace line coverage
name: Workspace coverage (weekly)
runs-on: sm-standard-4
# The instrumented build cannot reuse the regular CI cache (different
# RUSTFLAGS), so a cold run rebuilds the workspace before running the
# full suite. Two later exact-head runs exhausted 150 minutes before the
# report steps, so allow one additional 90-minute cold-run margin while
# keeping the calibration job bounded.
timeout-minutes: 240
# RUSTFLAGS), so a cold week rebuilds the workspace before running the
# full suite; give it double the test job's 60-minute budget.
timeout-minutes: 120
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
# Match the PR gate's nextest semantics (ci.yml runs `--profile ci`):
@@ -106,9 +91,7 @@ jobs:
cargo llvm-cov report --json --output-path target/llvm-cov/coverage.json
- name: Write per-crate summary
run: |
python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"
python3 scripts/check_security_coverage.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"
run: python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"
- name: Upload coverage artifact
if: always()
@@ -40,7 +40,7 @@ on:
schedule:
# 04:00 UTC nightly — staggered clear of fuzz/e2e-s3tests (02:00),
# stale (01:30) and performance-ab (06:00).
- cron: "29 4 * * *"
- cron: "0 4 * * *"
# Only alert-on-failure needs more than read access; it declares its own
# job-level `issues: write`.
@@ -75,7 +75,11 @@ jobs:
cache-save-if: ${{ github.ref == 'refs/heads/main' }}
install-build-packaging-tools: 'false'
# The STS dual-node test requires awscurl and fails if it is unavailable.
# awscurl lets the STS dual-node test actually exercise its path. Without
# it the test skips gracefully with a visible log line
# (`awscurl_available()` in crates/e2e_test/src/common.rs), so the lane
# still passes — installing it just upgrades that one test from skip to
# real coverage.
- name: Set up Python
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
@@ -83,7 +87,7 @@ jobs:
- name: Install awscurl
run: |
python3 -m pip install --user --upgrade pip "awscurl==0.44"
python3 -m pip install --user --upgrade pip awscurl
echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"
- name: Verify awscurl
@@ -192,12 +196,6 @@ jobs:
cache-save-if: 'false'
install-build-packaging-tools: 'false'
- name: Install and verify protocol socket oracle
run: |
sudo apt-get update -qq
sudo apt-get install -y -qq iproute2
ss -tn state CLOSE-WAIT >/dev/null
# The suite owns fixed protocol ports and serializes its internal cases.
- name: Verify protocol e2e membership
env:
+2 -101
View File
@@ -21,9 +21,6 @@
# suite and reports promotion candidates. Regressions, unclassified tests,
# incomplete execution, and infrastructure errors fail the job; classified
# failures for not-yet-implemented features remain informational.
# - Non-blocking upstream HEAD canary: collects current upstream node IDs and
# reports new, removed, duplicate, or overlapping classifications without
# making upstream drift a release gate.
# - Manual runs (workflow_dispatch): same, with configurable mode/scope.
#
# All test execution is delegated to scripts/s3-tests/run.sh (single source of
@@ -93,14 +90,10 @@ on:
description: "Optional pytest -m expression"
required: false
default: ""
testexpr:
description: "Optional pytest -k expression"
required: false
default: ""
schedule:
# Weekly full sweep (Sunday 02:00 UTC): full suite, run against BOTH the
# single-node and the 4-node distributed topologies (matrix below).
- cron: "19 2 * * 0"
- cron: "0 2 * * 0"
env:
# main user
@@ -123,7 +116,6 @@ env:
XDIST: ${{ github.event.inputs.xdist || '4' }}
MAXFAIL: ${{ github.event.inputs.maxfail || '0' }}
MARKEXPR: ${{ github.event.inputs.markexpr || '' }}
TESTEXPR: ${{ github.event.inputs.testexpr || '' }}
S3_SHARD_COUNT: ${{ github.event_name == 'schedule' && '4' || github.event.inputs.shard-count || '1' }}
TEST_TIMEOUT: "300"
@@ -181,14 +173,9 @@ jobs:
- name: Install Python tools
run: |
python3 -m pip install --user --upgrade pip "awscurl==0.44" "tox==4.60.0"
python3 -m pip install --user --upgrade pip awscurl tox
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
- name: Verify Python tools
run: |
test "$(python3 -c 'import importlib.metadata as m; print(m.version("awscurl"))')" = "0.44"
test "$(python3 -c 'import importlib.metadata as m; print(m.version("tox"))')" = "4.60.0"
- name: Enable buildx
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
@@ -282,20 +269,14 @@ jobs:
EOF
cat > haproxy.cfg <<'EOF'
global
log stdout format raw local0 info
defaults
mode http
log global
log-format '%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %tsc %HM %HP'
timeout connect 5s
timeout client 30s
timeout server 30s
frontend fe_s3
bind *:9000
option http-buffer-request
default_backend be_s3
backend be_s3
@@ -333,7 +314,6 @@ jobs:
XDIST="${XDIST}" \
MAXFAIL="${MAXFAIL}" \
MARKEXPR="${MARKEXPR}" \
TESTEXPR="${TESTEXPR}" \
./scripts/s3-tests/run.sh
- name: Publish compatibility report
@@ -362,85 +342,6 @@ jobs:
name: s3tests-${{ env.TEST_MODE }}-shard-${{ matrix.shard-index }}
path: artifacts/**
upstream-head-canary:
name: Upstream HEAD classification canary
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
continue-on-error: true
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: "3.12"
- name: Install collection tool
run: |
python3 -m pip install --user "tox==4.60.0"
python3 - <<'PY'
from importlib.metadata import version
assert version("tox") == "4.60.0"
PY
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
- name: Compare upstream HEAD classifications
id: upstream-compare
run: |
ARTIFACT_DIR="artifacts/s3tests-upstream-head"
UPSTREAM_DIR="${RUNNER_TEMP}/s3-tests-upstream"
mkdir -p "${ARTIFACT_DIR}"
git clone --depth 1 https://github.com/ceph/s3-tests.git "${UPSTREAM_DIR}"
git -C "${UPSTREAM_DIR}" rev-parse HEAD > "${ARTIFACT_DIR}/upstream-sha.txt"
cp "${UPSTREAM_DIR}/s3tests.conf.SAMPLE" "${UPSTREAM_DIR}/s3tests.conf"
(
cd "${UPSTREAM_DIR}"
S3TEST_CONF="${UPSTREAM_DIR}/s3tests.conf" tox -- \
-q --collect-only s3tests/functional/test_s3.py \
-m "not rustfs_never_marker"
) 2>&1 | tee "${ARTIFACT_DIR}/collect.log"
grep -E '^s3tests/functional/test_s3\.py::' \
"${ARTIFACT_DIR}/collect.log" > "${ARTIFACT_DIR}/collected-nodeids.txt"
python3 scripts/s3-tests/report_compat.py \
--lists-dir scripts/s3-tests \
--collected-nodeids "${ARTIFACT_DIR}/collected-nodeids.txt" \
--check-classifications-only 2>&1 | tee "${ARTIFACT_DIR}/classification-drift.txt"
- name: Publish canary report
if: always()
env:
CANARY_OUTCOME: ${{ steps.upstream-compare.outcome }}
run: |
{
echo "## ceph/s3-tests upstream HEAD canary"
echo
if [ -f artifacts/s3tests-upstream-head/upstream-sha.txt ]; then
echo "Upstream HEAD: $(cat artifacts/s3tests-upstream-head/upstream-sha.txt)"
fi
echo
echo '```text'
if [ -s artifacts/s3tests-upstream-head/classification-drift.txt ]; then
cat artifacts/s3tests-upstream-head/classification-drift.txt
elif [ "${CANARY_OUTCOME}" != "success" ]; then
echo "Canary did not complete; inspect the collection log artifact."
else
echo "No classification drift detected."
fi
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
- name: Upload canary artifacts
if: always() && env.ACT != 'true'
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: s3tests-upstream-head
path: artifacts/s3tests-upstream-head/**
retention-days: 14
alert-on-failure:
name: Alert on scheduled failure
needs: [s3tests]
+1 -1
View File
@@ -30,7 +30,7 @@ on:
- "Cargo.lock"
- ".github/workflows/fuzz.yml"
schedule:
- cron: "17 2 * * *"
- cron: "0 2 * * *"
workflow_dispatch:
inputs:
profile:
-18
View File
@@ -121,21 +121,3 @@ jobs:
cargo nextest run --run-ignored ignored-only --no-tests=fail \
-p "$INTEROP_PACKAGE" --features "$INTEROP_FEATURES" \
-E "$INTEROP_FILTER"
alert-on-failure:
name: Alert on scheduled failure
needs: [minio-interop]
if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
issues: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Open or update failure-tracking issue
uses: ./.github/actions/schedule-failure-issue
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
+10 -3
View File
@@ -45,6 +45,13 @@
# docker-capable self-hosted `dind-sm-standard-2` label was the alternative but
# has fewer cores and reintroduces fleet-state risk for no reliability gain.
# DISABLED. This workflow is switched off in the repository's Actions settings
# (state: disabled_manually) and does not run on any trigger, including its cron
# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
# reading this file, which has already misled at least one audit — hence this
# banner. Re-enabling is a UI action; anyone doing so should first check that the
# workflow still matches the current CI layout. See rustfs/backlog#1603.
#
name: mint
on:
@@ -63,13 +70,13 @@ on:
- core
- full
mint-image:
description: "Mint image reference (empty = pinned default)"
description: "Mint image reference"
required: false
default: ""
default: "minio/mint:edge"
schedule:
# Weekly, after the Sunday s3-tests full sweep (starts 02:00 UTC, up to
# 3h) has finished, so the two never contend for the same runner pool.
- cron: "41 6 * * 0"
- cron: "0 6 * * 0"
env:
S3_ACCESS_KEY: rustfsadmin-ci
+7 -24
View File
@@ -16,7 +16,7 @@ name: Nightly GNU Build
on:
schedule:
- cron: "7 0 * * *"
- cron: "0 0 * * *"
timezone: "Asia/Shanghai"
workflow_dispatch:
@@ -39,10 +39,11 @@ jobs:
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
steps:
- name: Checkout repository
- name: Checkout main branch
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
ref: main
- name: Setup Rust environment
uses: ./.github/actions/setup
@@ -88,10 +89,11 @@ jobs:
# either casing.
NO_PROXY: 127.0.0.1,localhost
steps:
- name: Checkout repository
- name: Checkout main branch
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
ref: main
- name: Setup Rust environment
uses: ./.github/actions/setup
@@ -176,10 +178,11 @@ jobs:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
NO_PROXY: 127.0.0.1,localhost
steps:
- name: Checkout repository
- name: Checkout main branch
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
ref: main
- name: Setup Rust environment
uses: ./.github/actions/setup
@@ -191,23 +194,3 @@ jobs:
- name: Run HA leader failover live checks (three-node Raft cluster in Docker)
run: bash scripts/test/vault_ha_kms_live.sh
alert-on-failure:
name: Alert on scheduled failure
needs: [build, kms-vault-lane, kms-vault-ha-failover]
if: >-
always() && github.event_name == 'schedule' &&
(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
issues: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Open or update failure-tracking issue
uses: ./.github/actions/schedule-failure-issue
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
+96 -73
View File
@@ -22,15 +22,22 @@
# correctness cost (e.g. the #4221 fsync durability fix) is recorded, not
# blocked (rustfs/backlog#935 correction 1).
# DISABLED. This workflow is switched off in the repository's Actions settings
# (state: disabled_manually) and does not run on any trigger, including its cron
# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
# reading this file, which has already misled at least one audit — hence this
# banner. Re-enabling is a UI action; anyone doing so should first check that the
# workflow still matches the current CI layout. See rustfs/backlog#1603.
#
name: Performance A/B
on:
schedule:
- cron: "31 6 * * *" # 06:31 UTC nightly, against main
- cron: "0 6 * * *" # 06:00 UTC nightly, against main
workflow_dispatch:
inputs:
duration:
description: "warp duration per round"
description: "warp duration per round (short by default to fit the double-build budget)"
required: false
default: "12s"
type: string
@@ -39,8 +46,12 @@ on:
required: false
default: false
type: boolean
push:
# Every main commit pre-builds and caches its release binary (perf-3) so the
# nightly A/B restores a ready baseline instead of paying the double build.
branches: [main]
permissions:
actions: read
contents: read
env:
@@ -48,19 +59,83 @@ env:
RUST_BACKTRACE: 1
jobs:
warp-ab:
name: Warp A/B budget gate
# perf-3: on every push to main, build the release binary once and cache it
# keyed by commit SHA (rustfs-baseline-<sha>). The warp-ab measurements
# restore this instead of paying the ~32min-per-side source
# build. That double build is what pushed the expanded 24-cell nightly past its
# ceiling — 2026-07-11..07-14 all cancelled on the 120min timeout. Incremental
# builds off the shared cargo cache keep each push cheap, and building on the
# same sm-standard-2 runner the A/B measures on guarantees the cached binary is
# ABI-identical. Do NOT source this from build.yml's per-merge artifact: those
# are cancelled ~7/8 of the time and are not a reliable baseline.
build-baseline-cache:
name: Build + cache baseline binary
if: github.event_name == 'push'
runs-on: sm-standard-2
# A normal nightly restores the last successful binary and builds only the
# candidate; daily access keeps that cache warm. A cache miss may build both
# and needs room for the A/B run plus artifact and cache publication.
timeout-minutes: 180
# Latest-wins: consumers only ever restore the binary for the *current*
# origin/main tip, so when pushes land faster than the ~65min build, a
# superseded build's output is dead weight — cancel it instead of stacking
# hour-long jobs on the shared runner pool. A skipped intermediate SHA at
# most costs one same-commit self-heal in the A/B job.
concurrency:
group: perf-baseline-build-main
cancel-in-progress: true
# #4806 put thin LTO + codegen-units=1 on [profile.release], pushing a
# single release build past 60min on this runner — every cache build on
# 2026-07-15 died on the old 60min ceiling ("exceeded the maximum execution
# time of 1h0m0s") and the cache never populated. The measured binary must
# keep the production profile, so the budget absorbs the build instead.
timeout-minutes: 100
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
fetch-depth: 0 # baseline may be an earlier successful scheduled head
- name: Setup Rust environment
uses: ./.github/actions/setup
with:
rust-version: stable
cache-shared-key: warp-ab-${{ hashFiles('**/Cargo.lock') }}
cache-save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Build release rustfs
run: cargo build --release --bin rustfs
- name: Stage binary for cache
run: |
set -euo pipefail
mkdir -p baseline-bin
cp target/release/rustfs baseline-bin/rustfs
- name: Cache baseline binary by SHA
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
with:
path: baseline-bin/rustfs
key: rustfs-baseline-${{ github.sha }}
warp-ab:
name: Warp A/B budget gate
# Always run on schedule / manual dispatch. Never on push — that event only
# feeds build-baseline-cache above.
if: >-
github.event_name == 'schedule' ||
github.event_name == 'workflow_dispatch'
runs-on: sm-standard-2
# With perf-3's cached baseline binary the common (cache-hit) nightly is
# measurement-only and finishes well under 50min. This ceiling stays
# generous only to absorb the same-commit cache-miss self-heal (~65min
# single build with the post-#4806 LTO profile + measurement). A timeout
# surfaces via the alert-on-failure job (it fires on cancelled/timed-out,
# not just failure). perf-6 recalibrates the budget once the noise study
# lands.
timeout-minutes: 120
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
fetch-depth: 0 # baseline is built from origin/main
- name: Setup Rust environment
uses: ./.github/actions/setup
@@ -88,55 +163,24 @@ jobs:
fi
echo "allow_regression=$allow" >> "$GITHUB_OUTPUT"
# A failed regression run must keep comparing against the last known-good
# scheduled head. Otherwise the next nightly would absorb the regression
# into its baseline and turn green without a fix.
- name: Find last successful scheduled baseline
id: scheduled_baseline
if: github.event_name == 'schedule'
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
with:
result-encoding: string
script: |
const { data } = await github.rest.actions.listWorkflowRuns({
owner: context.repo.owner,
repo: context.repo.repo,
workflow_id: "performance-ab.yml",
event: "schedule",
status: "success",
per_page: 1,
});
return data.workflow_runs[0]?.head_sha ?? "";
# Manual runs compare a selected ref with current main. Scheduled runs
# compare current main with the last successful scheduled head. With no
# history, the first run measures the candidate against itself and seeds
# that head only if the complete rig succeeds.
# perf-3: resolve the commits so the cache can be keyed by SHA. The
# baseline is origin/main; the candidate is the checked-out ref. On the
# nightly (checkout == main) they are the same commit, so one cached binary
# serves both phases and the run does zero source builds.
- name: Resolve baseline / candidate commits
id: commits
env:
SCHEDULED_BASELINE_SHA: ${{ steps.scheduled_baseline.outputs.result }}
run: |
set -euo pipefail
baseline_sha="$(git rev-parse origin/main)"
candidate_sha="$(git rev-parse HEAD)"
if [[ "${{ github.event_name }}" == "schedule" ]]; then
baseline_sha="${SCHEDULED_BASELINE_SHA:-$candidate_sha}"
else
baseline_sha="$(git rev-parse origin/main)"
fi
git cat-file -e "${baseline_sha}^{commit}"
if ! git merge-base --is-ancestor "$baseline_sha" "$candidate_sha"; then
echo "::error::baseline $baseline_sha is not an ancestor of candidate $candidate_sha; update the selected ref before comparing" >&2
exit 1
fi
echo "baseline_sha=$baseline_sha" >> "$GITHUB_OUTPUT"
echo "candidate_sha=$candidate_sha" >> "$GITHUB_OUTPUT"
echo "baseline commit: $baseline_sha"
echo "candidate commit: $candidate_sha"
# Exact-key restore of the candidate binary saved by its successful
# scheduled run. A miss leaves cache-hit unset and falls back to a source
# build of that known-good head.
# Exact-key restore of the baseline binary built by build-baseline-cache
# when origin/main last landed. A miss (binary evicted or not built yet)
# leaves cache-hit unset and the rig falls back to a source build.
- name: Restore cached baseline binary
id: baseline_cache
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
@@ -226,11 +270,11 @@ jobs:
elif [[ "$selfheal_built" == "true" ]]; then
base_src="source build (cache self-heal, saved as rustfs-baseline-$baseline_sha)"
else
base_src="isolated baseline source build (saved as rustfs-baseline-$baseline_sha)"
base_src="isolated origin/main source build (saved as rustfs-baseline-$baseline_sha)"
fi
if [[ "$candidate_sha" == "$baseline_sha" ]]; then
# No commits landed since the last successful baseline, so reuse
# the one binary for both phases and measure only rig drift.
# Nightly on main: the candidate is the same commit as the baseline,
# so reuse the one binary for both phases and skip all builds.
args+=(--candidate-bin "$base_bin")
cand_src="same binary as baseline (same commit)"
elif [[ "$candidate_built" == "true" ]]; then
@@ -318,23 +362,6 @@ jobs:
fi
} >> "$GITHUB_STEP_SUMMARY"
- name: Stage successful candidate baseline
if: >-
steps.ab.outputs.status == '0' &&
steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha
run: |
set -euo pipefail
cp candidate-bin/rustfs baseline-bin/rustfs
- name: Cache successful candidate baseline
if: >-
steps.ab.outputs.status == '0' &&
steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
with:
path: baseline-bin/rustfs
key: rustfs-baseline-${{ steps.commits.outputs.candidate_sha }}
# Scheduled failure alerting is handled by the alert-on-failure job below
# (perf-2 consuming ci-8's schedule-failure-issue composite action).
@@ -342,10 +369,6 @@ jobs:
if: always()
run: |
status="${{ steps.ab.outputs.status }}"
if [[ -z "$status" ]]; then
echo "::error::warp A/B setup failed before the rig ran. Check the first failed workflow step." >&2
exit 1
fi
if [[ "$status" != "0" ]]; then
echo "::error::warp A/B budget gate failed (exit $status). See the step summary / gate.md artifact." >&2
exit "$status"
+1 -1
View File
@@ -30,7 +30,7 @@ name: Runner Hygiene
on:
schedule:
- cron: "37 6 1 * *" # Monthly, 1st at 06:37 UTC
- cron: "0 6 1 * *" # Monthly, 1st at 06:00 UTC (after the daily audit cron)
workflow_dispatch:
permissions:
@@ -1,57 +0,0 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
name: Scheduled Validation Freshness
on:
schedule:
- cron: "47 23 * * *"
workflow_dispatch:
permissions:
contents: read
concurrency:
group: scheduled-validation-freshness
cancel-in-progress: false
jobs:
check-freshness:
name: Check scheduled validation freshness
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
actions: read
contents: read
issues: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Check latest scheduled runs
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set +e
python3 scripts/check_scheduled_validation_freshness.py \
--report "${RUNNER_TEMP}/scheduled-validation-freshness.md"
status=$?
cat "${RUNNER_TEMP}/scheduled-validation-freshness.md" >> "${GITHUB_STEP_SUMMARY}"
exit "${status}"
- name: Open or update freshness issue
if: failure()
uses: ./.github/actions/schedule-failure-issue
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
details-file: ${{ runner.temp }}/scheduled-validation-freshness.md
@@ -1,63 +0,0 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
name: Scheduled Validation Watchdog
on:
workflow_run:
workflows:
- "Security Audit"
- "Build and Release"
- "Continuous Integration"
- "coverage"
- "e2e-nightly"
- "e2e-s3tests"
- "Fuzz"
- "mint"
- "minio-interop"
- "Nightly GNU Build"
- "Performance A/B"
- "Runner Hygiene"
types: [completed]
permissions:
contents: read
jobs:
alert-on-incomplete-run:
name: Alert on incomplete scheduled run
if: >-
github.event.workflow_run.event == 'schedule' &&
github.event.workflow_run.conclusion != 'success' &&
github.event.workflow_run.conclusion != 'failure'
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
actions: read
contents: read
issues: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Open or update incomplete-run issue
uses: ./.github/actions/schedule-failure-issue
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
workflow-name: ${{ github.event.workflow_run.name }}
source-run-id: ${{ github.event.workflow_run.id }}
source-run-attempt: ${{ github.event.workflow_run.run_attempt }}
source-event: ${{ github.event.workflow_run.event }}
source-ref-name: ${{ github.event.workflow_run.head_branch }}
source-sha: ${{ github.event.workflow_run.head_sha }}
+1 -2
View File
@@ -30,8 +30,7 @@ make build-docker BUILD_OS=ubuntu22.04
- Crate membership: `Cargo.toml` `[workspace].members`
- Architecture, layering, crate map: [ARCHITECTURE.md](ARCHITECTURE.md)
- Migration guardrails & readiness contracts: [docs/architecture/](docs/architecture/README.md)
- CI workflow steps: `.github/workflows/`; event, timeout, and required-status
matrix: [docs/testing/ci-gates.md](docs/testing/ci-gates.md)
- CI gates: `.github/workflows/ci.yml` (source of truth; never copy its steps into docs)
- Test-layer taxonomy, per-layer entry commands, serial/nextest rules, flake
policy: [docs/testing/README.md](docs/testing/README.md)
- Tier/ILM transition debugging (xl.meta inspection, versionId tracing):
-2
View File
@@ -70,8 +70,6 @@ make pre-pr
> For the full test-layer taxonomy (unit / ecstore black-box / e2e / s3s-e2e / S3 compatibility / chaos / fuzz / bench), each layer's entry command, the naming conventions the migration gate depends on, and the serial/nextest rules, see [docs/testing/README.md](docs/testing/README.md).
> For the event, timeout, required-status, and local reproduction matrix, see [docs/testing/ci-gates.md](docs/testing/ci-gates.md).
### 🔒 Automated Pre-commit Hooks
#### What `make pre-commit` and `make pre-pr` actually run
Generated
+80 -132
View File
@@ -68,16 +68,16 @@ dependencies = [
[[package]]
name = "aes-gcm"
version = "0.11.1"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f2b8006a0c83f52b62ba44a97b58bf76fe2f70a329e588f67f89691d93d498f"
checksum = "fdf011db2e21ce0d575593d749db5554b47fed37aff429e4dc50bc91ac93a028"
dependencies = [
"aead",
"aes 0.9.2",
"cipher 0.5.2",
"ctr",
"ctutils",
"ghash",
"subtle",
"zeroize",
]
@@ -333,12 +333,6 @@ dependencies = [
"password-hash",
]
[[package]]
name = "array-init"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc"
[[package]]
name = "arrayvec"
version = "0.7.8"
@@ -819,12 +813,11 @@ dependencies = [
[[package]]
name = "async_zip"
version = "0.0.19"
version = "0.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb7f5f40e1eb30949a266fc900d37fd3267c7baf50c3705ac09c5d8ced5def63"
checksum = "0d8c50d65ce1b0e0cb65a785ff615f78860d7754290647d3b983208daa4f85e6"
dependencies = [
"async-compression",
"binrw",
"crc32fast",
"futures-lite",
"pin-project",
@@ -876,9 +869,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
[[package]]
name = "aws-config"
version = "1.11.0"
version = "1.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a767267da9e2c2e189b2f9df8b5657e850ecf5352644734ba130d4a57095cf1b"
checksum = "1b180a3c8b55960db3426d8964b8745e652466a1a49fe1a2eda828046d30b5e4"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -971,9 +964,9 @@ dependencies = [
[[package]]
name = "aws-sdk-kms"
version = "1.116.0"
version = "1.115.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "484ecdbea2a1cfc0e6eea69ce0a665f93913671b303ba40b2361b1d826544e7e"
checksum = "d5b034f8b7ceadb873d0bc607c30bb4b0be68e09a84c837174e7c2c6878ff882"
dependencies = [
"arc-swap",
"aws-credential-types",
@@ -997,9 +990,9 @@ dependencies = [
[[package]]
name = "aws-sdk-s3"
version = "1.143.0"
version = "1.142.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a0ade5433c9561daac7c0c6bc910f1240b4f8ec0d6148b0b463aac0691d747c9"
checksum = "f9e15a5c55e05f4b0b7e483160b3c85cccdf77cff02c95504f3e71d460855cd2"
dependencies = [
"arc-swap",
"aws-credential-types",
@@ -1034,9 +1027,9 @@ dependencies = [
[[package]]
name = "aws-sdk-sso"
version = "1.107.0"
version = "1.106.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "769b0abd0f89cfe11da5099986dd493e4f94347ce9a4562cb86ddecfe926b6c0"
checksum = "2d0efcee834347b6705eca3eea2defd88242f43774f55d7326604222e3c86260"
dependencies = [
"arc-swap",
"aws-credential-types",
@@ -1060,9 +1053,9 @@ dependencies = [
[[package]]
name = "aws-sdk-ssooidc"
version = "1.109.0"
version = "1.108.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f4075b8a2c8cda4076a3dcc43b9d6dabd93e0c2502abeaaf7e14aaead9bb312b"
checksum = "a59312a04cf19c962cfee32b64ecfee758f8786407ff6da5b30fff46ae96f201"
dependencies = [
"arc-swap",
"aws-credential-types",
@@ -1086,9 +1079,9 @@ dependencies = [
[[package]]
name = "aws-sdk-sts"
version = "1.112.0"
version = "1.111.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f582002918346a3e685be1b391c7bea155073088cea6bd4e4b7663df9e43b6c"
checksum = "120e7eb63457a9e547f9986fe3b273f77c43679da4d04f46359fa881c5e19b6e"
dependencies = [
"arc-swap",
"aws-credential-types",
@@ -1551,30 +1544,6 @@ dependencies = [
"serde",
]
[[package]]
name = "binrw"
version = "0.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ad120d555272286c1017d25165ab8bd74806f13fc85b258484ec7e4ce75458f"
dependencies = [
"array-init",
"binrw_derive",
"bytemuck",
]
[[package]]
name = "binrw_derive"
version = "0.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6df92e0e9baae4dc82c7bad7715ca40c0a5c71539057bf2ea04a5c29c980410b"
dependencies = [
"either",
"owo-colors",
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]]
name = "bit-set"
version = "0.8.0"
@@ -1662,9 +1631,9 @@ dependencies = [
[[package]]
name = "blocking"
version = "1.7.0"
version = "1.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a70e4329df6cb94385eed412ec92375c3cdd8a6e502493d1229b6414e4036dfa"
checksum = "e83f8d02be6967315521be875afa792a316e28d57b5a2d401897e2a7921b7f21"
dependencies = [
"async-channel",
"async-task",
@@ -1753,12 +1722,6 @@ version = "3.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
[[package]]
name = "bytemuck"
version = "1.25.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797"
[[package]]
name = "byteorder"
version = "1.5.0"
@@ -1895,9 +1858,9 @@ dependencies = [
[[package]]
name = "cc"
version = "1.4.4"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273"
checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d"
dependencies = [
"find-msvc-tools",
"jobserver",
@@ -2362,9 +2325,9 @@ dependencies = [
[[package]]
name = "crc32fast"
version = "1.5.1"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550"
checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
dependencies = [
"cfg-if",
]
@@ -2559,6 +2522,12 @@ dependencies = [
"subtle",
]
[[package]]
name = "cty"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35"
[[package]]
name = "curve25519-dalek"
version = "4.1.3"
@@ -3971,9 +3940,9 @@ dependencies = [
[[package]]
name = "either"
version = "1.18.0"
version = "1.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34"
checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d"
[[package]]
name = "elliptic-curve"
@@ -4507,7 +4476,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5"
dependencies = [
"polyval",
"zeroize",
]
[[package]]
@@ -5090,9 +5058,9 @@ dependencies = [
[[package]]
name = "hotpath"
version = "0.24.0"
version = "0.23.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2645642a23d4061ec15a4a6e74f851a3145c3125356846cfc7772ff9c6f2737"
checksum = "dce755d457a63bdd0c95e4c91511daad1b58b33209543b7f38027b676f387e5e"
dependencies = [
"arc-swap",
"async-channel",
@@ -5124,9 +5092,9 @@ dependencies = [
[[package]]
name = "hotpath-macros"
version = "0.24.0"
version = "0.23.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89a3d3cdf9b0d4d3d4f6d4a29798f3b9170401ba500eaa58dd8f890f926af0f1"
checksum = "a903af89a8429cb07790c3818bc15270b394f80af1bc254e5ccf9c7de2961770"
dependencies = [
"proc-macro2",
"quote",
@@ -5135,15 +5103,15 @@ dependencies = [
[[package]]
name = "hotpath-macros-meta"
version = "0.24.0"
version = "0.23.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e84cd2417fa60938241cf1cd6c03e09953f5c821122dc5da9b8f27975d136c5b"
checksum = "bcc0ab94ffbb2ee77f4a897df02b5a137a10cf24d69bda936e59aff4dd456e61"
[[package]]
name = "hotpath-meta"
version = "0.24.0"
version = "0.23.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d2c145b67b1a4e7bcefa918995e212c30a49a85f05cc5962fe1f717878d560b"
checksum = "053481f6cec8f775a3276c7f6e2f21123111d28261e4edc15ea7421c445964bb"
dependencies = [
"hotpath-macros-meta",
]
@@ -5410,9 +5378,9 @@ checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa"
[[package]]
name = "icu_provider"
version = "2.3.1"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73"
checksum = "92a7ed671a6aad807a8651a2e1782a6598fda9ce5185dd8158549e95a91c6428"
dependencies = [
"displaydoc",
"icu_locale_core",
@@ -5799,9 +5767,9 @@ dependencies = [
[[package]]
name = "keccak"
version = "0.2.2"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8f198d1db720e4940b5a493201d199d9f24f568f8f746bd13706243a2f71598"
checksum = "ffd9697dc4a9a62e2da93389f34400b77a28f0287711263cabb203b3ccb9c0e4"
dependencies = [
"cfg-if",
"cpufeatures 0.3.0",
@@ -6020,6 +5988,15 @@ version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
[[package]]
name = "libmimalloc-sys"
version = "0.1.49"
source = "git+https://github.com/xonatius/mimalloc_rust.git?rev=6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11#6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11"
dependencies = [
"cc",
"cty",
]
[[package]]
name = "libredox"
version = "0.1.20"
@@ -6113,9 +6090,9 @@ dependencies = [
[[package]]
name = "log"
version = "0.4.34"
version = "0.4.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
[[package]]
name = "lru"
@@ -6305,7 +6282,7 @@ dependencies = [
"hashbrown 0.16.1",
"indexmap 2.14.0",
"metrics",
"ordered-float 5.5.0",
"ordered-float 5.3.0",
"quanta",
"radix_trie",
"rand 0.9.5",
@@ -6420,6 +6397,14 @@ dependencies = [
"synstructure 0.13.2",
]
[[package]]
name = "mimalloc"
version = "0.1.52"
source = "git+https://github.com/xonatius/mimalloc_rust.git?rev=6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11#6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11"
dependencies = [
"libmimalloc-sys",
]
[[package]]
name = "mime"
version = "0.3.17"
@@ -7253,9 +7238,9 @@ dependencies = [
[[package]]
name = "ordered-float"
version = "5.5.0"
version = "5.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c7c9e0d9b23589f26070720bac724174bfec1083e82f7854cdd0267518343c0"
checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e"
dependencies = [
"num-traits",
]
@@ -7290,12 +7275,6 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e"
[[package]]
name = "owo-colors"
version = "4.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d"
[[package]]
name = "p12-keystore"
version = "0.2.1"
@@ -7838,7 +7817,6 @@ dependencies = [
"cpubits",
"cpufeatures 0.3.0",
"universal-hash",
"zeroize",
]
[[package]]
@@ -8309,16 +8287,6 @@ name = "quick-xml"
version = "0.41.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1"
dependencies = [
"memchr",
"serde",
]
[[package]]
name = "quick-xml"
version = "0.42.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b"
dependencies = [
"encoding_rs",
"memchr",
@@ -8983,9 +8951,9 @@ dependencies = [
[[package]]
name = "russh"
version = "0.63.0"
version = "0.62.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00cf00190c315093734a8d405225bd8773a219bc86538a9b73bfc51145b33995"
checksum = "9decb68e4e44e1079700e54f17c8f23806ec53d7e0db73ab1c71d9dabc666812"
dependencies = [
"aes 0.9.2",
"aws-lc-rs",
@@ -9194,11 +9162,13 @@ dependencies = [
"insta",
"jiff",
"libc",
"libmimalloc-sys",
"libsystemd",
"matchit 0.9.2",
"md-5 0.11.0",
"metrics",
"metrics-util",
"mimalloc",
"mime_guess",
"opentelemetry",
"opentelemetry_sdk",
@@ -9207,7 +9177,7 @@ dependencies = [
"percent-encoding",
"pin-project-lite",
"proptest",
"quick-xml 0.42.0",
"quick-xml",
"rand 0.10.2",
"rcgen",
"regex",
@@ -9234,8 +9204,6 @@ dependencies = [
"rustfs-lock",
"rustfs-log-analyzer",
"rustfs-madmin",
"rustfs-mimalloc",
"rustfs-mimalloc-sys",
"rustfs-notify",
"rustfs-object-capacity",
"rustfs-object-data-cache",
@@ -9473,7 +9441,7 @@ dependencies = [
"path-absolutize",
"pin-project-lite",
"proptest",
"quick-xml 0.42.0",
"quick-xml",
"rand 0.10.2",
"ratelimit",
"rcgen",
@@ -9619,7 +9587,6 @@ dependencies = [
"serde",
"serde_json",
"serial_test",
"sha2 0.11.0",
"temp-env",
"tempfile",
"thiserror 2.0.20",
@@ -9908,24 +9875,6 @@ dependencies = [
"tokio",
]
[[package]]
name = "rustfs-mimalloc"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a406f4aa07084301d485beec873af6dccc8e3f8762da244743df92038b1db1a6"
dependencies = [
"rustfs-mimalloc-sys",
]
[[package]]
name = "rustfs-mimalloc-sys"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3051b819175f58445d4c369a72f0ab88149f3885ba8bea2aff3be01f53fe7cd"
dependencies = [
"cc",
]
[[package]]
name = "rustfs-notify"
version = "1.0.0-rc.3"
@@ -9940,7 +9889,7 @@ dependencies = [
"jiff",
"metrics",
"percent-encoding",
"quick-xml 0.42.0",
"quick-xml",
"rayon",
"rustc-hash",
"rustfs-config",
@@ -10713,9 +10662,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"
[[package]]
name = "rustls-webpki"
version = "0.103.15"
version = "0.103.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2"
checksum = "0527518605e68109d875e248ea259b6758801cf165e4b2c2733ae3b51f12535a"
dependencies = [
"aws-lc-rs",
"ring",
@@ -10759,7 +10708,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
[[package]]
name = "s3s"
version = "0.15.0-alpha.1"
source = "git+https://github.com/rustfs/s3s.git?rev=e080e38c56a3b43acbacce55710d765a5ce9003d#e080e38c56a3b43acbacce55710d765a5ce9003d"
source = "git+https://github.com/rustfs/s3s.git?rev=ed70cb048cc4be168419d461cb9ac3c2c7fa6d5a#ed70cb048cc4be168419d461cb9ac3c2c7fa6d5a"
dependencies = [
"arc-swap",
"arrayvec",
@@ -10786,7 +10735,7 @@ dependencies = [
"nom 8.0.0",
"numeric_cast",
"pin-project-lite",
"quick-xml 0.41.0",
"quick-xml",
"regex",
"serde",
"serde_json",
@@ -12731,15 +12680,14 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]]
name = "uuid"
version = "1.25.0"
version = "1.24.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f053576934f05a761a402421fbbe3d425d9366f75f978806a037b3ca481abecc"
checksum = "2cefc03fd367c0c6d4305de1b312cf00248c4114f4a0418ce6a6af769e3b0bd9"
dependencies = [
"getrandom 0.4.3",
"js-sys",
"rand 0.10.2",
"serde_core",
"sha1_smol",
"wasm-bindgen",
]
@@ -13453,9 +13401,9 @@ dependencies = [
[[package]]
name = "zerovec-derive"
version = "0.11.6"
version = "0.11.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da"
checksum = "9f212a141d820099d57ffafb9569be9617a6f27d3dc881fbee8fb56642f917a9"
dependencies = [
"proc-macro2",
"quote",
+13 -13
View File
@@ -135,7 +135,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.3" }
# Async Runtime and Networking
async-channel = "2.5.0"
async_zip = { default-features = false, version = "0.0.19" }
async_zip = { default-features = false, version = "0.0.18" }
mysql_async = { default-features = false, version = "0.37" }
async-compression = { version = "0.4.43" }
async-recursion = "1.1.1"
@@ -178,7 +178,7 @@ byteorder = "1.5.0"
flatbuffers = "25.12.19"
form_urlencoded = "1.2.2"
prost = "0.14.4"
quick-xml = "0.42.0"
quick-xml = "0.41.0"
rmp = { version = "0.8.15" }
rmp-serde = { version = "1.3.1" }
serde = { version = "1.0.229" }
@@ -191,7 +191,7 @@ serde_urlencoded = "0.7.1"
# matching stable releases are not available yet, while previous stable lines
# have incompatible APIs. Keep them exact-pinned and monitor upstream for stable
# releases.
aes-gcm = { version = "=0.11.1" }
aes-gcm = { version = "=0.11.0" }
argon2 = { version = "=0.6.0-rc.8" }
blake2 = "=0.11.0-rc.6"
chacha20poly1305 = { version = "=0.11.0" }
@@ -227,11 +227,11 @@ arc-swap = "1.9.2"
astral-tokio-tar = "0.6.4"
atoi = "3.1.0"
atomic_enum = "0.3.0"
aws-config = { version = "1.11.0" }
aws-config = { version = "1.10.1" }
aws-credential-types = { version = "1.3.0" }
aws-sdk-kms = { default-features = false, version = "1.116.0" }
aws-sdk-s3 = { default-features = false, version = "1.143.0" }
aws-sdk-sts = { default-features = false, version = "1.112.0" }
aws-sdk-kms = { default-features = false, version = "1.115.0" }
aws-sdk-s3 = { default-features = false, version = "1.142.0" }
aws-sdk-sts = { default-features = false, version = "1.111.0" }
aws-smithy-http-client = { default-features = false, version = "1.4.0" }
aws-smithy-runtime-api = { version = "1.15.0" }
aws-smithy-types = { version = "1.6.2" }
@@ -291,7 +291,7 @@ rustify = { version = "0.7", default-features = false }
rustix = { version = "1.1.4" }
rust-embed = { version = "8.12.0" }
rustc-hash = { version = "2.1.3" }
s3s = { git = "https://github.com/rustfs/s3s.git", rev = "e080e38c56a3b43acbacce55710d765a5ce9003d" }
s3s = { git = "https://github.com/rustfs/s3s.git", rev = "ed70cb048cc4be168419d461cb9ac3c2c7fa6d5a" }
serial_test = "4.0.1"
shadow-rs = { default-features = false, version = "2.0.0" }
siphasher = "1.0.3"
@@ -314,7 +314,7 @@ tracing-subscriber = { version = "0.3.23" }
transform-stream = "0.3.1"
url = "2.5.8"
urlencoding = "2.1.3"
uuid = { version = "1.25.0" }
uuid = { version = "1.24.1" }
vaultrs = { version = "0.8.0" }
tar = "0.4.46"
walkdir = "2.5.0"
@@ -343,16 +343,16 @@ libunftp = { version = "0.23.0" }
unftp-core = "0.1.0"
suppaftp = { version = "10.0.2" }
rcgen = { version = "0.14.9", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
russh = { version = "0.63.0" }
russh = { version = "0.62.7" }
russh-sftp = "2.4.0"
# WebDAV
dav-server = "0.11.0"
# Performance Analysis and Memory Profiling
rustfs-mimalloc = { version = "0.5.0" }
rustfs-mimalloc-sys = { version = "0.5.0" }
hotpath = { version = "0.24.0", default-features = false }
mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11" }
libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11", features = ["extended"] }
hotpath = { version = "0.23.3", default-features = false }
# Snapshot testing for output format regression detection
insta = { version = "1.48" }
-160
View File
@@ -901,10 +901,6 @@ pub struct Metrics {
scanner_cycle_max_duration_millis: AtomicU64,
scanner_cycle_max_objects: AtomicU64,
scanner_cycle_max_directories: AtomicU64,
scanner_cycle_timeout_total: AtomicU64,
scanner_cycle_recovery_required_total: AtomicU64,
scanner_cycle_last_progress_age_seconds: AtomicU64,
scanner_leader_lease_without_progress: AtomicBool,
scanner_bitrot_cycle_enabled: AtomicBool,
scanner_bitrot_cycle_millis: AtomicU64,
scanner_checkpoint: Mutex<Option<ScannerCheckpointReport>>,
@@ -918,15 +914,7 @@ pub struct Metrics {
scanner_dirty_usage_last_cycle_dirty_buckets: AtomicU64,
scanner_dirty_usage_last_cycle_cleared_buckets: AtomicU64,
scanner_usage_last_save_unix_secs: AtomicU64,
scanner_usage_last_durable_success_unix_secs: AtomicU64,
scanner_usage_last_publication_unix_secs: AtomicU64,
scanner_usage_last_publication_state: Mutex<String>,
scanner_usage_last_publication_reason: Mutex<String>,
scanner_usage_last_save_result: AtomicU8,
scanner_usage_deferred_pending: AtomicBool,
scanner_usage_deferred_total: AtomicU64,
scanner_usage_last_deferred_unix_secs: AtomicU64,
scanner_usage_last_deferred_reason: Mutex<String>,
scanner_source_work: Vec<ScannerSourceWorkCounters>,
current_scan_cycle_source_work_start: Vec<ScannerSourceWorkCounters>,
last_scan_cycle_source_work: Vec<ScannerSourceWorkCounters>,
@@ -1224,22 +1212,6 @@ pub struct ScannerUsageFreshnessSnapshot {
pub last_usage_save_unix_secs: u64,
pub last_usage_save_result: String,
pub last_usage_save_result_code: u64,
#[serde(default)]
pub last_durable_success_unix_secs: u64,
#[serde(default)]
pub last_publication_unix_secs: u64,
#[serde(default)]
pub last_publication_state: String,
#[serde(default)]
pub last_publication_reason: String,
#[serde(default)]
pub deferred_pending: bool,
#[serde(default)]
pub deferred_total: u64,
#[serde(default)]
pub last_deferred_unix_secs: u64,
#[serde(default)]
pub last_deferred_reason: String,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
@@ -1398,14 +1370,6 @@ pub struct ScannerMetricsReport {
#[serde(default)]
pub cycle_max_directories: u64,
#[serde(default)]
pub cycle_timeout_total: u64,
#[serde(default)]
pub cycle_recovery_required_total: u64,
#[serde(default)]
pub cycle_last_progress_age: u64,
#[serde(default)]
pub leader_lease_without_progress: bool,
#[serde(default)]
pub bitrot_cycle_enabled: bool,
#[serde(default)]
pub bitrot_cycle_seconds: f64,
@@ -1466,9 +1430,6 @@ const OTEL_SCANNER_BUCKETS_SCANNED: &str = "rustfs_scanner_buckets_scanned_total
const OTEL_SCANNER_CYCLES: &str = "rustfs_scanner_cycles_total";
const OTEL_SCANNER_CYCLE_DURATION_SECONDS: &str = "rustfs_scanner_cycle_duration_seconds";
const OTEL_SCANNER_BUCKET_DRIVE_DURATION_SECONDS: &str = "rustfs_scanner_bucket_drive_duration_seconds";
const OTEL_SCANNER_CYCLE_TIMEOUT_TOTAL: &str = "rustfs_scanner_cycle_timeout_total";
const OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE: &str = "rustfs_scanner_cycle_last_progress_age";
const OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS: &str = "rustfs_scanner_leader_lease_without_progress";
fn scan_cycle_result_label(result: u8) -> &'static str {
match result {
@@ -1952,10 +1913,6 @@ impl Metrics {
scanner_cycle_max_duration_millis: AtomicU64::new(0),
scanner_cycle_max_objects: AtomicU64::new(0),
scanner_cycle_max_directories: AtomicU64::new(0),
scanner_cycle_timeout_total: AtomicU64::new(0),
scanner_cycle_recovery_required_total: AtomicU64::new(0),
scanner_cycle_last_progress_age_seconds: AtomicU64::new(0),
scanner_leader_lease_without_progress: AtomicBool::new(false),
scanner_bitrot_cycle_enabled: AtomicBool::new(false),
scanner_bitrot_cycle_millis: AtomicU64::new(0),
scanner_checkpoint: Mutex::new(None),
@@ -1969,15 +1926,7 @@ impl Metrics {
scanner_dirty_usage_last_cycle_dirty_buckets: AtomicU64::new(0),
scanner_dirty_usage_last_cycle_cleared_buckets: AtomicU64::new(0),
scanner_usage_last_save_unix_secs: AtomicU64::new(0),
scanner_usage_last_durable_success_unix_secs: AtomicU64::new(0),
scanner_usage_last_publication_unix_secs: AtomicU64::new(0),
scanner_usage_last_publication_state: Mutex::new(String::new()),
scanner_usage_last_publication_reason: Mutex::new(String::new()),
scanner_usage_last_save_result: AtomicU8::new(ScannerUsageSaveResult::Unknown as u8),
scanner_usage_deferred_pending: AtomicBool::new(false),
scanner_usage_deferred_total: AtomicU64::new(0),
scanner_usage_last_deferred_unix_secs: AtomicU64::new(0),
scanner_usage_last_deferred_reason: Mutex::new(String::new()),
scanner_source_work: ScannerWorkSource::all()
.iter()
.map(|_| ScannerSourceWorkCounters::default())
@@ -2302,44 +2251,6 @@ impl Metrics {
.store(unix_now_secs(), Ordering::Relaxed);
}
/// Record an intentional retryable usage publication deferral separately
/// from the last durable save result.
pub fn record_scanner_usage_deferred(&self, reason: impl Into<String>) {
let reason = reason.into();
self.record_scanner_usage_publication("deferred", reason.clone());
self.scanner_usage_deferred_pending.store(true, Ordering::Release);
self.scanner_usage_deferred_total.fetch_add(1, Ordering::Relaxed);
self.scanner_usage_last_deferred_unix_secs
.store(unix_now_secs(), Ordering::Relaxed);
let mut last_reason = match self.scanner_usage_last_deferred_reason.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
*last_reason = reason;
}
pub fn record_scanner_usage_durable_success(&self) {
self.record_scanner_usage_publication("success", "");
self.scanner_usage_last_durable_success_unix_secs
.store(unix_now_secs(), Ordering::Relaxed);
self.scanner_usage_deferred_pending.store(false, Ordering::Release);
}
pub fn record_scanner_usage_publication(&self, state: &str, reason: impl Into<String>) {
self.scanner_usage_last_publication_unix_secs
.store(unix_now_secs(), Ordering::Relaxed);
let mut publication_state = match self.scanner_usage_last_publication_state.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
*publication_state = state.to_string();
let mut publication_reason = match self.scanner_usage_last_publication_reason.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
*publication_reason = reason.into();
}
pub fn record_scanner_source_work(&self, source: ScannerWorkSource, work: ScannerSourceWorkUpdate) {
if let Some(counters) = self.scanner_source_work.get(source.index()) {
counters.add(work);
@@ -2501,29 +2412,12 @@ impl Metrics {
.store(cycle_max_objects.unwrap_or_default(), Ordering::Relaxed);
self.scanner_cycle_max_directories
.store(cycle_max_directories.unwrap_or_default(), Ordering::Relaxed);
self.scanner_leader_lease_without_progress.store(false, Ordering::Relaxed);
self.scanner_cycle_last_progress_age_seconds.store(0, Ordering::Relaxed);
metrics::gauge!(OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS).set(0.0);
metrics::gauge!(OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE).set(0.0);
self.scanner_bitrot_cycle_enabled
.store(bitrot_cycle.is_some(), Ordering::Relaxed);
self.scanner_bitrot_cycle_millis
.store(bitrot_cycle.map(duration_millis_saturated).unwrap_or_default(), Ordering::Relaxed);
}
pub fn record_scanner_cycle_timeout(&self, recovery_required: bool, progress_age: Duration) {
self.scanner_cycle_timeout_total.fetch_add(1, Ordering::Relaxed);
if recovery_required {
self.scanner_cycle_recovery_required_total.fetch_add(1, Ordering::Relaxed);
}
self.scanner_cycle_last_progress_age_seconds
.store(progress_age.as_secs(), Ordering::Relaxed);
self.scanner_leader_lease_without_progress.store(true, Ordering::Relaxed);
metrics::counter!(OTEL_SCANNER_CYCLE_TIMEOUT_TOTAL).increment(1);
metrics::gauge!(OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE).set(progress_age.as_secs_f64());
metrics::gauge!(OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS).set(1.0);
}
pub fn record_scanner_set_scan_state(&self, concurrency_limit: Option<usize>, queued: Option<usize>, active: Option<usize>) {
if let Some(concurrency_limit) = concurrency_limit {
self.scanner_set_scan_concurrency_limit
@@ -3362,23 +3256,6 @@ impl Metrics {
last_usage_save_unix_secs: self.scanner_usage_last_save_unix_secs.load(Ordering::Relaxed),
last_usage_save_result: usage_save_result.as_str().to_string(),
last_usage_save_result_code: usage_save_result as u8 as u64,
last_durable_success_unix_secs: self.scanner_usage_last_durable_success_unix_secs.load(Ordering::Relaxed),
last_publication_unix_secs: self.scanner_usage_last_publication_unix_secs.load(Ordering::Relaxed),
last_publication_state: match self.scanner_usage_last_publication_state.lock() {
Ok(state) => state.clone(),
Err(poisoned) => poisoned.into_inner().clone(),
},
last_publication_reason: match self.scanner_usage_last_publication_reason.lock() {
Ok(reason) => reason.clone(),
Err(poisoned) => poisoned.into_inner().clone(),
},
deferred_pending: self.scanner_usage_deferred_pending.load(Ordering::Acquire),
deferred_total: self.scanner_usage_deferred_total.load(Ordering::Relaxed),
last_deferred_unix_secs: self.scanner_usage_last_deferred_unix_secs.load(Ordering::Relaxed),
last_deferred_reason: match self.scanner_usage_last_deferred_reason.lock() {
Ok(reason) => reason.clone(),
Err(poisoned) => poisoned.into_inner().clone(),
},
};
m.throttle_idle_mode_enabled = self.scanner_throttle_idle_mode_enabled.load(Ordering::Relaxed);
m.throttle_sleep_factor = self.scanner_throttle_sleep_factor_micros.load(Ordering::Relaxed) as f64 / 1_000_000.0;
@@ -3388,10 +3265,6 @@ impl Metrics {
m.cycle_max_duration_seconds = self.scanner_cycle_max_duration_millis.load(Ordering::Relaxed) as f64 / 1000.0;
m.cycle_max_objects = self.scanner_cycle_max_objects.load(Ordering::Relaxed);
m.cycle_max_directories = self.scanner_cycle_max_directories.load(Ordering::Relaxed);
m.cycle_timeout_total = self.scanner_cycle_timeout_total.load(Ordering::Relaxed);
m.cycle_recovery_required_total = self.scanner_cycle_recovery_required_total.load(Ordering::Relaxed);
m.cycle_last_progress_age = self.scanner_cycle_last_progress_age_seconds.load(Ordering::Relaxed);
m.leader_lease_without_progress = self.scanner_leader_lease_without_progress.load(Ordering::Relaxed);
m.bitrot_cycle_enabled = self.scanner_bitrot_cycle_enabled.load(Ordering::Relaxed);
m.bitrot_cycle_seconds = self.scanner_bitrot_cycle_millis.load(Ordering::Relaxed) as f64 / 1000.0;
m.scan_checkpoint = match self.scanner_checkpoint.lock() {
@@ -4750,7 +4623,6 @@ mod tests {
metrics.record_scanner_dirty_usage_cycle_snapshot(1);
metrics.record_scanner_dirty_usage_cycle_clear(1, 1);
metrics.record_scanner_usage_save_result(ScannerUsageSaveResult::Success);
metrics.record_scanner_usage_deferred("data_movement");
let report = metrics.report().await;
@@ -4762,22 +4634,6 @@ mod tests {
assert!(report.usage_freshness.last_usage_save_unix_secs > 0);
assert_eq!(report.usage_freshness.last_usage_save_result, "success");
assert_eq!(report.usage_freshness.last_usage_save_result_code, 1);
assert!(report.usage_freshness.deferred_pending);
assert_eq!(report.usage_freshness.deferred_total, 1);
assert!(report.usage_freshness.last_deferred_unix_secs > 0);
assert_eq!(report.usage_freshness.last_deferred_reason, "data_movement");
metrics.record_scanner_usage_durable_success();
let report = metrics.report().await;
assert!(!report.usage_freshness.deferred_pending);
assert_eq!(report.usage_freshness.deferred_total, 1);
assert!(report.usage_freshness.last_durable_success_unix_secs > 0);
assert_eq!(report.usage_freshness.last_publication_state, "success");
metrics.record_scanner_usage_publication("no_update", "no_update");
let report = metrics.report().await;
assert_eq!(report.usage_freshness.last_publication_state, "no_update");
assert_eq!(report.usage_freshness.last_publication_reason, "no_update");
}
#[tokio::test]
@@ -5070,20 +4926,4 @@ mod tests {
assert!(!report.bitrot_cycle_enabled);
assert_eq!(report.bitrot_cycle_seconds, 0.0);
}
#[tokio::test]
async fn scanner_cycle_timeout_metrics_reset_for_a_new_cycle() {
let metrics = Metrics::new();
metrics.record_scanner_cycle_timeout(true, Duration::from_secs(17));
let timed_out = metrics.report().await;
assert_eq!(timed_out.cycle_timeout_total, 1);
assert_eq!(timed_out.cycle_last_progress_age, 17);
assert!(timed_out.leader_lease_without_progress);
metrics.record_scanner_cycle_config(Duration::from_secs(60), None, Some(Duration::from_secs(1)), None, None);
let current = metrics.report().await;
assert_eq!(current.cycle_timeout_total, 1);
assert_eq!(current.cycle_last_progress_age, 0);
assert!(!current.leader_lease_without_progress);
}
}
+11 -334
View File
@@ -23,32 +23,21 @@
//! unconsumed intents is the consumer's job (see `rustfs-heal`
//! `heal::mrf_queue`), mirroring MinIO's `.heal/mrf/list.bin`.
use std::collections::HashMap;
use std::collections::hash_map::RandomState;
use std::hash::{BuildHasher, Hash};
use std::sync::atomic::AtomicU64;
use std::sync::atomic::AtomicUsize;
use std::sync::{
Arc, Mutex, OnceLock,
Arc, OnceLock,
atomic::{AtomicBool, Ordering},
};
use std::time::{Duration, Instant};
use tokio::sync::mpsc;
use uuid::Uuid;
/// Bounded capacity of the global MRF channel. Backpressure is resolved by
/// dropping (and counting) intents, never by blocking the producer.
const MRF_CHANNEL_CAPACITY: usize = 8192;
const MRF_COALESCER_SHARDS: usize = 16;
const MRF_COALESCER_MAX_KEYS: usize = 8192;
const MRF_COALESCER_MAX_BYTES: usize = 16 * 1024 * 1024;
const MRF_COALESCER_TTL: Duration = Duration::from_secs(60);
const MRF_MAX_IDENTITY_COMPONENT: usize = 1024;
/// Why an intent was produced. Drives the heal priority mapping on the
/// consumer side (DecodeFailure -> Urgent, MetadataCorruption -> High,
/// PartialWrite -> Normal).
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum MrfKind {
/// Erasure decode failed while serving a read (read path).
DecodeFailure,
@@ -78,52 +67,12 @@ pub struct MrfIntent {
/// Version the intent targets, as raw UUID bytes.
pub version_id: Option<[u8; 16]>,
pub kind: MrfKind,
/// Stable erasure-set scope when the producer has it. Kept optional so
/// metadata corruption and legacy producers do not invent a scope.
pub scope: Option<MrfScope>,
/// Generation of the node-local ingress lease. It is not persisted in
/// the journal; replayed records acquire a fresh lease when re-enqueued.
pub lease: Option<MrfIngressLease>,
pub enqueued_at_ms: u64,
/// Times this intent has already been offered to the heal manager.
/// Dropped by the consumer once it reaches `MRF_MAX_ATTEMPTS`.
pub attempts: u8,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct MrfScope {
pub pool_index: u32,
pub set_index: u32,
}
/// Opaque generation used to release exactly the admission that created an
/// ingress entry. A generation prevents a late terminal callback from
/// deleting a newer retry for the same identity (ABA).
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct MrfIngressLease(u64);
impl MrfIngressLease {
const fn new(value: u64) -> Self {
Self(value)
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum MrfDropReason {
Disabled,
Uninitialized,
Full,
OversizedIdentity,
CoalescerFull,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum MrfIngressResult {
Enqueued,
Coalesced,
Dropped(MrfDropReason),
}
/// Consumer-side retry ceiling before an intent is given up on.
pub const MRF_MAX_ATTEMPTS: u8 = 3;
@@ -138,159 +87,6 @@ impl MrfIntent {
static GLOBAL_MRF_SENDER: OnceLock<mpsc::Sender<MrfIntent>> = OnceLock::new();
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
struct MrfIdentityKey {
kind: MrfKind,
bucket: Arc<str>,
object: Arc<str>,
version_id: Option<[u8; 16]>,
scope: Option<MrfScope>,
}
#[derive(Debug)]
struct IngressEntry {
lease: MrfIngressLease,
expires_at: Instant,
bytes: usize,
}
type MrfCoalescerShard = Mutex<HashMap<MrfIdentityKey, IngressEntry>>;
type MrfCoalescer = Box<[MrfCoalescerShard]>;
static MRF_COALESCER: OnceLock<MrfCoalescer> = OnceLock::new();
static NEXT_MRF_LEASE: AtomicU64 = AtomicU64::new(1);
static MRF_COALESCER_COUNT: AtomicUsize = AtomicUsize::new(0);
static MRF_COALESCER_BYTES: AtomicUsize = AtomicUsize::new(0);
static MRF_HASH_STATE: OnceLock<RandomState> = OnceLock::new();
fn coalescer() -> &'static [MrfCoalescerShard] {
MRF_COALESCER.get_or_init(|| {
(0..MRF_COALESCER_SHARDS)
.map(|_| Mutex::new(HashMap::new()))
.collect::<Vec<_>>()
.into_boxed_slice()
})
}
fn key_shard(key: &MrfIdentityKey) -> usize {
let hash = MRF_HASH_STATE.get_or_init(RandomState::new).hash_one(key);
usize::try_from(hash).unwrap_or(0) % MRF_COALESCER_SHARDS
}
fn canonical_version(version_id: Option<Uuid>) -> Option<[u8; 16]> {
version_id
.filter(|version| !version.is_nil())
.map(|version| *version.as_bytes())
}
fn canonical_identity(
kind: MrfKind,
version_id: Option<[u8; 16]>,
scope: Option<MrfScope>,
) -> (Option<[u8; 16]>, Option<MrfScope>) {
let version_id = version_id.filter(|bytes| *bytes != [0; 16]);
match kind {
MrfKind::MetadataCorruption => (None, None),
MrfKind::DecodeFailure | MrfKind::PartialWrite => (version_id, scope),
}
}
fn identity_estimated_bytes(key: &MrfIdentityKey) -> usize {
64usize
.saturating_add(key.bucket.len())
.saturating_add(key.object.len())
.saturating_add(key.version_id.map_or(0, |_| 16))
.saturating_add(key.scope.map_or(0, |_| 8))
}
fn reserve(counter: &AtomicUsize, limit: usize, amount: usize) -> bool {
let mut current = counter.load(Ordering::Relaxed);
loop {
let Some(next) = current.checked_add(amount) else {
return false;
};
if next > limit {
return false;
}
match counter.compare_exchange_weak(current, next, Ordering::Relaxed, Ordering::Relaxed) {
Ok(_) => return true,
Err(observed) => current = observed,
}
}
}
fn coalescer_admit(key: MrfIdentityKey) -> Result<MrfIngressLease, MrfIngressResult> {
let shard = key_shard(&key);
let mut entries = coalescer()[shard]
.lock()
.map_err(|_| MrfIngressResult::Dropped(MrfDropReason::CoalescerFull))?;
let now = Instant::now();
let before = entries.len();
let mut expired_bytes = 0usize;
entries.retain(|_, entry| {
if entry.expires_at > now {
true
} else {
expired_bytes = expired_bytes.saturating_add(entry.bytes);
false
}
});
let evicted = before.saturating_sub(entries.len());
if evicted > 0 {
MRF_COALESCER_COUNT.fetch_sub(evicted, Ordering::Relaxed);
MRF_COALESCER_BYTES.fetch_sub(expired_bytes, Ordering::Relaxed);
let evicted = u64::try_from(evicted).unwrap_or(u64::MAX);
metrics::counter!("rustfs_heal_mrf_coalescer_expired_total").increment(evicted);
metrics::counter!("rustfs_heal_mrf_coalescer_evictions_total").increment(evicted);
}
if entries.contains_key(&key) {
metrics::counter!("rustfs_heal_mrf_coalesced_total").increment(1);
return Err(MrfIngressResult::Coalesced);
}
let bytes = identity_estimated_bytes(&key);
let count_reserved = reserve(&MRF_COALESCER_COUNT, MRF_COALESCER_MAX_KEYS, 1);
let bytes_reserved = count_reserved && reserve(&MRF_COALESCER_BYTES, MRF_COALESCER_MAX_BYTES, bytes);
if !count_reserved || !bytes_reserved {
if count_reserved {
MRF_COALESCER_COUNT.fetch_sub(1, Ordering::Relaxed);
}
metrics::counter!("rustfs_heal_mrf_dropped_total", "reason" => "coalescer_full").increment(1);
return Err(MrfIngressResult::Dropped(MrfDropReason::CoalescerFull));
}
let lease = MrfIngressLease::new(NEXT_MRF_LEASE.fetch_add(1, Ordering::Relaxed));
if entries
.insert(
key,
IngressEntry {
lease,
expires_at: now + MRF_COALESCER_TTL,
bytes,
},
)
.is_some()
{
MRF_COALESCER_COUNT.fetch_sub(1, Ordering::Relaxed);
MRF_COALESCER_BYTES.fetch_sub(bytes, Ordering::Relaxed);
metrics::counter!("rustfs_heal_mrf_coalesced_total").increment(1);
return Err(MrfIngressResult::Coalesced);
}
Ok(lease)
}
fn coalescer_release(key: &MrfIdentityKey, lease: Option<MrfIngressLease>) {
let Some(lease) = lease else {
return;
};
if let Ok(mut entries) = coalescer()[key_shard(key)].lock() {
let should_remove = entries.get(key).is_some_and(|entry| entry.lease == lease);
if should_remove {
let bytes = entries.remove(key).map(|entry| entry.bytes).unwrap_or(0);
MRF_COALESCER_COUNT.fetch_sub(1, Ordering::Relaxed);
MRF_COALESCER_BYTES.fetch_sub(bytes, Ordering::Relaxed);
}
}
}
/// Delivery kill-switch, set from `RUSTFS_HEAL_MRF_ENABLE`. Producers check
/// this before touching the channel so the disabled path stays allocation- and
/// sync-free.
@@ -326,90 +122,21 @@ pub fn init_mrf_channel() -> Result<mpsc::Receiver<MrfIntent>, &'static str> {
/// This runs on IO error paths, so it stays synchronous and cheap: one
/// bounded allocation for the two `Arc<str>` handles plus the channel slot.
pub fn try_send_mrf_intent(kind: MrfKind, bucket: &str, object: &str, version_id: Option<Uuid>) -> bool {
matches!(
try_send_mrf_intent_typed(kind, bucket, object, version_id, None),
MrfIngressResult::Enqueued
)
}
/// Typed ingress result. `Coalesced` means an equivalent in-flight channel
/// intent already exists; it is not a second executable or durable admission.
pub fn try_send_mrf_intent_typed(
kind: MrfKind,
bucket: &str,
object: &str,
version_id: Option<Uuid>,
scope: Option<MrfScope>,
) -> MrfIngressResult {
if !mrf_delivery_enabled() {
return MrfIngressResult::Dropped(MrfDropReason::Disabled);
return false;
}
let Some(sender) = GLOBAL_MRF_SENDER.get() else {
return MrfIngressResult::Dropped(MrfDropReason::Uninitialized);
};
if bucket.len() > MRF_MAX_IDENTITY_COMPONENT || object.len() > MRF_MAX_IDENTITY_COMPONENT {
return MrfIngressResult::Dropped(MrfDropReason::OversizedIdentity);
}
let (version_id, scope) = canonical_identity(kind, canonical_version(version_id), scope);
let key = MrfIdentityKey {
kind,
bucket: Arc::from(bucket),
object: Arc::from(object),
version_id,
scope,
};
let lease = match coalescer_admit(key.clone()) {
Ok(lease) => lease,
Err(result) => return result,
return false;
};
let intent = MrfIntent {
bucket: key.bucket.clone(),
object: key.object.clone(),
version_id: key.version_id,
bucket: Arc::from(bucket),
object: Arc::from(object),
version_id: version_id.map(|vid| *vid.as_bytes()),
kind,
scope,
lease: Some(lease),
enqueued_at_ms: unix_now_ms(),
attempts: 0,
};
match sender.try_send(intent) {
Ok(()) => MrfIngressResult::Enqueued,
Err(mpsc::error::TrySendError::Full(_)) => {
coalescer_release(&key, Some(lease));
metrics::counter!("rustfs_heal_mrf_dropped_total", "reason" => "channel_full").increment(1);
MrfIngressResult::Dropped(MrfDropReason::Full)
}
Err(mpsc::error::TrySendError::Closed(_)) => {
coalescer_release(&key, Some(lease));
MrfIngressResult::Dropped(MrfDropReason::Uninitialized)
}
}
}
/// Release the ingress key once the consumer owns the intent.
pub fn release_mrf_intent(intent: &MrfIntent) {
release_mrf_identity(intent.kind, &intent.bucket, &intent.object, intent.version_id, intent.scope, intent.lease);
}
pub fn release_mrf_identity(
kind: MrfKind,
bucket: &str,
object: &str,
version_id: Option<[u8; 16]>,
scope: Option<MrfScope>,
lease: Option<MrfIngressLease>,
) {
let (version_id, scope) = canonical_identity(kind, version_id, scope);
coalescer_release(
&MrfIdentityKey {
kind,
bucket: Arc::from(bucket),
object: Arc::from(object),
version_id,
scope,
},
lease,
);
sender.try_send(intent).is_ok()
}
fn unix_now_ms() -> u64 {
@@ -417,8 +144,7 @@ fn unix_now_ms() -> u64 {
// failure would be a bug rather than something to handle here.
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.ok()
.and_then(|d| u64::try_from(d.as_millis()).ok())
.map(|d| d.as_millis() as u64)
.unwrap_or(0)
}
@@ -489,60 +215,12 @@ mod tests {
object: Arc::from("object"),
version_id: Some([0u8; 16]),
kind: MrfKind::DecodeFailure,
scope: None,
lease: None,
enqueued_at_ms: 0,
attempts: 0,
};
assert!(intent.estimated_bytes() >= intent.bucket.len() + intent.object.len());
}
#[test]
fn ingress_duplicate_identity_coalesces_and_releases_for_retry() {
let key = MrfIdentityKey {
kind: MrfKind::DecodeFailure,
bucket: Arc::from("ingress-test-bucket"),
object: Arc::from("ingress-test-object"),
version_id: Some([9; 16]),
scope: Some(MrfScope {
pool_index: 3,
set_index: 4,
}),
};
let lease = coalescer_admit(key.clone()).expect("first identity should be admitted");
for _ in 0..999 {
assert_eq!(coalescer_admit(key.clone()), Err(MrfIngressResult::Coalesced));
}
coalescer_release(&key, Some(lease));
let retry_lease = coalescer_admit(key.clone()).expect("released identity must admit a retry");
coalescer_release(&key, Some(retry_lease));
}
#[test]
fn ingress_identity_preserves_kind_scope_and_version_boundaries() {
let (nil_version, nil_scope) = canonical_identity(
MrfKind::DecodeFailure,
Some([0; 16]),
Some(MrfScope {
pool_index: 1,
set_index: 2,
}),
);
assert_eq!(nil_version, None, "nil UUID is the unversioned identity");
assert!(nil_scope.is_some());
let (metadata_version, metadata_scope) = canonical_identity(
MrfKind::MetadataCorruption,
Some([7; 16]),
Some(MrfScope {
pool_index: 1,
set_index: 2,
}),
);
assert_eq!(metadata_version, None);
assert_eq!(metadata_scope, None);
}
#[tokio::test]
async fn try_send_delivers_and_respects_capacity() {
let mut receiver = init_mrf_channel().expect("first initialization should succeed");
@@ -552,7 +230,6 @@ mod tests {
let intent = receiver.recv().await.expect("intent should arrive");
assert_eq!(intent.kind, MrfKind::DecodeFailure);
assert_eq!(intent.bucket.as_ref(), "b");
release_mrf_intent(&intent);
// Disable delivery: producers become no-ops.
set_mrf_delivery_enabled(false);
@@ -562,8 +239,8 @@ mod tests {
// Fill the bounded channel past capacity: excess intents are dropped,
// never blocking.
let mut accepted = 0;
for index in 0..(MRF_CHANNEL_CAPACITY + 64) {
if try_send_mrf_intent(MrfKind::PartialWrite, "b", &format!("o-{index}"), None) {
for _ in 0..(MRF_CHANNEL_CAPACITY + 64) {
if try_send_mrf_intent(MrfKind::PartialWrite, "b", "o", None) {
accepted += 1;
}
}
-6
View File
@@ -84,12 +84,6 @@ Current guidance:
- `RUSTFS_SCANNER_CYCLE_MAX_OBJECTS` (canonical)
- `RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES` (canonical)
Scanner cycle budget controls:
- When `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` is unset, the finite default is 1800 seconds (30 minutes), matching the scanner benchmark guidance.
- An explicit `0` preserves the compatibility behavior of an unbounded runtime budget. Object and directory budgets likewise remain unbounded when explicitly set to `0`.
- A timed-out cycle cancels cooperative scanner work, then fences its leader epoch before releasing the lease. An uncooperative I/O operation is dropped after the bounded shutdown window; its cursor is not claimed to be durable and the scanner reports `recovery-required` when the worker cannot stop cooperatively, the cycle state was not confirmed durable, or epoch fencing cannot be persisted.
## Mmap read environment aliases
- `RUSTFS_OBJECT_MMAP_READ_ENABLE` (canonical)
+2 -2
View File
@@ -60,9 +60,9 @@ pub const DEFAULT_RNG_SEED: Option<u64> = None; // None means random
/// Dedicated blocking thread pool for fsync/fdatasync operations.
/// When > 1, fsync operations are isolated from the main blocking pool to
/// prevent device-bound fsync from starving read operations (pread/stat/open).
/// Default 64 isolates fsync from the main blocking pool to prevent device-bound fsync from starving read I/O.
/// Default 0 means auto (no isolation, use main runtime).
pub const ENV_FSYNC_BLOCKING_THREADS: &str = "RUSTFS_RUNTIME_FSYNC_BLOCKING_THREADS";
pub const DEFAULT_FSYNC_BLOCKING_THREADS: usize = 64;
pub const DEFAULT_FSYNC_BLOCKING_THREADS: usize = 0;
// Dial9 Tokio Telemetry Default values
pub const DEFAULT_RUNTIME_DIAL9_ENABLED: bool = false; // Disabled by default
+3 -6
View File
@@ -143,12 +143,9 @@ pub const ENV_SCANNER_MAX_WAIT_SECS: &str = "RUSTFS_SCANNER_MAX_WAIT_SECS";
/// Default scanner speed preset.
pub const DEFAULT_SCANNER_SPEED: &str = "default";
/// Default scanner cycle runtime budget when no override is configured.
///
/// An explicit `0` remains the compatibility escape hatch for an unbounded
/// cycle. Keeping the unset default finite prevents a stalled scanner I/O
/// operation from holding the leader lease forever.
pub const DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS: u64 = 30 * 60;
/// Default scanner cycle runtime budget.
/// `0` keeps the existing unbounded per-cycle behavior.
pub const DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS: u64 = 0;
/// Default scanner per-cycle object budget.
/// `0` keeps the existing unbounded per-cycle behavior.
File diff suppressed because it is too large Load Diff
+10 -13
View File
@@ -123,7 +123,7 @@ via `create_s3_client(idx)` / `create_all_clients()`. See
| `find_available_port` | Random free port (isolation primitive) |
| `rustfs_binary_path` / `_with_features` | Locate/build the binary; honors `RUSTFS_BUILD_FEATURES` |
| `requested_rustfs_build_features` / `rustfs_build_feature_enabled` | Feature-gate a test to what the binary was built with |
| `execute_awscurl` / `awscurl_post` / `_get` / `_put` / `_delete` / `awscurl_post_sts_form_urlencoded` | Admin/STS API calls via `awscurl`; missing binaries are test failures |
| `awscurl_available` + `execute_awscurl` / `awscurl_post` / `_get` / `_put` / `_delete` / `awscurl_post_sts_form_urlencoded` | Admin/STS API calls via `awscurl` (skip gracefully when absent) |
| `replication_fast_env` | Env vars that shrink replication timers (from repl-4); pass to `start_rustfs_server_with_env` |
| `local_http_client` / `init_logging` | Loopback HTTP client; idempotent tracing init |
| `RustFSTestClusterEnvironment` (`new`/`start`/`start_node`/`stop_node`/`create_all_clients`) | Multi-node harness |
@@ -189,7 +189,7 @@ cargo nextest run --profile e2e-smoke -p e2e_test
cargo nextest run --profile e2e-full -p e2e_test
# Cluster fault nightly lane
cargo nextest run --profile e2e-nightly -p e2e_test
# Replication nightly lane; awscurl is required for STS paths
# Replication nightly lane; install awscurl so STS paths do not skip
cargo nextest run --profile e2e-repl-nightly -p e2e_test
# Fixed-port protocol nightly lane
RUSTFS_BUILD_FEATURES=ftps,webdav,sftp \
@@ -221,8 +221,9 @@ The `s3s-e2e` CI job selects a random `RUSTFS_TEST_PORT` (see the `e2e-tests`
job) to dodge this; local single-node tests already use random ports, so a
lingering orphan is usually the cause of a spurious bind failure.
**`awscurl` not found.** `awscurl`-dependent tests fail closed with a process
spawn error. Install the pinned CI version before running their profiles.
**`awscurl` not found.** `awscurl`-dependent tests skip gracefully with a
visible log line (`awscurl_available()`); install `awscurl` to actually run
them.
## Related
@@ -257,9 +258,10 @@ A test module may join the smoke filter only if every test in it is:
2. **Single-node** — spawns its own server via
`RustFSTestEnvironment`/`start_rustfs_server` on a random port with an
isolated temp dir. No `RustFSTestClusterEnvironment`, no fixed ports.
3. **Hermetic dependencies** — no pre-started server at `localhost:9000`, no
Vault, and no fixed protocol ports. Any required CLI must be pinned and
installed by the workflow; a missing CLI must fail the test.
3. **Dependency-free** — no pre-started server at `localhost:9000`, no Vault,
no fixed protocol ports. Tools that may be absent on the runner (e.g.
`awscurl`) are acceptable only when the test skips gracefully with a
visible log line (see `bucket_policy_check_test.rs`).
4. **Not `#[ignore]`** — ignored tests are activation work (backlog#1149
ci-13 / backlog#1148 ilm-3), not smoke candidates.
@@ -276,9 +278,4 @@ listed by `cargo nextest list -p e2e_test`. Regenerate it when adding or
moving e2e tests so acceptance numbers in the test-strategy issues
(backlog#1147#1155) stay auditable. When a profile membership change is
intentional, review its JSON listing before updating the matching
`.config/e2e-*-selection.txt` test-ID digest. Update only the platform that
produced the listing:
```bash
python3 scripts/check_test_wiring.py --update-profile e2e-full /path/to/listing.json linux
```
`.config/e2e-*-selection.txt` test-ID digest.
@@ -52,6 +52,10 @@ fn create_user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key:
#[tokio::test]
async fn test_bucket_policy_authenticated_user() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if !crate::common::awscurl_available() {
info!("Skipping test_bucket_policy_authenticated_user because awscurl is not available");
return Ok(());
}
info!("Starting test_bucket_policy_authenticated_user...");
let mut env = RustFSTestEnvironment::new().await?;
+16 -77
View File
@@ -15,27 +15,16 @@
use crate::common::RustFSTestClusterEnvironment;
use aws_sdk_s3::Client;
use aws_sdk_s3::error::SdkError;
use aws_sdk_s3::types::{CorsConfiguration, CorsRule};
use bytes::Bytes;
use std::sync::Arc;
use tokio::sync::Barrier;
use tracing::{info, warn};
const BUCKET: &str = "conditional-put-race-bucket";
const BUCKET_METADATA_RELOAD_BUCKET: &str = "bucket-metadata-reload-barrier";
async fn cleanup_object(client: &Client, key: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
client.delete_object().bucket(BUCKET).key(key).send().await?;
Ok(())
}
async fn assert_bucket_cors_missing(client: &Client) {
let result = client.get_bucket_cors().bucket(BUCKET_METADATA_RELOAD_BUCKET).send().await;
match result {
Err(SdkError::ServiceError(error)) => {
assert_eq!(error.err().meta().code(), Some("NoSuchCORSConfiguration"));
}
result => panic!("expected the peer to report a missing CORS configuration: {result:?}"),
async fn cleanup_object(client: &Client, key: &str) {
if let Err(e) = client.delete_object().bucket(BUCKET).key(key).send().await {
warn!("Failed to delete object '{}' from bucket '{}' during cleanup: {:?}", key, BUCKET, e);
}
}
@@ -82,13 +71,14 @@ async fn run_race_iteration(
test_key: &str,
iteration: usize,
) -> Result<usize, Box<dyn std::error::Error + Send + Sync>> {
cleanup_object(&clients[0], test_key).await?;
cleanup_object(&clients[0], test_key).await;
tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
match clients[0].head_object().bucket(BUCKET).key(test_key).send().await {
Ok(_) => return Err(format!("object still exists after cleanup in iteration {iteration}").into()),
Err(error) if error.as_service_error().is_some_and(|error| error.is_not_found()) => {}
Err(error) => return Err(format!("failed to verify cleanup in iteration {iteration}: {error:?}").into()),
let head_result = clients[0].head_object().bucket(BUCKET).key(test_key).send().await;
if head_result.is_ok() {
warn!("Warning: Object still exists after cleanup, skipping iteration {}", iteration);
return Ok(0);
}
info!("\n=== Iteration {} ===", iteration);
@@ -130,16 +120,14 @@ async fn run_race_iteration(
info!("Result: {} out of {} succeeded", success_count, clients.len());
if had_error {
return Err("one or more conditional PUTs failed unexpectedly".into());
}
if success_count > 1 {
info!(">>> RACE CONDITION DETECTED!");
} else if success_count == 1 {
info!(">>> Correct behavior: exactly 1 writer succeeded.");
} else if had_error {
return Err("all conditional PUTs failed (e.g. cluster/bucket not ready)".into());
} else {
return Err("no conditional PUT succeeded".into());
info!(">>> Unexpected: no writers succeeded.");
}
Ok(success_count)
@@ -179,7 +167,7 @@ async fn test_conditional_put_race_cluster() -> Result<(), Box<dyn std::error::E
}
}
cleanup_object(&clients[0], &test_key).await?;
cleanup_object(&clients[0], &test_key).await;
tokio::time::sleep(tokio::time::Duration::from_millis(50)).await;
}
@@ -189,7 +177,7 @@ async fn test_conditional_put_race_cluster() -> Result<(), Box<dyn std::error::E
info!("Total iterations: {}", iterations);
info!("Correct (1 winner): {}", correct_count);
info!("Race conditions: {}", races_detected);
info!("Failed iterations: {}", error_count);
info!("Errors (skipped): {}", error_count);
assert_eq!(races_detected, 0, "Race conditions detected: {}/{}", races_detected, iterations);
assert_eq!(
@@ -197,10 +185,6 @@ async fn test_conditional_put_race_cluster() -> Result<(), Box<dyn std::error::E
"{} iteration(s) failed due to errors (e.g. cluster not ready)",
error_count
);
assert_eq!(
correct_count, iterations,
"only {correct_count}/{iterations} iterations observed exactly one winner"
);
Ok(())
}
@@ -217,7 +201,7 @@ async fn test_conditional_put_basic_cluster() -> Result<(), Box<dyn std::error::
let client = cluster.create_s3_client(0)?;
let test_key = "basic-conditional-put";
cleanup_object(&client, test_key).await?;
cleanup_object(&client, test_key).await;
let result = client
.put_object()
@@ -249,51 +233,6 @@ async fn test_conditional_put_basic_cluster() -> Result<(), Box<dyn std::error::
assert_eq!(code, "PreconditionFailed");
}
cleanup_object(&client, test_key).await?;
Ok(())
}
#[tokio::test]
async fn test_bucket_cors_write_is_visible_on_peer_before_response() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
crate::common::init_logging();
let mut cluster = RustFSTestClusterEnvironment::new(2).await?;
cluster.start().await?;
cluster.create_test_bucket(BUCKET_METADATA_RELOAD_BUCKET).await?;
let writer = cluster.create_s3_client(0)?;
let reader = cluster.create_s3_client(1)?;
assert_bucket_cors_missing(&reader).await;
let rule = CorsRule::builder()
.allowed_methods("GET")
.allowed_origins("https://example.com")
.build()?;
let configuration = CorsConfiguration::builder().cors_rules(rule).build()?;
writer
.put_bucket_cors()
.bucket(BUCKET_METADATA_RELOAD_BUCKET)
.cors_configuration(configuration)
.send()
.await?;
let response = reader.get_bucket_cors().bucket(BUCKET_METADATA_RELOAD_BUCKET).send().await?;
let rules = response.cors_rules();
assert_eq!(
rules.len(),
1,
"peer should observe the committed CORS rule before the write response returns"
);
assert_eq!(rules[0].allowed_methods(), ["GET"]);
assert_eq!(rules[0].allowed_origins(), ["https://example.com"]);
writer
.delete_bucket_cors()
.bucket(BUCKET_METADATA_RELOAD_BUCKET)
.send()
.await?;
assert_bucket_cors_missing(&reader).await;
writer.delete_bucket().bucket(BUCKET_METADATA_RELOAD_BUCKET).send().await?;
cleanup_object(&client, test_key).await;
Ok(())
}
+7 -28
View File
@@ -494,20 +494,15 @@ fn awscurl_binary_path() -> PathBuf {
.unwrap_or_else(|| PathBuf::from("awscurl"))
}
fn verify_awscurl_path(path: &Path) -> std::io::Result<()> {
let output = Command::new(path).arg("--help").output()?;
if output.status.success() {
return Ok(());
pub fn awscurl_available() -> bool {
let path = awscurl_binary_path();
if path.components().count() > 1 || path.is_absolute() {
return path.is_file();
}
Err(std::io::Error::other(format!(
"awscurl prerequisite check failed: {}",
String::from_utf8_lossy(&output.stderr).trim()
)))
}
pub fn require_awscurl() -> std::io::Result<()> {
verify_awscurl_path(&awscurl_binary_path())
std::env::var_os("PATH")
.map(|paths| std::env::split_paths(&paths).any(|dir| dir.join(&path).is_file()))
.unwrap_or(false)
}
// Global initialization
@@ -1752,22 +1747,6 @@ mod tests {
assert_eq!(normalize_rustfs_build_features(" , "), None);
}
#[test]
fn missing_awscurl_is_a_prerequisite_failure() {
let missing = std::env::temp_dir().join(format!("missing-awscurl-{}", Uuid::new_v4()));
let error = verify_awscurl_path(&missing).expect_err("a missing awscurl binary must fail the test prerequisite");
assert_eq!(error.kind(), ErrorKind::NotFound);
}
#[test]
fn available_awscurl_client_passes_prerequisite_check() {
let executable = std::env::current_exe().expect("the test executable should have a path");
verify_awscurl_path(&executable).expect("an available client with a working help command should pass");
}
#[test]
fn capture_log_path_uses_temp_directory_basename() {
assert_eq!(
@@ -16,7 +16,9 @@
//! session policy** (`Policy` parameter) via `awscurl --service sts` with explicit
//! `Content-Type: application/x-www-form-urlencoded` on `POST /`.
use crate::common::{RustFSTestEnvironment, awscurl_delete, awscurl_post_sts_form_urlencoded, awscurl_put, init_logging};
use crate::common::{
RustFSTestEnvironment, awscurl_available, awscurl_delete, awscurl_post_sts_form_urlencoded, awscurl_put, init_logging,
};
use aws_sdk_s3::config::{Credentials, Region};
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{Delete, ObjectIdentifier, Tag, Tagging};
@@ -173,6 +175,11 @@ async fn cleanup_bucket_and_object(admin: &Client, bucket: &str, key: &str) {
#[tokio::test]
async fn test_e2e_iam_policy_existing_object_tag_get_object() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if !awscurl_available() {
info!("Skipping test_e2e_iam_policy_existing_object_tag_get_object: awscurl not available");
return Ok(());
}
let suffix = Uuid::new_v4();
let user = format!("e2eiamtag-{suffix}");
let user_secret = "longSecretKeyForTest123!";
@@ -226,6 +233,11 @@ async fn test_e2e_iam_policy_existing_object_tag_get_object() -> Result<(), Box<
#[tokio::test]
async fn test_e2e_bucket_policy_existing_object_tag_get_object() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if !awscurl_available() {
info!("Skipping test_e2e_bucket_policy_existing_object_tag_get_object: awscurl not available");
return Ok(());
}
let suffix = Uuid::new_v4();
let user = format!("e2ebptag-{suffix}");
let user_secret = "longSecretKeyForTest456!";
@@ -282,6 +294,11 @@ async fn test_e2e_bucket_policy_existing_object_tag_get_object() -> Result<(), B
#[tokio::test]
async fn test_e2e_sts_assume_role_session_policy_existing_object_tag() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if !awscurl_available() {
info!("Skipping test_e2e_sts_assume_role_session_policy_existing_object_tag: awscurl not available");
return Ok(());
}
let suffix = Uuid::new_v4();
let parent = format!("e2e-sts-par-{suffix}");
let parent_secret = "longSecretKeyForParentSts99!";
@@ -353,6 +370,11 @@ async fn test_e2e_sts_assume_role_session_policy_existing_object_tag() -> Result
#[tokio::test]
async fn test_e2e_sts_session_policy_delete_objects_object_prefix_only() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if !awscurl_available() {
info!("Skipping test_e2e_sts_session_policy_delete_objects_object_prefix_only: awscurl not available");
return Ok(());
}
let suffix = Uuid::new_v4();
let parent = format!("e2e-sts-del-par-{suffix}");
let parent_secret = "longSecretKeyForParentDelete99!";
+36 -118
View File
@@ -14,7 +14,7 @@
//! E2E tests for group management (fixes #2028).
use crate::common::{RustFSTestEnvironment, admin_ok, admin_request, init_logging};
use crate::common::{RustFSTestEnvironment, admin_request, awscurl_delete, awscurl_get, awscurl_put, init_logging};
use aws_sdk_s3::config::{Credentials, Region};
use aws_sdk_s3::{Client, Config};
use tracing::info;
@@ -83,6 +83,7 @@ async fn update_group_members_rejects_invalid_new_group_names() -> Result<(), Bo
/// Test that deleting a group with members fails, and deleting an empty group succeeds.
#[tokio::test(flavor = "multi_thread")]
#[ignore = "requires awscurl and spawns a real RustFS server"]
async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
@@ -90,58 +91,29 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std
env.start_rustfs_server(vec![]).await?;
// 1. Create a user
let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey=testuser1", env.url);
let user_body = serde_json::json!({
"secretKey": "testuser1secret",
"status": "enabled"
});
admin_ok(
&env,
http::Method::PUT,
"/rustfs/admin/v3/add-user?accessKey=testuser1",
Some(user_body.to_string()),
)
.await?;
awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
info!("Created testuser1");
// 2. Create a group with testuser1 as a member
let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
let add_member_body = serde_json::json!({
"group": "testgroup",
"members": ["testuser1"],
"isRemove": false,
"groupStatus": "enabled"
});
admin_ok(
&env,
http::Method::PUT,
"/rustfs/admin/v3/update-group-members",
Some(add_member_body.to_string()),
)
.await?;
awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
info!("Added testuser1 to testgroup");
// 3. Attempt to delete the group while it still has members — should fail
let (delete_status, delete_body) = admin_request(
&env.url,
http::Method::DELETE,
"/rustfs/admin/v3/group/testgroup",
None,
&env.access_key,
&env.secret_key,
)
.await?;
assert_eq!(
delete_status,
reqwest::StatusCode::BAD_REQUEST,
"deleting a non-empty group must return HTTP 400, body: {delete_body}"
);
assert!(
delete_body.contains("<Code>InvalidRequest</Code>"),
"deleting a non-empty group must return InvalidRequest, body: {delete_body}"
);
assert!(
delete_body.contains("<Message>group is not empty</Message>"),
"deleting a non-empty group returned an unexpected message: {delete_body}"
);
let delete_group_url = format!("{}/rustfs/admin/v3/group/testgroup", env.url);
let delete_result = awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await;
assert!(delete_result.is_err(), "deleting a non-empty group should fail");
info!("Delete of non-empty group correctly rejected");
// 4. Remove the member from the group
@@ -151,42 +123,17 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std
"isRemove": true,
"groupStatus": "enabled"
});
admin_ok(
&env,
http::Method::PUT,
"/rustfs/admin/v3/update-group-members",
Some(remove_member_body.to_string()),
)
.await?;
awscurl_put(&update_members_url, &remove_member_body.to_string(), &env.access_key, &env.secret_key).await?;
info!("Removed testuser1 from testgroup");
// 5. Delete the now-empty group — should succeed
admin_ok(&env, http::Method::DELETE, "/rustfs/admin/v3/group/testgroup", None).await?;
awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await?;
info!("Deleted empty testgroup successfully");
// 6. Verify the group no longer exists
let (get_status, get_body) = admin_request(
&env.url,
http::Method::GET,
"/rustfs/admin/v3/group?group=testgroup",
None,
&env.access_key,
&env.secret_key,
)
.await?;
assert_eq!(
get_status,
reqwest::StatusCode::NOT_FOUND,
"a deleted group must return HTTP 404, body: {get_body}"
);
assert!(
get_body.contains("<Code>NoSuchResource</Code>"),
"a deleted group must return NoSuchResource, body: {get_body}"
);
assert!(
get_body.contains("<Message>group &apos;testgroup&apos; does not exist</Message>"),
"a deleted group returned an unexpected message: {get_body}"
);
let get_group_url = format!("{}/rustfs/admin/v3/group?group=testgroup", env.url);
let get_result = awscurl_get(&get_group_url, &env.access_key, &env.secret_key).await;
assert!(get_result.is_err(), "group should no longer exist after deletion");
info!("Confirmed testgroup no longer exists");
Ok(())
@@ -195,6 +142,7 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std
/// Test that a user with only group membership (no explicit user policy) gets group policies
/// and can perform actions allowed by the group (regression test for #2028.1).
#[tokio::test(flavor = "multi_thread")]
#[ignore = "requires awscurl and spawns a real RustFS server"]
async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
@@ -212,56 +160,39 @@ async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn s
"Statement": [{
"Effect": "Allow",
"Action": ["s3:ListAllMyBuckets"],
"Resource": ["arn:aws:s3:::*"]
"Resource": ["*"]
}]
});
admin_ok(
&env,
http::Method::PUT,
&format!("/rustfs/admin/v3/add-canned-policy?name={policy_name}"),
Some(policy_doc.to_string()),
)
.await?;
let add_policy_url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", env.url, policy_name);
awscurl_put(&add_policy_url, &policy_doc.to_string(), &env.access_key, &env.secret_key).await?;
info!("Created canned policy {}", policy_name);
// 2. Create user with no explicit policy
let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, user_name);
let user_body = serde_json::json!({
"secretKey": user_secret,
"status": "enabled"
});
admin_ok(
&env,
http::Method::PUT,
&format!("/rustfs/admin/v3/add-user?accessKey={user_name}"),
Some(user_body.to_string()),
)
.await?;
awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
info!("Created user {} with no explicit policy", user_name);
// 3. Add user to group (creates group with this member; user_group_memberships must be updated)
let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
let add_member_body = serde_json::json!({
"group": group_name,
"members": [user_name],
"isRemove": false,
"groupStatus": "enabled"
});
admin_ok(
&env,
http::Method::PUT,
"/rustfs/admin/v3/update-group-members",
Some(add_member_body.to_string()),
)
.await?;
awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
info!("Added {} to group {}", user_name, group_name);
// 4. Attach policy to group
admin_ok(
&env,
http::Method::PUT,
&format!("/rustfs/admin/v3/set-user-or-group-policy?policyName={policy_name}&userOrGroup={group_name}&isGroup=true"),
Some(String::new()),
)
.await?;
let set_policy_url = format!(
"{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=true",
env.url, policy_name, group_name
);
awscurl_put(&set_policy_url, "", &env.access_key, &env.secret_key).await?;
info!("Attached policy {} to group {}", policy_name, group_name);
// 5. User with only group (no user policy) should be able to list buckets
@@ -278,6 +209,7 @@ async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn s
/// Test that after deleting a user who was the only member of a group, the group can be deleted
/// (regression test for #2028.2: delete group uses backend membership, not stale cache).
#[tokio::test(flavor = "multi_thread")]
#[ignore = "requires awscurl and spawns a real RustFS server"]
async fn test_delete_group_after_deleting_user() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
@@ -289,47 +221,33 @@ async fn test_delete_group_after_deleting_user() -> Result<(), Box<dyn std::erro
let group_name = "soledeletegroup";
// 1. Create user
let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, user_name);
let user_body = serde_json::json!({
"secretKey": user_secret,
"status": "enabled"
});
admin_ok(
&env,
http::Method::PUT,
&format!("/rustfs/admin/v3/add-user?accessKey={user_name}"),
Some(user_body.to_string()),
)
.await?;
awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
info!("Created user {}", user_name);
// 2. Add user to group
let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
let add_member_body = serde_json::json!({
"group": group_name,
"members": [user_name],
"isRemove": false,
"groupStatus": "enabled"
});
admin_ok(
&env,
http::Method::PUT,
"/rustfs/admin/v3/update-group-members",
Some(add_member_body.to_string()),
)
.await?;
awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
info!("Added {} to group {}", user_name, group_name);
// 3. Delete the user (backend and cache update so group membership becomes empty)
admin_ok(
&env,
http::Method::DELETE,
&format!("/rustfs/admin/v3/remove-user?accessKey={user_name}"),
None,
)
.await?;
let remove_user_url = format!("{}/rustfs/admin/v3/remove-user?accessKey={}", env.url, user_name);
awscurl_delete(&remove_user_url, &env.access_key, &env.secret_key).await?;
info!("Deleted user {}", user_name);
// 4. Deleting the group should succeed (backend has empty members; no stale cache)
admin_ok(&env, http::Method::DELETE, &format!("/rustfs/admin/v3/group/{group_name}"), None).await?;
let delete_group_url = format!("{}/rustfs/admin/v3/group/{}", env.url, group_name);
awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await?;
info!("Deleted group {} after user was removed", group_name);
Ok(())
@@ -380,24 +380,10 @@ mod tests {
cluster.start_node(1).await?;
let status_url = format!("{}/rustfs/admin/v3/background-heal/status", cluster.nodes[0].url);
let mut recovered = serde_json::Value::Null;
for _ in 0..60 {
let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
assert!(
!status_body.contains("MissingContentLength"),
"background heal status should not fail without an explicit Content-Length: {status_body}"
);
recovered = serde_json::from_str(&status_body)
.map_err(|err| format!("background heal status is not JSON ({err}): {status_body}"))?;
if recovered["clusterStatusComplete"] == serde_json::Value::Bool(true) {
break;
}
sleep(Duration::from_secs(1)).await;
}
assert_eq!(
recovered["clusterStatusComplete"],
serde_json::Value::Bool(true),
"cluster heal status should recover before root heal starts: {recovered}"
let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
assert!(
!status_body.contains("MissingContentLength"),
"background heal status should not fail without an explicit Content-Length: {status_body}"
);
let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
@@ -37,7 +37,7 @@ async fn test_bucket_default_sse_s3_put_object() -> Result<(), Box<dyn std::erro
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
kms_env.wait_for_kms_ready().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -159,7 +159,7 @@ async fn test_bucket_default_sse_kms_put_object() -> Result<(), Box<dyn std::err
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
kms_env.wait_for_kms_ready().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -278,7 +278,7 @@ async fn test_bucket_default_sse_kms_multipart_crc32() -> Result<(), Box<dyn std
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
kms_env.wait_for_kms_ready().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -475,7 +475,7 @@ async fn test_explicit_encryption_overrides_bucket_default() -> Result<(), Box<d
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
kms_env.wait_for_kms_ready().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -570,7 +570,7 @@ async fn test_sse_kms_without_key_id_populates_default() -> Result<(), Box<dyn s
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
kms_env.wait_for_kms_ready().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
+35 -107
View File
@@ -22,7 +22,9 @@
//! - KMS backend configuration (Local and Vault)
//! - SSE encryption testing utilities
use crate::common::{RustFSTestEnvironment, awscurl_get, awscurl_post, init_logging as common_init_logging, local_http_client};
use crate::common::{
RustFSTestEnvironment, awscurl_available, awscurl_get, awscurl_post, init_logging as common_init_logging, local_http_client,
};
use aws_sdk_s3::Client;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::ServerSideEncryption;
@@ -57,6 +59,15 @@ pub fn init_logging() {
// Additional KMS-specific logging configuration can be added here if needed
}
pub fn skip_if_kms_admin_tool_unavailable(test_name: &str) -> bool {
if awscurl_available() {
return false;
}
info!("Skipping {} because awscurl is not available in PATH", test_name);
true
}
pub fn sse_customer_key_md5_base64(key: &str) -> String {
let mut hasher = Md5::new();
hasher.update(key.as_bytes());
@@ -178,121 +189,34 @@ pub async fn wait_for_kms_ready(
access_key: &str,
secret_key: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
wait_for_kms_ready_with_timeout(base_url, access_key, secret_key, Duration::from_secs(5)).await
}
async fn wait_for_kms_ready_with_timeout(
base_url: &str,
access_key: &str,
secret_key: &str,
total_deadline: Duration,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let total_deadline = Duration::from_secs(5);
let start = tokio::time::Instant::now();
let deadline = start + total_deadline;
let mut backoff = Duration::from_millis(200);
let max_backoff = Duration::from_secs(1);
let mut first_attempt = true;
loop {
match tokio::time::timeout_at(deadline, get_kms_status(base_url, access_key, secret_key)).await {
Ok(Ok(status)) => {
let backend_status = serde_json::from_str::<serde_json::Value>(&status)
.ok()
.and_then(|value| value.get("backend_status")?.as_str().map(str::to_owned));
if backend_status.as_deref() == Some("healthy") {
info!("KMS is ready (status: {})", status);
return Ok(());
}
warn!(
backend_status = backend_status.as_deref().unwrap_or("missing"),
elapsed_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX),
"KMS not ready yet, retrying…"
);
if !first_attempt {
if start.elapsed() >= total_deadline {
return Err("KMS failed to become ready within 5 seconds".into());
}
Ok(Err(e)) => {
let elapsed_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
warn!(error = %e, elapsed_ms, "KMS not ready yet, retrying…");
}
Err(_) => return Err(format!("KMS failed to become ready within {} ms", total_deadline.as_millis()).into()),
sleep(backoff).await;
backoff = (backoff * 2).min(max_backoff);
}
first_attempt = false;
let now = tokio::time::Instant::now();
if now >= deadline {
return Err(format!("KMS failed to become ready within {} ms", total_deadline.as_millis()).into());
}
sleep((now + backoff).min(deadline) - now).await;
backoff = (backoff * 2).min(max_backoff);
}
}
#[cfg(test)]
mod readiness_tests {
use super::{wait_for_kms_ready, wait_for_kms_ready_with_timeout};
use std::sync::{
Arc,
atomic::{AtomicUsize, Ordering},
};
use std::time::Duration;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpListener;
#[tokio::test]
async fn kms_readiness_retries_http_success_until_backend_is_healthy() {
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind readiness test server");
let address = listener.local_addr().expect("read readiness test server address");
let requests = Arc::new(AtomicUsize::new(0));
let server_requests = Arc::clone(&requests);
let server = tokio::spawn(async move {
for backend_status in ["error", "healthy"] {
let (mut socket, _) = listener.accept().await.expect("accept readiness request");
let mut request = Vec::new();
let mut chunk = [0_u8; 1024];
while !request.windows(4).any(|window| window == b"\r\n\r\n") {
let read = socket.read(&mut chunk).await.expect("read readiness request");
if read == 0 {
break;
}
request.extend_from_slice(&chunk[..read]);
}
server_requests.fetch_add(1, Ordering::SeqCst);
let body = format!(r#"{{"backend_status":"{backend_status}"}}"#);
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
body.len()
);
socket.write_all(response.as_bytes()).await.expect("write readiness response");
match get_kms_status(base_url, access_key, secret_key).await {
Ok(status) => {
info!("KMS is ready (status: {})", status);
return Ok(());
}
});
wait_for_kms_ready(&format!("http://{address}"), "access-key", "secret-key")
.await
.expect("KMS should become ready after the healthy response");
let observed_requests = requests.load(Ordering::SeqCst);
server.abort();
assert_eq!(observed_requests, 2, "an HTTP 200 unhealthy status must be retried");
}
#[tokio::test]
async fn kms_readiness_deadline_covers_a_stalled_status_request() {
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind readiness test server");
let address = listener.local_addr().expect("read readiness test server address");
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.expect("accept readiness request");
let mut request = [0_u8; 1024];
let _ = socket.read(&mut request).await.expect("read readiness request");
std::future::pending::<()>().await;
});
let result = tokio::time::timeout(
Duration::from_secs(1),
wait_for_kms_ready_with_timeout(&format!("http://{address}"), "access-key", "secret-key", Duration::from_millis(50)),
)
.await
.expect("readiness helper must enforce its own deadline");
server.abort();
assert!(result.is_err(), "a stalled status request must not outlive the readiness deadline");
Err(e) => {
if start.elapsed() >= total_deadline {
return Err(format!("KMS did not become ready within 5 s: last error: {e}").into());
}
warn!(error = %e, elapsed_ms = start.elapsed().as_millis() as u64, "KMS not ready yet, retrying…");
}
}
}
}
@@ -479,6 +403,10 @@ pub async fn test_kms_key_management(
access_key: &str,
secret_key: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
if skip_if_kms_admin_tool_unavailable("test_kms_key_management") {
return Ok(());
}
info!("Testing KMS key management APIs");
// Test CreateKey
@@ -61,7 +61,7 @@ async fn test_metadata_replace_self_copy_of_sse_object_stays_decryptable() {
)
.await
.expect("failed to start RustFS with local KMS");
kms_env.wait_for_kms_ready().await.expect("KMS ready");
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
let client = kms_env.base_env.create_s3_client();
// Deliberately an UNVERSIONED bucket: that is the branch where the store layer can service
@@ -160,7 +160,7 @@ async fn test_metadata_replace_self_copy_dropping_sse_rewrites_plaintext() {
)
.await
.expect("failed to start RustFS with local KMS");
kms_env.wait_for_kms_ready().await.expect("KMS ready");
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
let client = kms_env.base_env.create_s3_client();
// Unversioned, and deliberately WITHOUT a bucket default-encryption rule, so the copy below
@@ -256,7 +256,7 @@ async fn test_metadata_replace_self_copy_under_bucket_default_sse_stays_decrypta
)
.await
.expect("failed to start RustFS with local KMS");
kms_env.wait_for_kms_ready().await.expect("KMS ready");
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
let client = kms_env.base_env.create_s3_client();
let bucket = "copy-object-self-copy-bucket-default-sse-test";
@@ -56,7 +56,7 @@ async fn test_self_copy_of_historical_sse_s3_version_is_readable() {
)
.await
.expect("failed to start RustFS with local KMS");
kms_env.wait_for_kms_ready().await.expect("KMS ready");
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
let client = kms_env.base_env.create_s3_client();
let bucket = "copy-object-version-restore-sse-test";
@@ -87,7 +87,7 @@ async fn test_head_reports_managed_metadata_for_sse_s3() -> Result<(), Box<dyn s
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key = kms_env.start_rustfs_for_local_kms().await?;
kms_env.wait_for_kms_ready().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -147,7 +147,7 @@ async fn test_head_reports_managed_metadata_for_sse_kms_and_copy() -> Result<(),
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
kms_env.wait_for_kms_ready().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -250,7 +250,7 @@ async fn test_multipart_upload_writes_encrypted_data() -> Result<(), Box<dyn std
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
kms_env.wait_for_kms_ready().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -24,6 +24,7 @@ use super::common::{
test_sse_kms_encryption, test_sse_s3_encryption,
};
use crate::common::{TEST_BUCKET, init_logging};
use tokio::time::{Duration, sleep};
use tracing::info;
/// Comprehensive test: Full KMS workflow with all encryption types
@@ -34,7 +35,7 @@ async fn test_comprehensive_kms_full_workflow() -> Result<(), Box<dyn std::error
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
kms_env.wait_for_kms_ready().await?;
sleep(Duration::from_secs(3)).await;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -102,7 +103,7 @@ async fn test_comprehensive_stress_test() -> Result<(), Box<dyn std::error::Erro
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
kms_env.wait_for_kms_ready().await?;
sleep(Duration::from_secs(3)).await;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -136,7 +137,7 @@ async fn test_comprehensive_key_isolation() -> Result<(), Box<dyn std::error::Er
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
kms_env.wait_for_kms_ready().await?;
sleep(Duration::from_secs(3)).await;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -207,7 +208,7 @@ async fn test_comprehensive_concurrent_operations() -> Result<(), Box<dyn std::e
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
kms_env.wait_for_kms_ready().await?;
sleep(Duration::from_secs(3)).await;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -252,7 +253,7 @@ async fn test_comprehensive_performance_benchmark() -> Result<(), Box<dyn std::e
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
kms_env.wait_for_kms_ready().await?;
sleep(Duration::from_secs(3)).await;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -44,7 +44,7 @@ async fn test_kms_zero_byte_file_encryption() -> Result<(), Box<dyn std::error::
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
kms_env.wait_for_kms_ready().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -117,7 +117,7 @@ async fn test_kms_single_byte_file_encryption() -> Result<(), Box<dyn std::error
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
kms_env.wait_for_kms_ready().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -209,7 +209,7 @@ async fn test_kms_multipart_boundary_conditions() -> Result<(), Box<dyn std::err
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
kms_env.wait_for_kms_ready().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -284,7 +284,7 @@ async fn test_kms_invalid_key_scenarios() -> Result<(), Box<dyn std::error::Erro
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
kms_env.wait_for_kms_ready().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -371,7 +371,7 @@ async fn test_kms_concurrent_encryption() -> Result<(), Box<dyn std::error::Erro
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
kms_env.wait_for_kms_ready().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
let s3_client = Arc::new(kms_env.base_env.create_s3_client());
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -478,7 +478,7 @@ async fn test_kms_key_validation_security() -> Result<(), Box<dyn std::error::Er
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
kms_env.wait_for_kms_ready().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -37,7 +37,7 @@ async fn test_kms_key_directory_unavailable() -> Result<(), Box<dyn std::error::
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
kms_env.wait_for_kms_ready().await?;
tokio::time::sleep(Duration::from_secs(3)).await;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -127,7 +127,7 @@ async fn test_kms_corrupted_key_files() -> Result<(), Box<dyn std::error::Error
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
kms_env.wait_for_kms_ready().await?;
tokio::time::sleep(Duration::from_secs(3)).await;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -218,7 +218,7 @@ async fn test_kms_multipart_upload_interruption() -> Result<(), Box<dyn std::err
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
kms_env.wait_for_kms_ready().await?;
tokio::time::sleep(Duration::from_secs(3)).await;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -401,7 +401,7 @@ async fn test_kms_resource_constraints() -> Result<(), Box<dyn std::error::Error
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
kms_env.wait_for_kms_ready().await?;
tokio::time::sleep(Duration::from_secs(3)).await;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
+9 -5
View File
@@ -20,7 +20,8 @@
//! - Complete encryption/decryption lifecycle
use super::common::{
LocalKMSTestEnvironment, get_kms_status, sse_customer_key_md5_base64, test_kms_key_management, test_sse_c_encryption,
LocalKMSTestEnvironment, get_kms_status, skip_if_kms_admin_tool_unavailable, sse_customer_key_md5_base64,
test_kms_key_management, test_sse_c_encryption,
};
use crate::common::{TEST_BUCKET, init_logging};
use tracing::{error, info};
@@ -28,6 +29,9 @@ use tracing::{error, info};
#[tokio::test]
async fn test_local_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_if_kms_admin_tool_unavailable("test_local_kms_end_to_end") {
return Ok(());
}
info!("Starting Local KMS End-to-End Test");
// Create LocalKMS test environment
@@ -42,7 +46,7 @@ async fn test_local_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + S
.expect("Failed to start RustFS with Local KMS");
// Wait a moment for RustFS to fully start up and initialize KMS
kms_env.wait_for_kms_ready().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);
@@ -123,7 +127,7 @@ async fn test_local_kms_key_isolation() {
.expect("Failed to start RustFS with Local KMS");
// Wait a moment for RustFS to fully start up and initialize KMS
kms_env.wait_for_kms_ready().await.expect("KMS ready");
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);
@@ -223,7 +227,7 @@ async fn test_local_kms_large_file() {
.expect("Failed to start RustFS with Local KMS");
// Wait a moment for RustFS to fully start up and initialize KMS
kms_env.wait_for_kms_ready().await.expect("KMS ready");
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);
@@ -305,7 +309,7 @@ async fn test_local_kms_multipart_upload() {
.expect("Failed to start RustFS with Local KMS");
// Wait for KMS initialization
kms_env.wait_for_kms_ready().await.expect("KMS ready");
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);
+20 -4
View File
@@ -19,11 +19,12 @@
//! multipart upload behaviour.
use crate::common::{TEST_BUCKET, init_logging};
use tokio::time::{Duration, sleep};
use tracing::{error, info};
use super::common::{
VAULT_KEY_NAME, VaultTestEnvironment, get_kms_status, sse_customer_key_md5_base64, start_kms,
test_all_multipart_encryption_types, test_error_scenarios, test_kms_key_management, test_sse_c_encryption,
VAULT_KEY_NAME, VaultTestEnvironment, get_kms_status, skip_if_kms_admin_tool_unavailable, sse_customer_key_md5_base64,
start_kms, test_all_multipart_encryption_types, test_error_scenarios, test_kms_key_management, test_sse_c_encryption,
test_sse_kms_encryption, test_sse_s3_encryption,
};
@@ -44,8 +45,8 @@ impl VaultKmsTestContext {
start_kms(&env.base_env.url, &env.base_env.access_key, &env.base_env.secret_key).await?;
// Wait for KMS to finish initialising.
super::common::wait_for_kms_ready(&env.base_env.url, &env.base_env.access_key, &env.base_env.secret_key).await?;
// Allow Vault to finish initialising token auth and transit engine.
sleep(Duration::from_secs(2)).await;
Ok(Self { env })
}
@@ -62,6 +63,9 @@ impl VaultKmsTestContext {
#[tokio::test]
async fn test_vault_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_if_kms_admin_tool_unavailable("test_vault_kms_end_to_end") {
return Ok(());
}
info!("Starting Vault KMS End-to-End Test with default key {}", VAULT_KEY_NAME);
let context = VaultKmsTestContext::new().await?;
@@ -114,6 +118,9 @@ async fn test_vault_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + S
#[tokio::test]
async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_isolation") {
return Ok(());
}
info!("Starting Vault KMS SSE-C key isolation test");
let context = VaultKmsTestContext::new().await?;
@@ -197,6 +204,9 @@ async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error
#[tokio::test]
async fn test_vault_kms_large_file() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_if_kms_admin_tool_unavailable("test_vault_kms_large_file") {
return Ok(());
}
info!("Starting Vault KMS large file SSE-S3 test");
let context = VaultKmsTestContext::new().await?;
@@ -258,6 +268,9 @@ async fn test_vault_kms_large_file() -> Result<(), Box<dyn std::error::Error + S
#[tokio::test]
async fn test_vault_kms_multipart_upload() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_if_kms_admin_tool_unavailable("test_vault_kms_multipart_upload") {
return Ok(());
}
info!("Starting Vault KMS multipart upload encryption suite");
let context = VaultKmsTestContext::new().await?;
@@ -285,6 +298,9 @@ async fn test_vault_kms_multipart_upload() -> Result<(), Box<dyn std::error::Err
#[tokio::test]
async fn test_vault_kms_key_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_operations") {
return Ok(());
}
info!("Starting Vault KMS key operations test (CRUD)");
let context = VaultKmsTestContext::new().await?;
+3
View File
@@ -39,6 +39,9 @@ mod kms_edge_cases_test;
#[cfg(test)]
mod kms_fault_recovery_test;
#[cfg(test)]
mod test_runner;
#[cfg(test)]
mod bucket_default_encryption_test;
@@ -33,7 +33,7 @@ async fn test_step1_basic_single_file_encryption() -> Result<(), Box<dyn std::er
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
kms_env.wait_for_kms_ready().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -89,7 +89,7 @@ async fn test_step2_basic_multipart_upload_without_encryption() -> Result<(), Bo
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
kms_env.wait_for_kms_ready().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -187,7 +187,7 @@ async fn test_step3_multipart_upload_with_sse_s3() -> Result<(), Box<dyn std::er
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
kms_env.wait_for_kms_ready().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -310,7 +310,7 @@ async fn test_step4_large_multipart_upload_with_encryption() -> Result<(), Box<d
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
kms_env.wait_for_kms_ready().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -435,7 +435,7 @@ async fn test_step5_all_encryption_types_multipart() -> Result<(), Box<dyn std::
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
kms_env.wait_for_kms_ready().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
+499
View File
@@ -0,0 +1,499 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
#![allow(dead_code)]
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Unified KMS test suite runner
//!
//! This module provides a unified interface for running KMS tests with categorization,
//! filtering, and comprehensive reporting capabilities.
use crate::common::init_logging;
use std::time::Instant;
use tokio::time::{Duration, sleep};
use tracing::{debug, error, info, warn};
/// Test category for organization and filtering
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum TestCategory {
CoreFunctionality,
MultipartEncryption,
EdgeCases,
FaultRecovery,
Comprehensive,
Performance,
}
impl TestCategory {
pub fn as_str(&self) -> &'static str {
match self {
TestCategory::CoreFunctionality => "core-functionality",
TestCategory::MultipartEncryption => "multipart-encryption",
TestCategory::EdgeCases => "edge-cases",
TestCategory::FaultRecovery => "fault-recovery",
TestCategory::Comprehensive => "comprehensive",
TestCategory::Performance => "performance",
}
}
}
/// Test definition with metadata
#[derive(Debug, Clone)]
pub struct TestDefinition {
pub name: String,
pub description: String,
pub category: TestCategory,
pub estimated_duration: Duration,
pub is_critical: bool,
}
impl TestDefinition {
pub fn new(
name: impl Into<String>,
description: impl Into<String>,
category: TestCategory,
estimated_duration: Duration,
is_critical: bool,
) -> Self {
Self {
name: name.into(),
description: description.into(),
category,
estimated_duration,
is_critical,
}
}
}
/// Test execution result
#[derive(Debug, Clone)]
pub struct TestResult {
pub test_name: String,
pub category: TestCategory,
pub success: bool,
pub duration: Duration,
pub error_message: Option<String>,
}
impl TestResult {
pub fn success(test_name: String, category: TestCategory, duration: Duration) -> Self {
Self {
test_name,
category,
success: true,
duration,
error_message: None,
}
}
pub fn failure(test_name: String, category: TestCategory, duration: Duration, error: String) -> Self {
Self {
test_name,
category,
success: false,
duration,
error_message: Some(error),
}
}
}
/// Comprehensive test suite configuration
#[derive(Debug, Clone)]
pub struct TestSuiteConfig {
pub categories: Vec<TestCategory>,
pub include_critical_only: bool,
pub max_duration: Option<Duration>,
pub parallel_execution: bool,
}
impl Default for TestSuiteConfig {
fn default() -> Self {
Self {
categories: vec![
TestCategory::CoreFunctionality,
TestCategory::MultipartEncryption,
TestCategory::EdgeCases,
TestCategory::FaultRecovery,
TestCategory::Comprehensive,
],
include_critical_only: false,
max_duration: None,
parallel_execution: false,
}
}
}
/// Unified KMS test suite runner
pub struct KMSTestSuite {
tests: Vec<TestDefinition>,
config: TestSuiteConfig,
}
impl KMSTestSuite {
/// Create a new test suite with default configuration
pub fn new() -> Self {
let tests = vec![
// Core Functionality Tests
TestDefinition::new(
"test_local_kms_end_to_end",
"End-to-end KMS test with all encryption types",
TestCategory::CoreFunctionality,
Duration::from_secs(60),
true,
),
TestDefinition::new(
"test_local_kms_key_isolation",
"Test KMS key isolation and security",
TestCategory::CoreFunctionality,
Duration::from_secs(45),
true,
),
// Multipart Encryption Tests
TestDefinition::new(
"test_local_kms_multipart_upload",
"Test large file multipart upload with encryption",
TestCategory::MultipartEncryption,
Duration::from_secs(120),
true,
),
TestDefinition::new(
"test_step1_basic_single_file_encryption",
"Basic single file encryption test",
TestCategory::MultipartEncryption,
Duration::from_secs(30),
false,
),
TestDefinition::new(
"test_step2_basic_multipart_upload_without_encryption",
"Basic multipart upload without encryption",
TestCategory::MultipartEncryption,
Duration::from_secs(45),
false,
),
TestDefinition::new(
"test_step3_multipart_upload_with_sse_s3",
"Multipart upload with SSE-S3 encryption",
TestCategory::MultipartEncryption,
Duration::from_secs(60),
true,
),
TestDefinition::new(
"test_step4_large_multipart_upload_with_encryption",
"Large file multipart upload with encryption",
TestCategory::MultipartEncryption,
Duration::from_secs(90),
false,
),
TestDefinition::new(
"test_step5_all_encryption_types_multipart",
"All encryption types multipart test",
TestCategory::MultipartEncryption,
Duration::from_secs(120),
true,
),
// Edge Cases Tests
TestDefinition::new(
"test_kms_zero_byte_file_encryption",
"Test encryption of zero-byte files",
TestCategory::EdgeCases,
Duration::from_secs(20),
false,
),
TestDefinition::new(
"test_kms_single_byte_file_encryption",
"Test encryption of single-byte files",
TestCategory::EdgeCases,
Duration::from_secs(20),
false,
),
TestDefinition::new(
"test_kms_multipart_boundary_conditions",
"Test multipart upload boundary conditions",
TestCategory::EdgeCases,
Duration::from_secs(45),
false,
),
TestDefinition::new(
"test_kms_invalid_key_scenarios",
"Test invalid key scenarios",
TestCategory::EdgeCases,
Duration::from_secs(30),
false,
),
TestDefinition::new(
"test_kms_concurrent_encryption",
"Test concurrent encryption operations",
TestCategory::EdgeCases,
Duration::from_secs(60),
false,
),
TestDefinition::new(
"test_kms_key_validation_security",
"Test key validation security",
TestCategory::EdgeCases,
Duration::from_secs(30),
false,
),
// Fault Recovery Tests
TestDefinition::new(
"test_kms_key_directory_unavailable",
"Test KMS when key directory is unavailable",
TestCategory::FaultRecovery,
Duration::from_secs(45),
false,
),
TestDefinition::new(
"test_kms_corrupted_key_files",
"Test KMS with corrupted key files",
TestCategory::FaultRecovery,
Duration::from_secs(30),
false,
),
TestDefinition::new(
"test_kms_multipart_upload_interruption",
"Test multipart upload interruption recovery",
TestCategory::FaultRecovery,
Duration::from_secs(60),
false,
),
TestDefinition::new(
"test_kms_resource_constraints",
"Test KMS under resource constraints",
TestCategory::FaultRecovery,
Duration::from_secs(90),
false,
),
// Comprehensive Tests
TestDefinition::new(
"test_comprehensive_kms_full_workflow",
"Full KMS workflow comprehensive test",
TestCategory::Comprehensive,
Duration::from_secs(300),
true,
),
TestDefinition::new(
"test_comprehensive_stress_test",
"KMS stress test with large datasets",
TestCategory::Comprehensive,
Duration::from_secs(400),
false,
),
TestDefinition::new(
"test_comprehensive_key_isolation",
"Comprehensive key isolation test",
TestCategory::Comprehensive,
Duration::from_secs(180),
false,
),
TestDefinition::new(
"test_comprehensive_concurrent_operations",
"Comprehensive concurrent operations test",
TestCategory::Comprehensive,
Duration::from_secs(240),
false,
),
TestDefinition::new(
"test_comprehensive_performance_benchmark",
"KMS performance benchmark test",
TestCategory::Comprehensive,
Duration::from_secs(360),
false,
),
];
Self {
tests,
config: TestSuiteConfig::default(),
}
}
/// Configure the test suite
pub fn with_config(mut self, config: TestSuiteConfig) -> Self {
self.config = config;
self
}
/// Filter tests based on category
pub fn filter_by_category(&self, category: &TestCategory) -> Vec<&TestDefinition> {
self.tests.iter().filter(|test| &test.category == category).collect()
}
/// Filter tests based on criticality
pub fn filter_critical_tests(&self) -> Vec<&TestDefinition> {
self.tests.iter().filter(|test| test.is_critical).collect()
}
/// Get test summary by category
pub fn get_category_summary(&self) -> std::collections::HashMap<TestCategory, Vec<&TestDefinition>> {
let mut summary = std::collections::HashMap::new();
for test in &self.tests {
summary.entry(test.category.clone()).or_insert_with(Vec::new).push(test);
}
summary
}
/// Run the complete test suite
pub async fn run_test_suite(&self) -> Vec<TestResult> {
init_logging();
info!("🚀 Starting unified KMS test suite");
let start_time = Instant::now();
let mut results = Vec::new();
// Filter tests based on configuration
let tests_to_run: Vec<&TestDefinition> = self
.tests
.iter()
.filter(|test| self.config.categories.contains(&test.category))
.filter(|test| !self.config.include_critical_only || test.is_critical)
.collect();
info!("📊 Test plan: {} test(s) scheduled", tests_to_run.len());
for (i, test) in tests_to_run.iter().enumerate() {
info!(" {}. {} ({})", i + 1, test.name, test.category.as_str());
}
// Execute tests
for (i, test_def) in tests_to_run.iter().enumerate() {
info!("🧪 Running test {}/{}: {}", i + 1, tests_to_run.len(), test_def.name);
info!(" 📝 Description: {}", test_def.description);
info!(" 🏷️ Category: {}", test_def.category.as_str());
info!(" ⏱️ Estimated duration: {:?}", test_def.estimated_duration);
let test_start = Instant::now();
let result = self.run_single_test(test_def).await;
let test_duration = test_start.elapsed();
match result {
Ok(_) => {
info!("✅ Test passed: {} ({:.2}s)", test_def.name, test_duration.as_secs_f64());
results.push(TestResult::success(test_def.name.clone(), test_def.category.clone(), test_duration));
}
Err(e) => {
error!("❌ Test failed: {} ({:.2}s): {}", test_def.name, test_duration.as_secs_f64(), e);
results.push(TestResult::failure(
test_def.name.clone(),
test_def.category.clone(),
test_duration,
e.to_string(),
));
}
}
// Add delay between tests to avoid resource conflicts
if i < tests_to_run.len() - 1 {
debug!("⏸️ Waiting two seconds before the next test...");
sleep(Duration::from_secs(2)).await;
}
}
let total_duration = start_time.elapsed();
self.print_test_summary(&results, total_duration);
results
}
/// Run a single test by dispatching to the appropriate test function
async fn run_single_test(&self, test_def: &TestDefinition) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
// This is a placeholder for test dispatch logic
// In a real implementation, this would dispatch to actual test functions
warn!("⚠️ Test '{}' is not implemented in the unified runner; skipping", test_def.name);
Ok(())
}
/// Print comprehensive test summary
fn print_test_summary(&self, results: &[TestResult], total_duration: Duration) {
info!("📊 KMS test suite summary");
info!("⏱️ Total duration: {:.2} seconds", total_duration.as_secs_f64());
info!("📈 Total tests: {}", results.len());
let passed = results.iter().filter(|r| r.success).count();
let failed = results.iter().filter(|r| !r.success).count();
info!("✅ Passed: {}", passed);
info!("❌ Failed: {}", failed);
info!("📊 Success rate: {:.1}%", (passed as f64 / results.len() as f64) * 100.0);
// Summary by category
let mut category_summary: std::collections::HashMap<TestCategory, (usize, usize)> = std::collections::HashMap::new();
for result in results {
let (total, passed_count) = category_summary.entry(result.category.clone()).or_insert((0, 0));
*total += 1;
if result.success {
*passed_count += 1;
}
}
info!("📊 Category summary:");
for (category, (total, passed_count)) in category_summary {
info!(
" 🏷️ {}: {}/{} ({:.1}%)",
category.as_str(),
passed_count,
total,
(passed_count as f64 / total as f64) * 100.0
);
}
// List failed tests
if failed > 0 {
warn!("❌ Failing tests:");
for result in results.iter().filter(|r| !r.success) {
warn!(" - {}: {}", result.test_name, result.error_message.as_deref().unwrap_or("Unknown error"));
}
}
}
}
/// Quick test suite for critical tests only
#[tokio::test]
async fn test_kms_critical_suite() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let config = TestSuiteConfig {
categories: vec![TestCategory::CoreFunctionality, TestCategory::MultipartEncryption],
include_critical_only: true,
max_duration: Some(Duration::from_secs(600)), // 10 minutes max
parallel_execution: false,
};
let suite = KMSTestSuite::new().with_config(config);
let results = suite.run_test_suite().await;
let failed_count = results.iter().filter(|r| !r.success).count();
if failed_count > 0 {
return Err(format!("Critical test suite failed: {failed_count} tests failed").into());
}
info!("✅ All critical tests passed");
Ok(())
}
/// Full comprehensive test suite
#[tokio::test]
async fn test_kms_full_suite() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let suite = KMSTestSuite::new();
let results = suite.run_test_suite().await;
let total_tests = results.len();
let failed_count = results.iter().filter(|r| !r.success).count();
let success_rate = ((total_tests - failed_count) as f64 / total_tests as f64) * 100.0;
info!("📊 Full suite success rate: {:.1}%", success_rate);
// Allow up to 10% failure rate for non-critical tests
if success_rate < 90.0 {
return Err(format!("Test suite success rate too low: {success_rate:.1}%").into());
}
info!("✅ Full test suite succeeded");
Ok(())
}
-5
View File
@@ -61,11 +61,6 @@ mod get_codec_streaming_compat_test;
#[cfg(test)]
mod version_id_regression_test;
// Receiver-side replication LWW (rustfs/backlog#1953): stale inbound
// replication metadata must not overwrite a newer local category state.
#[cfg(test)]
mod replication_lww_receiver_test;
// Data usage regression tests
#[cfg(test)]
mod data_usage_test;
@@ -41,6 +41,13 @@ async fn create_issue_3107_fixture(root: &Path) -> TestResult {
Ok(())
}
fn mc_available() -> bool {
Command::new("mc")
.arg("--version")
.output()
.is_ok_and(|output| output.status.success())
}
fn run_mc(args: &[&str]) -> TestResult {
let output = Command::new("mc").args(args).output()?;
if !output.status.success() {
@@ -68,7 +75,10 @@ fn count_files(root: &Path) -> usize {
async fn test_mc_mirror_small_bucket_completes_without_list_timeout() -> TestResult {
crate::common::init_logging();
info!("Starting issue #3107 mc mirror regression test");
run_mc(&["--version"])?;
if !mc_available() {
info!("Skipping issue #3107 mc mirror regression test because mc is not installed");
return Ok(());
}
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
@@ -4278,6 +4278,10 @@ async fn test_signed_put_object_extract_preserves_pax_metadata_and_version_id()
async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retention_conditions()
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if !crate::common::awscurl_available() {
return Ok(());
}
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
+12 -25
View File
@@ -16,7 +16,6 @@ use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_lo
use aws_sdk_s3::primitives::ByteStream;
use http::header::{CONTENT_TYPE, HOST};
use reqwest::StatusCode;
use rustfs_config::{ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, ENV_NOTIFY_ENABLE};
use rustfs_signer::pre_sign_v4;
use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
use s3s::Body;
@@ -977,8 +976,7 @@ async fn test_get_object_lambda_rejects_disabled_target() -> Result<(), Box<dyn
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")])
.await?;
env.start_rustfs_server(vec![]).await?;
let bucket = "object-lambda-e2e-disabled-target";
let key = "input.txt";
@@ -994,24 +992,17 @@ async fn test_get_object_lambda_rejects_disabled_target() -> Result<(), Box<dyn
.send()
.await?;
let queue_dir = format!("{}/disabled-target-queue", env.temp_dir);
tokio::fs::create_dir_all(&queue_dir).await?;
let config_url = format!("{}/rustfs/admin/v3/set-config-kv", env.url);
let directive = format!(
"notify_webhook:transformer enable=off endpoint=\"http://127.0.0.1:9/transform\" auth_token=\"secret-token\" queue_dir=\"{queue_dir}\""
);
let disable_response = signed_request(
http::Method::PUT,
&config_url,
&env.access_key,
&env.secret_key,
Some(directive.into_bytes()),
Some("text/plain"),
configure_webhook_target_with_key_values(
&env,
"transformer",
vec![
("endpoint", "http://127.0.0.1:9/transform".to_string()),
("auth_token", "secret-token".to_string()),
("enable", "off".to_string()),
],
)
.await?;
let disable_status = disable_response.status();
let disable_body = disable_response.text().await?;
assert_eq!(disable_status, StatusCode::OK, "failed to disable target: {disable_body}");
wait_for_target_visibility(&env, "transformer").await?;
let lambda_url = format!("{}/{}/{}?lambdaArn={}", env.url, bucket, key, urlencoding::encode(lambda_arn));
let response = signed_request(http::Method::GET, &lambda_url, &env.access_key, &env.secret_key, None, None).await?;
@@ -1030,8 +1021,7 @@ async fn test_configure_object_lambda_target_rejects_invalid_endpoint() -> Resul
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")])
.await?;
env.start_rustfs_server(vec![]).await?;
let bucket = "object-lambda-e2e-invalid-endpoint";
@@ -1074,8 +1064,7 @@ async fn test_configure_object_lambda_notify_webhook_rejects_response_header_tim
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")])
.await?;
env.start_rustfs_server(vec![]).await?;
let response = send_configure_webhook_target_request(
&env,
@@ -1184,8 +1173,6 @@ async fn test_listen_notification_fans_in_remote_node_events() -> Result<(), Box
init_logging();
let mut cluster = RustFSTestClusterEnvironment::new(2).await?;
cluster.set_env(ENV_NOTIFY_ENABLE, "true");
cluster.set_env(ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, "1");
cluster.start().await?;
let bucket = "listen-notification-cluster";
@@ -26,7 +26,7 @@ use aws_sdk_s3::config::{Credentials, Region};
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
use russh::client::{self, Handle};
use russh::keys::ssh_key::LineEnding;
use russh::keys::{Algorithm, PrivateKey, PublicKeyOrCertificate};
use russh::keys::{Algorithm, PrivateKey, PublicKey};
use russh_sftp::client::SftpSession;
use russh_sftp::protocol::OpenFlags;
use std::path::Path;
@@ -46,7 +46,7 @@ pub struct AcceptAnyServerKey;
impl client::Handler for AcceptAnyServerKey {
type Error = anyhow::Error;
async fn check_server_key(&mut self, _server_public_key: &PublicKeyOrCertificate) -> Result<bool, Self::Error> {
async fn check_server_key(&mut self, _server_public_key: &PublicKey) -> Result<bool, Self::Error> {
Ok(true)
}
}
+51
View File
@@ -18,6 +18,15 @@ use http::{Method, StatusCode};
use tokio::time::{Duration, sleep, timeout};
use tracing::{debug, info};
fn skip_without_awscurl() -> bool {
if crate::common::awscurl_available() {
return false;
}
info!("Skipping quota test because awscurl is not available");
true
}
/// Test environment setup for quota tests
pub struct QuotaTestEnv {
pub env: RustFSTestEnvironment,
@@ -267,6 +276,9 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_basic_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
// Create test bucket
@@ -308,6 +320,9 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_admission_aws_chunked_declared_encoding() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
@@ -356,6 +371,9 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_update_and_clear() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
@@ -388,6 +406,9 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_delete_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
@@ -421,6 +442,9 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_usage_tracking() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
@@ -456,6 +480,9 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_statistics() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
@@ -486,6 +513,9 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_check_api() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
@@ -523,6 +553,9 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_multiple_buckets() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
// Create two buckets in the same environment
@@ -560,6 +593,9 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_error_handling() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
@@ -592,6 +628,9 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_http_endpoints() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
@@ -650,6 +689,9 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_normal_user_permissions() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
@@ -702,6 +744,9 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_copy_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
@@ -744,6 +789,9 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_batch_delete() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
@@ -799,6 +847,9 @@ mod integration_tests {
#[tokio::test]
async fn test_quota_multipart_upload() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
if skip_without_awscurl() {
return Ok(());
}
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
+162 -123
View File
@@ -1,26 +1,52 @@
#![cfg(test)]
use crate::common::{RustFSTestEnvironment, TEST_BUCKET, init_logging};
use aws_config::meta::region::RegionProviderChain;
use aws_sdk_s3::Client;
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
use aws_sdk_s3::config::{Credentials, Region};
use aws_sdk_s3::error::SdkError;
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
use bytes::Bytes;
use std::error::Error;
use std::fmt::Debug;
type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
const ENDPOINT: &str = "http://localhost:9000";
const ACCESS_KEY: &str = "rustfsadmin";
const SECRET_KEY: &str = "rustfsadmin";
const BUCKET: &str = "api-test";
fn assert_s3_error_code<T, E>(result: Result<T, SdkError<E>>, expected: &str)
where
T: Debug,
E: ProvideErrorMetadata + Debug,
{
let error = result.expect_err("conditional request must fail");
assert_eq!(
error.as_service_error().and_then(ProvideErrorMetadata::code),
Some(expected),
"unexpected conditional request error: {error:?}"
async fn create_aws_s3_client() -> Result<Client, Box<dyn Error>> {
let region_provider = RegionProviderChain::default_provider().or_else(Region::new("us-east-1"));
let shared_config = aws_config::defaults(aws_config::BehaviorVersion::latest())
.region(region_provider)
.credentials_provider(Credentials::new(ACCESS_KEY, SECRET_KEY, None, None, "static"))
.endpoint_url(ENDPOINT)
.load()
.await;
let client = Client::from_conf(
aws_sdk_s3::Config::from(&shared_config)
.to_builder()
.force_path_style(true)
.build(),
);
Ok(client)
}
/// Setup test bucket, creating it if it doesn't exist
async fn setup_test_bucket(client: &Client) -> Result<(), Box<dyn Error>> {
match client.create_bucket().bucket(BUCKET).send().await {
Ok(_) => {}
Err(SdkError::ServiceError(e)) => {
let e = e.into_err();
let error_code = e.meta().code().unwrap_or("");
if !error_code.eq("BucketAlreadyExists") {
return Err(e.into());
}
}
Err(e) => {
return Err(e.into());
}
}
Ok(())
}
/// Generate test data of specified size
@@ -34,12 +60,7 @@ fn generate_test_data(size: usize) -> Vec<u8> {
}
/// Upload an object and return its ETag
async fn upload_object_with_metadata(
client: &Client,
bucket: &str,
key: &str,
data: &[u8],
) -> Result<String, Box<dyn Error + Send + Sync>> {
async fn upload_object_with_metadata(client: &Client, bucket: &str, key: &str, data: &[u8]) -> Result<String, Box<dyn Error>> {
let response = client
.put_object()
.bucket(bucket)
@@ -48,164 +69,188 @@ async fn upload_object_with_metadata(
.send()
.await?;
response
.e_tag()
.map(str::to_owned)
.ok_or_else(|| std::io::Error::other("put object response did not include an ETag").into())
let etag = response.e_tag().unwrap_or("").to_string();
Ok(etag)
}
async fn object_body(client: &Client, key: &str) -> Result<Bytes, Box<dyn Error + Send + Sync>> {
let response = client.get_object().bucket(TEST_BUCKET).key(key).send().await?;
Ok(response.body.collect().await?.into_bytes())
/// Cleanup test objects from bucket
async fn cleanup_objects(client: &Client, bucket: &str, keys: &[&str]) {
for key in keys {
let _ = client.delete_object().bucket(bucket).key(*key).send().await;
}
}
/// Generate unique test object key
fn generate_test_key(prefix: &str) -> String {
use std::time::{SystemTime, UNIX_EPOCH};
let timestamp = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos();
format!("{prefix}-{timestamp}")
}
#[tokio::test]
async fn test_conditional_put_okay() -> TestResult {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
env.create_test_bucket(TEST_BUCKET).await?;
let client = env.create_s3_client();
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_conditional_put_okay() -> Result<(), Box<dyn std::error::Error>> {
let client = create_aws_s3_client().await?;
setup_test_bucket(&client).await?;
let test_key = "conditional-put-ok";
let test_key = generate_test_key("conditional-put-ok");
let initial_data = generate_test_data(1024); // 1KB test data
let matching_data = generate_test_data(2048); // 2KB updated data
let non_matching_data = generate_test_data(3072); // 3KB updated data
let updated_data = generate_test_data(2048); // 2KB updated data
// Upload initial object and get its ETag
let initial_etag = upload_object_with_metadata(&client, TEST_BUCKET, test_key, &initial_data).await?;
let initial_etag = upload_object_with_metadata(&client, BUCKET, &test_key, &initial_data).await?;
// Test 1: PUT with matching If-Match condition (should succeed)
client
let response1 = client
.put_object()
.bucket(TEST_BUCKET)
.key(test_key)
.body(Bytes::from(matching_data.clone()).into())
.bucket(BUCKET)
.key(&test_key)
.body(Bytes::from(updated_data.clone()).into())
.if_match(&initial_etag)
.send()
.await?;
assert_eq!(object_body(&client, test_key).await?.as_ref(), matching_data);
.await;
assert!(response1.is_ok(), "PUT with matching If-Match should succeed");
// Test 2: PUT with non-matching If-None-Match condition (should succeed)
let fake_etag = "\"fake-etag-12345\"";
client
let response2 = client
.put_object()
.bucket(TEST_BUCKET)
.key(test_key)
.body(Bytes::from(non_matching_data.clone()).into())
.bucket(BUCKET)
.key(&test_key)
.body(Bytes::from(updated_data.clone()).into())
.if_none_match(fake_etag)
.send()
.await?;
assert_eq!(object_body(&client, test_key).await?.as_ref(), non_matching_data);
.await;
assert!(response2.is_ok(), "PUT with non-matching If-None-Match should succeed");
// Cleanup
cleanup_objects(&client, BUCKET, &[&test_key]).await;
Ok(())
}
#[tokio::test]
async fn test_conditional_put_failed() -> TestResult {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
env.create_test_bucket(TEST_BUCKET).await?;
let client = env.create_s3_client();
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_conditional_put_failed() -> Result<(), Box<dyn std::error::Error>> {
let client = create_aws_s3_client().await?;
setup_test_bucket(&client).await?;
let test_key = "conditional-put-failed";
let test_key = generate_test_key("conditional-put-failed");
let initial_data = generate_test_data(1024);
let updated_data = generate_test_data(2048);
// Upload initial object and get its ETag
let initial_etag = upload_object_with_metadata(&client, TEST_BUCKET, test_key, &initial_data).await?;
let initial_etag = upload_object_with_metadata(&client, BUCKET, &test_key, &initial_data).await?;
// Test 1: PUT with non-matching If-Match condition (should fail with 412)
let fake_etag = "\"fake-etag-should-not-match\"";
let response1 = client
.put_object()
.bucket(TEST_BUCKET)
.key(test_key)
.bucket(BUCKET)
.key(&test_key)
.body(Bytes::from(updated_data.clone()).into())
.if_match(fake_etag)
.send()
.await;
assert_s3_error_code(response1, "PreconditionFailed");
assert_eq!(object_body(&client, test_key).await?.as_ref(), initial_data);
assert!(response1.is_err(), "PUT with non-matching If-Match should fail");
if let Err(e) = response1 {
if let SdkError::ServiceError(e) = e {
let e = e.into_err();
let error_code = e.meta().code().unwrap_or("");
assert_eq!("PreconditionFailed", error_code);
} else {
panic!("Unexpected error: {e:?}");
}
}
// Test 2: PUT with matching If-None-Match condition (should fail with 412)
let response2 = client
.put_object()
.bucket(TEST_BUCKET)
.key(test_key)
.bucket(BUCKET)
.key(&test_key)
.body(Bytes::from(updated_data.clone()).into())
.if_none_match(&initial_etag)
.send()
.await;
assert_s3_error_code(response2, "PreconditionFailed");
assert_eq!(object_body(&client, test_key).await?.as_ref(), initial_data);
assert!(response2.is_err(), "PUT with matching If-None-Match should fail");
if let Err(e) = response2 {
if let SdkError::ServiceError(e) = e {
let e = e.into_err();
let error_code = e.meta().code().unwrap_or("");
assert_eq!("PreconditionFailed", error_code);
} else {
panic!("Unexpected error: {e:?}");
}
}
// Cleanup - only need to clean up the initial object since failed PUTs shouldn't create objects
cleanup_objects(&client, BUCKET, &[&test_key]).await;
Ok(())
}
#[tokio::test]
async fn test_conditional_put_when_object_does_not_exist() -> TestResult {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
env.create_test_bucket(TEST_BUCKET).await?;
let client = env.create_s3_client();
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_conditional_put_when_object_does_not_exist() -> Result<(), Box<dyn std::error::Error>> {
let client = create_aws_s3_client().await?;
setup_test_bucket(&client).await?;
let key = "conditional-put-missing";
let key = "some_key";
cleanup_objects(&client, BUCKET, &[key]).await;
// When the object does not exist, the If-Match condition should always fail
let response1 = client
.put_object()
.bucket(TEST_BUCKET)
.bucket(BUCKET)
.key(key)
.body(Bytes::from(generate_test_data(1024)).into())
.if_match("*")
.send()
.await;
assert_s3_error_code(response1, "NoSuchKey");
assert!(response1.is_err());
if let Err(e) = response1 {
if let SdkError::ServiceError(e) = e {
let e = e.into_err();
let error_code = e.meta().code().unwrap_or("");
assert_eq!("NoSuchKey", error_code);
} else {
panic!("Unexpected error: {e:?}");
}
}
// When the object does not exist, the If-None-Match condition should be able to succeed
let created_data = generate_test_data(1024);
client
let response2 = client
.put_object()
.bucket(TEST_BUCKET)
.bucket(BUCKET)
.key(key)
.body(Bytes::from(created_data.clone()).into())
.body(Bytes::from(generate_test_data(1024)).into())
.if_none_match("*")
.send()
.await?;
assert_eq!(object_body(&client, key).await?.as_ref(), created_data);
.await;
assert!(response2.is_ok());
cleanup_objects(&client, BUCKET, &[key]).await;
Ok(())
}
#[tokio::test]
async fn test_conditional_multi_part_upload() -> TestResult {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
env.create_test_bucket(TEST_BUCKET).await?;
let client = env.create_s3_client();
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_conditional_multi_part_upload() -> Result<(), Box<dyn std::error::Error>> {
let client = create_aws_s3_client().await?;
setup_test_bucket(&client).await?;
let test_key = "conditional-multipart-upload";
let test_key = generate_test_key("multipart-upload-ok");
let test_data = generate_test_data(1024);
let initial_etag = upload_object_with_metadata(&client, TEST_BUCKET, test_key, &test_data).await?;
let initial_etag = upload_object_with_metadata(&client, BUCKET, &test_key, &test_data).await?;
let part_size = 5 * 1024 * 1024; // 5MB per part (minimum for multipart)
let num_parts = 3;
let mut parts = Vec::new();
let mut expected_data = Vec::with_capacity(part_size * usize::try_from(num_parts)?);
// Initiate multipart upload
let initiate_response = client
.create_multipart_upload()
.bucket(TEST_BUCKET)
.key(test_key)
.send()
.await?;
let initiate_response = client.create_multipart_upload().bucket(BUCKET).key(&test_key).send().await?;
let upload_id = initiate_response
.upload_id()
@@ -213,13 +258,12 @@ async fn test_conditional_multi_part_upload() -> TestResult {
// Upload parts
for part_number in 1..=num_parts {
let part_data = vec![u8::try_from(part_number)?; part_size];
expected_data.extend_from_slice(&part_data);
let part_data = generate_test_data(part_size);
let upload_part_response = client
.upload_part()
.bucket(TEST_BUCKET)
.key(test_key)
.bucket(BUCKET)
.key(&test_key)
.upload_id(upload_id)
.part_number(part_number)
.body(Bytes::from(part_data).into())
@@ -242,62 +286,57 @@ async fn test_conditional_multi_part_upload() -> TestResult {
// Test 1: Multipart upload with wildcard If-None-Match, should fail
let complete_response = client
.complete_multipart_upload()
.bucket(TEST_BUCKET)
.key(test_key)
.bucket(BUCKET)
.key(&test_key)
.upload_id(upload_id)
.multipart_upload(completed_upload.clone())
.if_none_match("*")
.send()
.await;
assert_s3_error_code(complete_response, "PreconditionFailed");
assert!(complete_response.is_err());
// Test 2: Multipart upload with matching If-None-Match, should fail
let complete_response = client
.complete_multipart_upload()
.bucket(TEST_BUCKET)
.key(test_key)
.bucket(BUCKET)
.key(&test_key)
.upload_id(upload_id)
.multipart_upload(completed_upload.clone())
.if_none_match(initial_etag.clone())
.send()
.await;
assert_s3_error_code(complete_response, "PreconditionFailed");
assert!(complete_response.is_err());
// Test 3: Multipart upload with unmatching If-Match, should fail
let complete_response = client
.complete_multipart_upload()
.bucket(TEST_BUCKET)
.key(test_key)
.bucket(BUCKET)
.key(&test_key)
.upload_id(upload_id)
.multipart_upload(completed_upload.clone())
.if_match("\"abcdef\"")
.send()
.await;
assert_s3_error_code(complete_response, "PreconditionFailed");
let staged_parts = client
.list_parts()
.bucket(TEST_BUCKET)
.key(test_key)
.upload_id(upload_id)
.send()
.await?;
assert_eq!(staged_parts.parts().len(), usize::try_from(num_parts)?);
assert!(complete_response.is_err());
// Test 4: Multipart upload with matching If-Match, should succeed
client
let complete_response = client
.complete_multipart_upload()
.bucket(TEST_BUCKET)
.key(test_key)
.bucket(BUCKET)
.key(&test_key)
.upload_id(upload_id)
.multipart_upload(completed_upload)
.multipart_upload(completed_upload.clone())
.if_match(initial_etag)
.send()
.await?;
assert_eq!(object_body(&client, test_key).await?.as_ref(), expected_data);
.await;
assert!(complete_response.is_ok());
// Cleanup
cleanup_objects(&client, BUCKET, &[&test_key]).await;
Ok(())
}
+153 -174
View File
@@ -13,228 +13,207 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::common::RustFSTestEnvironment;
use crate::common::workspace_root;
use crate::storage_api::node_interact::{
TonicInterceptor, VolumeInfo, WalkDirOptions, gen_tonic_signature_interceptor, node_service_time_out_client,
};
use aws_sdk_s3::primitives::ByteStream;
use futures::future::join_all;
use rmp_serde::{Deserializer, Serializer};
use rustfs_filemeta::MetaCacheEntry;
use rustfs_filemeta::{MetaCacheEntry, MetacacheReader, MetacacheWriter};
use rustfs_protos::proto_gen::node_service::WalkDirRequest;
use rustfs_protos::{
models::{PingBody, PingBodyBuilder},
proto_gen::node_service::{ListVolumesRequest, LocalStorageInfoRequest, MakeVolumeRequest, PingRequest, ReadAllRequest},
proto_gen::node_service::{
ListVolumesRequest, LocalStorageInfoRequest, MakeVolumeRequest, PingRequest, PingResponse, ReadAllRequest,
},
};
use serde::{Deserialize, Serialize};
use std::error::Error;
use std::io::Cursor;
use std::path::PathBuf;
use tokio::spawn;
use tonic::Request;
use tonic::codegen::tokio_stream::StreamExt;
type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
const TEST_RPC_SECRET: &str = "rustfs-internode-signature-e2e-secret";
const CLUSTER_ADDR: &str = "http://localhost:9000";
fn signature_interceptor() -> TonicInterceptor {
TonicInterceptor::Signature(gen_tonic_signature_interceptor())
}
fn rpc_client_error(error: Box<dyn Error>) -> std::io::Error {
std::io::Error::other(error.to_string())
}
async fn start_server() -> Result<RustFSTestEnvironment, Box<dyn Error + Send + Sync>> {
let _ = rustfs_credentials::set_global_rpc_secret(TEST_RPC_SECRET.to_string());
let effective = rustfs_credentials::try_get_rpc_token().expect("RPC secret must resolve in the test process");
assert_eq!(effective, TEST_RPC_SECRET, "the test process uses an unexpected RPC secret");
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server_without_cleanup_with_env(&[
("RUSTFS_RPC_SECRET", TEST_RPC_SECRET),
("RUSTFS_INTERNODE_RPC_SIGNATURE_STRICT", "false"),
("RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT", "false"),
("RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT", "false"),
("RUST_LOG", "error"),
])
.await?;
Ok(env)
}
#[tokio::test]
async fn ping() -> TestResult {
let env = start_server().await?;
#[ignore = "requires running RustFS server at localhost:9000"]
async fn ping() -> Result<(), Box<dyn Error>> {
let mut fbb = flatbuffers::FlatBufferBuilder::new();
let payload = fbb.create_vector(b"hello world");
let mut builder = PingBodyBuilder::new(&mut fbb);
builder.add_payload(payload);
let root = builder.finish();
fbb.finish(root, None);
let mut client = node_service_time_out_client(&env.url, signature_interceptor())
.await
.map_err(rpc_client_error)?;
let response = client
.ping(Request::new(PingRequest {
version: 1,
body: bytes::Bytes::copy_from_slice(fbb.finished_data()),
}))
.await?
.into_inner();
let finished_data = fbb.finished_data();
let decoded_payload = flatbuffers::root::<PingBody>(finished_data);
assert!(decoded_payload.is_ok());
// Create client
let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
// Construct PingRequest
let request = Request::new(PingRequest {
version: 1,
body: bytes::Bytes::copy_from_slice(finished_data),
});
// Send request and get response
let response: PingResponse = client.ping(request).await?.into_inner();
// Print response
let ping_response_body = flatbuffers::root::<PingBody>(&response.body);
if let Err(e) = ping_response_body {
eprintln!("{e}");
} else {
println!("ping_resp:body(flatbuffer): {ping_response_body:?}");
}
assert_eq!(response.version, 1);
let body = flatbuffers::root::<PingBody>(&response.body)?;
assert_eq!(body.payload().expect("ping response must contain a payload").bytes(), b"hello, caller");
Ok(())
}
#[tokio::test]
async fn make_volume() -> TestResult {
let env = start_server().await?;
let mut client = node_service_time_out_client(&env.url, signature_interceptor())
.await
.map_err(rpc_client_error)?;
let response = client
.make_volume(Request::new(MakeVolumeRequest {
disk: env.temp_dir.clone(),
volume: "node-rpc-volume".to_string(),
}))
.await?
.into_inner();
#[ignore = "requires running RustFS server at localhost:9000"]
async fn make_volume() -> Result<(), Box<dyn Error>> {
let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
let request = Request::new(MakeVolumeRequest {
disk: "data".to_string(),
volume: "dandan".to_string(),
});
assert!(response.success, "make_volume failed: {:?}", response.error);
assert!(std::path::Path::new(&env.temp_dir).join("node-rpc-volume").is_dir());
let response = client.make_volume(request).await?.into_inner();
if response.success {
println!("success");
} else {
println!("failed: {:?}", response.error);
}
Ok(())
}
#[tokio::test]
async fn list_volumes() -> TestResult {
let env = start_server().await?;
let mut client = node_service_time_out_client(&env.url, signature_interceptor())
.await
.map_err(rpc_client_error)?;
let created = client
.make_volume(Request::new(MakeVolumeRequest {
disk: env.temp_dir.clone(),
volume: "node-rpc-listed-volume".to_string(),
}))
.await?
.into_inner();
assert!(created.success, "make_volume failed: {:?}", created.error);
#[ignore = "requires running RustFS server at localhost:9000"]
async fn list_volumes() -> Result<(), Box<dyn Error>> {
let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
let request = Request::new(ListVolumesRequest {
disk: "data".to_string(),
});
let response = client
.list_volumes(Request::new(ListVolumesRequest {
disk: env.temp_dir.clone(),
}))
.await?
.into_inner();
assert!(response.success, "list_volumes failed: {:?}", response.error);
let volumes = response
let response = client.list_volumes(request).await?.into_inner();
let volume_infos: Vec<VolumeInfo> = response
.volume_infos
.iter()
.map(|json| serde_json::from_str::<VolumeInfo>(json))
.collect::<Result<Vec<_>, _>>()?;
assert!(volumes.iter().any(|volume| volume.name == "node-rpc-listed-volume"));
.into_iter()
.filter_map(|json_str| serde_json::from_str::<VolumeInfo>(&json_str).ok())
.collect();
println!("{volume_infos:?}");
Ok(())
}
#[tokio::test]
async fn walk_dir() -> TestResult {
let env = start_server().await?;
let s3 = env.create_s3_client();
let bucket = "node-rpc-walk-bucket";
let key = "prefix/object.txt";
env.create_test_bucket(bucket).await?;
s3.put_object()
.bucket(bucket)
.key(key)
.body(ByteStream::from_static(b"walk payload"))
.send()
.await?;
#[ignore = "requires running RustFS server at localhost:9000"]
async fn walk_dir() -> Result<(), Box<dyn Error>> {
println!("walk_dir");
// TODO: use writer
let opts = WalkDirOptions {
bucket: bucket.to_string(),
bucket: "dandan".to_owned(),
base_dir: "".to_owned(),
recursive: true,
..Default::default()
};
let mut encoded = Vec::new();
opts.serialize(&mut Serializer::new(&mut encoded))?;
let mut client = node_service_time_out_client(&env.url, signature_interceptor())
.await
.map_err(rpc_client_error)?;
let mut stream = client
.walk_dir(Request::new(WalkDirRequest {
disk: env.temp_dir.clone(),
walk_dir_options: encoded.into(),
}))
.await?
.into_inner();
let (rd, mut wr) = tokio::io::duplex(1024);
let mut buf = Vec::new();
opts.serialize(&mut Serializer::new(&mut buf))?;
let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
let disk_path = std::env::var_os("RUSTFS_DISK_PATH").map(PathBuf::from).unwrap_or_else(|| {
let mut path = workspace_root();
path.push("target");
path.push(if cfg!(debug_assertions) { "debug" } else { "release" });
path.push("data");
path
});
let request = Request::new(WalkDirRequest {
disk: disk_path.to_string_lossy().into_owned(),
walk_dir_options: buf.into(),
});
let mut response = client.walk_dir(request).await?.into_inner();
let mut entries = Vec::new();
while let Some(response) = stream.next().await {
let response = response?;
assert!(response.success, "walk_dir failed: {:?}", response.error_info);
entries.push(serde_json::from_str::<MetaCacheEntry>(&response.meta_cache_entry)?);
let job1 = spawn(async move {
let mut out = MetacacheWriter::new(&mut wr);
loop {
match response.next().await {
Some(Ok(resp)) => {
if !resp.success {
println!("{}", resp.error_info.unwrap_or_else(|| "".to_string()));
}
let entry = serde_json::from_str::<MetaCacheEntry>(&resp.meta_cache_entry)
.map_err(|_e| std::io::Error::other(format!("Unexpected response: {response:?}")))
.unwrap();
out.write_obj(&entry).await.unwrap();
}
None => {
let _ = out.close().await;
break;
}
_ => {
println!("Unexpected response: {response:?}");
let _ = out.close().await;
break;
}
}
}
});
let job2 = spawn(async move {
let mut reader = MetacacheReader::new(rd);
while let Ok(Some(entry)) = reader.peek().await {
println!("{entry:?}");
}
});
join_all(vec![job1, job2]).await;
Ok(())
}
#[tokio::test]
#[ignore = "requires running RustFS server at localhost:9000"]
async fn read_all() -> Result<(), Box<dyn Error>> {
let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
let request = Request::new(ReadAllRequest {
disk: "data".to_string(),
volume: "ff".to_string(),
path: "format.json".to_string(),
});
let response = client.read_all(request).await?.into_inner();
let volume_infos = response.data;
println!("{}", response.success);
println!("{volume_infos:?}");
Ok(())
}
#[tokio::test]
#[ignore = "requires running RustFS server at localhost:9000"]
async fn storage_info() -> Result<(), Box<dyn Error>> {
let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
let request = Request::new(LocalStorageInfoRequest { metrics: true });
let response = client.local_storage_info(request).await?.into_inner();
if !response.success {
println!("{:?}", response.error_info);
return Ok(());
}
assert!(
entries.iter().any(|entry| entry.name == key),
"walk_dir did not return {key}: {entries:?}"
);
Ok(())
}
#[tokio::test]
async fn read_all() -> TestResult {
let env = start_server().await?;
let mut client = node_service_time_out_client(&env.url, signature_interceptor())
.await
.map_err(rpc_client_error)?;
let volume = "node-rpc-read-volume";
let created = client
.make_volume(Request::new(MakeVolumeRequest {
disk: env.temp_dir.clone(),
volume: volume.to_string(),
}))
.await?
.into_inner();
assert!(created.success, "make_volume failed: {:?}", created.error);
tokio::fs::write(std::path::Path::new(&env.temp_dir).join(volume).join("payload.bin"), b"read payload").await?;
let response = client
.read_all(Request::new(ReadAllRequest {
disk: env.temp_dir.clone(),
volume: volume.to_string(),
path: "payload.bin".to_string(),
}))
.await?
.into_inner();
assert!(response.success, "read_all failed: {:?}", response.error);
assert_eq!(response.data.as_ref(), b"read payload");
Ok(())
}
#[tokio::test]
async fn storage_info() -> TestResult {
let env = start_server().await?;
let mut client = node_service_time_out_client(&env.url, signature_interceptor())
.await
.map_err(rpc_client_error)?;
let response = client
.local_storage_info(Request::new(LocalStorageInfoRequest { metrics: true }))
.await?
.into_inner();
assert!(response.success, "local_storage_info failed: {:?}", response.error_info);
let mut decoder = Deserializer::new(Cursor::new(response.storage_info));
let storage_info: rustfs_madmin::StorageInfo = Deserialize::deserialize(&mut decoder)?;
let expected_disk = std::fs::canonicalize(&env.temp_dir)?;
assert!(!storage_info.disks.is_empty(), "local_storage_info returned no disks");
assert!(
storage_info
.disks
.iter()
.any(|disk| std::path::Path::new(&disk.drive_path) == expected_disk),
"local_storage_info did not include the configured disk: {:?}",
storage_info.disks
);
let info = response.storage_info;
let mut buf = Deserializer::new(Cursor::new(info));
let storage_info: rustfs_madmin::StorageInfo = Deserialize::deserialize(&mut buf).unwrap();
println!("{storage_info:?}");
Ok(())
}
+10 -422
View File
@@ -13,8 +13,9 @@
// limitations under the License.
use crate::common::{
RustFSTestEnvironment, admin_create_user, awscurl_post_sts_form_urlencoded, init_logging, local_http_client,
replication_fast_env, rustfs_binary_path, signed_request, signed_request_with_client, signed_request_with_session_token,
RustFSTestEnvironment, admin_create_user, awscurl_available, awscurl_post_sts_form_urlencoded, init_logging,
local_http_client, replication_fast_env, rustfs_binary_path, signed_request, signed_request_with_client,
signed_request_with_session_token,
};
use crate::fake_s3_target::{
FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, FaultAction as FakeTargetFault, Operation as FakeTargetOperation,
@@ -56,7 +57,7 @@ use rustfs_madmin::{
AddServiceAccountReq, ListServiceAccountsResp, PeerInfo, PeerSite, ReplicateAddStatus, ReplicateEditStatus,
ReplicateRemoveStatus, SRRemoveReq, SRResyncOpStatus, SRStatusInfo, SiteReplicationInfo, SyncStatus,
};
use s3s::header::{X_AMZ_REPLICATION_STATUS, X_AMZ_TAGGING};
use s3s::header::X_AMZ_REPLICATION_STATUS;
use sha2::{Digest, Sha256};
use std::collections::BTreeMap;
use std::convert::Infallible;
@@ -2022,7 +2023,6 @@ async fn forward_replication_proxy_request(
client: &reqwest::Client,
request_count: &AtomicU64,
mut replication_enabled: watch::Receiver<bool>,
mut held_tagging: watch::Receiver<Option<String>>,
) -> Response<Full<bytes::Bytes>> {
let (parts, body) = request.into_parts();
let is_replication = parts
@@ -2036,17 +2036,6 @@ async fn forward_replication_proxy_request(
return proxy_error_response("replication gate closed");
}
}
// Content-keyed hold: park only the replication request whose
// `x-amz-tagging` matches the held value, letting every other delivery
// through, so a test can make one specific (stale) delivery the last
// write the backend sees.
if let Some(tagging) = parts.headers.get(X_AMZ_TAGGING).and_then(|value| value.to_str().ok()) {
while held_tagging.borrow().as_deref() == Some(tagging) {
if held_tagging.changed().await.is_err() {
return proxy_error_response("replication tag hold closed");
}
}
}
}
let Some(path_and_query) = parts.uri.path_and_query() else {
@@ -2081,26 +2070,12 @@ async fn start_replication_counting_proxy(
backend_url: &str,
tasks: &mut JoinSet<()>,
) -> Result<(String, Arc<AtomicU64>, watch::Sender<bool>), Box<dyn Error + Send + Sync>> {
let (proxy_url, request_count, replication_enabled, _held_tagging) =
start_replication_counting_proxy_with_tag_hold(backend_url, tasks).await?;
Ok((proxy_url, request_count, replication_enabled))
}
/// [`start_replication_counting_proxy`] plus a content-keyed hold: while the
/// returned `watch::Sender<Option<String>>` holds `Some(tagging)`, replication
/// requests whose `x-amz-tagging` equals `tagging` are parked (and still
/// counted); all other traffic flows. Send `None` to release them.
async fn start_replication_counting_proxy_with_tag_hold(
backend_url: &str,
tasks: &mut JoinSet<()>,
) -> Result<(String, Arc<AtomicU64>, watch::Sender<bool>, watch::Sender<Option<String>>), Box<dyn Error + Send + Sync>> {
let listener = TcpListener::bind("127.0.0.1:0").await?;
let proxy_url = format!("http://{}", listener.local_addr()?);
let backend_url = backend_url.to_string();
let request_count = Arc::new(AtomicU64::new(0));
let task_request_count = request_count.clone();
let (replication_enabled, task_replication_enabled) = watch::channel(true);
let (held_tagging, task_held_tagging) = watch::channel(None);
tasks.spawn(async move {
let client = local_http_client();
let mut connections = JoinSet::new();
@@ -2112,14 +2087,12 @@ async fn start_replication_counting_proxy_with_tag_hold(
let client = client.clone();
let request_count = task_request_count.clone();
let replication_enabled = task_replication_enabled.clone();
let held_tagging = task_held_tagging.clone();
connections.spawn(async move {
let service = service_fn(move |request| {
let backend_url = backend_url.clone();
let client = client.clone();
let request_count = request_count.clone();
let replication_enabled = replication_enabled.clone();
let held_tagging = held_tagging.clone();
async move {
Ok::<_, Infallible>(
forward_replication_proxy_request(
@@ -2128,7 +2101,6 @@ async fn start_replication_counting_proxy_with_tag_hold(
&client,
&request_count,
replication_enabled,
held_tagging,
)
.await,
)
@@ -2141,7 +2113,7 @@ async fn start_replication_counting_proxy_with_tag_hold(
}
}
});
Ok((proxy_url, request_count, replication_enabled, held_tagging))
Ok((proxy_url, request_count, replication_enabled))
}
async fn site_replication_remove(
@@ -6977,395 +6949,6 @@ async fn test_site_replication_active_active_converges_without_loops_real_dual_n
}
}
/// Replication status a site reports for one object version via HEAD
/// (`x-amz-replication-status`), or `None` when the header is absent.
async fn head_replication_status(
client: &Client,
bucket: &str,
key: &str,
version_id: &str,
) -> Result<Option<String>, Box<dyn Error + Send + Sync>> {
let head = client
.head_object()
.bucket(bucket)
.key(key)
.version_id(version_id)
.send()
.await?;
Ok(head.replication_status().map(|status| status.as_str().to_string()))
}
/// Poll one site until the version's replication status is one of `expected`.
async fn wait_for_version_replication_status(
client: &Client,
bucket: &str,
key: &str,
version_id: &str,
expected: &[&str],
site: &str,
) -> Result<String, Box<dyn Error + Send + Sync>> {
let deadline = tokio::time::Instant::now() + Duration::from_secs(60);
loop {
let last = head_replication_status(client, bucket, key, version_id).await?;
if let Some(status) = last.as_deref()
&& expected.contains(&status)
{
return Ok(status.to_string());
}
if tokio::time::Instant::now() >= deadline {
return Err(format!(
"{site}: {bucket}/{key}?versionId={version_id} replication status {last:?} never reached {expected:?}"
)
.into());
}
sleep(Duration::from_millis(200)).await;
}
}
async fn put_single_tag(
client: &Client,
bucket: &str,
key: &str,
version_id: &str,
tag_key: &str,
tag_value: &str,
) -> Result<(), Box<dyn Error + Send + Sync>> {
client
.put_object_tagging()
.bucket(bucket)
.key(key)
.version_id(version_id)
.tagging(
aws_sdk_s3::types::Tagging::builder()
.tag_set(aws_sdk_s3::types::Tag::builder().key(tag_key).value(tag_value).build()?)
.build()?,
)
.send()
.await?;
Ok(())
}
async fn get_single_tag(
client: &Client,
bucket: &str,
key: &str,
version_id: &str,
tag_key: &str,
) -> Result<Option<String>, Box<dyn Error + Send + Sync>> {
let tagging = client
.get_object_tagging()
.bucket(bucket)
.key(key)
.version_id(version_id)
.send()
.await?;
Ok(tagging
.tag_set()
.iter()
.find(|tag| tag.key() == tag_key)
.map(|tag| tag.value().to_string()))
}
/// Poll one site until the version's `tag_key` equals `expected`.
async fn wait_for_single_tag(
client: &Client,
bucket: &str,
key: &str,
version_id: &str,
tag_key: &str,
expected: &str,
site: &str,
) -> Result<(), Box<dyn Error + Send + Sync>> {
let deadline = tokio::time::Instant::now() + Duration::from_secs(60);
loop {
let observed = get_single_tag(client, bucket, key, version_id, tag_key).await?;
if observed.as_deref() == Some(expected) {
return Ok(());
}
if tokio::time::Instant::now() >= deadline {
return Err(format!(
"{site}: {bucket}/{key}?versionId={version_id} tag {tag_key}={observed:?} never became {expected}"
)
.into());
}
sleep(Duration::from_millis(200)).await;
}
}
/// Tag key the dual-node LWW scenario edits on both sites.
const LWW_TAG_KEY: &str = "owner";
/// Assert the version's [`LWW_TAG_KEY`] stays `expected` on both sites for a
/// full quiet window (no late stale delivery flips it back).
async fn assert_tag_stable_on_both_sites(
site_a_client: &Client,
site_b_client: &Client,
bucket: &str,
key: &str,
version_id: &str,
expected: &str,
quiet: Duration,
) -> Result<(), Box<dyn Error + Send + Sync>> {
let deadline = tokio::time::Instant::now() + quiet;
loop {
let on_a = get_single_tag(site_a_client, bucket, key, version_id, LWW_TAG_KEY).await?;
let on_b = get_single_tag(site_b_client, bucket, key, version_id, LWW_TAG_KEY).await?;
assert_eq!(on_a.as_deref(), Some(expected), "site A tag {LWW_TAG_KEY} regressed from the LWW winner");
assert_eq!(on_b.as_deref(), Some(expected), "site B tag {LWW_TAG_KEY} regressed from the LWW winner");
if tokio::time::Instant::now() >= deadline {
return Ok(());
}
sleep(Duration::from_millis(250)).await;
}
}
/// Wait until the counting proxy in front of a site has admitted `expected`
/// replication requests in total (requests held by a closed gate still count).
async fn wait_for_proxy_replication_requests(
counter: &AtomicU64,
expected: u64,
site: &str,
) -> Result<(), Box<dyn Error + Send + Sync>> {
let deadline = tokio::time::Instant::now() + Duration::from_secs(60);
loop {
let observed = counter.load(Ordering::Relaxed);
if observed >= expected {
return Ok(());
}
if tokio::time::Instant::now() >= deadline {
return Err(format!("{site} proxy saw {observed} replication requests, expected at least {expected}").into());
}
sleep(Duration::from_millis(25)).await;
}
}
/// rustfs/backlog#1953 (audit A4/P1-6): receiver-side LWW for replicated
/// metadata categories, exercised end to end over the real dual-node
/// active-active site-replication control plane — sender, worker, status
/// bookkeeping and persisted failure recovery all participate (the single-server
/// `replication_lww_receiver_test` only injects authorized replication PUTs).
///
/// Scenario on one versioned object:
/// 1. reciprocal tag edits in real order (A then B) converge both sites on the
/// newer tag and leave the author COMPLETED / the receiver REPLICA;
/// 2. out-of-order delivery: A's edit is held at B's inbound proxy while B
/// authors a newer edit that reaches A first; releasing the stale delivery
/// must NOT roll B back — both sites settle on B's value and stay there
/// through a quiet window, with no FAILED/PENDING status left behind;
/// 3. persisted retry: B is stopped, A's delivery reaches FAILED, A restarts,
/// then B returns and the scanner-replayed edit converges both sites forward.
/// Durable metadata-MRF serialization/reconstruction is covered separately by
/// `metadata_mrf_roundtrip_preserves_tags_and_admitted_targets`.
#[tokio::test]
async fn test_site_replication_tagging_lww_converges_active_active_real_dual_node() -> TestResult {
init_logging();
match tokio::time::timeout(Duration::from_secs(420), async {
// The scanner is fast for the final persisted-failure recovery phase.
// Step 2 finishes and proves a quiet stable winner before that phase,
// so a later scanner pass cannot mask its stale-delivery assertion.
let mut site_env = replication_fast_env();
site_env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
site_env.extend_from_slice(FAST_SCANNER_ENV);
let mut site_a_env = RustFSTestEnvironment::new().await?;
site_a_env.start_rustfs_server_with_env(vec![], &site_env).await?;
let mut site_b_env = RustFSTestEnvironment::new().await?;
site_b_env.start_rustfs_server_with_env(vec![], &site_env).await?;
let mut proxy_tasks = JoinSet::new();
let (site_a_proxy, site_a_replication_requests, _site_a_replication_enabled, site_a_held_tagging) =
start_replication_counting_proxy_with_tag_hold(&site_a_env.url, &mut proxy_tasks).await?;
let (site_b_proxy, site_b_replication_requests, _site_b_replication_enabled, site_b_held_tagging) =
start_replication_counting_proxy_with_tag_hold(&site_b_env.url, &mut proxy_tasks).await?;
let site_a_client = site_a_env.create_s3_client();
let site_b_client = site_b_env.create_s3_client();
let bucket = "site-repl-tag-lww";
let key = "lww.txt";
let add_status = site_replication_add(
&site_a_env,
&[
PeerSite {
name: "lww-site-a".to_string(),
endpoint: site_a_env.url.clone(),
access_key: site_a_env.access_key.clone(),
secret_key: site_a_env.secret_key.clone(),
..Default::default()
},
PeerSite {
name: "lww-site-b".to_string(),
endpoint: site_b_env.url.clone(),
access_key: site_b_env.access_key.clone(),
secret_key: site_b_env.secret_key.clone(),
..Default::default()
},
],
)
.await?;
assert!(add_status.success, "unexpected site add result: {add_status:?}");
let site_info = wait_for_site_replication_enabled(&site_a_env, 2).await?;
wait_for_site_replication_enabled(&site_b_env, 2).await?;
// Route both directions through the counting proxies so inbound
// replication to B can be held (out-of-order delivery) and observed.
for (env_url, proxy_url, label) in [(&site_a_env.url, &site_a_proxy, "A"), (&site_b_env.url, &site_b_proxy, "B")] {
let mut peer = site_info
.sites
.iter()
.find(|peer| peer.endpoint == *env_url)
.ok_or_else(|| format!("site {label} peer missing from replication info"))?
.clone();
peer.endpoint = proxy_url.clone();
peer.sync_state = SyncStatus::Enable;
let edit = site_replication_edit(&site_a_env, "", &peer).await?;
assert!(edit.success, "unexpected site {label} endpoint edit: {edit:?}");
}
for env in [&site_a_env, &site_b_env] {
wait_for_site_replication_info(env, |info| {
info.sites.iter().any(|peer| peer.endpoint == site_a_proxy)
&& info.sites.iter().any(|peer| peer.endpoint == site_b_proxy)
})
.await?;
}
site_a_client.create_bucket().bucket(bucket).send().await?;
wait_for_bucket_on_target(&site_b_client, bucket).await?;
let version_id = site_a_client
.put_object()
.bucket(bucket)
.key(key)
.body(ByteStream::from_static(b"tag lww payload"))
.send()
.await?
.version_id()
.ok_or("site A PUT omitted version ID")?
.to_string();
wait_for_replicated_object(&site_b_client, bucket, key, "tag lww payload").await?;
wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["COMPLETED"], "site A").await?;
wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["REPLICA"], "site B").await?;
wait_for_proxy_replication_requests(&site_b_replication_requests, 1, "site B").await?;
// --- 1. reciprocal edits in real order: A then B ----------------------
put_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY, "a1").await?;
wait_for_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY, "a1", "site B").await?;
wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["COMPLETED"], "site A").await?;
wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["REPLICA"], "site B").await?;
wait_for_proxy_replication_requests(&site_b_replication_requests, 2, "site B").await?;
put_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY, "b1").await?;
wait_for_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY, "b1", "site A").await?;
wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["COMPLETED"], "site B").await?;
wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["REPLICA"], "site A").await?;
assert_tag_stable_on_both_sites(&site_a_client, &site_b_client, bucket, key, &version_id, "b1", Duration::from_secs(3))
.await?;
// --- 2. concurrent edits, stale delivery last ------------------------
// Both sites edit the same version while each other's delivery is
// parked at the peer's inbound proxy (content-keyed: only the
// `owner=a2` / `owner=b2` replication PUTs wait, everything else
// flows). B's edit is the newer one. Releasing A's stale `a2` first
// makes it the last write B sees while A itself still holds `a2`, so
// nothing A could re-deliver carries the winner: only receiver-side
// LWW on B can keep `b2`. Releasing `b2` afterwards converges A.
site_b_held_tagging.send(Some("owner=a2".to_string()))?;
site_a_held_tagging.send(Some("owner=b2".to_string()))?;
let a2_parked_at = site_b_replication_requests.load(Ordering::Relaxed) + 1;
let b2_parked_at = site_a_replication_requests.load(Ordering::Relaxed) + 1;
put_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY, "a2").await?;
wait_for_proxy_replication_requests(&site_b_replication_requests, a2_parked_at, "site B").await?;
sleep(Duration::from_millis(50)).await;
put_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY, "b2").await?;
wait_for_proxy_replication_requests(&site_a_replication_requests, b2_parked_at, "site A").await?;
assert_eq!(
get_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY)
.await?
.as_deref(),
Some("a2")
);
assert_eq!(
get_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY)
.await?
.as_deref(),
Some("b2")
);
// Release the stale a2 delivery onto B: the newer local b2 must
// survive, and the delivery itself must still succeed (A reaches
// COMPLETED instead of looping through MRF with the stale value).
site_b_held_tagging.send(None)?;
wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["COMPLETED"], "site A").await?;
let stale_deadline = tokio::time::Instant::now() + Duration::from_secs(3);
loop {
assert_eq!(
get_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY)
.await?
.as_deref(),
Some("b2"),
"a stale inbound delivery rolled back site B's newer tag (receiver-side LWW regression)"
);
if tokio::time::Instant::now() >= stale_deadline {
break;
}
sleep(Duration::from_millis(250)).await;
}
// Release b2 onto A: the newer edit wins there and both sites settle.
// B's own version may legitimately read REPLICA here: the stale inbound
// a2 write re-labelled it as a replica write (keeping B's tags); what
// must not remain is PENDING/FAILED.
site_a_held_tagging.send(None)?;
wait_for_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY, "b2", "site A").await?;
wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["COMPLETED", "REPLICA"], "site B")
.await?;
assert_tag_stable_on_both_sites(&site_a_client, &site_b_client, bucket, key, &version_id, "b2", Duration::from_secs(4))
.await?;
for (client, site) in [(&site_a_client, "site A"), (&site_b_client, "site B")] {
let status = head_replication_status(client, bucket, key, &version_id).await?;
assert!(
matches!(status.as_deref(), Some("COMPLETED" | "REPLICA")),
"{site} must not be left PENDING/FAILED after the concurrent edits: {status:?}"
);
}
// --- 3. persisted FAILED state survives a source restart ------------
site_b_env.stop_server();
put_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY, "a3").await?;
wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["FAILED"], "site A").await?;
site_a_env.restart_server_preserving_data(vec![], &site_env).await?;
wait_for_site_replication_enabled(&site_a_env, 2).await?;
site_b_env.restart_server_preserving_data(vec![], &site_env).await?;
wait_for_site_replication_enabled(&site_b_env, 2).await?;
wait_for_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY, "a3", "site B").await?;
wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["COMPLETED"], "site A").await?;
wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["REPLICA"], "site B").await?;
assert_tag_stable_on_both_sites(&site_a_client, &site_b_client, bucket, key, &version_id, "a3", Duration::from_secs(3))
.await?;
// The object itself never forked: one version on each side.
tokio::time::timeout(
Duration::from_secs(70),
assert_replication_converged(&site_a_client, bucket, &site_b_client, bucket),
)
.await??;
let state = list_replication_state(&site_a_client, bucket).await?;
assert_eq!(state.len(), 1, "tag edits must not create new object versions: {state:?}");
assert_eq!(state[0].version_id, version_id);
proxy_tasks.abort_all();
Ok(())
})
.await
{
Ok(result) => result,
Err(_) => Err("site replication tagging LWW test timed out".into()),
}
}
#[tokio::test]
async fn test_site_replication_replicates_policy_backed_user_access_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {
init_logging();
@@ -7698,6 +7281,11 @@ async fn test_site_replication_replicates_multiple_service_accounts_real_dual_no
async fn test_site_replication_replicates_service_accounts_created_from_sts_session_real_dual_node() -> TestResult {
init_logging();
if !awscurl_available() {
eprintln!("Skipping STS site replication service-account test because awscurl is unavailable");
return Ok(());
}
let mut source_env = RustFSTestEnvironment::new().await?;
source_env
.start_rustfs_server_with_env(vec![], LOOPBACK_REPLICATION_TARGET_ENV)
@@ -1,155 +0,0 @@
#![cfg(test)]
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Receiver-side replication LWW over the wire (rustfs/backlog#1953, audit
//! A4/P1-6).
//!
//! In an active-active topology both sites' metadata states arrive at the
//! peer as authorized replication PUTs carrying per-category source
//! timestamps (`x-rustfs-source-replication-tagging-timestamp` header
//! family). Before the fix the receiver applied them unconditionally, so a
//! stale delivery overwrote a newer local state and the two sites diverged
//! permanently while both reported COMPLETED. This test drives one live
//! `rustfs` server with simulated inbound replication PUTs for the same
//! object version and asserts the newer tagging state wins regardless of
//! delivery order, while a stale delivery still succeeds at the object level
//! (a failure would loop through MRF re-delivering the stale value).
//!
//! The real dual-site path (sender, worker, status bookkeeping, MRF replay)
//! is covered by
//! `replication_extension_test::test_site_replication_tagging_lww_converges_active_active_real_dual_node`;
//! this file stays as the fast, single-process receiver check.
use crate::common::{RustFSTestEnvironment, init_logging};
use aws_sdk_s3::Client;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
const HDR_SOURCE_REPLICATION_REQUEST: &str = "x-rustfs-source-replication-request";
const HDR_SOURCE_VERSION_ID: &str = "x-rustfs-source-version-id";
const HDR_SOURCE_MTIME: &str = "x-rustfs-source-mtime";
const HDR_SOURCE_TAGGING_TIMESTAMP: &str = "x-rustfs-source-replication-tagging-timestamp";
const SOURCE_MTIME: &str = "2026-01-01T00:00:00Z";
const T_STALE: &str = "2026-01-01T00:00:01Z";
const T_LOCAL: &str = "2026-02-01T00:00:00Z";
const T_NEWER: &str = "2026-03-01T00:00:00Z";
/// Simulated inbound authorized replication PUT: same object version, tags and
/// the source-authored tagging timestamp carried in transport headers.
async fn inbound_replication_put(
client: &Client,
bucket: &str,
key: &str,
version_id: &str,
tags: &str,
tagging_timestamp: &str,
) -> TestResult {
let version_id = version_id.to_string();
let tagging_timestamp = tagging_timestamp.to_string();
client
.put_object()
.bucket(bucket)
.key(key)
.body(ByteStream::from_static(b"lww-e2e-body"))
.tagging(tags)
.customize()
.mutate_request(move |req| {
req.headers_mut().insert(HDR_SOURCE_REPLICATION_REQUEST, "true");
req.headers_mut().insert(HDR_SOURCE_VERSION_ID, version_id.clone());
req.headers_mut().insert(HDR_SOURCE_MTIME, SOURCE_MTIME);
req.headers_mut()
.insert(HDR_SOURCE_TAGGING_TIMESTAMP, tagging_timestamp.clone());
})
.send()
.await?;
Ok(())
}
async fn tag_value(client: &Client, bucket: &str, key: &str, version_id: &str, tag_key: &str) -> Option<String> {
let tagging = client
.get_object_tagging()
.bucket(bucket)
.key(key)
.version_id(version_id)
.send()
.await
.expect("object tagging should be readable");
tagging
.tag_set()
.iter()
.find(|tag| tag.key() == tag_key)
.map(|tag| tag.value().to_string())
}
#[tokio::test(flavor = "multi_thread")]
async fn receiver_lww_keeps_newer_tags_across_delivery_orders() -> TestResult {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
let client = env.create_s3_client();
let bucket = "replication-lww-receiver";
let key = "object";
client.create_bucket().bucket(bucket).send().await?;
client
.put_bucket_versioning()
.bucket(bucket)
.versioning_configuration(
VersioningConfiguration::builder()
.status(BucketVersioningStatus::Enabled)
.build(),
)
.send()
.await?;
// First delivery establishes version V with tags stamped T_LOCAL.
let version_id = uuid::Uuid::new_v4().to_string();
inbound_replication_put(&client, bucket, key, &version_id, "site=local", T_LOCAL).await?;
assert_eq!(
tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
Some("local"),
"the first delivery must establish the tagged version"
);
// A stale delivery (older source timestamp) must succeed at the object
// level but must NOT overwrite the newer tags.
inbound_replication_put(&client, bucket, key, &version_id, "site=stale", T_STALE).await?;
assert_eq!(
tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
Some("local"),
"a stale inbound delivery must not overwrite newer tags (rustfs/backlog#1953)"
);
// A newer delivery still converges the version onto the newest state.
inbound_replication_put(&client, bucket, key, &version_id, "site=newer", T_NEWER).await?;
assert_eq!(
tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
Some("newer"),
"a newer inbound delivery must overwrite older tags"
);
client
.delete_object()
.bucket(bucket)
.key(key)
.version_id(&version_id)
.send()
.await?;
env.delete_test_bucket(bucket).await.ok();
Ok(())
}
@@ -21,11 +21,12 @@
//! - SSRF prevention (internal/private endpoints rejected for tiering)
//! - Race condition handling (concurrent writes converge without corruption)
use crate::common::{RustFSTestEnvironment, awscurl_put, init_logging, require_awscurl};
use crate::common::{RustFSTestEnvironment, awscurl_available, awscurl_put, init_logging};
use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart, Tag, Tagging};
use std::error::Error;
use tracing::info;
/// Oversized tagging payloads must be rejected by the per-object tag limit.
///
@@ -224,12 +225,16 @@ async fn test_concurrent_object_operations() -> Result<(), Box<dyn Error + Send
/// outcome — the internal endpoint is not accepted — is asserted here.
///
/// The admin API is exercised via signed `awscurl` requests, matching the
/// pattern used by the other admin-API E2E tests in this crate. The full E2E
/// lane installs and verifies the pinned `awscurl` prerequisite.
/// pattern used by the other admin-API E2E tests in this crate; the test is
/// skipped when `awscurl` is not installed.
#[tokio::test]
async fn test_tiering_url_validation() -> Result<(), Box<dyn Error + Send + Sync>> {
init_logging();
require_awscurl()?;
if !awscurl_available() {
info!("Skipping tiering URL validation test because awscurl is not available");
return Ok(());
}
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
@@ -15,6 +15,7 @@
use crate::common::{RustFSTestClusterEnvironment, init_logging};
use aws_sdk_s3::error::SdkError;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::CompletedMultipartUpload;
use tokio::time::{Duration, sleep};
use tracing::info;
use uuid::Uuid;
@@ -42,18 +43,32 @@ async fn list_parts_reports_missing_upload(
}
}
async fn multipart_listing_reports_missing_upload(
async fn complete_reports_missing_upload(
client: &aws_sdk_s3::Client,
bucket: &str,
key: &str,
upload_id: &str,
) -> Result<bool, Box<dyn std::error::Error + Send + Sync>> {
let result = client.list_multipart_uploads().bucket(bucket).prefix(key).send().await?;
Ok(!result
.uploads()
.iter()
.any(|upload| upload.key() == Some(key) && upload.upload_id() == Some(upload_id)))
let result = client
.complete_multipart_upload()
.bucket(bucket)
.key(key)
.upload_id(upload_id)
.multipart_upload(CompletedMultipartUpload::builder().build())
.send()
.await;
match result {
Ok(_) => Ok(false),
Err(SdkError::ServiceError(err)) => {
let code = err.err().meta().code().unwrap_or("");
if code == "NoSuchUpload" {
Ok(true)
} else {
Err(format!("unexpected complete_multipart_upload service error: code={code}, err={err:?}").into())
}
}
Err(err) => Err(format!("unexpected complete_multipart_upload error: {err:?}").into()),
}
}
async fn wait_for_cleanup_on_all_nodes(
@@ -66,8 +81,8 @@ async fn wait_for_cleanup_on_all_nodes(
let mut all_cleaned = true;
for (idx, client) in clients.iter().enumerate() {
let list_parts_missing = list_parts_reports_missing_upload(client, bucket, key, upload_id).await?;
let listing_missing = multipart_listing_reports_missing_upload(client, bucket, key, upload_id).await?;
if !(list_parts_missing && listing_missing) {
let complete_missing = complete_reports_missing_upload(client, bucket, key, upload_id).await?;
if !(list_parts_missing && complete_missing) {
info!("stale multipart still visible on node {} at attempt {}", idx, attempt + 1);
all_cleaned = false;
break;
@@ -131,10 +146,6 @@ async fn test_stale_multipart_cleanup_removes_incomplete_upload_across_cluster()
1,
"multipart upload should be visible before background cleanup"
);
assert!(
!multipart_listing_reports_missing_upload(&clients[2], CLEANUP_BUCKET, &key, &upload_id).await?,
"multipart upload listing should contain the upload before background cleanup"
);
wait_for_cleanup_on_all_nodes(&clients, CLEANUP_BUCKET, &key, &upload_id).await?;
+19 -23
View File
@@ -161,7 +161,7 @@ pub mod bucket {
pub mod objectlock_sys {
pub use crate::bucket::object_lock::objectlock_sys::{
BucketObjectLockSys, ObjectLockBlockReason, add_years, check_object_lock_for_deletion,
check_retention_for_modification, is_retention_active, replication_write_may_pass_worm_gate,
check_retention_for_modification, is_retention_active,
};
}
}
@@ -187,24 +187,22 @@ pub mod bucket {
pub use crate::bucket::replication::{
BucketReplicationResyncStatus, BucketReplicationStat, BucketReplicationStats, BucketStats,
DeleteReplicationConfigSnapshot, DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool, InQueueMetric,
MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts, OperatorRuleContract,
REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
REPLICATE_INCOMING_DELETE, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS,
REPLICATION_WRITABLE_FIELDS, ReplicateDecision, ReplicateObjectInfo, ReplicationBatchAdmission, ReplicationConfig,
MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION,
REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, REPLICATE_INCOMING_DELETE,
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
ReplicateDecision, ReplicateObjectInfo, ReplicationBatchAdmission, ReplicationConfig,
ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationDeleteScheduleInput,
ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge, ReplicationObjectIO,
ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge,
ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage, ReplicationTargetValidationError,
ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, TargetReplicationResyncStatus,
VersionPurgeStatusType, XferStats, assign_site_replication_rule_priorities, commit_force_delete_intent,
complete_force_delete_intent, delete_replication_state_from_config, delete_replication_version_id,
get_global_replication_pool, get_global_replication_stats, get_proxy_targets, init_background_replication,
invalid_replication_config_status_field, is_site_replication_role, is_site_replication_rule,
merge_incoming_replication_config, merge_user_replication_config, persist_force_delete_intent,
read_durable_mrf_backlog, replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map,
replication_target_arn_deployment_id, replication_target_arns, resync_start_conflict_id,
should_remove_replication_target, should_schedule_delete_replication, should_use_existing_delete_replication_info,
should_use_existing_delete_replication_source, site_replication_rule_deployment_id,
VersionPurgeStatusType, XferStats, commit_force_delete_intent, complete_force_delete_intent,
delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool,
get_global_replication_stats, get_proxy_targets, init_background_replication,
invalid_replication_config_status_field, persist_force_delete_intent, read_durable_mrf_backlog,
replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map, replication_target_arns,
resync_start_conflict_id, should_remove_replication_target, should_schedule_delete_replication,
should_use_existing_delete_replication_info, should_use_existing_delete_replication_source,
unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
version_purge_status_to_filemeta,
};
@@ -319,6 +317,8 @@ pub mod config {
}
pub mod data_usage {
#[cfg(feature = "test-util")]
pub use crate::data_usage::seed_bucket_usage_memory_for_test;
pub use crate::data_usage::{
DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache,
@@ -330,8 +330,6 @@ pub mod data_usage {
remove_bucket_usage_from_backend, replace_bucket_usage_memory_from_info, store_compression_total_in_backend,
store_data_usage_in_backend,
};
#[cfg(feature = "test-util")]
pub use crate::data_usage::{get_bucket_usage_memory, seed_bucket_usage_memory_for_test};
}
pub mod disk {
@@ -458,13 +456,11 @@ pub mod rpc {
ScannerBucketListing, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, build_put_file_auth_trailer,
check_and_record_signed_rpc_nonce, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, sign_ns_scanner_capability,
sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer,
verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
verify_tonic_rpc_signature_with_bootstrap,
normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_put_file_capability,
sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
tonic_rpc_auth_failure_reason, verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature,
verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest,
verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
};
}
+2 -79
View File
@@ -58,8 +58,8 @@ use rustfs_config::{DEFAULT_TRUST_LEAF_CERT_AS_CA, ENV_TRUST_LEAF_CERT_AS_CA, RU
use rustfs_utils::egress::{OutboundUrlError, validate_outbound_url};
use rustfs_utils::http::{
AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_MODE,
AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_OBJECT_TAGGING_LOWER, AMZ_STORAGE_CLASS, AMZ_WEBSITE_REDIRECT_LOCATION, is_amz_header,
is_minio_header, is_rustfs_header, is_standard_header, is_storageclass_header,
AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_STORAGE_CLASS, AMZ_WEBSITE_REDIRECT_LOCATION, is_amz_header, is_minio_header,
is_rustfs_header, is_standard_header, is_storageclass_header,
};
use rustfs_utils::http::{
SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_PROXY_REQUEST,
@@ -1774,22 +1774,6 @@ impl PutObjectOptions {
Self::insert_checked(&mut header, AMZ_BUCKET_REPLICATION_STATUS, self.internal.replication_status.as_str());
}
// MinIO PutObjectOptions.Header parity: object tags travel on the
// `x-amz-tagging` header (form-urlencoded). `replication_put_object_options`
// fills `user_tags` from the source version; without this header the
// whole-object transport delivered a tagless replica, so tag edits
// never reached the peer and the receiver-side LWW comparison
// (rustfs/backlog#1953) had nothing to judge.
if !self.user_tags.is_empty() {
let mut tags: Vec<(&String, &String)> = self.user_tags.iter().collect();
tags.sort();
let mut encoded = url::form_urlencoded::Serializer::new(String::new());
for (key, value) in tags {
encoded.append_pair(key, value);
}
Self::insert_checked(&mut header, AMZ_OBJECT_TAGGING_LOWER, &encoded.finish());
}
for (k, v) in &self.user_metadata {
let Ok(header_value) = HeaderValue::from_str(v) else {
warn!("skipping user metadata header with invalid value: {}", k);
@@ -3211,29 +3195,6 @@ mod tests {
}
}
#[test]
fn put_object_headers_carry_user_tags_on_x_amz_tagging() {
// rustfs/backlog#1953: tag edits replicate through the whole-object
// transport, so the source tags must travel on x-amz-tagging.
let mut opts = PutObjectOptions::default();
opts.user_tags.insert("owner".to_string(), "site a".to_string());
opts.user_tags.insert("env".to_string(), "prod".to_string());
let header = opts.header();
let tagging = header
.get(AMZ_OBJECT_TAGGING_LOWER)
.expect("user tags must be transported on x-amz-tagging")
.to_str()
.expect("tag header must be ASCII");
// Deterministic key order; values are form-urlencoded.
assert_eq!(tagging, "env=prod&owner=site+a");
assert!(
PutObjectOptions::default().header().get(AMZ_OBJECT_TAGGING_LOWER).is_none(),
"a tagless source must not send an empty x-amz-tagging header"
);
}
#[test]
fn put_object_headers_omit_unset_replication_timestamps() {
// UNIX_EPOCH means "never modified on the source"; sending it would
@@ -3464,44 +3425,6 @@ mod tests {
assert!(mutexes.contains_key("second"));
}
#[tokio::test]
async fn update_all_targets_publishes_disable_proxy_on_target_client() {
// The read-proxy selector (replication_proxy::get_proxy_targets) skips
// targets whose TargetClient carries disable_proxy — the persisted
// per-target opt-out must survive client publication.
let sys = BucketTargetSys::default();
let target = |arn: &str, disable_proxy: bool| BucketTarget {
arn: arn.to_string(),
endpoint: "192.168.1.10:9000".to_string(),
target_bucket: "target-bucket".to_string(),
region: "us-east-1".to_string(),
disable_proxy,
credentials: Some(Credentials {
access_key: "access".to_string(),
secret_key: "secret".to_string(),
session_token: None,
expiration: None,
}),
..Default::default()
};
let targets = BucketTargets {
targets: vec![target("arn:proxied", false), target("arn:opted-out", true)],
};
sys.update_all_targets("bucket", Some(&targets)).await;
let proxied = sys
.get_remote_target_client("bucket", "arn:proxied")
.await
.expect("client should be published");
assert!(!proxied.disable_proxy);
let opted_out = sys
.get_remote_target_client("bucket", "arn:opted-out")
.await
.expect("client should be published");
assert!(opted_out.disable_proxy, "disable_proxy must reach the published TargetClient");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn target_updates_serialize_client_build_through_publication_per_bucket() {
let sys = Arc::new(BucketTargetSys::default());
@@ -3739,55 +3739,17 @@ impl ManualTransitionRunReport {
}
pub fn merge_scan_report_preserving_worker(&mut self, scan_report: &ManualTransitionRunReport) {
let previous = self.clone();
let resumed_after_checkpoint = previous.continuation_token.is_some() && scan_report.scanned < previous.scanned;
let mut tier_failure_by_reason = self.tier_failure_by_reason.clone();
for (reason, count) in &scan_report.tier_failure_by_reason {
let current = tier_failure_by_reason.get(reason).copied().unwrap_or_default();
let merged = if resumed_after_checkpoint {
current.saturating_add(*count)
} else {
current.max(*count)
};
tier_failure_by_reason.insert(*reason, merged);
tier_failure_by_reason.insert(*reason, current.max(*count));
}
let transition_completed = self.transition_completed;
let transition_failed = self.transition_failed;
*self = scan_report.clone();
if resumed_after_checkpoint {
self.scanned = previous.scanned.saturating_add(scan_report.scanned);
self.eligible = previous.eligible.saturating_add(scan_report.eligible);
self.enqueued = previous.enqueued.saturating_add(scan_report.enqueued);
self.dry_run_eligible = previous.dry_run_eligible.saturating_add(scan_report.dry_run_eligible);
self.skipped_not_transition = previous
.skipped_not_transition
.saturating_add(scan_report.skipped_not_transition);
self.skipped_tier = previous.skipped_tier.saturating_add(scan_report.skipped_tier);
self.skipped_delete_marker = previous
.skipped_delete_marker
.saturating_add(scan_report.skipped_delete_marker);
self.skipped_directory = previous.skipped_directory.saturating_add(scan_report.skipped_directory);
self.skipped_replication = previous.skipped_replication.saturating_add(scan_report.skipped_replication);
self.skipped_already_transitioned = previous
.skipped_already_transitioned
.saturating_add(scan_report.skipped_already_transitioned);
self.skipped_already_in_flight = previous
.skipped_already_in_flight
.saturating_add(scan_report.skipped_already_in_flight);
self.skipped_queue_full = previous.skipped_queue_full.saturating_add(scan_report.skipped_queue_full);
self.skipped_queue_closed = previous.skipped_queue_closed.saturating_add(scan_report.skipped_queue_closed);
self.skipped_queue_timeout = previous
.skipped_queue_timeout
.saturating_add(scan_report.skipped_queue_timeout);
self.tier_failure = previous.tier_failure.saturating_add(scan_report.tier_failure);
}
self.lifecycle_config_found = previous.lifecycle_config_found || scan_report.lifecycle_config_found;
self.truncated_by_limit = previous.truncated_by_limit || scan_report.truncated_by_limit;
self.truncated_by_duration = previous.truncated_by_duration || scan_report.truncated_by_duration;
self.cancelled = previous.cancelled || scan_report.cancelled;
self.transition_completed = transition_completed;
self.transition_failed = transition_failed;
self.tier_failure = self.tier_failure.saturating_add(transition_failed);
self.tier_failure = scan_report.tier_failure.saturating_add(transition_failed);
self.tier_failure_by_reason = tier_failure_by_reason;
}
@@ -3803,10 +3765,7 @@ struct ManualTransitionContinuationToken {
version_marker: Option<String>,
}
pub(super) fn encode_manual_transition_continuation_token(
marker: Option<String>,
version_marker: Option<String>,
) -> Option<String> {
fn encode_manual_transition_continuation_token(marker: Option<String>, version_marker: Option<String>) -> Option<String> {
if marker.is_none() && version_marker.is_none() {
return None;
}
@@ -9177,7 +9136,6 @@ mod tests {
assert_eq!(loaded.report.scanned, 37);
assert_eq!(loaded.report.eligible, 11);
assert_eq!(loaded.report.enqueued, 5);
assert_eq!(loaded.cursor_revision, Some(37));
assert!(loaded.lease_expires_at_unix_nanos > 0);
let token = loaded
.report
@@ -9196,30 +9154,6 @@ mod tests {
assert_eq!(admission.lease_id, loaded.lease_id);
assert_eq!(admission.lease_expires_at_unix_nanos, loaded.lease_expires_at_unix_nanos);
let mut same_marker_report = report.clone();
same_marker_report.scanned += 1;
persist_manual_transition_page_checkpoint(
&checkpoint_options,
&same_marker_report,
Some("logs/page-end".to_string()),
Some("opaque-next-version".to_string()),
)
.await
.expect("same-marker version checkpoint should persist through the durable progress sink");
let same_marker_checkpointed = load_manual_transition_job_record(ecstore.clone(), job_id)
.await
.expect("same-marker version checkpoint should reload");
assert_eq!(same_marker_checkpointed.cursor_revision, Some(38));
let (_, version_marker) = decode_manual_transition_continuation_token(
same_marker_checkpointed
.report
.continuation_token
.as_deref()
.expect("same-marker version checkpoint should persist a cursor"),
)
.expect("same-marker version cursor should decode");
assert_eq!(version_marker.as_deref(), Some("opaque-next-version"));
create_test_bucket(&ecstore, &bucket).await;
let lifecycle_xml = format!(
r#"<?xml version="1.0" encoding="UTF-8"?>
@@ -9276,7 +9210,6 @@ mod tests {
assert_eq!(checkpointed.report.scanned, 1000);
assert_eq!(checkpointed.report.eligible, 1000);
assert_eq!(checkpointed.report.dry_run_eligible, 1000);
assert_eq!(checkpointed.cursor_revision, Some(1000));
let token = checkpointed
.report
.continuation_token
File diff suppressed because it is too large Load Diff
@@ -20,16 +20,10 @@ use serde::{Deserialize, Serialize};
use time::OffsetDateTime;
use uuid::Uuid;
#[cfg(test)]
use crate::bucket::lifecycle::bucket_lifecycle_ops::encode_manual_transition_continuation_token;
use crate::bucket::lifecycle::bucket_lifecycle_ops::{
ManualTransitionQueueSnapshot, ManualTransitionRunOptions, ManualTransitionRunReport,
};
use crate::bucket::lifecycle::config_boundary;
use crate::bucket::lifecycle::durable_namespace::{
MANUAL_TRANSITION_JOB_NAMESPACE, MANUAL_TRANSITION_SCOPE_NAMESPACE, MANUAL_TRANSITION_TASK_NAMESPACE,
MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE,
};
use crate::disk::RUSTFS_META_BUCKET;
use crate::error::{Error, Result as EcstoreResult};
use crate::object_api::ObjectOptions;
@@ -40,10 +34,10 @@ use crate::store::ECStore;
pub const MANUAL_TRANSITION_JOB_SCHEMA: &str = "rustfs-manual-transition-job-v1";
pub const MANUAL_TRANSITION_TASK_SCHEMA: &str = "rustfs-manual-transition-task-v1";
pub const MANUAL_TRANSITION_WORKER_RESULT_SCHEMA: &str = "rustfs-manual-transition-worker-result-v1";
pub const MANUAL_TRANSITION_JOB_RECORD_PREFIX: &str = MANUAL_TRANSITION_JOB_NAMESPACE.prefix;
pub const MANUAL_TRANSITION_SCOPE_RECORD_PREFIX: &str = MANUAL_TRANSITION_SCOPE_NAMESPACE.prefix;
pub const MANUAL_TRANSITION_TASK_PREFIX: &str = MANUAL_TRANSITION_TASK_NAMESPACE.prefix;
pub const MANUAL_TRANSITION_WORKER_RESULT_PREFIX: &str = MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE.prefix;
pub const MANUAL_TRANSITION_JOB_RECORD_PREFIX: &str = "ilm/manual-transition/jobs";
pub const MANUAL_TRANSITION_SCOPE_RECORD_PREFIX: &str = "ilm/manual-transition/scopes";
pub const MANUAL_TRANSITION_TASK_PREFIX: &str = "ilm/manual-transition/tasks";
pub const MANUAL_TRANSITION_WORKER_RESULT_PREFIX: &str = "ilm/manual-transition/results";
pub const MAX_MANUAL_TRANSITION_JOB_RECORD_SIZE: usize = 64 * 1024;
pub const MAX_MANUAL_TRANSITION_TASK_RECORD_SIZE: usize = 16 * 1024;
pub const MAX_MANUAL_TRANSITION_WORKER_RESULT_RECORD_SIZE: usize = 8 * 1024;
@@ -201,8 +195,6 @@ pub struct ManualTransitionJobRecord {
pub updated_at_unix_nanos: i128,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub completed_at_unix_nanos: Option<i128>,
#[serde(default, skip_serializing)]
pub cursor_revision: Option<u64>,
pub report: ManualTransitionRunReport,
pub queue_snapshot: ManualTransitionQueueSnapshot,
#[serde(default, skip_serializing_if = "Option::is_none")]
@@ -232,7 +224,6 @@ impl ManualTransitionJobRecord {
created_at_unix_nanos: now,
updated_at_unix_nanos: now,
completed_at_unix_nanos: None,
cursor_revision: None,
report: ManualTransitionRunReport {
bucket: bucket.to_string(),
prefix: options.prefix.clone(),
@@ -247,7 +238,7 @@ impl ManualTransitionJobRecord {
pub fn complete(&mut self, report: ManualTransitionRunReport, queue_snapshot: ManualTransitionQueueSnapshot) {
self.scan_completed = true;
self.merge_scan_report(&report);
self.report.merge_scan_report_preserving_worker(&report);
self.queue_snapshot = queue_snapshot;
self.error = None;
self.mark_terminal_if_worker_drained();
@@ -325,7 +316,7 @@ impl ManualTransitionJobRecord {
}
}
self.queue_snapshot = queue_snapshot;
self.advance_updated_at();
self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
self.mark_terminal_if_worker_drained();
}
@@ -368,14 +359,14 @@ impl ManualTransitionJobRecord {
self.report.tier_failure = scan_tier_failure.saturating_add(transition_failed);
self.report.tier_failure_by_reason = scan_tier_failure_by_reason;
self.queue_snapshot = queue_snapshot;
self.advance_updated_at();
self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
self.mark_terminal_if_worker_drained();
true
}
pub fn mark_cancel_requested(&mut self) {
self.cancel_requested = true;
self.advance_updated_at();
self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
}
pub fn claim_recovery_lease(&mut self, owner_id: impl Into<String>, queue_snapshot: ManualTransitionQueueSnapshot) {
@@ -389,7 +380,7 @@ impl ManualTransitionJobRecord {
pub fn abandon_recovery_lease(&mut self, lease_id: Uuid) {
if self.state == ManualTransitionJobState::Running && self.lease_id == lease_id {
self.lease_expires_at_unix_nanos = 0;
self.advance_updated_at();
self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
}
}
@@ -407,7 +398,7 @@ impl ManualTransitionJobRecord {
pub fn renew_lease(&mut self, queue_snapshot: ManualTransitionQueueSnapshot) {
let now = OffsetDateTime::now_utc().unix_timestamp_nanos();
self.updated_at_unix_nanos = self.updated_at_unix_nanos.saturating_add(1).max(now);
self.updated_at_unix_nanos = now;
self.lease_expires_at_unix_nanos = manual_transition_job_lease_expires_at(now);
self.queue_snapshot = queue_snapshot;
}
@@ -447,16 +438,11 @@ impl ManualTransitionJobRecord {
pub fn update_running_progress(&mut self, report: ManualTransitionRunReport, queue_snapshot: ManualTransitionQueueSnapshot) {
if self.state == ManualTransitionJobState::Running {
self.merge_scan_report(&report);
self.report.merge_scan_report_preserving_worker(&report);
self.renew_lease(queue_snapshot);
}
}
fn merge_scan_report(&mut self, report: &ManualTransitionRunReport) {
self.report.merge_scan_report_preserving_worker(report);
self.cursor_revision = manual_transition_cursor_revision(&self.report);
}
pub fn mark_unknown_if_unowned(&mut self) {
if self.state == ManualTransitionJobState::Running {
self.state = ManualTransitionJobState::Unknown;
@@ -477,13 +463,9 @@ impl ManualTransitionJobRecord {
}
fn mark_updated_terminal(&mut self) {
self.advance_updated_at();
self.completed_at_unix_nanos = Some(self.updated_at_unix_nanos);
}
fn advance_updated_at(&mut self) {
let now = OffsetDateTime::now_utc().unix_timestamp_nanos();
self.updated_at_unix_nanos = self.updated_at_unix_nanos.saturating_add(1).max(now);
self.updated_at_unix_nanos = now;
self.completed_at_unix_nanos = Some(now);
}
fn mark_terminal_if_worker_drained(&mut self) {
@@ -561,7 +543,6 @@ impl ManualTransitionJobRecord {
if job.state == ManualTransitionJobState::Cancelled && job.cancel_requested {
job.report.cancelled = true;
}
job.cursor_revision = manual_transition_cursor_revision(&job.report);
job.validate()?;
Ok(job)
}
@@ -1128,8 +1109,7 @@ pub fn manual_transition_scope_record_object_name(scope_key: &str) -> Result<Str
pub async fn save_manual_transition_job_record(api: Arc<ECStore>, job: &ManualTransitionJobRecord) -> EcstoreResult<()> {
let object = manual_transition_job_record_object_name(job.job_id).map_err(manual_transition_job_store_error)?;
let data = job.encode().map_err(manual_transition_job_store_error)?;
config_boundary::save_config(api.clone(), &object, data.clone()).await?;
api.record_durable_ilm_decommission_progress(&object, &data).await
config_boundary::save_config(api, &object, data).await
}
pub async fn load_manual_transition_job_record(api: Arc<ECStore>, job_id: Uuid) -> EcstoreResult<ManualTransitionJobRecord> {
@@ -1162,9 +1142,9 @@ pub async fn save_manual_transition_job_record_if_current(
let object = manual_transition_job_record_object_name(job.job_id).map_err(manual_transition_job_store_error)?;
let data = job.encode().map_err(manual_transition_job_store_error)?;
config_boundary::save_config_with_opts_quiet(
api.clone(),
api,
&object,
data.clone(),
data,
&ObjectOptions {
max_parity: true,
http_preconditions: Some(HTTPPreconditions {
@@ -1174,8 +1154,7 @@ pub async fn save_manual_transition_job_record_if_current(
..Default::default()
},
)
.await?;
api.record_durable_ilm_decommission_progress(&object, &data).await
.await
}
/// Applies a job-record mutation with optimistic concurrency control.
@@ -1613,9 +1592,9 @@ pub async fn save_manual_transition_scope_admission_if_absent(
let object = manual_transition_scope_record_object_name(&admission.scope_key).map_err(manual_transition_job_store_error)?;
let data = serde_json::to_vec(admission).map_err(Error::other)?;
config_boundary::save_config_with_opts(
api.clone(),
api,
&object,
data.clone(),
data,
&ObjectOptions {
max_parity: true,
http_preconditions: Some(HTTPPreconditions {
@@ -1625,8 +1604,7 @@ pub async fn save_manual_transition_scope_admission_if_absent(
..Default::default()
},
)
.await?;
api.record_durable_ilm_decommission_progress(&object, &data).await
.await
}
pub async fn load_manual_transition_scope_admission(
@@ -1664,9 +1642,9 @@ pub async fn save_manual_transition_scope_admission_if_current(
let object = manual_transition_scope_record_object_name(&admission.scope_key).map_err(manual_transition_job_store_error)?;
let data = serde_json::to_vec(admission).map_err(Error::other)?;
match config_boundary::save_config_with_opts(
api.clone(),
api,
&object,
data.clone(),
data,
&ObjectOptions {
max_parity: true,
http_preconditions: Some(HTTPPreconditions {
@@ -1682,8 +1660,7 @@ pub async fn save_manual_transition_scope_admission_if_current(
Err(Error::PreconditionFailed)
}
result => result,
}?;
api.record_durable_ilm_decommission_progress(&object, &data).await
}
}
pub async fn claim_manual_transition_scope_admission(
@@ -1976,15 +1953,13 @@ pub async fn delete_manual_transition_scope_admission_if_current(
job_id: Uuid,
lease_id: Uuid,
) -> EcstoreResult<bool> {
let (admission, etag) = match load_manual_transition_scope_admission_with_etag(api.clone(), scope_key).await {
Ok((admission, etag)) if admission.job_id == job_id && admission.lease_id == lease_id => (admission, etag),
let etag = match load_manual_transition_scope_admission_with_etag(api.clone(), scope_key).await {
Ok((admission, etag)) if admission.job_id == job_id && admission.lease_id == lease_id => etag,
Ok(_) => return Ok(false),
Err(Error::ConfigNotFound) => return Ok(true),
Err(err) => return Err(err),
};
let object = manual_transition_scope_record_object_name(scope_key).map_err(manual_transition_job_store_error)?;
let data = serde_json::to_vec(&admission).map_err(Error::other)?;
api.record_durable_ilm_decommission_terminal(&object, &data).await?;
match config_boundary::delete_config_if_match(api, &object, &etag).await {
Ok(()) | Err(Error::ConfigNotFound) => Ok(true),
Err(Error::PreconditionFailed) => Ok(false),
@@ -1996,11 +1971,6 @@ fn manual_transition_job_store_error(err: ManualTransitionJobError) -> Error {
Error::other(err)
}
fn manual_transition_cursor_revision(report: &ManualTransitionRunReport) -> Option<u64> {
report.continuation_token.as_ref()?;
(report.scanned > 0).then_some(report.scanned)
}
pub fn manual_transition_scope_admission_lease_expired(admission: &ManualTransitionScopeAdmission) -> bool {
OffsetDateTime::now_utc().unix_timestamp_nanos() > admission.lease_expires_at_unix_nanos
}
@@ -2240,76 +2210,6 @@ mod tests {
);
}
#[test]
fn manual_transition_job_scan_progress_accumulates_resumed_checkpoint_counters() {
let options = ManualTransitionRunOptions::default();
let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER);
let first_token =
encode_manual_transition_continuation_token(Some("logs/page-a".to_string()), Some("version-a".to_string()));
record.update_running_progress(
ManualTransitionRunReport {
bucket: "bucket".to_string(),
lifecycle_config_found: true,
scanned: 1000,
eligible: 800,
enqueued: 50,
dry_run_eligible: 10,
skipped_not_transition: 2,
skipped_tier: 3,
skipped_delete_marker: 4,
skipped_directory: 5,
skipped_replication: 6,
skipped_already_transitioned: 7,
skipped_already_in_flight: 8,
skipped_queue_full: 9,
skipped_queue_closed: 10,
skipped_queue_timeout: 11,
tier_failure: 12,
truncated_by_duration: true,
continuation_token: first_token,
..Default::default()
},
ManualTransitionQueueSnapshot::default(),
);
let next_token =
encode_manual_transition_continuation_token(Some("logs/page-b".to_string()), Some("version-b".to_string()));
record.update_running_progress(
ManualTransitionRunReport {
bucket: "bucket".to_string(),
scanned: 3,
eligible: 2,
enqueued: 1,
dry_run_eligible: 1,
skipped_not_transition: 1,
tier_failure: 1,
continuation_token: next_token.clone(),
..Default::default()
},
ManualTransitionQueueSnapshot::default(),
);
assert_eq!(record.report.scanned, 1003);
assert_eq!(record.report.eligible, 802);
assert_eq!(record.report.enqueued, 51);
assert_eq!(record.report.dry_run_eligible, 11);
assert_eq!(record.report.skipped_not_transition, 3);
assert_eq!(record.report.skipped_tier, 3);
assert_eq!(record.report.skipped_delete_marker, 4);
assert_eq!(record.report.skipped_directory, 5);
assert_eq!(record.report.skipped_replication, 6);
assert_eq!(record.report.skipped_already_transitioned, 7);
assert_eq!(record.report.skipped_already_in_flight, 8);
assert_eq!(record.report.skipped_queue_full, 9);
assert_eq!(record.report.skipped_queue_closed, 10);
assert_eq!(record.report.skipped_queue_timeout, 11);
assert_eq!(record.report.tier_failure, 13);
assert!(record.report.lifecycle_config_found);
assert!(record.report.truncated_by_duration);
assert_eq!(record.report.continuation_token, next_token);
assert_eq!(record.cursor_revision, Some(1003));
}
#[test]
fn manual_transition_job_apply_worker_result_counts_preserves_existing_failure_reasons() {
let options = ManualTransitionRunOptions::default();
@@ -2677,98 +2577,6 @@ mod tests {
assert!(decoded.report.tier_failure_by_reason.is_empty());
}
#[test]
fn manual_transition_job_record_derives_revision_from_legacy_cursor() {
let options = ManualTransitionRunOptions::default();
let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER);
let continuation_token =
encode_manual_transition_continuation_token(Some("logs/page-a".to_string()), Some("version-a".to_string()));
record.update_running_progress(
ManualTransitionRunReport {
bucket: "bucket".to_string(),
scanned: 9,
continuation_token,
..Default::default()
},
ManualTransitionQueueSnapshot::default(),
);
let encoded = record.encode().expect("job record should encode");
let mut value: serde_json::Value = serde_json::from_slice(&encoded).expect("encoded job should be json");
value["job"]
.as_object_mut()
.expect("job should be object")
.remove("cursor_revision");
let record_bytes = serde_json::to_vec(&value["job"]).expect("legacy job should encode");
value["content_sha256"] = serde_json::Value::String(hex_sha256(&record_bytes, ToOwned::to_owned));
let legacy = serde_json::to_vec(&value).expect("legacy envelope should encode");
let decoded = ManualTransitionJobRecord::decode(record.job_id, &legacy).expect("legacy job should decode");
assert_eq!(decoded.cursor_revision, Some(9));
}
#[test]
fn manual_transition_job_record_omits_cursor_revision_for_old_readers() {
#[allow(dead_code)]
#[derive(serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct LegacyPersistedManualTransitionJobRecord {
schema: String,
content_sha256: String,
job: LegacyManualTransitionJobRecord,
}
#[allow(dead_code)]
#[derive(serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct LegacyManualTransitionJobRecord {
job_id: Uuid,
scope_key: String,
bucket: String,
prefix: String,
tier: Option<String>,
dry_run: bool,
max_objects: Option<u64>,
max_duration: Option<std::time::Duration>,
owner_id: String,
lease_id: Uuid,
lease_expires_at_unix_nanos: i128,
state: ManualTransitionJobState,
scan_completed: bool,
cancel_requested: bool,
created_at_unix_nanos: i128,
updated_at_unix_nanos: i128,
completed_at_unix_nanos: Option<i128>,
report: ManualTransitionRunReport,
queue_snapshot: ManualTransitionQueueSnapshot,
error: Option<String>,
}
let options = ManualTransitionRunOptions::default();
let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER);
let continuation_token =
encode_manual_transition_continuation_token(Some("logs/page-a".to_string()), Some("version-a".to_string()));
record.update_running_progress(
ManualTransitionRunReport {
bucket: "bucket".to_string(),
scanned: 7,
continuation_token: continuation_token.clone(),
..Default::default()
},
ManualTransitionQueueSnapshot::default(),
);
assert_eq!(record.cursor_revision, Some(7));
let encoded = record.encode().expect("job record should encode");
let value: serde_json::Value = serde_json::from_slice(&encoded).expect("encoded job should be json");
assert!(value["job"].get("cursor_revision").is_none());
let legacy: LegacyPersistedManualTransitionJobRecord =
serde_json::from_slice(&encoded).expect("old reader should accept new job record");
assert_eq!(legacy.job.job_id, record.job_id);
assert_eq!(legacy.job.report.continuation_token, continuation_token);
}
#[test]
fn manual_transition_job_record_rejects_unknown_report_fields() {
let options = ManualTransitionRunOptions::default();
@@ -16,7 +16,6 @@ pub mod bucket_lifecycle_audit;
pub mod bucket_lifecycle_ops;
mod config_boundary;
pub mod core;
mod durable_namespace;
pub mod evaluator;
pub mod manual_transition_job;
mod metadata_boundary;
@@ -32,8 +31,3 @@ pub mod tier_free_version_recovery;
pub mod tier_last_day_stats;
pub mod tier_sweeper;
pub mod transition_transaction;
pub(crate) use durable_namespace::{
DurableIlmRecordCheckpoint, ILM_META_PREFIX, ValidatedDurableIlmRecord, classify_durable_ilm_record,
validate_durable_ilm_record,
};
@@ -20,7 +20,6 @@ use tokio_util::sync::CancellationToken;
use tracing::{debug, warn};
use crate::bucket::lifecycle::config_boundary;
use crate::bucket::lifecycle::durable_namespace::TIER_DELETE_JOURNAL_NAMESPACE;
use crate::bucket::lifecycle::runtime_boundary;
use crate::bucket::lifecycle::tier_sweeper::{
Jentry, TierDeleteJournalState, TierDeleteSourceIdentity,
@@ -50,7 +49,7 @@ const TIER_DELETE_JOURNAL_VERSION: u8 = 2;
const TIER_DELETE_JOURNAL_EXACT_VERSION: u8 = 3;
const TIER_DELETE_JOURNAL_STATE_VERSION: u8 = 4;
const TIER_DELETE_JOURNAL_TRANSACTION_VERSION: u8 = 5;
pub(crate) const TIER_DELETE_JOURNAL_PREFIX: &str = TIER_DELETE_JOURNAL_NAMESPACE.prefix;
pub(crate) const TIER_DELETE_JOURNAL_PREFIX: &str = "ilm/tier-delete-journal/";
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
@@ -433,21 +432,6 @@ async fn process_committed_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jen
)
.await?;
}
let path = tier_delete_journal_object_name(je);
let data = encode_tier_delete_journal_entry(je).map_err(std::io::Error::other)?;
let target_pool_indices = api
.record_durable_ilm_decommission_terminal_target_pools(&path, &data)
.await
.map_err(std::io::Error::other)?;
if let Some(target_pool_indices) = target_pool_indices {
for target_pool_idx in target_pool_indices {
match config_boundary::delete_config(api.pools[target_pool_idx].clone(), &path).await {
Ok(()) | Err(Error::ConfigNotFound) => {}
Err(err) => return Err(std::io::Error::other(err)),
}
}
return Ok(());
}
remove_tier_delete_journal_entry(api, je).await
}
@@ -21,7 +21,6 @@ use tracing::{debug, warn};
use uuid::Uuid;
use crate::bucket::lifecycle::config_boundary;
use crate::bucket::lifecycle::durable_namespace::TRANSITION_TRANSACTION_NAMESPACE;
use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
use crate::bucket::lifecycle::tier_sweeper::{
delete_confirmed_transition_candidate_exact_with_lease_idempotent,
@@ -43,7 +42,7 @@ const TRANSITION_TRANSACTION_RECOVERY_INTERVAL: Duration = Duration::from_secs(6
const TRANSITION_TRANSACTION_RECOVERY_TIMEOUT: Duration = Duration::from_secs(300);
pub const TRANSITION_TRANSACTION_SCHEMA: &str = "rustfs-transition-transaction-v1";
pub const TRANSITION_TRANSACTION_PREFIX: &str = "ilm/transition-transactions";
pub const TRANSITION_TRANSACTION_RECORD_PREFIX: &str = TRANSITION_TRANSACTION_NAMESPACE.prefix;
pub const TRANSITION_TRANSACTION_RECORD_PREFIX: &str = "ilm/transition-transactions/records";
pub const MAX_TRANSITION_TRANSACTION_SIZE: usize = 64 * 1024;
pub type Result<T> = std::result::Result<T, TransitionTransactionError>;
@@ -585,8 +584,7 @@ pub(crate) async fn save_transition_transaction_record(
let object =
transition_transaction_record_object_name(transaction.transaction_id).map_err(transition_transaction_store_error)?;
let data = transaction.encode().map_err(transition_transaction_store_error)?;
config_boundary::save_config(api.clone(), &object, data.clone()).await?;
api.record_durable_ilm_decommission_progress(&object, &data).await
config_boundary::save_config(api, &object, data).await
}
pub(crate) async fn load_transition_transaction_record(
@@ -598,14 +596,8 @@ pub(crate) async fn load_transition_transaction_record(
TransitionTransaction::decode(transaction_id, &data).map_err(transition_transaction_store_error)
}
pub(crate) async fn delete_transition_transaction_record(
api: Arc<ECStore>,
transaction: &TransitionTransaction,
) -> EcstoreResult<()> {
let object =
transition_transaction_record_object_name(transaction.transaction_id).map_err(transition_transaction_store_error)?;
let data = transaction.encode().map_err(transition_transaction_store_error)?;
api.record_durable_ilm_decommission_terminal(&object, &data).await?;
pub(crate) async fn delete_transition_transaction_record(api: Arc<ECStore>, transaction_id: Uuid) -> EcstoreResult<()> {
let object = transition_transaction_record_object_name(transaction_id).map_err(transition_transaction_store_error)?;
match config_boundary::delete_config(api, &object).await {
Ok(()) | Err(Error::ConfigNotFound) => Ok(()),
Err(err) => Err(err),
@@ -821,7 +813,7 @@ pub async fn finalize_missing_transition_transaction_for_operator(
if probe != TransitionOperatorProbe::Missing {
return Err(TransitionOperatorError::CandidateNotMissing(probe));
}
delete_transition_transaction_record(api, &transaction)
delete_transition_transaction_record(api, transaction_id)
.await
.map_err(TransitionOperatorError::Store)
}
@@ -857,22 +849,22 @@ pub async fn process_transition_transaction_record(
match transaction.state {
TransitionTransactionState::Uploaded => {
delete_transition_remote_candidate(api.clone(), transaction).await?;
delete_transition_transaction_record(api, transaction).await?;
delete_transition_transaction_record(api, transaction.transaction_id).await?;
Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
}
TransitionTransactionState::CleanupPending => match local_commit_matches_transaction(api.clone(), transaction).await {
Ok(true) => {
delete_transition_transaction_record(api, transaction).await?;
delete_transition_transaction_record(api, transaction.transaction_id).await?;
Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
}
Ok(false) => {
delete_transition_remote_candidate(api.clone(), transaction).await?;
delete_transition_transaction_record(api, transaction).await?;
delete_transition_transaction_record(api, transaction.transaction_id).await?;
Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
}
Err(err) if transition_source_is_missing(&err) => {
delete_transition_remote_candidate(api.clone(), transaction).await?;
delete_transition_transaction_record(api, transaction).await?;
delete_transition_transaction_record(api, transaction.transaction_id).await?;
Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
}
Err(err) => Err(err),
@@ -880,7 +872,7 @@ pub async fn process_transition_transaction_record(
TransitionTransactionState::LocalCommitStarted => {
match local_commit_matches_transaction(api.clone(), transaction).await {
Ok(true) => {
delete_transition_transaction_record(api, transaction).await?;
delete_transition_transaction_record(api, transaction.transaction_id).await?;
Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
}
Ok(false) => Ok(TransitionTransactionRecoveryOutcome::Retained),
@@ -889,7 +881,7 @@ pub async fn process_transition_transaction_record(
}
}
TransitionTransactionState::AbortedNoRemote | TransitionTransactionState::Committed => {
delete_transition_transaction_record(api, transaction).await?;
delete_transition_transaction_record(api, transaction.transaction_id).await?;
Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
}
TransitionTransactionState::UploadOutcomeUnknown => recover_unknown_upload_outcome(api, transaction).await,
@@ -915,7 +907,7 @@ async fn recover_unknown_upload_outcome(
.map_err(Error::other)?
{
TransitionCandidateProbe::Missing => {
delete_transition_transaction_record(api, transaction).await?;
delete_transition_transaction_record(api, transaction.transaction_id).await?;
Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
}
TransitionCandidateProbe::UnversionedPresent => {
@@ -933,7 +925,7 @@ async fn recover_unknown_upload_outcome(
)
.await
.map_err(Error::other)?;
delete_transition_transaction_record(api, transaction).await?;
delete_transition_transaction_record(api, transaction.transaction_id).await?;
Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
}
TransitionCandidateProbe::VersionedPresent(version_id) => {
@@ -966,7 +958,7 @@ async fn cleanup_recovered_unknown_upload_candidate(
.map_err(transition_transaction_store_error)?;
save_transition_transaction_record(api.clone(), &cleanup).await?;
delete_transition_remote_candidate(api.clone(), &cleanup).await?;
delete_transition_transaction_record(api, &cleanup).await?;
delete_transition_transaction_record(api, cleanup.transaction_id).await?;
Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
}
@@ -15,7 +15,7 @@
use crate::bucket::metadata_sys::{ObjectLockConfigState, get_object_lock_config, get_object_lock_config_state};
use crate::bucket::object_lock::objectlock;
use crate::error::{Error, Result, StorageError};
use crate::object_api::{ObjectInfo, ObjectOptions};
use crate::object_api::ObjectInfo;
use s3s::dto::{Date, DefaultRetention, ObjectLockConfiguration, ObjectLockLegalHoldStatus, ObjectLockRetentionMode};
use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE};
use std::sync::Arc;
@@ -136,50 +136,12 @@ pub fn add_years(dt: OffsetDateTime, years: i32) -> OffsetDateTime {
/// Check if an object has legal hold enabled.
/// Returns true if legal hold is ON.
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
fn has_legal_hold(user_defined: &std::collections::HashMap<String, String>) -> bool {
let lhold = objectlock::get_object_legalhold_meta(user_defined);
matches!(lhold.status, Some(ref st) if st.as_str() == ObjectLockLegalHoldStatus::ON)
}
/// Whether an authorized replication write (`ObjectOptions::replication_request`)
/// may overwrite a locked destination version.
///
/// The source's lock state governs a replica (MinIO `checkPutObjectLockAllowed`
/// skips the existing-version check for replicas), and a source-side hold
/// release or retention change reaches this site only through this write. The
/// overwrite is allowed only when the write carries the source timestamp of
/// every category that currently locks the version, so receiver-side LWW
/// (`merge_replication_metadata_lww`) judges each of them: a category locked
/// more recently here is kept, otherwise the source's newer state wins. A write
/// without that timestamp carries no source decision for the category — the
/// metadata replace would lift the lock unjudged — so it stays WORM-rejected.
///
/// The locking categories come from the same authoritative evaluation as the
/// commit-time WORM gate (`check_object_lock_for_deletion_with_state`): the
/// bucket default retention locks a version that carries no explicit
/// retention keys, so it is judged here too rather than read off the keys.
/// Malformed persisted lock metadata or a non-authoritative bucket
/// configuration is an error, never a pass.
pub fn replication_write_may_pass_worm_gate(
state: &ObjectLockConfigState,
obj_info: &ObjectInfo,
opts: &ObjectOptions,
) -> Result<bool> {
if !opts.replication_request {
return Ok(false);
}
if obj_info.delete_marker {
// Delete markers are never locked (same as the WORM gate).
return Ok(true);
}
let config = object_lock_config_from_state(state)?;
if legal_hold_locks(obj_info)? && opts.replication_legalhold_timestamp.is_none() {
return Ok(false);
}
let retention_locked = active_retention(config, obj_info)?.is_some();
Ok(!(retention_locked && opts.replication_retention_timestamp.is_none()))
}
/// Check if an object is locked based on its metadata.
/// This is a common function used by both lifecycle evaluation and deletion checks.
///
@@ -277,101 +239,69 @@ pub(crate) fn check_object_lock_for_deletion_with_config(
return Ok(None);
}
if legal_hold_locks(obj_info)? {
return Ok(Some(ObjectLockBlockReason::LegalHold));
if let Some(status) = obj_info.user_defined.get(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()) {
if status.eq_ignore_ascii_case(ObjectLockLegalHoldStatus::ON) {
return Ok(Some(ObjectLockBlockReason::LegalHold));
}
if !status.eq_ignore_ascii_case(ObjectLockLegalHoldStatus::OFF) {
return Err(Error::other("persisted object legal-hold metadata is invalid"));
}
}
if let Some((mode_str, retain_until)) = active_retention(config, obj_info)?
&& let Some(reason) = check_retention_blocks_deletion(mode_str, Some(retain_until), bypass_governance)
{
return Ok(Some(reason));
}
Ok(None)
}
/// A cleared retention / legal hold is persisted as empty strings (the MinIO
/// on-disk shape, `parse_object_lock_retention`); read it as "no lock" rather
/// than as corrupt metadata.
fn persisted_lock_value<'a>(obj_info: &'a ObjectInfo, key: &str) -> Option<&'a String> {
obj_info.user_defined.get(key).filter(|value| !value.is_empty())
}
/// Whether the version's persisted legal hold is ON. Any other non-empty
/// value than ON/OFF is malformed metadata and fails closed.
fn legal_hold_locks(obj_info: &ObjectInfo) -> Result<bool> {
let Some(status) = persisted_lock_value(obj_info, X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()) else {
return Ok(false);
};
if status.eq_ignore_ascii_case(ObjectLockLegalHoldStatus::ON) {
return Ok(true);
}
if !status.eq_ignore_ascii_case(ObjectLockLegalHoldStatus::OFF) {
return Err(Error::other("persisted object legal-hold metadata is invalid"));
}
Ok(false)
}
/// The retention that currently locks the version, if any: the explicit
/// persisted retention when the keys are present, otherwise the bucket
/// default retention computed from the version's modification time. Returns
/// `(mode, retain_until)` only while the retention is still active.
fn active_retention<'a>(
config: Option<&'a ObjectLockConfiguration>,
obj_info: &ObjectInfo,
) -> Result<Option<(&'a str, OffsetDateTime)>> {
let mode = persisted_lock_value(obj_info, X_AMZ_OBJECT_LOCK_MODE.as_str());
let retain_until = persisted_lock_value(obj_info, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str());
match (mode, retain_until) {
(None, None) => {}
let mode = obj_info.user_defined.get(X_AMZ_OBJECT_LOCK_MODE.as_str());
let retain_until = obj_info.user_defined.get(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str());
let explicit_ret = match (mode, retain_until) {
(None, None) => None,
(Some(mode), Some(retain_until)) => {
let mode =
objectlock::parse_ret_mode(mode).ok_or_else(|| Error::other("persisted object retention mode is invalid"))?;
let retain_until = OffsetDateTime::parse(retain_until, &time::format_description::well_known::Iso8601::DEFAULT)
.map(Date::from)
.map_err(|_| Error::other("persisted object retention date is invalid"))?;
let mode_str = match mode.as_str() {
ObjectLockRetentionMode::COMPLIANCE => ObjectLockRetentionMode::COMPLIANCE,
ObjectLockRetentionMode::GOVERNANCE => ObjectLockRetentionMode::GOVERNANCE,
_ => return Err(Error::other("persisted object retention mode is invalid")),
};
return Ok(is_retention_active(mode_str, Some(&retain_until)).then(|| (mode_str, OffsetDateTime::from(retain_until))));
Some((mode, retain_until))
}
_ => return Err(Error::other("persisted object retention metadata is incomplete")),
};
if let Some((mode, retain_until)) = &explicit_ret {
let mode_str = mode.as_str();
if is_retention_active(mode_str, Some(retain_until))
&& let Some(reason) =
check_retention_blocks_deletion(mode_str, Some(OffsetDateTime::from(retain_until.clone())), bypass_governance)
{
return Ok(Some(reason));
}
}
let Some(default_retention) = config.and_then(|config| config.rule.as_ref()?.default_retention.as_ref()) else {
return Ok(None);
};
let Some(mode) = &default_retention.mode else {
return Ok(None);
};
let mode_str = mode.as_str();
if mode_str != ObjectLockRetentionMode::COMPLIANCE && mode_str != ObjectLockRetentionMode::GOVERNANCE {
return Ok(None);
}
// Calculate retention expiration date from object modification time
let mod_time = obj_info
.mod_time
.ok_or_else(|| Error::other("persisted object modification time is missing"))?;
let now = objectlock::utc_now_ntp();
let retain_until = if let Some(days) = default_retention.days {
mod_time.saturating_add(time::Duration::days(i64::from(days)))
} else {
let years = default_retention
.years
.ok_or_else(|| Error::other("persisted bucket Object Lock retention period is invalid"))?;
add_years(mod_time, years)
};
Ok((retain_until.unix_timestamp() > now.unix_timestamp()).then_some((mode_str, retain_until)))
}
if explicit_ret.is_none()
&& let Some(default_retention) = config.and_then(|config| config.rule.as_ref()?.default_retention.as_ref())
&& let Some(mode) = &default_retention.mode
{
let mode_str = mode.as_str();
if mode_str == ObjectLockRetentionMode::COMPLIANCE || mode_str == ObjectLockRetentionMode::GOVERNANCE {
// Calculate retention expiration date from object modification time
let mod_time = obj_info
.mod_time
.ok_or_else(|| Error::other("persisted object modification time is missing"))?;
let now = objectlock::utc_now_ntp();
let retain_until = if let Some(days) = default_retention.days {
mod_time.saturating_add(time::Duration::days(i64::from(days)))
} else {
let years = default_retention
.years
.ok_or_else(|| Error::other("persisted bucket Object Lock retention period is invalid"))?;
add_years(mod_time, years)
};
fn object_lock_config_from_state(state: &ObjectLockConfigState) -> Result<Option<&ObjectLockConfiguration>> {
match state {
ObjectLockConfigState::Configured { config, .. } => Ok(Some(config)),
ObjectLockConfigState::ConfirmedAbsent => Ok(None),
ObjectLockConfigState::Fabricated => Err(Error::other("bucket Object Lock metadata is not authoritative")),
if retain_until.unix_timestamp() > now.unix_timestamp()
&& let Some(reason) = check_retention_blocks_deletion(mode_str, Some(retain_until), bypass_governance)
{
return Ok(Some(reason));
}
}
}
Ok(None)
}
pub(crate) fn check_object_lock_for_deletion_with_state(
@@ -379,7 +309,13 @@ pub(crate) fn check_object_lock_for_deletion_with_state(
obj_info: &ObjectInfo,
bypass_governance: bool,
) -> Result<Option<ObjectLockBlockReason>> {
check_object_lock_for_deletion_with_config(object_lock_config_from_state(state)?, obj_info, bypass_governance)
match state {
ObjectLockConfigState::Configured { config, .. } => {
check_object_lock_for_deletion_with_config(Some(config), obj_info, bypass_governance)
}
ObjectLockConfigState::ConfirmedAbsent => check_object_lock_for_deletion_with_config(None, obj_info, bypass_governance),
ObjectLockConfigState::Fabricated => Err(Error::other("bucket Object Lock metadata is not authoritative")),
}
}
/// Compatibility wrapper for callers that predate fallible metadata lookup.
@@ -550,210 +486,6 @@ mod tests {
}
}
fn replication_opts(hold_ts: bool, retention_ts: bool) -> ObjectOptions {
ObjectOptions {
replication_request: true,
replication_legalhold_timestamp: hold_ts.then_some(OffsetDateTime::UNIX_EPOCH),
replication_retention_timestamp: retention_ts.then_some(OffsetDateTime::UNIX_EPOCH),
..Default::default()
}
}
fn lock_metadata(entries: &[&[(&str, &str)]]) -> std::collections::HashMap<String, String> {
entries
.iter()
.flat_map(|entries| entries.iter())
.map(|(key, value)| (key.to_string(), value.to_string()))
.collect()
}
fn lock_object_info(user_defined: std::collections::HashMap<String, String>) -> ObjectInfo {
ObjectInfo {
user_defined: Arc::new(user_defined),
mod_time: Some(OffsetDateTime::now_utc()),
..Default::default()
}
}
/// A replication write passes the WORM gate only when it carries the
/// source timestamp of every category that currently locks the version.
#[test]
fn replication_write_passes_worm_gate_only_with_every_locking_category_timestamp() {
use rustfs_utils::http::headers::{
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
};
let hold = [(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, "ON")];
let retention = [
(AMZ_OBJECT_LOCK_MODE_LOWER, "GOVERNANCE"),
(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, "2099-01-01T00:00:00Z"),
];
let expired = [
(AMZ_OBJECT_LOCK_MODE_LOWER, "COMPLIANCE"),
(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, "2000-01-01T00:00:00Z"),
];
let absent = ObjectLockConfigState::ConfirmedAbsent;
let passes = |state: &ObjectLockConfigState, entries: &[&[(&str, &str)]], opts: &ObjectOptions| {
replication_write_may_pass_worm_gate(state, &lock_object_info(lock_metadata(entries)), opts)
.expect("well-formed lock metadata must be judged")
};
assert!(passes(&absent, &[&hold, &retention], &replication_opts(true, true)));
assert!(!passes(&absent, &[&hold, &retention], &replication_opts(true, false)));
assert!(!passes(&absent, &[&hold, &retention], &replication_opts(false, true)));
assert!(passes(&absent, &[&hold], &replication_opts(true, false)));
assert!(!passes(&absent, &[&hold], &replication_opts(false, true)));
assert!(passes(&absent, &[&retention], &replication_opts(false, true)));
assert!(!passes(&absent, &[&retention], &replication_opts(true, false)));
// Expired retention and a released hold no longer lock anything.
assert!(passes(
&absent,
&[&expired, &[(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, "OFF")]],
&replication_opts(false, false)
));
// Never for a non-replication write, whatever it carries.
let local = ObjectOptions {
replication_request: false,
..replication_opts(true, true)
};
assert!(!passes(&absent, &[&hold], &local));
}
/// The bucket default retention locks a version that carries no explicit
/// retention keys (`check_object_lock_for_deletion_with_config` judges it
/// from the modification time), so the replication bypass must demand the
/// retention source timestamp for it too — a tagging-only replication
/// write must not overwrite the default-protected version unjudged.
#[test]
fn replication_write_under_bucket_default_retention_requires_retention_timestamp() {
use rustfs_utils::http::headers::{AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER};
for mode in [ObjectLockRetentionMode::COMPLIANCE, ObjectLockRetentionMode::GOVERNANCE] {
let state = ObjectLockConfigState::Configured {
config: default_retention_config(mode),
updated_at: OffsetDateTime::now_utc(),
};
let no_keys = lock_object_info(std::collections::HashMap::new());
assert!(
check_object_lock_for_deletion_with_state(&state, &no_keys, false)
.expect("default retention must be judged")
.is_some(),
"{mode}: the gate must report the default retention lock"
);
let tagging_only = ObjectOptions {
replication_request: true,
replication_tagging_timestamp: Some(OffsetDateTime::UNIX_EPOCH),
..Default::default()
};
assert!(
!replication_write_may_pass_worm_gate(&state, &no_keys, &tagging_only).expect("judged"),
"{mode}: a tagging-only replication write must not pass the default retention lock"
);
assert!(
replication_write_may_pass_worm_gate(&state, &no_keys, &replication_opts(false, true)).expect("judged"),
"{mode}: the retention source timestamp lets LWW judge the default retention"
);
// Default retention plus a legal hold: both categories need a timestamp.
let held = lock_object_info(lock_metadata(&[&[(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, "ON")]]));
assert!(!replication_write_may_pass_worm_gate(&state, &held, &replication_opts(false, true)).expect("judged"));
assert!(!replication_write_may_pass_worm_gate(&state, &held, &replication_opts(true, false)).expect("judged"));
assert!(replication_write_may_pass_worm_gate(&state, &held, &replication_opts(true, true)).expect("judged"));
// A version whose default retention has already expired (old
// mod_time) is not locked by the default any more.
let expired_default = ObjectInfo {
mod_time: Some(make_datetime(2000, 1, 1)),
..lock_object_info(std::collections::HashMap::new())
};
assert!(replication_write_may_pass_worm_gate(&state, &expired_default, &tagging_only).expect("judged"));
// A delete marker is never locked, so there is nothing to judge.
let delete_marker = ObjectInfo {
delete_marker: true,
..lock_object_info(std::collections::HashMap::new())
};
assert!(replication_write_may_pass_worm_gate(&state, &delete_marker, &tagging_only).expect("judged"));
// Cleared (empty) explicit keys fall back to the bucket default.
let cleared = lock_object_info(lock_metadata(&[&[(AMZ_OBJECT_LOCK_MODE_LOWER, "")]]));
assert!(!replication_write_may_pass_worm_gate(&state, &cleared, &tagging_only).expect("judged"));
}
}
/// The replication bypass never judges from a non-authoritative bucket
/// state or malformed persisted lock metadata; both are errors, not a pass.
#[test]
fn replication_write_worm_gate_fails_closed_on_unverifiable_lock_state() {
use rustfs_utils::http::headers::AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER;
let opts = replication_opts(true, true);
let err = replication_write_may_pass_worm_gate(
&ObjectLockConfigState::Fabricated,
&lock_object_info(std::collections::HashMap::new()),
&opts,
)
.expect_err("fabricated bucket lock metadata must not be judged");
assert!(err.to_string().contains("not authoritative"));
let malformed = lock_object_info(lock_metadata(&[&[(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, "MAYBE")]]));
let err = replication_write_may_pass_worm_gate(&ObjectLockConfigState::ConfirmedAbsent, &malformed, &opts)
.expect_err("malformed legal hold must not be judged");
assert!(err.to_string().contains("legal-hold"));
let state = ObjectLockConfigState::Configured {
config: default_retention_config(ObjectLockRetentionMode::COMPLIANCE),
updated_at: OffsetDateTime::now_utc(),
};
let no_mod_time = ObjectInfo::default();
let err = replication_write_may_pass_worm_gate(&state, &no_mod_time, &opts)
.expect_err("default retention without a modification time must not be judged");
assert!(err.to_string().contains("modification time"));
}
/// A local PutObjectRetention / PutObjectLegalHold "clear" persists the
/// lock keys as empty strings (the MinIO on-disk shape, see
/// `parse_object_lock_retention`); that is "no lock", not corruption, and
/// must not wedge later explicit-version PUTs or deletes
/// (rustfs/backlog#1953).
#[test]
fn deletion_treats_cleared_empty_lock_metadata_as_unlocked() {
use rustfs_utils::http::headers::{
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
};
let cases: [(&str, &[&str]); 3] = [
(
"cleared retention",
&[AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER],
),
("cleared legal hold", &[AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER]),
(
"all cleared",
&[
AMZ_OBJECT_LOCK_MODE_LOWER,
AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER,
],
),
];
for (case, keys) in cases {
let user_defined = keys.iter().map(|key| (key.to_string(), String::new())).collect();
let obj_info = ObjectInfo {
user_defined: Arc::new(user_defined),
..Default::default()
};
let result = check_object_lock_for_deletion_with_config(None, &obj_info, false);
assert!(matches!(result, Ok(None)), "{case}: empty lock keys must read as unlocked: {result:?}");
}
}
#[test]
fn deletion_rejects_invalid_persisted_legal_hold_metadata() {
let mut user_defined = std::collections::HashMap::new();
+5 -8
View File
@@ -44,14 +44,11 @@ mod replication_versioning_boundary;
mod runtime_boundary;
pub use replication_config_boundary::{
ObjectOpts, OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS,
REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS,
REPLICATION_WRITABLE_FIELDS, ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationTargetValidationError,
assign_site_replication_rule_priorities, invalid_replication_config_status_field, is_site_replication_role,
is_site_replication_rule, merge_incoming_replication_config, merge_user_replication_config,
replication_target_arn_deployment_id, replication_target_arns, should_remove_replication_target,
site_replication_rule_deployment_id, unsupported_replication_config_field, validate_replication_config_structure,
validate_replication_config_target_arns,
ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationTargetValidationError,
invalid_replication_config_status_field, replication_target_arns, should_remove_replication_target,
unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
};
pub(crate) use replication_filemeta_boundary::version_purge_statuses_map;
pub use replication_filemeta_boundary::{
@@ -13,12 +13,9 @@
// limitations under the License.
pub use rustfs_replication::{
ObjectOpts, OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS,
REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS,
REPLICATION_WRITABLE_FIELDS, ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationRuleExt,
ReplicationTargetValidationError, assign_site_replication_rule_priorities, invalid_replication_config_status_field,
is_site_replication_role, is_site_replication_rule, merge_incoming_replication_config, merge_user_replication_config,
replication_target_arn_deployment_id, replication_target_arns, should_remove_replication_target,
site_replication_rule_deployment_id, unsupported_replication_config_field, validate_replication_config_structure,
validate_replication_config_target_arns,
ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationRuleExt, ReplicationTargetValidationError,
invalid_replication_config_status_field, replication_target_arns, should_remove_replication_target,
unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
};
@@ -2855,7 +2855,7 @@ fn replicate_object_info_from_object_info(
.map(|v| OffsetDateTime::parse(&v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH));
let mut rstate = oi.replication_state();
rstate.replicate_decision_str = dsc.to_string();
let asz = oi.get_actual_size_or_physical();
let asz = oi.get_actual_size().unwrap_or_default();
let ssec = replication_object_is_ssec_encrypted(&oi.user_defined);
let checksum = if ssec { oi.checksum.clone() } else { None };
@@ -4194,30 +4194,6 @@ mod tests {
assert_eq!(ri.checksum, Some(checksum));
}
#[test]
fn metadata_mrf_roundtrip_preserves_tags_and_admitted_targets() {
let target = "arn:rustfs:replication:target-a";
let object = ObjectInfo {
bucket: "source".to_string(),
name: "object".to_string(),
version_id: Some(Uuid::new_v4()),
user_tags: Arc::new("owner=a3".to_string()),
..Default::default()
};
let live =
replicate_object_info_from_object_info(object.clone(), test_replicate_decision(&[target]), ReplicationType::Metadata);
let persisted = live.to_mrf_entry();
let encoded = encode_mrf_file(std::slice::from_ref(&persisted)).expect("metadata MRF entry should encode");
let decoded = decode_mrf_file(&encoded).expect("metadata MRF entry should decode");
assert_eq!(decoded[0].op, MrfOpKind::Metadata);
assert_eq!(decoded[0].target_arns, vec![target.to_string()]);
let replayed = admitted_mrf_replicate_object(object, &decoded[0], ReplicationType::Metadata);
assert_eq!(replayed.op_type, ReplicationType::Metadata);
assert_eq!(replayed.user_tags, "owner=a3");
assert_eq!(replayed.admitted_target_arns(), vec![target.to_string()]);
}
#[tokio::test]
async fn mrf_save_admission_waits_for_capacity_instead_of_dropping() {
let (tx, mut rx) = mpsc::channel(1);
@@ -76,8 +76,7 @@ use metrics::counter;
use rmp_serde;
use rustfs_s3_types::EventName;
use rustfs_utils::http::{
AMZ_BUCKET_REPLICATION_STATUS, AMZ_TAGGING_DIRECTIVE, SUFFIX_REPLICATION_RESET, SUFFIX_REPLICATION_STATUS,
has_internal_suffix, insert_str,
AMZ_TAGGING_DIRECTIVE, SUFFIX_REPLICATION_RESET, SUFFIX_REPLICATION_STATUS, has_internal_suffix, insert_str,
};
use rustfs_utils::{DEFAULT_SIP_HASH_KEY, get_env_usize, sip_hash};
#[cfg(test)]
@@ -175,14 +174,6 @@ fn has_raw_status(err: &SdkError<HeadObjectError>, status: u16) -> bool {
err.raw_response().is_some_and(|r| r.status().as_u16() == status)
}
fn metadata_requires_existing_target(op_type: ReplicationType, object_info: &ObjectInfo) -> bool {
op_type == ReplicationType::Metadata
&& object_info
.user_defined
.get(AMZ_BUCKET_REPLICATION_STATUS)
.is_some_and(|status| status.eq_ignore_ascii_case(ReplicationStatusType::Replica.as_str()))
}
const METRIC_VERSION_IDENTITY_DRIFT_TOTAL: &str = "rustfs_replication_version_identity_drift_total";
/// Targets that already produced a version-identity-drift warning this
@@ -1421,7 +1412,7 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
};
let mut replication_state = oi.replication_state();
replication_state.replicate_decision_str = dsc.to_string();
let actual_size = oi.get_actual_size_or_physical();
let actual_size = oi.get_actual_size().unwrap_or_default();
Ok(ReplicateObjectInfo {
name: oi.name.clone(),
@@ -3503,7 +3494,6 @@ async fn resolve_replicate_all_action(
start_time,
ssec_audit_required,
} = ctx;
let require_existing_target = metadata_requires_existing_target(roi.op_type, &object_info);
let replication_action;
match head_object_for_worker(tgt_client.as_ref(), &tgt_client.bucket, object, roi.version_id.map(|v| v.to_string())).await {
Ok(oi) => {
@@ -3565,13 +3555,7 @@ async fn resolve_replicate_all_action(
// Version-ID format mismatch: retry without versionId and compare ETags.
match head_object_fallback(tgt_client, object).await {
Ok(Some(oi)) => {
let etags_match = replication_etags_match(object_info.etag.as_deref(), oi.e_tag.as_deref());
if require_existing_target && !etags_match {
rinfo.error = Some("replica metadata target does not contain matching object data".to_string());
rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
return None;
}
replication_action = if etags_match {
replication_action = if replication_etags_match(object_info.etag.as_deref(), oi.e_tag.as_deref()) {
if ssec_audit_required
&& !settle_ssec_passthrough_evidence(&oi, tgt_client, bucket, object, rinfo).await
{
@@ -3584,11 +3568,6 @@ async fn resolve_replicate_all_action(
};
}
Ok(None) => {
if require_existing_target {
rinfo.error = Some("replica metadata target does not contain this object version".to_string());
rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
return None;
}
replication_action = ReplicationAction::All;
}
Err(e2) => {
@@ -3614,12 +3593,7 @@ async fn resolve_replicate_all_action(
return None;
}
}
} else if e.as_service_error().is_some_and(|se| se.is_not_found()) || has_raw_status(&e, 404) {
if require_existing_target {
rinfo.error = Some("replica metadata target does not contain this object version".to_string());
rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
return None;
}
} else if e.as_service_error().is_some_and(|se| se.is_not_found()) {
replication_action = ReplicationAction::All;
} else {
rinfo.error = Some(e.to_string());
@@ -3894,7 +3868,6 @@ async fn replicate_object_with_multipart<S: ReplicationObjectIO>(ctx: MultipartR
actual_size,
object_info.etag.clone().unwrap_or_default(),
object_info.mod_time,
&put_opts.internal,
),
)
.await
@@ -3948,113 +3921,6 @@ mod tests {
})
}
fn spawn_head_status_server(status: u16) -> (String, std::thread::JoinHandle<()>) {
use std::io::{Read, Write};
let listener = std::net::TcpListener::bind(("127.0.0.1", 0)).expect("test HTTP listener should bind");
let endpoint = format!("http://{}", listener.local_addr().expect("test HTTP listener should have an address"));
let handle = std::thread::spawn(move || {
let (mut stream, _) = listener.accept().expect("test HTTP client should connect");
let mut request = [0_u8; 8192];
let bytes_read = stream.read(&mut request).expect("test HTTP request should be read");
assert!(bytes_read > 0, "test HTTP request should not be empty");
assert!(request[..bytes_read].starts_with(b"HEAD "), "replication comparison must use HEAD");
write!(stream, "HTTP/1.1 {status} Test\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
.expect("test HTTP response should be written");
});
(endpoint, handle)
}
#[tokio::test]
async fn replica_metadata_missing_target_stops_before_full_put() {
let (endpoint, server) = spawn_head_status_server(404);
let target = test_target_client(endpoint);
let roi = ReplicateObjectInfo {
bucket: "source".to_string(),
name: "object".to_string(),
version_id: Some(Uuid::new_v4()),
op_type: ReplicationType::Metadata,
// Normal metadata writes replace REPLICA with per-target PENDING
// before constructing the worker request.
replication_status: ReplicationStatusType::Pending,
..Default::default()
};
let object_info = ObjectInfo {
bucket: roi.bucket.clone(),
name: roi.name.clone(),
version_id: roi.version_id,
etag: Some("source-etag".to_string()),
user_defined: Arc::new(HashMap::from([(
AMZ_BUCKET_REPLICATION_STATUS.to_string(),
ReplicationStatusType::Replica.as_str().to_string(),
)])),
..Default::default()
};
let mut rinfo = replicate_all_target_info(&roi, &target);
let action = resolve_replicate_all_action(
ReplicateAllActionContext {
roi: &roi,
tgt_client: &target,
bucket: &roi.bucket,
object: &roi.name,
start_time: OffsetDateTime::now_utc(),
ssec_audit_required: false,
},
object_info,
&mut rinfo,
)
.await;
assert!(action.is_none(), "missing replica metadata targets must not reach the payload PUT path");
assert_eq!(rinfo.replication_status, ReplicationStatusType::Failed);
assert_eq!(
rinfo.error.as_deref(),
Some("replica metadata target does not contain this object version")
);
server.join().expect("test HTTP server should finish");
}
#[tokio::test]
async fn source_metadata_missing_target_rebuilds_object() {
let (endpoint, server) = spawn_head_status_server(404);
let target = test_target_client(endpoint);
let roi = ReplicateObjectInfo {
bucket: "source".to_string(),
name: "object".to_string(),
version_id: Some(Uuid::new_v4()),
op_type: ReplicationType::Metadata,
replication_status: ReplicationStatusType::Pending,
..Default::default()
};
let object_info = ObjectInfo {
bucket: roi.bucket.clone(),
name: roi.name.clone(),
version_id: roi.version_id,
etag: Some("source-etag".to_string()),
..Default::default()
};
let mut rinfo = replicate_all_target_info(&roi, &target);
let action = resolve_replicate_all_action(
ReplicateAllActionContext {
roi: &roi,
tgt_client: &target,
bucket: &roi.bucket,
object: &roi.name,
start_time: OffsetDateTime::now_utc(),
ssec_audit_required: false,
},
object_info,
&mut rinfo,
)
.await;
assert!(matches!(action, Some((ReplicationAction::All, _))));
assert!(rinfo.error.is_none());
server.join().expect("test HTTP server should finish");
}
async fn register_test_target(target: &Arc<TargetClient>) {
ReplicationTargetStore::register_test_target(target).await;
}
@@ -389,7 +389,7 @@ fn replication_source_object(object_info: &ObjectInfo) -> ReplicationSourceObjec
.map(|mod_time| OffsetDateTime::from_unix_timestamp(mod_time.unix_timestamp()).unwrap_or(mod_time)),
version_id: object_info.version_id.map(|version_id| version_id.to_string()),
etag: object_info.etag.as_deref(),
actual_size: object_info.get_actual_size_or_physical(),
actual_size: object_info.get_actual_size().unwrap_or_default(),
delete_marker: object_info.delete_marker,
content_type: object_info.content_type.as_deref(),
content_encoding: object_info.content_encoding.as_deref(),
@@ -472,7 +472,6 @@ pub(crate) fn replication_complete_multipart_options(
actual_size: String,
source_etag: String,
source_mtime: Option<OffsetDateTime>,
source_internal: &AdvancedPutOptions,
) -> PutObjectOptions {
let mut user_metadata = HashMap::new();
insert_header_map(&mut user_metadata, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, actual_size);
@@ -485,14 +484,6 @@ pub(crate) fn replication_complete_multipart_options(
// mtime must degrade to epoch so header() suppresses the header
// instead of asserting the replication time as the object's mtime.
source_mtime: source_mtime.unwrap_or(OffsetDateTime::UNIX_EPOCH),
// Carry the per-category LWW timestamps on the complete request as
// well: the receiver's CompleteMultipartUpload options builder
// parses the same headers, so the multipart transport gets the
// same receiver-side LWW as the single-PUT transport
// (rustfs/backlog#1953). Epoch values keep the headers suppressed.
tagging_timestamp: source_internal.tagging_timestamp,
retention_timestamp: source_internal.retention_timestamp,
legalhold_timestamp: source_internal.legalhold_timestamp,
replication_status: ReplicationStatusType::Replica,
replication_request: true,
..Default::default()
@@ -551,20 +542,6 @@ mod tests {
assert!(replication_target_head_is_newer_null_version(&source, &target));
}
#[test]
fn replication_source_uses_physical_size_for_unknown_compressed_object() {
let mut metadata = HashMap::new();
rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string());
let source = ObjectInfo {
size: 128,
actual_size: -1,
user_defined: Arc::new(metadata),
..Default::default()
};
assert_eq!(replication_source_object(&source).actual_size, 128);
}
#[test]
fn replication_target_head_content_matches_compare_etag_only() {
let source = ObjectInfo {
@@ -672,39 +649,20 @@ mod tests {
#[test]
fn replication_complete_multipart_options_sets_actual_size() {
let source_mtime = OffsetDateTime::from_unix_timestamp(1_716_170_000).expect("valid test timestamp");
let source_internal = AdvancedPutOptions {
tagging_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_100).expect("valid test timestamp"),
retention_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_200).expect("valid test timestamp"),
legalhold_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_300).expect("valid test timestamp"),
..Default::default()
};
let options = replication_complete_multipart_options(
"1024".to_string(),
"0123456789abcdef0123456789abcdef-3".to_string(),
Some(source_mtime),
&source_internal,
);
assert_eq!(options.internal.source_etag, "0123456789abcdef0123456789abcdef-3");
assert_eq!(options.internal.source_mtime, source_mtime);
// The complete request must carry the same per-category LWW timestamps
// as the initiate request; the receiver reads them from the complete
// headers (rustfs/backlog#1953).
assert_eq!(options.internal.tagging_timestamp, source_internal.tagging_timestamp);
assert_eq!(options.internal.retention_timestamp, source_internal.retention_timestamp);
assert_eq!(options.internal.legalhold_timestamp, source_internal.legalhold_timestamp);
// Absent source mtime must degrade to epoch (header suppressed), not
// the AdvancedPutOptions default of now_utc() — that default would
// stamp the replication time as the replica's mtime and break the
// multipart HEAD convergence. Unset category timestamps stay epoch so
// header() keeps suppressing them.
let options_no_mtime =
replication_complete_multipart_options("1024".to_string(), String::new(), None, &AdvancedPutOptions::default());
// multipart HEAD convergence.
let options_no_mtime = replication_complete_multipart_options("1024".to_string(), String::new(), None);
assert_eq!(options_no_mtime.internal.source_mtime.unix_timestamp(), 0);
assert_eq!(options_no_mtime.internal.tagging_timestamp.unix_timestamp(), 0);
assert_eq!(options_no_mtime.internal.retention_timestamp.unix_timestamp(), 0);
assert_eq!(options_no_mtime.internal.legalhold_timestamp.unix_timestamp(), 0);
assert_eq!(
get_header_map(&options.user_metadata, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE).as_deref(),
@@ -32,10 +32,6 @@ pub struct Credentials {
pub access_key: String,
#[serde(rename = "secretKey")]
pub secret_key: String,
// The aliases accept madmin's JSON tags (MinIO-written bucket-targets
// metadata and mc request bodies) without changing the snake_case
// persisted/peer wire format this struct serializes to.
#[serde(alias = "sessionToken")]
pub session_token: Option<String>,
pub expiration: Option<Timestamp>,
}
@@ -206,14 +202,12 @@ pub struct BucketTarget {
#[serde(default)]
pub region: String,
// madmin-go v3.0.109 tags this `bandwidthlimit`; `bandwidth` is a legacy
// alias kept for inputs written before the madmin tag was verified.
#[serde(alias = "bandwidthlimit", alias = "bandwidth", default)]
#[serde(alias = "bandwidth", default)]
pub bandwidth_limit: i64,
#[serde(rename = "replicationSync", default)]
pub replication_sync: bool,
#[serde(alias = "storageclass", default)]
#[serde(default)]
pub storage_class: String,
#[serde(rename = "skipTlsVerify", default)]
pub skip_tls_verify: bool,
@@ -226,7 +220,7 @@ pub struct BucketTarget {
#[serde(rename = "resetBeforeDate", with = "time::serde::rfc3339::option", default)]
pub reset_before_date: Option<OffsetDateTime>,
#[serde(alias = "resetID", default)]
#[serde(default)]
pub reset_id: String,
#[serde(rename = "totalDowntime", with = "duration_seconds", default)]
pub total_downtime: Duration,
@@ -239,7 +233,7 @@ pub struct BucketTarget {
#[serde(default)]
pub latency: LatencyStat,
#[serde(alias = "deploymentID", default)]
#[serde(default)]
pub deployment_id: String,
#[serde(default)]
@@ -537,85 +531,6 @@ mod tests {
assert_eq!(value["totalDowntime"], 90);
}
#[test]
fn bucket_target_persisted_wire_keys_stay_snake_case() {
// bucket-targets.json (persisted via `serde_json::to_vec(&BucketTargets)`
// in the admin set/remove handlers) and the msgpack struct-map form
// (`BucketTargets::marshal_msg`) both come straight from this struct's
// serde field names. madmin naming is applied only in the admin
// response layer (`remote_target_admin_json`); renaming here would
// silently break every existing deployment's persisted metadata.
let targets = BucketTargets {
targets: vec![BucketTarget {
credentials: Some(Credentials {
access_key: "ak".to_string(),
secret_key: "sk".to_string(),
session_token: Some("token".to_string()),
expiration: None,
}),
bandwidth_limit: 5,
storage_class: "STANDARD".to_string(),
reset_id: "reset-1".to_string(),
deployment_id: "deploy-1".to_string(),
..Default::default()
}],
};
let json = serde_json::to_value(&targets).expect("targets should serialize to JSON");
let msgpack: serde_json::Value =
rmp_serde::from_slice(&targets.marshal_msg().expect("targets should marshal to msgpack"))
.expect("msgpack struct map should decode into a JSON value");
for (wire, entry) in [("JSON", &json["targets"][0]), ("msgpack", &msgpack["targets"][0])] {
assert_eq!(entry["bandwidth_limit"], 5, "{wire} key `bandwidth_limit` must stay");
assert_eq!(entry["storage_class"], "STANDARD", "{wire} key `storage_class` must stay");
assert_eq!(entry["reset_id"], "reset-1", "{wire} key `reset_id` must stay");
assert_eq!(entry["deployment_id"], "deploy-1", "{wire} key `deployment_id` must stay");
assert_eq!(entry["credentials"]["session_token"], "token", "{wire} key `session_token` must stay");
}
}
#[test]
fn minio_written_bucket_targets_json_populates_madmin_named_fields() {
// A MinIO-written bucket-targets.json carries madmin's JSON tags
// (`bandwidthlimit`, `storageclass`, `resetID`, `deploymentID`,
// `credentials.sessionToken` — madmin-go v3.0.109 bucket-targets.go).
// On migration these must land in the matching fields instead of
// silently defaulting (backlog#1951).
let targets: BucketTargets = serde_json::from_value(serde_json::json!({
"targets": [{
"sourcebucket": "src",
"endpoint": "minio.example:9000",
"credentials": {
"accessKey": "ak",
"secretKey": "sk",
"sessionToken": "minio-session-token"
},
"targetbucket": "dst",
"type": "replication",
"replicationSync": true,
"bandwidthlimit": 107374182400i64,
"storageclass": "STANDARD",
"resetID": "reset-789",
"deploymentID": "deploy-123"
}]
}))
.expect("MinIO-written bucket-targets.json must deserialize");
let target = &targets.targets[0];
assert_eq!(target.bandwidth_limit, 107374182400);
assert_eq!(target.storage_class, "STANDARD");
assert_eq!(target.reset_id, "reset-789");
assert_eq!(target.deployment_id, "deploy-123");
assert_eq!(
target
.credentials
.as_ref()
.and_then(|credentials| credentials.session_token.as_deref()),
Some("minio-session-token")
);
}
#[test]
fn test_bucket_target_debug_redacts_credentials() {
let target = BucketTarget {
+13 -68
View File
@@ -94,7 +94,6 @@ const REPLAY_CACHE_AUTO_MEMORY_PERCENT: u64 = 13;
const REPLAY_CACHE_AUTO_RPC_RPS_PER_CPU: usize = 4096;
const REPLAY_CACHE_AUTO_MAX_CAPACITY: usize = 33_554_432;
const NS_SCANNER_CAPABILITY_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-capability-v3";
const NS_SCANNER_TIER_REGISTRY_GENERATION_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-tier-registry-generation-v1";
pub const TONIC_RPC_PREFIX: &str = "/node_service.NodeService";
static INTERNODE_RPC_SIGNATURE_STRICT: LazyLock<bool> = LazyLock::new(|| {
get_env_bool(
@@ -637,79 +636,40 @@ pub fn verify_put_file_capability(challenge: Uuid, server_epoch: Uuid, version:
.map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid put_file capability proof"))
}
fn update_ns_scanner_capability_mac(
mac: &mut HmacSha256,
challenge: Uuid,
server_epoch: Uuid,
supports_tier_registry_generation: bool,
) {
fn update_ns_scanner_capability_mac(mac: &mut HmacSha256, challenge: Uuid, server_epoch: Uuid) {
mac.update(NS_SCANNER_CAPABILITY_AUTH_DOMAIN);
mac.update(&NS_SCANNER_PROTOCOL_VERSION.to_be_bytes());
mac.update(challenge.as_bytes());
mac.update(server_epoch.as_bytes());
if supports_tier_registry_generation {
// The optional response capability is part of the authenticated
// scope. A proxy cannot turn an old/unsupported peer into a worker
// that receives generation-fenced scanner work.
mac.update(NS_SCANNER_TIER_REGISTRY_GENERATION_AUTH_DOMAIN);
}
}
fn generate_ns_scanner_capability_proof(
secret: &str,
challenge: Uuid,
server_epoch: Uuid,
supports_tier_registry_generation: bool,
) -> std::io::Result<Vec<u8>> {
fn generate_ns_scanner_capability_proof(secret: &str, challenge: Uuid, server_epoch: Uuid) -> std::io::Result<Vec<u8>> {
if challenge.is_nil() || server_epoch.is_nil() {
return Err(std::io::Error::other("Invalid namespace scanner capability scope"));
}
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch, supports_tier_registry_generation);
update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch);
Ok(mac.finalize().into_bytes().to_vec())
}
fn verify_ns_scanner_capability_proof(
secret: &str,
challenge: Uuid,
server_epoch: Uuid,
proof: &[u8],
supports_tier_registry_generation: bool,
) -> std::io::Result<()> {
fn verify_ns_scanner_capability_proof(secret: &str, challenge: Uuid, server_epoch: Uuid, proof: &[u8]) -> std::io::Result<()> {
if challenge.is_nil() || server_epoch.is_nil() {
return Err(std::io::Error::other("Invalid namespace scanner capability scope"));
}
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch, supports_tier_registry_generation);
update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch);
mac.verify_slice(proof)
.map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid namespace scanner capability proof"))
}
pub fn sign_ns_scanner_capability(challenge: Uuid, server_epoch: Uuid) -> std::io::Result<Vec<u8>> {
sign_ns_scanner_capability_with_tier_registry_generation(challenge, server_epoch, false)
generate_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch)
}
pub fn verify_ns_scanner_capability(challenge: Uuid, server_epoch: Uuid, proof: &[u8]) -> std::io::Result<()> {
verify_ns_scanner_capability_with_tier_registry_generation(challenge, server_epoch, proof, false)
}
pub fn sign_ns_scanner_capability_with_tier_registry_generation(
challenge: Uuid,
server_epoch: Uuid,
supports_tier_registry_generation: bool,
) -> std::io::Result<Vec<u8>> {
generate_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch, supports_tier_registry_generation)
}
pub fn verify_ns_scanner_capability_with_tier_registry_generation(
challenge: Uuid,
server_epoch: Uuid,
proof: &[u8],
supports_tier_registry_generation: bool,
) -> std::io::Result<()> {
verify_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch, proof, supports_tier_registry_generation)
verify_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch, proof)
}
#[derive(Clone, Copy)]
@@ -1749,28 +1709,13 @@ mod tests {
let secret = "test-scanner-capability-secret";
let challenge = Uuid::new_v4();
let server_epoch = Uuid::new_v4();
let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false)
.expect("capability proof should be generated");
let proof =
generate_ns_scanner_capability_proof(secret, challenge, server_epoch).expect("capability proof should be generated");
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false).is_ok());
assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof, false).is_err());
assert!(verify_ns_scanner_capability_proof(secret, challenge, Uuid::new_v4(), &proof, false).is_err());
assert!(verify_ns_scanner_capability_proof("different-secret", challenge, server_epoch, &proof, false).is_err());
}
#[test]
fn namespace_scanner_capability_proof_binds_tier_registry_generation_support() {
let secret = "test-scanner-capability-secret";
let challenge = Uuid::new_v4();
let server_epoch = Uuid::new_v4();
let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, true)
.expect("generation capability proof should be generated");
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, true).is_ok());
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false).is_err());
let legacy = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false)
.expect("legacy capability proof should be generated");
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &legacy, true).is_err());
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof).is_ok());
assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof).is_err());
assert!(verify_ns_scanner_capability_proof(secret, challenge, Uuid::new_v4(), &proof).is_err());
assert!(verify_ns_scanner_capability_proof("different-secret", challenge, server_epoch, &proof).is_err());
}
/// Security regression for GHSA-r5qv-rc46-hv8q (internode RPC fail-closed,
@@ -13,18 +13,17 @@
// limitations under the License.
use crate::cluster::rpc::{
build_auth_headers, build_put_file_auth_trailer, verify_ns_scanner_capability_with_tier_registry_generation,
verify_put_file_capability,
build_auth_headers, build_put_file_auth_trailer, verify_ns_scanner_capability, verify_put_file_capability,
};
use crate::disk::error::{Error, Result};
use crate::disk::{FileReader, FileWriter};
use crate::storage_api_contracts::internode::{
NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY, NS_SCANNER_LEADER_EPOCH_QUERY,
NS_SCANNER_PROTOCOL_VERSION, NS_SCANNER_PROTOCOL_VERSION_QUERY, NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY,
NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY,
NsScannerCapabilityResponse, PUT_FILE_AUTH_QUERY, PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_CHALLENGE_QUERY,
PUT_FILE_CAPABILITY_QUERY, PUT_FILE_CAPABILITY_VERSION, PUT_FILE_NONCE_QUERY, PUT_FILE_SERVER_EPOCH_QUERY,
PutFileCapabilityResponse, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, NsScannerCapabilityResponse, PUT_FILE_AUTH_QUERY,
PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_CHALLENGE_QUERY, PUT_FILE_CAPABILITY_QUERY, PUT_FILE_CAPABILITY_VERSION,
PUT_FILE_NONCE_QUERY, PUT_FILE_SERVER_EPOCH_QUERY, PutFileCapabilityResponse, WALK_DIR_BODY_SHA256_QUERY,
WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
};
use async_trait::async_trait;
use http::{HeaderMap, HeaderValue, Method, header::CONTENT_TYPE};
@@ -138,12 +137,6 @@ fn put_file_capability_status_is_legacy(status: u16) -> bool {
status == 404
}
fn ns_scanner_capability_error_allows_legacy(error: &Error) -> bool {
[400, 404, 405, 426]
.into_iter()
.any(|status| error.is_internode_http_status(status))
}
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
#[allow(
dead_code,
@@ -227,7 +220,6 @@ pub struct NsScannerStreamRequest {
#[derive(Debug, Clone)]
pub struct NsScannerCapabilityRequest {
pub endpoint: String,
pub supports_tier_registry_generation: bool,
}
/// Data-plane stream opener used by `RemoteDisk`.
@@ -260,15 +252,6 @@ pub trait InternodeDataTransport: Send + Sync + std::fmt::Debug {
async fn probe_ns_scanner(&self, _request: NsScannerCapabilityRequest) -> Result<Uuid> {
Err(Error::MethodNotAllowed)
}
async fn probe_ns_scanner_capability(&self, request: NsScannerCapabilityRequest) -> Result<NsScannerCapabilityResponse> {
let server_epoch = self.probe_ns_scanner(request).await?;
Ok(NsScannerCapabilityResponse {
version: NS_SCANNER_PROTOCOL_VERSION,
server_epoch,
proof: Vec::new(),
supports_tier_registry_generation: None,
})
}
// Interface facet nobody calls yet: every transport implements both, but no
// caller negotiates on them. Kept for the internode transport split
// (backlog#1350); deleting them would delete the seam and six impls.
@@ -352,44 +335,27 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
}
async fn probe_ns_scanner(&self, request: NsScannerCapabilityRequest) -> Result<Uuid> {
Ok(self.probe_ns_scanner_capability(request).await?.server_epoch)
}
async fn probe_ns_scanner_capability(&self, request: NsScannerCapabilityRequest) -> Result<NsScannerCapabilityResponse> {
if request.supports_tier_registry_generation {
return match self.probe_ns_scanner_capability_once(&request).await {
Ok(response) => Ok(response),
Err(marked_error) if ns_scanner_capability_error_allows_legacy(&marked_error) => {
// A v3 peer may reject the additive query marker, ignore
// it, or return its legacy proof. Retry once without the
// marker and only downgrade after that legacy response is
// authenticated; an unverified epoch is never trusted.
let legacy_request = NsScannerCapabilityRequest {
endpoint: request.endpoint.clone(),
supports_tier_registry_generation: false,
};
match self.probe_ns_scanner_capability_once(&legacy_request).await {
Ok(mut response) => {
response.supports_tier_registry_generation = None;
Ok(response)
}
Err(legacy_error) if ns_scanner_capability_error_allows_legacy(&legacy_error) => {
// Some old deployments expose only the legacy
// protocol response (or advertise 426). Treat
// the pair as an explicit unsupported result so
// the scanner can use its coordinator fallback.
Err(Error::MethodNotAllowed)
}
Err(_) => Err(marked_error),
}
}
// A server failure, network failure, or authentication error
// is not evidence of an old parser. Do not issue an
// unauthenticated legacy probe or silently downgrade.
Err(marked_error) => Err(marked_error),
};
let challenge = Uuid::new_v4();
let url = build_ns_scanner_capability_url(&request, challenge);
let mut headers = msgpack_headers();
build_auth_headers(&url, &Method::GET, &mut headers)?;
let reader = HttpReader::new(url, Method::GET, headers, None).await?;
let mut body = Vec::new();
reader
.take(u64::try_from(NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE + 1).unwrap_or(u64::MAX))
.read_to_end(&mut body)
.await?;
if body.is_empty() || body.len() > NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE {
return Err(Error::other("invalid remote namespace scanner capability response size"));
}
self.probe_ns_scanner_capability_once(&request).await
let response: NsScannerCapabilityResponse =
rmp_serde::from_slice(&body).map_err(|_| Error::other("invalid remote namespace scanner capability response"))?;
if response.version != NS_SCANNER_PROTOCOL_VERSION || response.server_epoch.is_nil() {
return Err(Error::other("incompatible remote namespace scanner capability response"));
}
verify_ns_scanner_capability(challenge, response.server_epoch, &response.proof)
.map_err(|err| Error::other(format!("remote namespace scanner capability authentication failed: {err}")))?;
Ok(response.server_epoch)
}
fn name(&self) -> &'static str {
@@ -402,53 +368,6 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
}
impl TcpHttpInternodeDataTransport {
async fn probe_ns_scanner_capability_once(
&self,
request: &NsScannerCapabilityRequest,
) -> Result<NsScannerCapabilityResponse> {
let challenge = Uuid::new_v4();
let url = build_ns_scanner_capability_url(request, challenge);
let mut headers = msgpack_headers();
build_auth_headers(&url, &Method::GET, &mut headers)?;
let reader = HttpReader::new(url, Method::GET, headers, None).await?;
let mut body = Vec::new();
reader
.take(u64::try_from(NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE + 1).unwrap_or(u64::MAX))
.read_to_end(&mut body)
.await?;
if body.is_empty() || body.len() > NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE {
return Err(Error::other("invalid remote namespace scanner capability response size"));
}
let mut response: NsScannerCapabilityResponse =
rmp_serde::from_slice(&body).map_err(|_| Error::other("invalid remote namespace scanner capability response"))?;
if response.version != NS_SCANNER_PROTOCOL_VERSION || response.server_epoch.is_nil() {
return Err(Error::other("incompatible remote namespace scanner capability response"));
}
if let Err(err) = verify_ns_scanner_capability_with_tier_registry_generation(
challenge,
response.server_epoch,
&response.proof,
request.supports_tier_registry_generation,
) {
// A permissive older peer can ignore the additive marker and
// return a valid legacy-scope proof with HTTP 200. Accept that
// response only after independently authenticating the legacy
// scope; all other verification failures remain fail-closed.
if request.supports_tier_registry_generation && ns_scanner_capability_legacy_proof_is_valid(challenge, &response) {
response.supports_tier_registry_generation = None;
return Ok(response);
}
return Err(Error::other(format!("remote namespace scanner capability authentication failed: {err}")));
}
// The proof authenticates the requested capability scope, not the
// optional response field. Derive the client-facing bit from that
// verified scope so an intermediary cannot strip or rewrite the field
// and force a silent downgrade after a successful generation-bound
// handshake.
normalize_ns_scanner_capability_response(&mut response, request.supports_tier_registry_generation);
Ok(response)
}
async fn put_file_auth_capability(&self, endpoint: &str) -> Result<Option<Uuid>> {
resolve_put_file_auth_capability(endpoint, || async {
tokio::time::timeout(PUT_FILE_CAPABILITY_PROBE_TIMEOUT, self.probe_put_file_auth(endpoint))
@@ -730,14 +649,6 @@ fn build_walk_dir_url(request: &WalkDirStreamRequest) -> String {
)
}
fn normalize_ns_scanner_capability_response(response: &mut NsScannerCapabilityResponse, requested_generation_support: bool) {
response.supports_tier_registry_generation = requested_generation_support.then_some(true);
}
fn ns_scanner_capability_legacy_proof_is_valid(challenge: Uuid, response: &NsScannerCapabilityResponse) -> bool {
verify_ns_scanner_capability_with_tier_registry_generation(challenge, response.server_epoch, &response.proof, false).is_ok()
}
fn build_ns_scanner_url(request: &NsScannerStreamRequest) -> String {
let body_sha256 = hex_simd::encode_to_string(Sha256::digest(&request.body), hex_simd::AsciiCase::Lower);
format!(
@@ -764,18 +675,13 @@ fn build_ns_scanner_url(request: &NsScannerStreamRequest) -> String {
fn build_ns_scanner_capability_url(request: &NsScannerCapabilityRequest, challenge: Uuid) -> String {
format!(
"{}{}?{}={}&{}={}{}",
"{}{}?{}={}&{}={}",
request.endpoint,
NS_SCANNER_PATH,
NS_SCANNER_PROTOCOL_VERSION_QUERY,
NS_SCANNER_PROTOCOL_VERSION,
NS_SCANNER_CAPABILITY_CHALLENGE_QUERY,
challenge,
if request.supports_tier_registry_generation {
format!("&{}=true", NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY)
} else {
String::new()
}
challenge
)
}
@@ -888,7 +794,6 @@ mod tests {
let probe_err = transport
.probe_ns_scanner(NsScannerCapabilityRequest {
endpoint: "http://node1:9000".to_string(),
supports_tier_registry_generation: false,
})
.await
.expect_err("legacy transport should report namespace scanner as unsupported");
@@ -1482,7 +1387,6 @@ mod tests {
let url = build_ns_scanner_capability_url(
&NsScannerCapabilityRequest {
endpoint: "http://node1:9000".to_string(),
supports_tier_registry_generation: false,
},
challenge,
);
@@ -1495,85 +1399,6 @@ mod tests {
);
}
#[test]
fn ns_scanner_capability_url_marks_generation_support_only_when_requested() {
let challenge = Uuid::new_v4();
let url = build_ns_scanner_capability_url(
&NsScannerCapabilityRequest {
endpoint: "http://node1:9000".to_string(),
supports_tier_registry_generation: true,
},
challenge,
);
assert!(url.contains(&format!("&{}=true", NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY)));
}
#[test]
fn ns_scanner_capability_legacy_fallback_requires_explicit_compatibility_status() {
for status in [400, 404, 405, 426] {
let error = Error::from(rustfs_rio::new_test_internode_http_io_error(
rustfs_rio::InternodeHttpErrorKind::HttpStatus(http::StatusCode::from_u16(status).expect("test status")),
));
assert!(
ns_scanner_capability_error_allows_legacy(&error),
"status {status} should permit legacy retry"
);
}
let marked_server_error = Error::from(rustfs_rio::new_test_internode_http_io_error(
rustfs_rio::InternodeHttpErrorKind::HttpStatus(http::StatusCode::INTERNAL_SERVER_ERROR),
));
let network_error = Error::from(rustfs_rio::new_test_internode_http_io_error(
rustfs_rio::InternodeHttpErrorKind::ConnectionRefused,
));
let authentication_error = Error::other("remote namespace scanner capability authentication failed");
assert!(!ns_scanner_capability_error_allows_legacy(&marked_server_error));
assert!(!ns_scanner_capability_error_allows_legacy(&network_error));
assert!(!ns_scanner_capability_error_allows_legacy(&authentication_error));
}
#[test]
fn authenticated_ns_scanner_capability_ignores_unprotected_response_bit() {
let mut response = NsScannerCapabilityResponse {
version: NS_SCANNER_PROTOCOL_VERSION,
server_epoch: Uuid::new_v4(),
proof: Vec::new(),
supports_tier_registry_generation: None,
};
normalize_ns_scanner_capability_response(&mut response, true);
assert_eq!(response.supports_tier_registry_generation, Some(true));
response.supports_tier_registry_generation = Some(false);
normalize_ns_scanner_capability_response(&mut response, false);
assert_eq!(response.supports_tier_registry_generation, None);
}
#[test]
fn ns_scanner_capability_accepts_only_authenticated_legacy_scope_after_marker_mismatch() {
crate::runtime::sources::ensure_test_rpc_secret();
let challenge = Uuid::new_v4();
let response = NsScannerCapabilityResponse {
version: NS_SCANNER_PROTOCOL_VERSION,
server_epoch: Uuid::new_v4(),
proof: crate::cluster::rpc::sign_ns_scanner_capability(challenge, Uuid::new_v4())
.expect("placeholder proof should be generated"),
supports_tier_registry_generation: None,
};
// A proof bound to a different challenge cannot authorize the legacy
// fallback, even though the response has the expected shape.
assert!(!ns_scanner_capability_legacy_proof_is_valid(challenge, &response));
let server_epoch = response.server_epoch;
let valid_response = NsScannerCapabilityResponse {
proof: crate::cluster::rpc::sign_ns_scanner_capability(challenge, server_epoch)
.expect("legacy proof should be generated"),
..response
};
assert!(ns_scanner_capability_legacy_proof_is_valid(challenge, &valid_response));
}
#[test]
fn transport_config_defaults_to_tcp_http() {
let transport = build_internode_data_transport(None).unwrap();
+2 -3
View File
@@ -35,9 +35,8 @@ pub use http_auth::{
TONIC_RPC_PREFIX, build_auth_headers, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, gen_signature_headers,
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
set_tonic_mutation_body_digest, set_tonic_rolling_canonical_body_digest, set_tonic_rolling_mutation_body_digest,
sign_ns_scanner_capability, sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability,
sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer,
sign_ns_scanner_capability, sign_put_file_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge,
tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_put_file_auth_trailer,
verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
verify_tonic_rpc_signature_with_bootstrap,
@@ -85,7 +85,6 @@ const PEER_REST_RECOVERY_MAX_ATTEMPTS: u32 = 60;
const PEER_REST_RECOVERY_MAX_BACKOFF: Duration = Duration::from_secs(30);
const SCANNER_ACTIVITY_MAX_MESSAGE_SIZE: usize = 1024;
const REPLICATION_STATS_MAX_MESSAGE_SIZE: usize = 8 * 1024 * 1024;
const BUCKET_METADATA_RELOAD_TIMEOUT: Duration = Duration::from_secs(5);
/// Error for a peer that reported `success = false` without an `error_info` payload.
///
@@ -1090,9 +1089,7 @@ impl PeerRestClient {
.await?
.max_decoding_message_size(BACKGROUND_HEAL_STATUS_MAX_MESSAGE_SIZE);
let response = match client
.background_heal_status(Request::new(BackgroundHealStatusRequest {
protocol_version: rustfs_protos::BACKGROUND_HEAL_STATUS_PROTOCOL_VERSION,
}))
.background_heal_status(Request::new(BackgroundHealStatusRequest::default()))
.await
{
Ok(response) => response.into_inner(),
@@ -1331,38 +1328,27 @@ impl PeerRestClient {
}
pub async fn load_bucket_metadata(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
let result = tokio::time::timeout(BUCKET_METADATA_RELOAD_TIMEOUT, async {
let result = self.load_bucket_metadata_once(bucket, scanner_maintenance_change).await;
if let Err(err) = &result
&& Self::is_network_like_error(err)
{
self.prepare_retry().await;
return self.load_bucket_metadata_once(bucket, scanner_maintenance_change).await;
self.finalize_result(
async {
let mut client = self.get_client().await?;
let mut request = Request::new(LoadBucketMetadataRequest {
bucket: bucket.to_string(),
scanner_maintenance_change,
});
set_tonic_mutation_body_digest(&mut request)?;
let response = client.load_bucket_metadata(request).await?.into_inner();
if !response.success {
if let Some(msg) = response.error_info {
return Err(Error::other(msg));
}
return Err(peer_failure_without_details("load_bucket_metadata", Some(bucket)));
}
Ok(())
}
result
})
.await,
)
.await
.unwrap_or_else(|_| Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))));
self.finalize_result(result).await
}
async fn load_bucket_metadata_once(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
let mut client = self.get_client().await?;
let mut request = Request::new(LoadBucketMetadataRequest {
bucket: bucket.to_string(),
scanner_maintenance_change,
});
set_tonic_mutation_body_digest(&mut request)?;
request.set_timeout(BUCKET_METADATA_RELOAD_TIMEOUT);
let response = client.load_bucket_metadata(request).await?.into_inner();
if !response.success {
if let Some(msg) = response.error_info {
return Err(Error::other(msg));
}
return Err(peer_failure_without_details("load_bucket_metadata", Some(bucket)));
}
Ok(())
}
pub async fn delete_bucket_metadata(&self, bucket: &str) -> Result<()> {
+27 -232
View File
@@ -42,19 +42,18 @@ use futures::lock::Mutex;
use metrics::counter;
use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
use rustfs_io_metrics::internode_metrics::{
INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE, INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE,
INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE,
INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE, INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP,
INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE, INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE,
INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP,
};
use rustfs_protos::ChannelClass;
use rustfs_protos::evict_failed_connection;
use rustfs_protos::proto_gen::node_service::RenamePartRequest;
use rustfs_protos::proto_gen::node_service::{
BatchReadVersionRequest, BatchReadVersionResponse, CheckPartsRequest, DeletePathsRequest, DeleteRequest,
DeleteVersionRequest, DeleteVersionsRequest, DeleteVersionsResponse, DeleteVolumeRequest, DiskInfoRequest, ListDirRequest,
ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, PreparePartTransactionRequest, ReadAllRequest,
ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest,
RenameDataRequest, RenameFileRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest,
DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, DiskInfoRequest, ListDirRequest, ListVolumesRequest,
MakeVolumeRequest, MakeVolumesRequest, PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest,
ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest,
RenameFileRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest,
SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
WriteMetadataRequest, node_service_client::NodeServiceClient,
};
@@ -99,7 +98,6 @@ const NS_SCANNER_CAPABILITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
const REMOTE_DISK_READ_RETRY_BASE_BACKOFF: Duration = Duration::from_millis(50);
const ENV_RUSTFS_METADATA_BATCH_READ: &str = "RUSTFS_METADATA_BATCH_READ";
const LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC: &str = "RUSTFS_BATCH_METADATA_RPC";
const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE";
const BATCH_METADATA_RPC_OFF: &str = "off";
const BATCH_METADATA_RPC_AUTO: &str = "auto";
const BATCH_METADATA_RPC_ON: &str = "on";
@@ -114,28 +112,6 @@ const EVENT_REMOTE_DISK_RPC: &str = "remote_disk_rpc";
const SNAPSHOT_LEASE_PROTOCOL_VERSION: u32 = 1;
pub const REMOTE_SNAPSHOT_LEASE_TTL: Duration = Duration::from_secs(60);
fn decode_delete_versions_errors(response: DeleteVersionsResponse, expected_len: usize) -> Vec<Option<Error>> {
if !response.item_errors.is_empty() {
if response.item_errors.len() != expected_len {
return vec![Some(Error::other("malformed delete_versions item errors")); expected_len];
}
return response
.item_errors
.into_iter()
.map(|error| (error.code != 0).then(|| error.into()))
.collect();
}
if response.errors.len() != expected_len {
return vec![Some(Error::other("malformed delete_versions errors")); expected_len];
}
response
.errors
.into_iter()
.map(|error| (!error.is_empty()).then(|| Error::other(error)))
.collect()
}
fn snapshot_lease_token_from_response(response: SnapshotLeaseResponse) -> Result<SnapshotLeaseToken> {
if !response.success {
return Err(response.error.unwrap_or_default().into());
@@ -204,8 +180,7 @@ fn parse_batch_metadata_rpc_mode(raw: &str) -> BatchMetadataRpcMode {
}
fn batch_metadata_rpc_mode_from_env() -> BatchMetadataRpcMode {
rustfs_utils::get_env_opt_str(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE)
.or_else(|| rustfs_utils::get_env_opt_str(ENV_RUSTFS_METADATA_BATCH_READ))
rustfs_utils::get_env_opt_str(ENV_RUSTFS_METADATA_BATCH_READ)
.or_else(|| rustfs_utils::get_env_opt_str(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC))
.as_deref()
.map(parse_batch_metadata_rpc_mode)
@@ -781,16 +756,14 @@ impl RemoteDisk {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
let probe = self.data_transport.probe_ns_scanner_capability(NsScannerCapabilityRequest {
let probe = self.data_transport.probe_ns_scanner(NsScannerCapabilityRequest {
endpoint: self.endpoint.grid_host(),
supports_tier_registry_generation: true,
});
let result = timeout(NS_SCANNER_CAPABILITY_PROBE_TIMEOUT, probe)
.await
.map_err(|_| DiskError::other("remote namespace scanner capability probe timed out"))?;
match result {
Ok(response) if response.supports_tier_registry_generation == Some(true) => Ok(Some(response.server_epoch)),
Ok(_) => Ok(None),
Ok(server_epoch) => Ok(Some(server_epoch)),
// RUSTFS_COMPAT_TODO(ns-scanner-rpc-v3): old peers and legacy transports lack the authenticated startup-epoch handshake. Remove after every supported peer implements namespace scanner protocol v3.
Err(DiskError::MethodNotAllowed) => Ok(None),
Err(err)
@@ -1831,12 +1804,6 @@ fn record_read_version_stage(stage: &'static str, started_at: Option<Instant>) {
}
}
fn record_batch_read_version_stage(stage: &'static str, started_at: Option<Instant>) {
if let Some(started_at) = started_at {
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_stage(stage, started_at.elapsed());
}
}
/// Aggregate encoded size (bytes) of a `ReadMultiple` response, preferring the msgpack payloads
/// and falling back to the JSON compatibility strings. Used to size the RPC for the payload
/// histogram / large-payload alerting (grpc-optimization P0 instrumentation).
@@ -1947,27 +1914,6 @@ fn decode_batch_read_version_response_items(
Ok(batch_read_version_resps)
}
fn batch_read_version_request_payload_len(req: &BatchReadVersionReq, req_json: &str, req_bin: &[u8]) -> usize {
req.items
.iter()
.fold(req_json.len().saturating_add(req_bin.len()), |total, item| {
total
.saturating_add(item.org_volume.len())
.saturating_add(item.volume.len())
.saturating_add(item.path.len())
.saturating_add(item.version_id.len())
})
}
fn batch_read_version_response_payload_len(response: &BatchReadVersionResponse) -> usize {
response
.batch_read_version_resps
.iter()
.map(String::len)
.sum::<usize>()
.saturating_add(response.batch_read_version_resps_bin.iter().map(Bytes::len).sum::<usize>())
}
fn validate_decoded_file_info(file_info: &FileInfo) -> Result<()> {
file_info.validate_for_metadata_read().map_err(Into::into)
}
@@ -2460,6 +2406,8 @@ impl DiskAPI for RemoteDisk {
return errors;
}
// TODO(backlog): replace string errors with typed `StorageError` variants
let result = self
.execute_with_timeout(
|| async {
@@ -2491,7 +2439,17 @@ impl DiskAPI for RemoteDisk {
}
return errors;
}
decode_delete_versions_errors(response, versions.len())
response
.errors
.iter()
.map(|error| {
if error.is_empty() {
None
} else {
Some(Error::other(error.to_string()))
}
})
.collect()
}
#[tracing::instrument(level = "trace", skip_all)]
@@ -2869,19 +2827,14 @@ impl DiskAPI for RemoteDisk {
state = "started",
"Remote disk RPC started"
);
let batch_read_version_attribution_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
let encode_started = read_version_stage_timer(batch_read_version_attribution_enabled);
let batch_read_version_req = compat_json(&req)?;
let batch_read_version_req_bin = encode_msgpack(&req)?;
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE, encode_started);
let request_payload_bytes = batch_read_version_attribution_enabled
.then(|| batch_read_version_request_payload_len(&req, &batch_read_version_req, &batch_read_version_req_bin));
let batch_result = self
.execute_with_timeout_for_op(
"batch_read_version",
move || async move {
let disk = self.disk_ref().await;
let disk_len = disk.len();
let mut client = self
.get_bulk_client()
.await
@@ -2892,20 +2845,9 @@ impl DiskAPI for RemoteDisk {
batch_read_version_req_bin: batch_read_version_req_bin.into(),
});
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_request();
if let Some(request_payload_bytes) = request_payload_bytes {
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_sent_bytes(
request_payload_bytes.saturating_add(disk_len),
);
}
let rpc_started = read_version_stage_timer(batch_read_version_attribution_enabled);
let response = match client.batch_read_version(request).await {
Ok(response) => {
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
response.into_inner()
}
Ok(response) => response.into_inner(),
Err(status) if status.code() == Code::Unimplemented => {
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
if mode.should_fallback_on_unimplemented() {
record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_FALLBACK_UNIMPLEMENTED);
warn!(
@@ -2922,7 +2864,6 @@ impl DiskAPI for RemoteDisk {
}
record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_UNSUPPORTED_NO_FALLBACK);
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
warn!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
@@ -2935,33 +2876,14 @@ impl DiskAPI for RemoteDisk {
);
return Err(Error::from(status));
}
Err(status) => {
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
return Err(Error::from(status));
}
Err(status) => return Err(Error::from(status)),
};
if !response.success {
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
return Err(response.error.unwrap_or_default().into());
}
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_recv_bytes(
batch_read_version_response_payload_len(&response),
);
let decode_started = read_version_stage_timer(batch_read_version_attribution_enabled);
match decode_batch_read_version_response_items(response, &self.endpoint) {
Ok(batch_read_version_resps) => {
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE, decode_started);
Ok(Some(batch_read_version_resps))
}
Err(err) => {
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE, decode_started);
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
Err(err)
}
}
decode_batch_read_version_response_items(response, &self.endpoint).map(Some)
},
get_max_timeout_duration(),
)
@@ -3838,63 +3760,6 @@ mod tests {
static INIT: Once = Once::new();
#[test]
fn delete_versions_response_preserves_typed_item_errors() {
let errors = decode_delete_versions_errors(
DeleteVersionsResponse {
success: true,
errors: vec!["file not found".to_string(), String::new()],
error: None,
item_errors: vec![
rustfs_protos::proto_gen::node_service::Error {
code: DiskError::FileNotFound.to_u32(),
error_info: "file not found".to_string(),
},
rustfs_protos::proto_gen::node_service::Error::default(),
],
},
2,
);
assert!(matches!(errors.as_slice(), [Some(DiskError::FileNotFound), None]));
}
#[test]
fn delete_versions_response_accepts_legacy_string_errors() {
let errors = decode_delete_versions_errors(
DeleteVersionsResponse {
success: true,
errors: vec!["legacy error".to_string(), String::new()],
error: None,
item_errors: Vec::new(),
},
2,
);
assert_eq!(errors.len(), 2);
assert_eq!(errors[0].as_ref().map(ToString::to_string).as_deref(), Some("io error legacy error"));
assert!(errors[1].is_none());
}
#[test]
fn delete_versions_response_rejects_misaligned_item_errors() {
let errors = decode_delete_versions_errors(
DeleteVersionsResponse {
success: true,
errors: vec!["file not found".to_string()],
error: None,
item_errors: vec![rustfs_protos::proto_gen::node_service::Error {
code: DiskError::FileNotFound.to_u32(),
error_info: "file not found".to_string(),
}],
},
2,
);
assert_eq!(errors.len(), 2);
assert!(errors.iter().all(Option::is_some));
}
#[test]
fn disk_mutation_digest_marks_rolling_compatibility() {
let mut request = Request::new(());
@@ -4042,21 +3907,10 @@ mod tests {
NsScannerProbe(NsScannerCapabilityRequest),
}
#[derive(Debug, Clone)]
#[derive(Debug, Clone, Default)]
struct RecordingInternodeDataTransport {
calls: Arc<StdMutex<Vec<RecordedTransportCall>>>,
ns_scanner_probe_status: Arc<StdMutex<Option<u16>>>,
ns_scanner_generation_support: Arc<StdMutex<Option<bool>>>,
}
impl Default for RecordingInternodeDataTransport {
fn default() -> Self {
Self {
calls: Arc::default(),
ns_scanner_probe_status: Arc::default(),
ns_scanner_generation_support: Arc::new(StdMutex::new(Some(true))),
}
}
}
#[derive(Clone, Debug)]
@@ -4276,15 +4130,6 @@ mod tests {
Self {
calls: Arc::default(),
ns_scanner_probe_status: Arc::new(StdMutex::new(Some(status))),
ns_scanner_generation_support: Arc::new(StdMutex::new(Some(true))),
}
}
fn with_ns_scanner_generation_support(support: Option<bool>) -> Self {
Self {
calls: Arc::default(),
ns_scanner_probe_status: Arc::default(),
ns_scanner_generation_support: Arc::new(StdMutex::new(support)),
}
}
@@ -4709,7 +4554,6 @@ mod tests {
} else {
"file version not found".to_string()
},
error_code: if success { 0 } else { DiskError::FileVersionNotFound.to_u32() },
}
}
@@ -4829,7 +4673,6 @@ mod tests {
fn batch_metadata_rpc_mode_uses_documented_env_before_legacy_alias() {
temp_env::with_vars(
[
(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, None::<&str>),
(ENV_RUSTFS_METADATA_BATCH_READ, Some("auto")),
(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("on")),
],
@@ -4839,25 +4682,10 @@ mod tests {
);
}
#[test]
fn batch_metadata_rpc_mode_uses_get_coalescer_env_before_batch_env() {
temp_env::with_vars(
[
(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, Some("on")),
(ENV_RUSTFS_METADATA_BATCH_READ, Some("off")),
(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("off")),
],
|| {
assert_eq!(batch_metadata_rpc_mode_from_env(), BatchMetadataRpcMode::On);
},
);
}
#[test]
fn batch_metadata_rpc_mode_falls_back_to_legacy_env_alias() {
temp_env::with_vars(
[
(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, None::<&str>),
(ENV_RUSTFS_METADATA_BATCH_READ, None::<&str>),
(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("on")),
],
@@ -4967,23 +4795,6 @@ mod tests {
Ok(Uuid::from_u128(1))
}
async fn probe_ns_scanner_capability(
&self,
request: NsScannerCapabilityRequest,
) -> Result<crate::storage_api_contracts::internode::NsScannerCapabilityResponse> {
let server_epoch = self.probe_ns_scanner(request).await?;
let supports_tier_registry_generation = *self
.ns_scanner_generation_support
.lock()
.expect("namespace scanner generation support lock poisoned");
Ok(crate::storage_api_contracts::internode::NsScannerCapabilityResponse {
version: crate::storage_api_contracts::internode::NS_SCANNER_PROTOCOL_VERSION,
server_epoch,
proof: Vec::new(),
supports_tier_registry_generation,
})
}
fn name(&self) -> &'static str {
"recording"
}
@@ -6796,22 +6607,6 @@ mod tests {
}
}
#[tokio::test]
async fn test_remote_disk_namespace_scanner_capability_falls_back_without_generation_support() {
for support in [None, Some(false)] {
let transport = RecordingInternodeDataTransport::with_ns_scanner_generation_support(support);
let remote_disk = new_remote_disk_with_transport(Arc::new(transport)).await;
assert_eq!(
remote_disk
.ns_scanner_server_epoch()
.await
.expect("missing generation support should be classified as unsupported"),
None
);
}
}
#[tokio::test]
async fn test_remote_disk_namespace_scanner_capability_rejects_legacy_transport() {
let remote_disk = new_remote_disk_with_transport(Arc::new(RetryingOpenReadInternodeDataTransport::default())).await;
@@ -14,10 +14,9 @@
use rustfs_io_metrics::internode_metrics::{
INTERNODE_MSGPACK_CODEC_JSON, INTERNODE_MSGPACK_CODEC_MSGPACK, INTERNODE_MSGPACK_DIRECTION_RESPONSE,
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
INTERNODE_OPERATION_GRPC_READ_VERSION, INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_OPERATION_PUT_FILE_STREAM,
INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_TRANSPORT_BACKEND_GRPC, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
global_internode_metrics,
INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE, INTERNODE_OPERATION_GRPC_READ_VERSION,
INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM,
INTERNODE_TRANSPORT_BACKEND_GRPC, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, global_internode_metrics,
};
use std::time::Duration;
@@ -94,59 +93,6 @@ pub(crate) fn record_remote_disk_grpc_read_version_request() {
);
}
pub(crate) fn record_remote_disk_grpc_batch_read_version_request() {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
}
global_internode_metrics().record_outgoing_request_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
INTERNODE_TRANSPORT_BACKEND_GRPC,
);
}
pub(crate) fn record_remote_disk_grpc_batch_read_version_stage(stage: &'static str, duration: Duration) {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
}
global_internode_metrics().record_stage_duration_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
INTERNODE_TRANSPORT_BACKEND_GRPC,
stage,
duration,
);
}
pub(crate) fn record_remote_disk_grpc_batch_read_version_error() {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
}
global_internode_metrics()
.record_error_for_operation_and_backend(INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, INTERNODE_TRANSPORT_BACKEND_GRPC);
}
pub(crate) fn record_remote_disk_grpc_batch_read_version_sent_bytes(bytes: usize) {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
}
global_internode_metrics().record_sent_bytes_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
INTERNODE_TRANSPORT_BACKEND_GRPC,
bytes,
);
}
pub(crate) fn record_remote_disk_grpc_batch_read_version_recv_bytes(bytes: usize) {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
}
global_internode_metrics().record_recv_bytes_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
INTERNODE_TRANSPORT_BACKEND_GRPC,
bytes,
);
record_grpc_payload_size(INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, bytes);
}
pub(crate) fn record_remote_disk_grpc_read_version_error() {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
-19
View File
@@ -406,25 +406,6 @@ where
Ok(data)
}
pub(crate) async fn read_config_limited_preserve_empty<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<Vec<u8>>
where
S: EcstoreObjectIO,
{
let (data, _obj) = read_config_limited_preserve_empty_with_metadata(api, file, max_bytes).await?;
Ok(data)
}
pub(crate) async fn read_config_limited_preserve_empty_with_metadata<S>(
api: Arc<S>,
file: &str,
max_bytes: usize,
) -> Result<(Vec<u8>, ObjectInfo)>
where
S: EcstoreObjectIO,
{
read_config_with_metadata_inner(api, file, &ObjectOptions::default(), true, Some(max_bytes)).await
}
/// Read an existing config object without treating an empty payload as absent.
/// Callers that validate their own payload format need to distinguish corruption
/// from `ConfigNotFound`.
+1 -4
View File
@@ -248,13 +248,10 @@ impl Config {
let shard_size = shard_size as usize;
// Keep the historical two-data-shard object budget while preventing
// wider EC layouts from multiplying the maximum inline object size.
// Use div_ceil to match the shard_file_size calculation (which also uses
// div_ceil), avoiding a 1-byte rounding discrepancy that prevents inline
// for objects right at the threshold.
let inline_block = if self.initialized && self.inline_block_explicit {
self.inline_block
} else {
DEFAULT_INLINE_OBJECT_BUDGET.div_ceil(data_shards).min(DEFAULT_INLINE_BLOCK)
(DEFAULT_INLINE_OBJECT_BUDGET / data_shards).min(DEFAULT_INLINE_BLOCK)
};
if versioned {
File diff suppressed because it is too large Load Diff
+68 -83
View File
@@ -21,7 +21,7 @@ use crate::storage_api_contracts::{
bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions},
multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo},
object::{DeleteAccounting, DeletedObject, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
object::{DeletedObject, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
range::HTTPRangeSpec,
};
use crate::{
@@ -414,66 +414,6 @@ fn apply_delete_objects_results(
}
}
fn apply_delete_accounting_results(
accounting: &mut [Option<DeleteAccounting>],
set_objects: &[DelObj],
set_accounting: &[Option<DeleteAccounting>],
) {
for (obj, value) in set_objects.iter().zip(set_accounting.iter()) {
accounting[obj.orig_idx] = value.clone();
}
}
impl Sets {
pub(crate) async fn delete_objects_with_accounting(
&self,
bucket: &str,
objects: Vec<ObjectToDelete>,
opts: ObjectOptions,
) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
let mut del_objects = vec![DeletedObject::default(); objects.len()];
let mut del_errs = vec![None; objects.len()];
let mut accounting = vec![None; objects.len()];
let mut set_obj_map = HashMap::new();
for (i, obj) in objects.iter().enumerate() {
let idx = self.get_hashed_set_index(obj.object_name.as_str());
set_obj_map.entry(idx).or_insert_with(Vec::new).push(DelObj {
orig_idx: i,
obj: obj.clone(),
});
}
let max_concurrent = set_obj_map.len().min(num_cpus::get()).max(1);
let semaphore = Arc::new(tokio::sync::Semaphore::new(max_concurrent));
let mut futures = FuturesUnordered::new();
let bucket = bucket.to_owned();
for (set_index, set_objects) in set_obj_map {
let disks = self.get_disks(set_index);
let objects = set_objects.iter().map(|entry| entry.obj.clone()).collect::<Vec<_>>();
let bucket = bucket.clone();
let opts = opts.clone();
let semaphore = semaphore.clone();
futures.push(async move {
let _permit = semaphore
.acquire_owned()
.await
.expect("delete_objects semaphore should remain open");
let (deleted, errors, accounting) = disks.delete_objects_with_accounting(&bucket, objects, opts).await;
(set_objects, deleted, errors, accounting)
});
}
while let Some((set_objects, deleted, errors, set_accounting)) = futures.next().await {
apply_delete_objects_results(&mut del_objects, &mut del_errs, &set_objects, &deleted, errors);
apply_delete_accounting_results(&mut accounting, &set_objects, &set_accounting);
}
(del_objects, del_errs, accounting)
}
}
#[async_trait::async_trait]
impl crate::storage_api_contracts::object::ObjectIO for Sets {
type Error = Error;
@@ -715,8 +655,65 @@ impl crate::storage_api_contracts::object::ObjectOperations for Sets {
objects: Vec<ObjectToDelete>,
opts: ObjectOptions,
) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
let (deleted, errors, _) = self.delete_objects_with_accounting(bucket, objects, opts).await;
(deleted, errors)
// Default return value
let mut del_objects = vec![DeletedObject::default(); objects.len()];
let mut del_errs = Vec::with_capacity(objects.len());
for _ in 0..objects.len() {
del_errs.push(None)
}
let mut set_obj_map = HashMap::new();
// hash key
for (i, obj) in objects.iter().enumerate() {
let idx = self.get_hashed_set_index(obj.object_name.as_str());
if !set_obj_map.contains_key(&idx) {
set_obj_map.insert(
idx,
vec![DelObj {
// set_idx: idx,
orig_idx: i,
obj: obj.clone(),
}],
);
} else if let Some(val) = set_obj_map.get_mut(&idx) {
val.push(DelObj {
// set_idx: idx,
orig_idx: i,
obj: obj.clone(),
});
}
}
let max_concurrent = set_obj_map.len().min(num_cpus::get()).max(1);
let semaphore = Arc::new(tokio::sync::Semaphore::new(max_concurrent));
let mut futures = FuturesUnordered::new();
let bucket = bucket.to_string();
for (k, v) in set_obj_map {
let disks = self.get_disks(k);
let objs: Vec<ObjectToDelete> = v.iter().map(|v| v.obj.clone()).collect();
let bucket = bucket.clone();
let opts = opts.clone();
let semaphore = semaphore.clone();
futures.push(async move {
let _permit = semaphore
.acquire_owned()
.await
.expect("delete_objects semaphore should remain open");
let (dobjects, errs) = disks.delete_objects(&bucket, objs, opts).await;
(v, dobjects, errs)
});
}
while let Some((v, dobjects, errs)) = futures.next().await {
apply_delete_objects_results(&mut del_objects, &mut del_errs, &v, &dobjects, errs);
}
(del_objects, del_errs)
}
#[tracing::instrument(skip(self))]
@@ -988,11 +985,14 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for Sets {
}
}
impl Sets {
pub(crate) async fn heal_format_with_fence<F>(&self, dry_run: bool, fence_lost: F) -> Result<(HealResultItem, Option<Error>)>
where
F: Fn() -> bool + Send + Sync,
{
#[async_trait::async_trait]
impl crate::storage_api_contracts::heal::HealOperations for Sets {
type Error = Error;
type HealResultItem = HealResultItem;
type HealOptions = HealOpts;
#[tracing::instrument(skip(self))]
async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
let (disks, init_errs) = init_storage_disks_with_errors(
&self.endpoints.endpoints,
&DiskOption {
@@ -1065,9 +1065,6 @@ impl Sets {
// Save new formats `format.json` on unformatted disks.
for (index, (fm, disk)) in tmp_new_formats.iter_mut().zip(disks.iter()).enumerate() {
if fm.is_some() && disk.is_some() {
if fence_lost() {
return Ok((res, Some(StorageError::SlowDown)));
}
if let Err(err) = save_format_file(disk, fm).await {
if let Some(disk) = disk.as_ref() {
let _ = disk.close().await;
@@ -1101,18 +1098,6 @@ impl Sets {
}
Ok((res, None))
}
}
#[async_trait::async_trait]
impl crate::storage_api_contracts::heal::HealOperations for Sets {
type Error = Error;
type HealResultItem = HealResultItem;
type HealOptions = HealOpts;
#[tracing::instrument(skip(self))]
async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
self.heal_format_with_fence(dry_run, || false).await
}
#[tracing::instrument(skip(self))]
async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
let mut result = HealResultItem {
+1 -1
View File
@@ -813,7 +813,7 @@ pub(crate) fn is_equivalent_data_movement_metadata(
.all(|(key, value)| source.user_defined.get(key) == Some(value))
}
pub(crate) fn is_equivalent_data_movement_object_identity(
fn is_equivalent_data_movement_object_identity(
source: &ObjectInfo,
target: &ObjectInfo,
compare_mod_time: bool,
+41 -477
View File
@@ -73,16 +73,6 @@ struct CachedBucketUsage {
// mutation. A strictly later generation is required before the mutation
// evidence can be discarded.
pending_scanner_position: Option<(u64, u64)>,
// Deletes are visible to admin immediately, but quota admission keeps
// them pending until a complete scanner generation reconciles the set.
// This marker intentionally remains process-local: the delete request
// updates this overlay before the scanner writes a durable snapshot. If
// the process restarts first, loading the persisted complete snapshot
// restores the pre-reconciliation (larger) baseline, which is
// conservative for quota admission. A persisted post-delete snapshot is
// necessarily a complete scanner reconciliation and therefore creates a
// fresh cache entry with no pending hold.
pending_negative_delta: u64,
}
type UsageMemoryCache = Arc<RwLock<HashMap<String, CachedBucketUsage>>>;
@@ -388,12 +378,8 @@ pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store:
"nonconverged data usage observations cannot replace the quota-authoritative snapshot",
));
}
let Some(expected_publication_epoch) = store.scanner_data_usage_publication_epoch().await else {
return Err(Error::other("data usage publication is blocked by data movement"));
};
// Prevent older data from overwriting newer persisted stats
let existing_snapshot = load_data_usage_snapshot(store.clone()).await;
if let Ok((existing, source)) = existing_snapshot
if let Ok((existing, source)) = load_data_usage_snapshot(store.clone()).await
&& source.is_authoritative()
&& let Some(reason) = stale_data_usage_persist_reason_for_source(&data_usage_info, &existing, source, SystemTime::now())
{
@@ -404,31 +390,19 @@ pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store:
return Ok(());
}
save_data_usage_in_backend(data_usage_info, store, expected_publication_epoch).await
save_data_usage_in_backend(data_usage_info, store).await
}
async fn save_data_usage_in_backend(
data_usage_info: DataUsageInfo,
store: Arc<ECStore>,
expected_publication_epoch: u64,
) -> Result<(), Error> {
async fn save_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<ECStore>) -> Result<(), Error> {
let data =
serde_json::to_vec(&data_usage_info).map_err(|e| Error::other(format!("Failed to serialize data usage info: {e}")))?;
// Save to backend using the same mechanism as original code
let Some((publication_guard, publication_epoch)) = store.scanner_data_usage_publication_admission_guard().await else {
return Err(Error::other("data usage publication is blocked by data movement"));
};
if publication_epoch != expected_publication_epoch {
return Err(Error::other("data usage publication epoch changed before save"));
}
crate::config::com::save_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH, data)
.await
.map_err(Error::other)?;
drop(publication_guard);
cleanup_observed_data_usage_after_authoritative_save_with_publication(store.as_ref(), &data_usage_info, Some(store.as_ref()))
.await;
cleanup_observed_data_usage_after_authoritative_save(store.as_ref(), &data_usage_info).await;
// Invalidate the cached snapshot so readers observe the new save on their
// next request instead of waiting out the remaining TTL. The next cached
@@ -465,24 +439,11 @@ impl ObservedDataUsageSnapshotCleanup for ECStore {
}
}
async fn cleanup_observed_data_usage_after_authoritative_save_with_publication<S>(
store: &S,
authoritative: &DataUsageInfo,
publication_store: Option<&ECStore>,
) where
async fn cleanup_observed_data_usage_after_authoritative_save<S>(store: &S, authoritative: &DataUsageInfo)
where
S: EcstoreObjectIO + ObservedDataUsageSnapshotCleanup + ?Sized,
{
let observed_read_epoch = match publication_store {
Some(publication_store) => {
let Some(epoch) = publication_store.scanner_data_usage_publication_epoch().await else {
return;
};
Some(epoch)
}
None => None,
};
let observed_snapshot = load_data_usage_for_bucket_removal(store, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await;
let (observed, revision) = match observed_snapshot {
let (observed, revision) = match load_data_usage_for_bucket_removal(store, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await {
Ok(Some(snapshot)) => snapshot,
Ok(None) => return,
Err(err) => {
@@ -498,19 +459,6 @@ async fn cleanup_observed_data_usage_after_authoritative_save_with_publication<S
return;
}
let publication_guard = match publication_store {
Some(publication_store) => {
let Some((guard, publication_epoch)) = publication_store.scanner_data_usage_publication_admission_guard().await
else {
return;
};
if observed_read_epoch.is_some_and(|expected| expected != publication_epoch) {
return;
}
Some(guard)
}
None => None,
};
match store.delete_observed_data_usage_snapshot(&revision).await {
Ok(()) | Err(Error::ConfigNotFound | Error::FileNotFound | Error::ObjectNotFound(_, _) | Error::PreconditionFailed) => {}
Err(err) => {
@@ -521,15 +469,6 @@ async fn cleanup_observed_data_usage_after_authoritative_save_with_publication<S
);
}
}
drop(publication_guard);
}
#[cfg(test)]
async fn cleanup_observed_data_usage_after_authoritative_save<S>(store: &S, authoritative: &DataUsageInfo)
where
S: EcstoreObjectIO + ObservedDataUsageSnapshotCleanup + ?Sized,
{
cleanup_observed_data_usage_after_authoritative_save_with_publication(store, authoritative, None).await;
}
fn set_buckets_count_from_usage(data_usage_info: &mut DataUsageInfo) {
@@ -570,7 +509,7 @@ pub async fn remove_bucket_usage_from_backend(store: Arc<ECStore>, bucket: &str)
pub(crate) async fn remove_bucket_usage_for_namespace_change(store: &ECStore, bucket: &str) -> Result<(), Error> {
prepare_bucket_usage_for_namespace_change(bucket, None).await?;
remove_bucket_usage_from_backend_with_guard_fenced(store, bucket, None).await
remove_bucket_usage_from_backend_with_guard(store, bucket, None).await
}
pub(crate) async fn prepare_bucket_usage_for_namespace_change(
@@ -593,7 +532,6 @@ pub(crate) async fn prepare_bucket_usage_for_namespace_change(
Ok(())
}
#[cfg(test)]
pub(crate) async fn remove_bucket_usage_from_backend_with_guard<S>(
store: &S,
bucket: &str,
@@ -603,24 +541,6 @@ where
S: EcstoreObjectIO + ?Sized,
{
let result = remove_bucket_usage_from_backend_with_store_and_guard(store, bucket, guard).await;
invalidate_bucket_usage_snapshot_caches(guard, bucket).await?;
result
}
pub(crate) async fn remove_bucket_usage_from_backend_with_guard_fenced(
store: &ECStore,
bucket: &str,
guard: Option<&rustfs_lock::NamespaceLockGuard>,
) -> Result<(), Error> {
let result = remove_bucket_usage_from_backend_with_store_and_guard_and_publication(store, bucket, guard, Some(store)).await;
invalidate_bucket_usage_snapshot_caches(guard, bucket).await?;
result
}
async fn invalidate_bucket_usage_snapshot_caches(
guard: Option<&rustfs_lock::NamespaceLockGuard>,
bucket: &str,
) -> Result<(), Error> {
let mut snapshot_cache = data_usage_snapshot_cache().write().await;
ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cache invalidation")?;
clear_data_usage_snapshot_cache(&mut snapshot_cache);
@@ -628,7 +548,7 @@ async fn invalidate_bucket_usage_snapshot_caches(
let mut admin_snapshot_cache = admin_data_usage_snapshot_cache().write().await;
ensure_bucket_namespace_guard(guard, bucket, "admin data usage snapshot cache invalidation")?;
clear_admin_data_usage_snapshot_cache(&mut admin_snapshot_cache);
Ok(())
result
}
async fn load_data_usage_for_bucket_removal<S>(store: &S, object: &str) -> Result<Option<(DataUsageInfo, String)>, Error>
@@ -687,83 +607,48 @@ fn ensure_bucket_namespace_guard(
Ok(())
}
#[cfg(test)]
async fn remove_bucket_usage_from_backend_with_store_and_guard<S>(
store: &S,
bucket: &str,
guard: Option<&rustfs_lock::NamespaceLockGuard>,
) -> Result<(), Error>
where
S: EcstoreObjectIO + ?Sized,
{
remove_bucket_usage_from_backend_with_store_and_guard_and_publication(store, bucket, guard, None).await
}
async fn remove_bucket_usage_from_backend_with_store_and_guard_and_publication<S>(
store: &S,
bucket: &str,
guard: Option<&rustfs_lock::NamespaceLockGuard>,
publication_store: Option<&ECStore>,
) -> Result<(), Error>
where
S: EcstoreObjectIO + ?Sized,
{
ensure_bucket_namespace_guard(guard, bucket, "data usage primary cleanup")?;
let primary_seed_epoch = match publication_store {
Some(publication_store) => Some(
publication_store
.scanner_data_usage_publication_epoch()
.await
.ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?,
),
None => None,
};
let primary_seed = load_data_usage_seed_for_missing_primary(store).await?;
remove_bucket_usage_from_object_with_retries_and_publication(
remove_bucket_usage_from_object_with_retries(
store,
DATA_USAGE_OBJ_NAME_PATH.as_str(),
bucket,
DATA_USAGE_REMOVE_CAS_RETRIES,
primary_seed.as_ref(),
Some(&primary_seed),
guard,
publication_store.map(|store| (store, primary_seed_epoch)),
)
.await?;
ensure_bucket_namespace_guard(guard, bucket, "data usage backup cleanup")?;
let backup_seed_epoch = match publication_store {
Some(publication_store) => Some(
publication_store
.scanner_data_usage_publication_epoch()
.await
.ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?,
),
None => None,
};
let backup_seed = load_data_usage_for_bucket_removal(store, DATA_USAGE_OBJ_NAME_PATH.as_str())
.await?
.map(|(data_usage_info, _)| data_usage_info)
.or_else(|| primary_seed.clone());
remove_bucket_usage_from_object_with_retries_and_publication(
.map_or(primary_seed, |(data_usage_info, _)| data_usage_info);
remove_bucket_usage_from_object_with_retries(
store,
DATA_USAGE_OBJ_BACKUP_PATH.as_str(),
bucket,
DATA_USAGE_REMOVE_CAS_RETRIES,
backup_seed.as_ref(),
Some(&backup_seed),
guard,
publication_store.map(|store| (store, backup_seed_epoch)),
)
.await?;
ensure_bucket_namespace_guard(guard, bucket, "observed data usage cleanup")?;
if let Err(err) = remove_bucket_usage_from_object_with_retries_and_publication(
if let Err(err) = remove_bucket_usage_from_object_with_retries(
store,
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
bucket,
DATA_USAGE_REMOVE_CAS_RETRIES,
None,
guard,
publication_store.map(|store| (store, None)),
)
.await
{
@@ -777,21 +662,12 @@ where
LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(),
LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str(),
] {
remove_bucket_usage_from_object_with_retries_and_publication(
store,
object,
bucket,
DATA_USAGE_REMOVE_CAS_RETRIES,
None,
guard,
publication_store.map(|store| (store, None)),
)
.await?;
remove_bucket_usage_from_object_with_retries(store, object, bucket, DATA_USAGE_REMOVE_CAS_RETRIES, None, guard).await?;
}
Ok(())
}
async fn load_data_usage_seed_for_missing_primary<S>(store: &S) -> Result<Option<DataUsageInfo>, Error>
async fn load_data_usage_seed_for_missing_primary<S>(store: &S) -> Result<DataUsageInfo, Error>
where
S: EcstoreObjectIO + ?Sized,
{
@@ -804,13 +680,12 @@ where
if !authoritative {
data_usage_info.usage_snapshot_complete = false;
}
return Ok(Some(data_usage_info));
return Ok(data_usage_info);
}
}
Ok(None)
Ok(DataUsageInfo::default())
}
#[cfg(test)]
async fn remove_bucket_usage_from_object_with_retries<S>(
store: &S,
object: &str,
@@ -819,42 +694,12 @@ async fn remove_bucket_usage_from_object_with_retries<S>(
missing_seed: Option<&DataUsageInfo>,
guard: Option<&rustfs_lock::NamespaceLockGuard>,
) -> Result<(), Error>
where
S: EcstoreObjectIO + ?Sized,
{
remove_bucket_usage_from_object_with_retries_and_publication(store, object, bucket, cas_retries, missing_seed, guard, None)
.await
}
async fn remove_bucket_usage_from_object_with_retries_and_publication<S>(
store: &S,
object: &str,
bucket: &str,
cas_retries: usize,
missing_seed: Option<&DataUsageInfo>,
guard: Option<&rustfs_lock::NamespaceLockGuard>,
publication: Option<(&ECStore, Option<u64>)>,
) -> Result<(), Error>
where
S: EcstoreObjectIO + ?Sized,
{
for attempt in 0..=cas_retries {
ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cleanup")?;
let read_epoch = match publication {
Some((publication_store, expected_publication_epoch)) => {
let epoch = publication_store
.scanner_data_usage_publication_epoch()
.await
.ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?;
if expected_publication_epoch.is_some_and(|expected| expected != epoch) {
return Err(Error::other("data usage publication epoch changed before snapshot read"));
}
Some(epoch)
}
None => None,
};
let loaded_snapshot = load_data_usage_for_bucket_removal(store, object).await?;
let (mut data_usage_info, revision) = match loaded_snapshot {
let (mut data_usage_info, revision) = match load_data_usage_for_bucket_removal(store, object).await? {
Some((data_usage_info, revision)) => (data_usage_info, Some(revision)),
None => match missing_seed {
Some(data_usage_info) => (data_usage_info.clone(), None),
@@ -878,22 +723,6 @@ where
},
};
ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot commit")?;
let publication_guard = match publication {
Some((publication_store, expected_publication_epoch)) => {
let Some((guard, publication_epoch)) = publication_store.scanner_data_usage_publication_admission_guard().await
else {
return Err(Error::other("data usage publication is blocked by data movement"));
};
if expected_publication_epoch
.or(read_epoch)
.is_some_and(|expected| expected != publication_epoch)
{
return Err(Error::other("data usage publication epoch changed before snapshot commit"));
}
Some(guard)
}
None => None,
};
let save_result = store
.put_object(
RUSTFS_META_BUCKET,
@@ -906,7 +735,6 @@ where
},
)
.await;
drop(publication_guard);
match save_result {
Ok(_) => return Ok(()),
Err(err) => {
@@ -1120,12 +948,7 @@ async fn load_observed_data_usage_snapshot(store: Arc<ECStore>) -> Option<DataUs
};
match parse_usage_snapshot(&data) {
Ok(info)
if info.usage_snapshot_converged == Some(false)
&& (info.is_complete_bucket_usage_snapshot() || info.is_valid_partial_snapshot()) =>
{
Some(info)
}
Ok(info) if info.usage_snapshot_converged == Some(false) && info.is_complete_bucket_usage_snapshot() => Some(info),
Ok(_) => {
error!(
event = "data_usage_snapshot_load_failed",
@@ -1170,7 +993,7 @@ async fn load_admin_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataU
}
fn discard_incomplete_bucket_usage(data_usage_info: &mut DataUsageInfo) {
if !data_usage_info.is_complete_bucket_usage_snapshot() && !data_usage_info.usage_snapshot_partial {
if !data_usage_info.is_complete_bucket_usage_snapshot() {
data_usage_info.usage_snapshot_complete = false;
data_usage_info.buckets_usage.clear();
data_usage_info.bucket_sizes.clear();
@@ -1568,37 +1391,7 @@ impl BucketUsageAccumulator {
}
pub fn quota_object_size(object: &ObjectInfo) -> Result<u64, Error> {
// A compressed object may carry -1 while the transformed size is unknown
// (legacy streaming sentinel). In that case the persisted physical size
// is still a valid accounting floor; every other negative value is corrupt.
// An explicit negative `actual-size` metadata value is corrupt, however:
// the sentinel is only valid in the in-memory/object-part field written by
// the legacy streaming path, not as a persisted declared size.
let compressed = object.is_compressed();
if object.actual_size < -1 || (object.actual_size == -1 && !compressed) {
return Err(Error::PartMissingOrCorrupt);
}
if object
.parts
.iter()
.any(|part| part.actual_size < -1 || (part.actual_size < 0 && !compressed))
{
return Err(Error::PartMissingOrCorrupt);
}
let declared_actual_size = rustfs_utils::http::get_str(&object.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE)
.filter(|value| !value.is_empty());
if declared_actual_size
.as_deref()
.and_then(|value| value.parse::<i64>().ok())
.is_some_and(|size| size < 0)
{
return Err(Error::PartMissingOrCorrupt);
}
let logical_size = match object.get_actual_size().map_err(Error::other)? {
size if size == -1 && compressed && declared_actual_size.is_none() => None,
size if size >= 0 => Some(u64::try_from(size).map_err(|_| Error::PartMissingOrCorrupt)?),
_ => return Err(Error::PartMissingOrCorrupt),
};
let logical_size = u64::try_from(object.get_actual_size().map_err(Error::other)?).map_err(|_| Error::PartMissingOrCorrupt)?;
let persisted_part_size = if object.parts.is_empty() {
u64::try_from(object.size).map_err(|_| Error::PartMissingOrCorrupt)?
} else {
@@ -1606,8 +1399,12 @@ pub fn quota_object_size(object: &ObjectInfo) -> Result<u64, Error> {
// Compressed streaming objects persist -1 when the transformed
// part size is unknown. The physical part size remains a valid
// quota floor; reject only non-negative values that overflow.
let actual_size = if part.actual_size == -1 {
0
let actual_size = if part.actual_size < 0 {
if object.is_compressed() {
0
} else {
return Err(Error::PartMissingOrCorrupt);
}
} else {
u64::try_from(part.actual_size).map_err(|_| Error::PartMissingOrCorrupt)?
};
@@ -1615,7 +1412,7 @@ pub fn quota_object_size(object: &ObjectInfo) -> Result<u64, Error> {
total.checked_add(part_size).ok_or(Error::PartMissingOrCorrupt)
})?
};
Ok(logical_size.unwrap_or(0).max(persisted_part_size))
Ok(logical_size.max(persisted_part_size))
}
type UsageVersionPage = StorageListObjectVersionsInfo<ObjectInfo>;
@@ -1820,7 +1617,6 @@ fn cached_bucket_usage_from_backend(usage: BucketUsageInfo, updated_at: SystemTi
dirty: false,
stale_snapshot_pending: false,
pending_scanner_position: None,
pending_negative_delta: 0,
}
}
@@ -1834,7 +1630,6 @@ fn cached_bucket_usage_now(usage: BucketUsageInfo) -> CachedBucketUsage {
dirty: false,
stale_snapshot_pending: false,
pending_scanner_position: None,
pending_negative_delta: 0,
}
}
@@ -1987,7 +1782,6 @@ pub async fn record_bucket_object_delete_memory(bucket: &str, deleted_size: u64,
.or_insert_with(|| cached_bucket_usage_now(BucketUsageInfo::default()));
entry.usage.size = entry.usage.size.saturating_sub(deleted_size);
entry.pending_negative_delta = entry.pending_negative_delta.saturating_add(deleted_size);
if removed_current_object {
entry.usage.objects_count = entry.usage.objects_count.saturating_sub(1);
entry.usage.versions_count = entry.usage.versions_count.saturating_sub(1);
@@ -2043,7 +1837,7 @@ pub async fn get_bucket_usage_memory(bucket: &str) -> Option<u64> {
cache
.get(bucket)
.filter(|cached| cached.authoritative)
.map(|cached| cached.usage.size.saturating_add(cached.pending_negative_delta))
.map(|cached| cached.usage.size)
}
async fn update_usage_cache_if_needed() {
@@ -2448,8 +2242,6 @@ pub async fn init_compression_total_memory_from_backend(store: Arc<ECStore>) {
#[cfg(test)]
mod tests {
use super::*;
use crate::layout::endpoints::EndpointServerPools;
use crate::runtime::instance::InstanceContext;
use crate::storage_api_contracts::object::ObjectIO as _;
use rustfs_data_usage::BucketUsageInfo;
use rustfs_lock::{LocalClient, LockRequest, LockType, NamespaceLock, ObjectKey};
@@ -2472,7 +2264,6 @@ mod tests {
error_after_commit_put: Option<usize>,
advance_time_on_put: Option<Duration>,
advance_time_after_get: Option<(UsageObjectSlot, Duration)>,
advance_publication_epoch_after_get: Option<(UsageObjectSlot, Arc<InstanceContext>)>,
advance_time_before_put: Option<(usize, Duration)>,
advance_time_after_put: Option<(usize, Duration)>,
put_count: usize,
@@ -2550,21 +2341,10 @@ mod tests {
}
_ => None,
};
let advance_publication_epoch = match state.advance_publication_epoch_after_get {
Some((expected_slot, ref ctx)) if expected_slot == slot => {
let ctx = Arc::clone(ctx);
state.advance_publication_epoch_after_get = None;
Some(ctx)
}
_ => None,
};
drop(state);
if let Some(duration) = advance {
tokio::time::advance(duration).await;
}
if let Some(ctx) = advance_publication_epoch {
ctx.advance_data_movement_operation_epoch();
}
Ok(crate::object_api::GetObjectReader {
stream: Box::new(Cursor::new(data)),
object_info: ObjectInfo {
@@ -2765,23 +2545,6 @@ mod tests {
.to_string()
}
fn build_publication_store(ctx: Arc<InstanceContext>) -> Arc<ECStore> {
let endpoint_pools = EndpointServerPools::default();
Arc::new(ECStore {
id: uuid::Uuid::new_v4(),
disk_map: HashMap::new(),
pools: Vec::new(),
peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoint_pools, ctx.clone()),
pool_meta: RwLock::new(crate::core::pools::PoolMeta::default()),
rebalance_meta: RwLock::new(None),
decommission_cancelers: RwLock::new(Vec::new()),
start_gate: TokioMutex::new(()),
pool_meta_save_gate: TokioMutex::new(()),
ctx,
bucket_fence_registry: Arc::default(),
})
}
#[test]
fn data_usage_cache_absence_covers_the_variants_that_actually_arrive() {
// `to_object_err` rewrites the raw storage variants before they reach
@@ -3154,45 +2917,6 @@ mod tests {
assert_eq!(selected.usage_snapshot_converged, Some(true));
}
#[test]
fn persisted_authoritative_stalls_but_memory_overlay_remains_visible() {
let authoritative = DataUsageInfo {
last_update: Some(SystemTime::UNIX_EPOCH),
scanner_epoch: Some(4),
scanner_cycle: Some(10),
usage_snapshot_complete: true,
..Default::default()
};
let mut partial = authoritative.clone();
partial.last_update = Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1));
partial.scanner_cycle = Some(11);
partial.usage_snapshot_complete = false;
partial.usage_snapshot_partial = true;
partial.usage_snapshot_converged = Some(false);
partial.usage_snapshot_authoritative_baseline = Some(authoritative.snapshot_identity());
partial.usage_snapshot_set_states = vec![rustfs_data_usage::DataUsageSnapshotSetState {
pool_index: 0,
set_index: 0,
scanner_cycle: Some(10),
scanner_epoch: Some(4),
scan_plan_digest: Some([1; 32]),
complete: false,
tombstone: false,
}];
partial.buckets_usage.insert(
"bucket".to_string(),
BucketUsageInfo {
size: 100,
..Default::default()
},
);
partial.buckets_count = 1;
let (selected, _) = select_admin_data_usage_snapshot(authoritative, true, Some(partial));
assert!(selected.usage_snapshot_partial);
assert_eq!(selected.buckets_usage.get("bucket").map(|usage| usage.size), Some(100));
}
#[tokio::test]
async fn authoritative_save_cleanup_removes_observed_snapshot_best_effort() {
let store = UsageCasStore::default();
@@ -3596,80 +3320,6 @@ mod tests {
);
}
#[test]
fn quota_object_size_accepts_compressed_unknown_actual_size_sentinel() {
let mut metadata = HashMap::new();
rustfs_utils::http::insert_str(
&mut metadata,
rustfs_utils::http::SUFFIX_COMPRESSION,
"klauspost/compress/s2".to_string(),
);
let object = ObjectInfo {
size: 400,
actual_size: -1,
user_defined: Arc::new(metadata),
..Default::default()
};
assert_eq!(quota_object_size(&object).expect("compressed sentinel is valid"), 400);
}
#[test]
fn quota_object_size_rejects_compressed_part_sum_overflow() {
let mut metadata = HashMap::new();
rustfs_utils::http::insert_str(
&mut metadata,
rustfs_utils::http::SUFFIX_COMPRESSION,
"klauspost/compress/s2".to_string(),
);
let object = ObjectInfo {
size: 1,
user_defined: Arc::new(metadata),
parts: Arc::new(vec![
rustfs_filemeta::ObjectPartInfo {
actual_size: i64::MAX,
..Default::default()
},
rustfs_filemeta::ObjectPartInfo {
actual_size: 1,
..Default::default()
},
]),
..Default::default()
};
assert!(matches!(quota_object_size(&object), Err(Error::Io(_))));
}
#[test]
fn quota_object_size_rejects_negative_values_other_than_the_compressed_sentinel() {
let mut metadata = HashMap::new();
rustfs_utils::http::insert_str(
&mut metadata,
rustfs_utils::http::SUFFIX_COMPRESSION,
"klauspost/compress/s2".to_string(),
);
let corrupt_object = ObjectInfo {
size: 400,
actual_size: -2,
user_defined: Arc::new(metadata.clone()),
..Default::default()
};
assert!(matches!(quota_object_size(&corrupt_object), Err(Error::PartMissingOrCorrupt)));
let corrupt_part = ObjectInfo {
size: 400,
user_defined: Arc::new(metadata),
parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo {
size: 400,
actual_size: -2,
..Default::default()
}]),
..Default::default()
};
assert!(matches!(quota_object_size(&corrupt_part), Err(Error::PartMissingOrCorrupt)));
}
#[tokio::test]
#[serial]
async fn live_bucket_usage_refreshes_are_coalesced_only_while_in_flight() {
@@ -4526,15 +4176,7 @@ mod tests {
.expect("namespace lock acquisition should not fail")
.expect("namespace lock should be acquired"),
);
let snapshot = data_usage_info_for_test(BUCKET, 2, 84, SystemTime::now());
let encoded = serde_json::to_vec(&snapshot).expect("usage snapshot should encode");
let store = Arc::new(UsageCasStore {
state: Mutex::new(UsageCasState {
object: Some((encoded.clone(), 1)),
backup_object: Some((encoded, 1)),
..Default::default()
}),
});
let store = Arc::new(UsageCasStore::default());
let successor = data_usage_info_for_test(BUCKET, 7, 294, SystemTime::now());
let mut snapshot_cache = data_usage_snapshot_cache().write().await;
*snapshot_cache = Some(CachedDataUsageSnapshot {
@@ -4633,17 +4275,21 @@ mod tests {
}
#[tokio::test]
async fn remove_bucket_usage_does_not_synthesize_authoritative_snapshot_when_all_missing() {
async fn remove_bucket_usage_creates_primary_and_backup_fences_when_missing() {
let store = Arc::new(UsageCasStore::default());
remove_bucket_usage_from_backend_with_store(store.as_ref(), "bucket-a")
.await
.expect("bucket removal should remain a no-op without a usage baseline");
.expect("bucket removal should create both usage fences");
let state = store.state.lock().await;
assert_eq!(state.put_count, 0);
assert!(state.object.is_none());
assert!(state.backup_object.is_none());
assert_eq!(state.put_count, 2);
for (data, revision) in [state.object.as_ref(), state.backup_object.as_ref()].into_iter().flatten() {
let saved = serde_json::from_slice::<DataUsageInfo>(data).expect("saved usage snapshot should decode");
assert_eq!(*revision, 1);
assert!(saved.last_update.is_some());
assert!(!data_usage_contains_bucket(&saved, "bucket-a"));
}
}
#[tokio::test]
@@ -4795,39 +4441,6 @@ mod tests {
assert_eq!(backup_err, Error::PreconditionFailed);
}
#[tokio::test]
async fn remove_bucket_usage_rejects_movement_epoch_flip_between_read_and_commit() {
let ctx = Arc::new(InstanceContext::new());
let publication_store = build_publication_store(ctx.clone());
let snapshot = data_usage_info_for_test("bucket-a", 2, 84, SystemTime::now());
let store = Arc::new(UsageCasStore {
state: Mutex::new(UsageCasState {
object: Some((serde_json::to_vec(&snapshot).expect("usage snapshot should encode"), 1)),
advance_publication_epoch_after_get: Some((UsageObjectSlot::Primary, ctx)),
..Default::default()
}),
});
let expected_epoch = publication_store
.scanner_data_usage_publication_epoch()
.await
.expect("idle publication store should admit the initial read");
let err = remove_bucket_usage_from_object_with_retries_and_publication(
store.as_ref(),
DATA_USAGE_OBJ_NAME_PATH.as_str(),
"bucket-a",
0,
None,
None,
Some((publication_store.as_ref(), Some(expected_epoch))),
)
.await
.expect_err("a movement epoch flip during the read must fence the commit");
assert!(err.to_string().contains("epoch changed"));
assert_eq!(store.state.lock().await.put_count, 0);
}
#[tokio::test]
async fn remove_bucket_usage_confirms_ambiguous_committed_final_attempt() {
let initial = data_usage_info_for_test("bucket-a", 2, 84, SystemTime::now());
@@ -4952,55 +4565,6 @@ mod tests {
);
}
#[tokio::test]
#[serial]
async fn partial_usage_is_observational_not_authoritative_for_quota() {
clear_usage_memory_cache_for_test().await;
let mut partial = data_usage_info_for_test("bucket-a", 10, 100, SystemTime::now());
partial.usage_snapshot_complete = false;
partial.usage_snapshot_partial = true;
replace_bucket_usage_memory_from_info(&partial).await;
assert_eq!(get_bucket_usage_memory("bucket-a").await, None);
}
#[tokio::test]
#[serial]
async fn stale_quota_uses_complete_baseline_plus_positive_deltas() {
clear_usage_memory_cache_for_test().await;
let baseline = data_usage_info_for_test("bucket-a", 1, 100, SystemTime::now());
replace_bucket_usage_memory_from_info(&baseline).await;
record_bucket_object_write_memory("bucket-a", None, 25).await;
assert_eq!(get_bucket_usage_memory("bucket-a").await, Some(125));
}
#[tokio::test]
#[serial]
async fn negative_delta_waits_for_set_reconciliation() {
clear_usage_memory_cache_for_test().await;
let baseline = data_usage_info_for_test("bucket-a", 1, 100, SystemTime::UNIX_EPOCH + Duration::from_secs(100));
replace_bucket_usage_memory_from_info(&baseline).await;
record_bucket_object_delete_memory("bucket-a", 25, true).await;
assert_eq!(get_bucket_usage_memory("bucket-a").await, Some(100));
// Simulate a process restart: the request-path overlay is gone, but
// the persisted authoritative snapshot is still the pre-reconciliation
// baseline. Quota must remain conservative until a complete scanner
// result proves the delete.
clear_usage_memory_cache_for_test().await;
replace_bucket_usage_memory_from_info(&baseline).await;
assert_eq!(get_bucket_usage_memory("bucket-a").await, Some(100));
let reconciled = data_usage_info_for_test("bucket-a", 0, 75, SystemTime::UNIX_EPOCH + Duration::from_secs(101));
replace_bucket_usage_memory_from_info(&reconciled).await;
assert_eq!(get_bucket_usage_memory("bucket-a").await, Some(75));
}
#[tokio::test]
#[serial]
async fn memory_overlay_counts_versioned_overwrite_as_new_version() {
+6 -77
View File
@@ -8000,15 +8000,10 @@ impl DiskAPI for LocalDisk {
use std::io::Write as _;
let file_path = self.io_get_object_path(volume, path)?;
let lock_path = file_path.with_extension("rustfs-cas.lock");
let path = path.to_string();
let sync_metadata = effective_durability(volume).syncs_commit_metadata();
return Ok(tokio::task::spawn_blocking(move || {
// A persistent directory lock bounds metadata growth. Removing
// per-target lock files can split flock ownership across inodes.
let lock_path = file_path
.parent()
.ok_or_else(|| std::io::Error::new(ErrorKind::InvalidInput, "conditional file has no parent"))?
.join(".rustfs-cas.lock");
let lock = std::fs::OpenOptions::new()
.create(true)
.truncate(false)
@@ -8073,25 +8068,7 @@ impl DiskAPI for LocalDisk {
.map_err(DiskError::from)??);
}
#[cfg(windows)]
{
let file_path = self.io_get_object_path(volume, path)?;
let sync_metadata = effective_durability(volume).syncs_commit_metadata();
let publication_root = self.publication_root.clone();
return Ok(tokio::task::spawn_blocking(move || {
os::compare_and_update_control_file(
&file_path,
expected.as_deref(),
replacement.as_deref(),
sync_metadata,
&publication_root,
)
})
.await
.map_err(DiskError::from)??);
}
#[cfg(not(any(unix, windows)))]
#[cfg(not(unix))]
{
let _ = (volume, path, expected, replacement);
Err(DiskError::MethodNotAllowed)
@@ -8963,7 +8940,6 @@ impl DiskAPI for LocalDisk {
)
.await?;
out.close().await?;
Ok(())
}
@@ -21847,9 +21823,9 @@ mod test {
assert!(matches!(results[1].as_ref().unwrap_err(), DiskError::Io(_)));
}
#[cfg(any(unix, windows))]
#[cfg(unix)]
#[tokio::test]
async fn windows_and_unix_conditional_file_update_never_deletes_a_new_owner() {
async fn conditional_file_update_never_deletes_a_new_owner() {
use tempfile::tempdir;
let dir = tempdir().expect("temp dir should be created");
@@ -21880,18 +21856,8 @@ mod test {
disk.read_all(RUSTFS_META_BUCKET, HEALING_MARKER_PATH)
.await
.expect("new owner marker should remain"),
owner_b.clone()
owner_b
);
assert_eq!(
disk.compare_and_update_file(RUSTFS_META_BUCKET, HEALING_MARKER_PATH, Some(owner_b), None)
.await
.expect("current owner should remove marker"),
ConditionalFileUpdate::Updated
);
assert!(matches!(
disk.read_all(RUSTFS_META_BUCKET, HEALING_MARKER_PATH).await,
Err(DiskError::FileNotFound)
));
}
#[cfg(unix)]
@@ -21906,10 +21872,7 @@ mod test {
let marker_path = disk
.get_object_path(RUSTFS_META_BUCKET, HEALING_MARKER_PATH)
.expect("marker path should resolve");
let lock_path = marker_path
.parent()
.expect("marker path should have a parent")
.join(".rustfs-cas.lock");
let lock_path = marker_path.with_extension("rustfs-cas.lock");
let lock = std::fs::OpenOptions::new()
.create(true)
.truncate(false)
@@ -21930,40 +21893,6 @@ mod test {
assert!(matches!(err, DiskError::Io(ref err) if err.kind() == ErrorKind::WouldBlock));
}
#[cfg(windows)]
#[tokio::test]
async fn windows_conditional_file_update_returns_would_block_when_marker_lock_is_contended() {
let dir = tempfile::tempdir().expect("temp dir should be created");
let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
ensure_test_volume(&disk, RUSTFS_META_BUCKET).await;
let marker_path = disk
.get_object_path(RUSTFS_META_BUCKET, HEALING_MARKER_PATH)
.expect("marker path should resolve");
let lock_path = marker_path
.parent()
.expect("marker path should have a parent")
.join(".rustfs-cas.lock");
let lock = std::fs::OpenOptions::new()
.create(true)
.truncate(false)
.read(true)
.write(true)
.open(lock_path)
.expect("marker lock should open");
lock.try_lock().expect("marker lock should be held");
let err = tokio::time::timeout(
Duration::from_secs(1),
disk.compare_and_update_file(RUSTFS_META_BUCKET, HEALING_MARKER_PATH, None, Some(Bytes::from_static(b"owner"))),
)
.await
.expect("contended conditional update must not block")
.expect_err("contended conditional update must retry");
assert!(matches!(err, DiskError::Io(ref err) if err.kind() == ErrorKind::WouldBlock));
}
#[cfg(target_os = "linux")]
#[tokio::test]
async fn replacement_io_paths_stay_under_the_mount_lease() {
+25 -69
View File
@@ -44,8 +44,6 @@ pub const PART_TRANSACTION_ROLLBACK: &str = "rollback";
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_DISK: &str = "disk";
const EVENT_DISK_PART_ERR_UNCLASSIFIED: &str = "disk_part_err_unclassified";
const ENV_BATCH_READ_VERSION_SERVER_PARALLELISM: &str = "RUSTFS_BATCH_READ_VERSION_SERVER_PARALLELISM";
const BATCH_READ_VERSION_SERVER_PARALLELISM: usize = 4;
pub fn part_transaction_path(part_path: &str) -> String {
match part_path.rsplit_once('/') {
@@ -64,7 +62,6 @@ use bytes::Bytes;
use endpoint::Endpoint;
use error::DiskError;
use error::{Error, Result};
use futures::stream::{self, StreamExt};
use local::LocalDisk;
use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
use rustfs_madmin::info_commands::DiskMetrics;
@@ -420,14 +417,6 @@ impl DiskAPI for Disk {
}
}
#[tracing::instrument(level = "trace", skip_all)]
async fn batch_read_version(&self, req: BatchReadVersionReq) -> Result<Vec<BatchReadVersionResp>> {
match self {
Disk::Local(local_disk) => local_disk.batch_read_version(req).await,
Disk::Remote(remote_disk) => remote_disk.batch_read_version(req).await,
}
}
#[tracing::instrument(level = "trace", skip_all)]
async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo> {
match self {
@@ -1039,47 +1028,36 @@ where
D: DiskAPI + ?Sized,
{
validate_batch_read_version_item_count(req.items.len())?;
let parallelism = batch_read_version_server_parallelism();
let mut responses = stream::iter(req.items.into_iter().enumerate())
.map(|(index, item)| async move {
match disk
.read_version(&item.org_volume, &item.volume, &item.path, &item.version_id, &req.opts)
.await
{
Ok(file_info) => BatchReadVersionResp {
index,
path: item.path,
version_id: item.version_id,
success: true,
file_info,
error: String::new(),
error_code: 0,
},
Err(err) => BatchReadVersionResp {
index,
path: item.path,
version_id: item.version_id,
success: false,
file_info: FileInfo::default(),
error: err.to_string(),
error_code: err.to_u32(),
},
}
})
.buffer_unordered(parallelism)
.collect::<Vec<_>>()
.await;
responses.sort_unstable_by_key(|response| response.index);
let mut responses = Vec::with_capacity(req.items.len());
for (index, item) in req.items.iter().enumerate() {
let response = match disk
.read_version(&item.org_volume, &item.volume, &item.path, &item.version_id, &req.opts)
.await
{
Ok(file_info) => BatchReadVersionResp {
index,
path: item.path.clone(),
version_id: item.version_id.clone(),
success: true,
file_info,
error: String::new(),
},
Err(err) => BatchReadVersionResp {
index,
path: item.path.clone(),
version_id: item.version_id.clone(),
success: false,
file_info: FileInfo::default(),
error: err.to_string(),
},
};
responses.push(response);
}
Ok(responses)
}
fn batch_read_version_server_parallelism() -> usize {
rustfs_utils::get_env_usize(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, BATCH_READ_VERSION_SERVER_PARALLELISM)
.clamp(1, BATCH_READ_VERSION_MAX_ITEMS)
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct CheckPartsResp {
pub results: Vec<usize>,
@@ -1344,8 +1322,6 @@ pub struct BatchReadVersionResp {
pub success: bool,
pub file_info: FileInfo,
pub error: String,
#[serde(default)]
pub error_code: u32,
}
pub fn validate_batch_read_version_item_count(item_count: usize) -> Result<()> {
@@ -1441,26 +1417,6 @@ mod tests {
assert!(!partial_valid_location.valid());
}
#[test]
fn batch_read_version_server_parallelism_defaults_to_conservative_four() {
temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, None::<&str>, || {
assert_eq!(batch_read_version_server_parallelism(), 4);
});
}
#[test]
fn batch_read_version_server_parallelism_honors_env_with_bounds() {
temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("8"), || {
assert_eq!(batch_read_version_server_parallelism(), 8);
});
temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("0"), || {
assert_eq!(batch_read_version_server_parallelism(), 1);
});
temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("9999"), || {
assert_eq!(batch_read_version_server_parallelism(), BATCH_READ_VERSION_MAX_ITEMS);
});
}
/// Test FileInfoVersions find_version_index
#[test]
fn test_file_info_versions_find_version_index() {
-85
View File
@@ -12,8 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#[cfg(windows)]
use crate::disk::ConditionalFileUpdate;
use crate::disk::error::DiskError;
use crate::disk::error::Result;
use crate::disk::error_conv::to_file_error;
@@ -3460,89 +3458,6 @@ fn read_windows_relative_file(file_path: &Path, parent_guard: &ExistingBaseDirec
Ok(Some(data))
}
#[cfg(windows)]
pub(crate) fn compare_and_update_control_file(
file_path: &Path,
expected: Option<&[u8]>,
replacement: Option<&[u8]>,
sync_metadata: bool,
publication_root: &PublicationRoot,
) -> io::Result<ConditionalFileUpdate> {
use windows_sys::{
Wdk::Storage::FileSystem::{
FILE_NON_DIRECTORY_FILE, FILE_OPEN, FILE_OPEN_IF, FILE_OPEN_REPARSE_POINT, FILE_SYNCHRONOUS_IO_NONALERT,
},
Win32::Storage::FileSystem::{
DELETE, FILE_ATTRIBUTE_NORMAL, FILE_READ_ATTRIBUTES, FILE_SHARE_READ, FILE_SHARE_WRITE, FILE_WRITE_DATA, SYNCHRONIZE,
},
};
let parent = file_path
.parent()
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "conditional file has no parent"))?;
let parent_guard = lock_windows_directory_tree(parent, Some(parent), publication_root)?;
let lock = open_windows_relative(
parent_guard.last_handle()?,
std::ffi::OsStr::new(".rustfs-cas.lock"),
SYNCHRONIZE | FILE_READ_ATTRIBUTES | FILE_WRITE_DATA,
FILE_SHARE_READ | FILE_SHARE_WRITE,
FILE_OPEN_IF,
FILE_NON_DIRECTORY_FILE | FILE_OPEN_REPARSE_POINT | FILE_SYNCHRONOUS_IO_NONALERT,
FILE_ATTRIBUTE_NORMAL,
true,
)?;
validate_windows_owned_file(&lock)?;
match lock.as_file().try_lock() {
Ok(()) => {}
Err(std::fs::TryLockError::WouldBlock) => return Err(io::Error::from(io::ErrorKind::WouldBlock)),
Err(std::fs::TryLockError::Error(err)) => return Err(err),
}
let current = read_windows_relative_file(file_path, &parent_guard)?;
let matches = match (&current, expected) {
(None, None) => true,
(Some(current), Some(expected)) => current.as_slice() == expected,
_ => false,
};
if !matches {
return Ok(match current {
None => ConditionalFileUpdate::Missing,
Some(_) => ConditionalFileUpdate::Mismatch,
});
}
match replacement {
Some(replacement) => RenameDestinationPathGuard {
directory: parent.to_path_buf(),
_directory_guard: parent_guard,
}
.write_file_for_path_access(file_path, replacement, sync_metadata, sync_metadata)?,
None => {
let file_name = file_path
.file_name()
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "conditional file must have a name"))?;
let file = open_windows_relative(
parent_guard.last_handle()?,
file_name,
DELETE | SYNCHRONIZE | FILE_READ_ATTRIBUTES,
FILE_SHARE_READ,
FILE_OPEN,
FILE_NON_DIRECTORY_FILE | FILE_OPEN_REPARSE_POINT | FILE_SYNCHRONOUS_IO_NONALERT,
0,
true,
)?;
validate_windows_owned_file(&file)?;
set_windows_file_delete_on_close(&file, true)?;
drop(file);
if sync_metadata {
fsync_dir_std(parent)?;
}
}
}
Ok(ConditionalFileUpdate::Updated)
}
#[cfg(windows)]
fn open_windows_directory_component(
parent: &WindowsDirectoryHandle,
+6 -38
View File
@@ -784,24 +784,6 @@ pub(crate) fn create_deferred_bitrot_reader_with_stripe_handle(
///
/// # Returns
/// A Result containing the BitrotWriterWrapper or an error
/// Size hint handed to `DiskAPI::create_file` for a bitrot-wrapped shard.
///
/// A known length is grown by one checksum per shard so the on-disk file size
/// matches what the bitrot writer emits. A negative length is the
/// unknown-size sentinel (`HashReader::SIZE_PRESERVE_LAYER`, used by SSE and
/// compression) and must be preserved: `RemoteDisk::create_file` forwards it
/// in the `put_file_stream` query, and the receiver only treats `size > 0` as
/// a fixed body length when locating the authenticated trailer. Clamping it
/// to `0` would claim an empty body and misframe the stream. `0` stays `0`
/// because a genuinely empty object still means an empty body.
fn bitrot_create_file_size(length: i64, shard_size: usize, checksum_algo: &HashAlgorithm) -> i64 {
if length <= 0 {
return length;
}
let length = length as usize;
(length.div_ceil(shard_size) * checksum_algo.size() + length) as i64
}
pub async fn create_bitrot_writer(
is_inline_buffer: bool,
disk: Option<&DiskStore>,
@@ -814,7 +796,12 @@ pub async fn create_bitrot_writer(
let writer = if is_inline_buffer {
CustomWriter::new_inline_buffer()
} else if let Some(disk) = disk {
let length = bitrot_create_file_size(length, shard_size, &checksum_algo);
let length = if length > 0 {
let length = length as usize;
(length.div_ceil(shard_size) * checksum_algo.size() + length) as i64
} else {
0
};
let file = disk.create_file("", volume, path, length).await?;
#[cfg(feature = "hotpath")]
@@ -833,25 +820,6 @@ mod tests {
use rustfs_rio::ChunkReader;
use std::collections::VecDeque;
#[test]
fn bitrot_create_file_size_grows_known_length_by_checksums() {
// 10 bytes over 4-byte shards = 3 shards, each followed by a 32-byte hash.
assert_eq!(bitrot_create_file_size(10, 4, &HashAlgorithm::HighwayHash256), 10 + 3 * 32);
assert_eq!(bitrot_create_file_size(10, 4, &HashAlgorithm::None), 10);
}
#[test]
fn bitrot_create_file_size_keeps_empty_and_unknown_distinct() {
assert_eq!(bitrot_create_file_size(0, 4, &HashAlgorithm::HighwayHash256), 0);
// SSE/compression streams advertise SIZE_PRESERVE_LAYER (-1); the remote
// put_file_stream receiver relies on a non-positive size to parse the auth
// trailer from the stream tail, so the sentinel must survive untouched.
assert_eq!(
bitrot_create_file_size(rustfs_rio::HashReader::SIZE_PRESERVE_LAYER, 4, &HashAlgorithm::HighwayHash256),
rustfs_rio::HashReader::SIZE_PRESERVE_LAYER
);
}
struct TestChunkReader {
chunks: VecDeque<Bytes>,
}
-8
View File
@@ -81,14 +81,6 @@ pub fn shutdown_background_monitors() {
cluster::rpc::shutdown_background_monitors();
}
/// Publish that the process is ready to serve user-object GET traffic.
///
/// Experimental metadata coalescing is allowed to run only after this point so
/// startup and internal metadata reads keep the original per-disk path.
pub fn mark_get_metadata_read_version_coalescing_service_ready() {
runtime::global::mark_get_metadata_read_version_coalescing_service_ready();
}
#[cfg(test)]
mod rio_tests {
#[test]
+4 -34
View File
@@ -699,9 +699,6 @@ impl ObjectInfo {
}
pub fn get_actual_size(&self) -> std::io::Result<i64> {
if self.actual_size < -1 || (self.actual_size == -1 && !self.is_compressed()) {
return Err(std::io::Error::other("invalid negative actual size"));
}
if self.actual_size > 0 {
return Ok(self.actual_size);
}
@@ -713,25 +710,10 @@ impl ObjectInfo {
let size = size_str.parse::<i64>().map_err(|e| std::io::Error::other(e.to_string()))?;
return Ok(size);
}
if self.actual_size == -1 && self.parts.is_empty() {
return Ok(-1);
}
let mut actual_size = 0_i64;
let mut unknown = false;
for part in self.parts.iter() {
match part.actual_size {
-1 => unknown = true,
size if size >= 0 => {
actual_size = actual_size
.checked_add(size)
.ok_or_else(|| std::io::Error::other("compressed actual size overflow"))?;
}
_ => return Err(std::io::Error::other("invalid negative compressed part size")),
}
}
if unknown {
return Ok(-1);
}
let mut actual_size = 0;
self.parts.iter().for_each(|part| {
actual_size += part.actual_size;
});
if actual_size == 0 && actual_size != self.size {
return Err(std::io::Error::other(format!("invalid decompressed size {} {}", actual_size, self.size)));
}
@@ -746,18 +728,6 @@ impl ObjectInfo {
Ok(self.size)
}
/// Returns a non-negative size for client and replication boundaries.
///
/// Compressed legacy metadata can retain the internal `-1` unknown-size
/// sentinel. Those boundaries cannot emit a negative length, so they use
/// the persisted physical size while quota accounting keeps the sentinel
/// distinction in [`crate::data_usage::quota_object_size`].
pub fn get_actual_size_or_physical(&self) -> i64 {
self.get_actual_size()
.map(|size| if size >= 0 { size } else { self.size.max(0) })
.unwrap_or_else(|_| self.size.max(0))
}
pub fn from_file_info(fi: &FileInfo, bucket: &str, object: &str, versioned: bool) -> ObjectInfo {
let mut version_id = fi.version_id;
+1 -14
View File
@@ -25,10 +25,7 @@ use lazy_static::lazy_static;
use rustfs_lock::client::LockClient;
use std::{
collections::HashMap,
sync::{
Arc, OnceLock,
atomic::{AtomicBool, Ordering},
},
sync::{Arc, OnceLock},
time::SystemTime,
};
use tokio::sync::{OnceCell, RwLock};
@@ -40,16 +37,6 @@ pub const DISK_MIN_INODES: u64 = 1000;
pub const DISK_FILL_FRACTION: f64 = 0.99;
pub const DISK_RESERVE_FRACTION: f64 = 0.15;
static GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY: AtomicBool = AtomicBool::new(false);
pub(crate) fn mark_get_metadata_read_version_coalescing_service_ready() {
GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY.store(true, Ordering::Release);
}
pub(crate) fn get_metadata_read_version_coalescing_service_ready() -> bool {
GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY.load(Ordering::Acquire)
}
// Global singletons for backward compatibility with MinIO port.
// These should be migrated to AppContext over time.
// See issue #730 for migration plan.
+1 -78
View File
@@ -52,18 +52,11 @@ use crate::services::tier::tier::TierConfigMgr;
use rustfs_lock::{GlobalLockManager, get_global_lock_manager};
use s3s::region::Region;
use std::collections::{HashMap, HashSet};
use std::sync::{
Arc, OnceLock,
atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering},
};
use std::sync::{Arc, OnceLock};
use tokio::sync::{OnceCell, RwLock};
use tokio_util::sync::CancellationToken;
use uuid::Uuid;
const SCANNER_PUBLICATION_STATE_UNKNOWN: u8 = 0;
const SCANNER_PUBLICATION_STATE_ALLOWED: u8 = 1;
const SCANNER_PUBLICATION_STATE_BLOCKED: u8 = 2;
/// Runtime state owned by a single `ECStore` instance.
///
/// This is intentionally minimal in the first migration slice; subsequent
@@ -167,22 +160,6 @@ pub struct InstanceContext {
/// workers (scanner/heal/tier/lifecycle) without touching another instance.
/// Replaces the process-global cancel-token static.
background_cancel_token: OnceLock<CancellationToken>,
/// Serializes data-movement transitions with scanner publication commits.
/// Readers are held across one publication commit; movement transitions
/// take the writer at their durable state commit boundary.
data_movement_operation_gate: Arc<RwLock<()>>,
/// Monotonic admission epoch paired with the operation gate. A
/// publication admitted before a movement transition must never be
/// mistaken for one admitted after the transition.
data_movement_operation_epoch: AtomicU64,
/// Once the admission epoch reaches its reserved terminal value, no new
/// publication may be admitted. Keeping this state separate from the
/// saturating counter prevents an unchanged `u64::MAX` value from being
/// mistaken for a fresh epoch after overflow.
data_movement_operation_epoch_exhausted: AtomicBool,
/// Last storage-owned movement snapshot observed under the operation
/// gate. SetDisks cache writers fail closed until ECStore refreshes it.
scanner_publication_state: AtomicU8,
/// Resolves object-encryption material at the application boundary.
object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
@@ -223,10 +200,6 @@ impl InstanceContext {
local_disk_set_drives: Arc::new(RwLock::new(Vec::new())),
bucket_metadata_sys: std::sync::Mutex::new(None),
background_cancel_token: OnceLock::new(),
data_movement_operation_gate: Arc::new(RwLock::new(())),
data_movement_operation_epoch: AtomicU64::new(0),
data_movement_operation_epoch_exhausted: AtomicBool::new(false),
scanner_publication_state: AtomicU8::new(SCANNER_PUBLICATION_STATE_UNKNOWN),
object_encryption_resolver: OnceLock::new(),
tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()),
@@ -245,56 +218,6 @@ impl InstanceContext {
self.lock_manager.clone()
}
pub(crate) fn data_movement_operation_gate(&self) -> Arc<RwLock<()>> {
Arc::clone(&self.data_movement_operation_gate)
}
pub(crate) fn data_movement_operation_epoch(&self) -> u64 {
self.data_movement_operation_epoch.load(Ordering::Acquire)
}
pub(crate) fn data_movement_operation_epoch_exhausted(&self) -> bool {
self.data_movement_operation_epoch_exhausted.load(Ordering::Acquire)
}
pub(crate) fn scanner_publication_state_allowed(&self) -> bool {
!self.data_movement_operation_epoch_exhausted()
&& self.scanner_publication_state.load(Ordering::Acquire) == SCANNER_PUBLICATION_STATE_ALLOWED
}
pub(crate) fn set_scanner_publication_state(&self, blocked: bool) {
self.scanner_publication_state.store(
if blocked {
SCANNER_PUBLICATION_STATE_BLOCKED
} else {
SCANNER_PUBLICATION_STATE_ALLOWED
},
Ordering::Release,
);
}
pub(crate) fn advance_data_movement_operation_epoch(&self) -> u64 {
self.scanner_publication_state
.store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release);
let _ = self
.data_movement_operation_epoch
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |epoch| Some(epoch.saturating_add(1)));
let result = self.data_movement_operation_epoch.load(Ordering::Acquire);
if result == u64::MAX {
self.data_movement_operation_epoch_exhausted.store(true, Ordering::Release);
}
result
}
#[cfg(test)]
pub(crate) fn set_data_movement_operation_epoch_for_test(&self, epoch: u64) {
self.data_movement_operation_epoch.store(epoch, Ordering::Release);
self.data_movement_operation_epoch_exhausted
.store(epoch == u64::MAX, Ordering::Release);
self.scanner_publication_state
.store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release);
}
/// Install the application-owned object-encryption resolver once.
pub fn set_object_encryption_resolver(
&self,
@@ -220,14 +220,6 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo
last_usage_save_unix_secs: metrics.usage_freshness.last_usage_save_unix_secs,
last_usage_save_result: metrics.usage_freshness.last_usage_save_result,
last_usage_save_result_code: metrics.usage_freshness.last_usage_save_result_code,
last_durable_success_unix_secs: metrics.usage_freshness.last_durable_success_unix_secs,
last_publication_unix_secs: metrics.usage_freshness.last_publication_unix_secs,
last_publication_state: metrics.usage_freshness.last_publication_state,
last_publication_reason: metrics.usage_freshness.last_publication_reason,
deferred_pending: metrics.usage_freshness.deferred_pending,
deferred_total: metrics.usage_freshness.deferred_total,
last_deferred_unix_secs: metrics.usage_freshness.last_deferred_unix_secs,
last_deferred_reason: metrics.usage_freshness.last_deferred_reason,
},
maintenance_control: MadminScannerMaintenanceControlSnapshot {
primary_control: metrics.maintenance_control.primary_control,
@@ -264,10 +256,6 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo
cycle_max_duration_seconds: metrics.cycle_max_duration_seconds,
cycle_max_objects: metrics.cycle_max_objects,
cycle_max_directories: metrics.cycle_max_directories,
cycle_timeout_total: metrics.cycle_timeout_total,
cycle_recovery_required_total: metrics.cycle_recovery_required_total,
cycle_last_progress_age: metrics.cycle_last_progress_age,
leader_lease_without_progress: metrics.leader_lease_without_progress,
bitrot_cycle_enabled: metrics.bitrot_cycle_enabled,
bitrot_cycle_seconds: metrics.bitrot_cycle_seconds,
scan_checkpoint: metrics.scan_checkpoint.map(|checkpoint| MadminScannerCheckpointReport {
@@ -623,10 +611,6 @@ mod test {
current_started: chrono_to_jiff_timestamp(current_started),
last_cycle_partial_source: "usage".to_string(),
last_cycle_partial_source_code: 1,
cycle_timeout_total: 3,
cycle_recovery_required_total: 2,
cycle_last_progress_age: 17,
leader_lease_without_progress: true,
partial_cycles_by_source: vec![rustfs_common::metrics::ScannerSourceCycleSnapshot {
source: "usage".to_string(),
cycles: 2,
@@ -638,10 +622,6 @@ mod test {
assert_eq!(scanner.current_started, chrono_to_jiff_timestamp(current_started));
assert_eq!(scanner.last_cycle_partial_source, "usage");
assert_eq!(scanner.last_cycle_partial_source_code, 1);
assert_eq!(scanner.cycle_timeout_total, 3);
assert_eq!(scanner.cycle_recovery_required_total, 2);
assert_eq!(scanner.cycle_last_progress_age, 17);
assert!(scanner.leader_lease_without_progress);
let usage = scanner
.partial_cycles_by_source
.iter()
@@ -824,14 +804,6 @@ mod test {
last_usage_save_unix_secs: 12,
last_usage_save_result: "success".to_string(),
last_usage_save_result_code: 1,
last_durable_success_unix_secs: 13,
last_publication_unix_secs: 14,
last_publication_state: "published".to_string(),
last_publication_reason: "complete".to_string(),
deferred_pending: true,
deferred_total: 15,
last_deferred_unix_secs: 16,
last_deferred_reason: "data_movement".to_string(),
},
..Default::default()
});
@@ -844,14 +816,6 @@ mod test {
assert_eq!(scanner.usage_freshness.last_usage_save_unix_secs, 12);
assert_eq!(scanner.usage_freshness.last_usage_save_result, "success");
assert_eq!(scanner.usage_freshness.last_usage_save_result_code, 1);
assert_eq!(scanner.usage_freshness.last_durable_success_unix_secs, 13);
assert_eq!(scanner.usage_freshness.last_publication_unix_secs, 14);
assert_eq!(scanner.usage_freshness.last_publication_state, "published");
assert_eq!(scanner.usage_freshness.last_publication_reason, "complete");
assert!(scanner.usage_freshness.deferred_pending);
assert_eq!(scanner.usage_freshness.deferred_total, 15);
assert_eq!(scanner.usage_freshness.last_deferred_unix_secs, 16);
assert_eq!(scanner.usage_freshness.last_deferred_reason, "data_movement");
}
#[test]

Some files were not shown because too many files have changed in this diff Show More