mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-07 12:35:54 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 965480ca65 |
@@ -1,2 +0,0 @@
|
||||
sha256-linux=4696a43b167ac608b3b8677027c9fe9fdac3396d37c8cca11dce531c720ac6d2
|
||||
sha256-darwin=9785867929047dfd8c6f768e0d2b1e0a8fdba85216f4a4139093b1619d03ff07
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=53b05ac745905809d3828c6994bdd8ecf9d20b2b61a8a9d80fe15eb62f932193
|
||||
sha256-linux=7c892afa4b9d1591b46bd79c976b647109a277284fddb3b98edced4b0297eda2
|
||||
sha256-darwin=a881fd7d3f5cb94654221ca85b8b30cce1b95e608824a55a15339cbc294e6d34
|
||||
sha256-linux=a2933d83dfe74ffa03410a0959333a1c48288b8469ca9f17273d449d7510c24b
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=5db88c6fec94d4f269c7d9cfc128bd2adc27b3d7021127e2fa0b1daccc5f900f
|
||||
sha256=a2542dc86bbff56b2177efc621785c56fa7e8d813b209b7d935e1e41a9f0ad15
|
||||
|
||||
@@ -40,21 +40,6 @@
|
||||
"invariant": "corrupt-part-arrays",
|
||||
"suite": "rustfs-filemeta",
|
||||
"name": "filemeta::test::crc_valid_but_part_arrays_corrupt_into_fileinfo_errors_not_panics"
|
||||
},
|
||||
{
|
||||
"invariant": "odm-source-contract-s3",
|
||||
"suite": "rustfs",
|
||||
"name": "on_demand_migration::source_client::tests::s3_backend_satisfies_the_shared_backend_contract"
|
||||
},
|
||||
{
|
||||
"invariant": "odm-source-contract-azure",
|
||||
"suite": "rustfs",
|
||||
"name": "on_demand_migration::azure::tests::azure_backend_satisfies_the_shared_backend_contract"
|
||||
},
|
||||
{
|
||||
"invariant": "odm-source-contract-gcs",
|
||||
"suite": "rustfs",
|
||||
"name": "on_demand_migration::gcs::tests::gcs_native_backend_satisfies_the_shared_backend_contract"
|
||||
}
|
||||
],
|
||||
"fixtures": [
|
||||
|
||||
@@ -31,7 +31,6 @@ script-tests: ## Run shell script tests
|
||||
./scripts/test_object_batch_bench_enhanced.sh
|
||||
./scripts/test_hotpath_warp_ab_gate.sh
|
||||
./scripts/test_hotpath_warp_abba.sh
|
||||
./scripts/test_scanner_validation_harness.sh
|
||||
./scripts/test_exact_1mib_handoff_abba.sh
|
||||
./scripts/test_pinned_paired_abba_bench.sh
|
||||
./scripts/test_manual_transition_runbooks.sh
|
||||
@@ -47,7 +46,6 @@ script-tests: ## Run shell script tests
|
||||
bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test
|
||||
./scripts/validate_object_data_cache_cold_stampede.sh --self-test
|
||||
./scripts/run_scanner_heal_evidence_case.sh --self-test
|
||||
|
||||
.PHONY: test
|
||||
test: core-deps script-tests ## Run all tests (needs cargo-nextest; RUSTFS_ALLOW_CARGO_TEST_FALLBACK=1 to override)
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
# Committed floor for the number of tests selected by the migration-critical
|
||||
# CI gate (see scripts/check_migration_gate_count.sh, backlog#1153 infra-12).
|
||||
#
|
||||
# The floor equals the exact count of rustfs-ecstore --lib tests, with the
|
||||
# test-util feature enabled, matching the gate filter (name substrings:
|
||||
# data_movement, rebalance, decommission, source_cleanup, delete_marker) at
|
||||
# the time this file was last updated.
|
||||
# The floor equals the exact count of rustfs-ecstore --lib tests matching the
|
||||
# gate filter (name substrings: data_movement, rebalance, decommission,
|
||||
# source_cleanup, delete_marker) at the time this file was last updated.
|
||||
# CI fails if the selected count drops below this number, so renames or
|
||||
# removals that thin the gate must update this file in the same PR.
|
||||
# Adding tests does not require a bump, but bumping keeps the guard tight.
|
||||
946
|
||||
571
|
||||
|
||||
@@ -183,13 +183,6 @@ test-group = 'e2e-reliability'
|
||||
filter = 'package(e2e_test) & test(/^inline_fast_path_cluster_test::/)'
|
||||
test-group = 'e2e-inline-boundaries'
|
||||
|
||||
# 4-node 4-drive distributed Actions suite: each case starts four rustfs
|
||||
# processes and up to sixteen data directories. Serialize across nextest's
|
||||
# process boundary so several 4x4 clusters never overlap.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(e2e_test) & test(/^distributed::/)'
|
||||
test-group = 'e2e-cluster-nightly'
|
||||
|
||||
# Vault KMS tests share the fixed dev-server port 8200. serial_test's #[serial]
|
||||
# does not cross nextest process boundaries, so keep every Vault-backed test in
|
||||
# one group.
|
||||
@@ -533,27 +526,6 @@ path = "junit.xml"
|
||||
filter = 'package(e2e_test)'
|
||||
test-group = 'e2e-cluster-nightly'
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# e2e-distributed profile — 4-node 4-disk Actions suite
|
||||
# ---------------------------------------------------------------------------
|
||||
# Storage-sensitive PR / nightly / dispatch lane owned by
|
||||
# .github/workflows/e2e-distributed.yml.
|
||||
# Each case starts four rustfs processes (and for site replication, two
|
||||
# clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY.
|
||||
# Serialized via e2e-cluster-nightly with no retries.
|
||||
[profile.e2e-distributed]
|
||||
default-filter = 'package(e2e_test) & test(/^distributed::/)'
|
||||
fail-fast = false
|
||||
# Decommission / rebalance cases poll for up to 180s with little stdout.
|
||||
slow-timeout = { period = "120s", terminate-after = 6 }
|
||||
|
||||
[profile.e2e-distributed.junit]
|
||||
path = "junit.xml"
|
||||
|
||||
[[profile.e2e-distributed.overrides]]
|
||||
filter = 'package(e2e_test)'
|
||||
test-group = 'e2e-cluster-nightly'
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# e2e-odm-interop profile — on-demand migration provider interop lane (ODM-20)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -614,10 +586,6 @@ path = "junit.xml"
|
||||
# cluster-fault lane. heal_erasure_disk_rebuild is intentionally not
|
||||
# excluded here because backlog#2213 promotes core heal rebuild coverage to
|
||||
# this merge/main lane while retaining nightly coverage.
|
||||
# * distributed:: — 4-node 4-disk Actions suite (S3, lock, versioning,
|
||||
# replication, quota, observability, expand/decommission/rebalance, site
|
||||
# replication, chaos, upgrade history/IAM). Owns [profile.e2e-distributed] and
|
||||
# .github/workflows/e2e-distributed.yml.
|
||||
# * on_demand_migration::interop_test — the ODM-20 provider interoperability
|
||||
# cases, which are meaningless without a source: they run in the dedicated
|
||||
# [profile.e2e-odm-interop] lane below, where the workflow points them at a
|
||||
@@ -639,7 +607,6 @@ default-filter = """
|
||||
package(e2e_test)
|
||||
& !test(/^protocols::/)
|
||||
& !test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|degraded_listing_availability_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
|
||||
& !test(/^distributed::/)
|
||||
& !test(/^replication_extension_test::/)
|
||||
& !test(/^replication_target_matrix_test::/)
|
||||
& !test(/^on_demand_migration::(concurrency_test|fault_test|interop_test|real_source_test)::/)
|
||||
|
||||
@@ -1,56 +0,0 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"cases": {
|
||||
"background-target-restart": {
|
||||
"gate": "G14",
|
||||
"task": "W21",
|
||||
"lane": "e2e-nightly",
|
||||
"suite": "e2e_test",
|
||||
"name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_remote_shards_after_background_target_restart",
|
||||
"oracle": "background-target-restart.json",
|
||||
"evidence": "process-restart",
|
||||
"unclean_shutdown_marker": false,
|
||||
"min_objects": 9,
|
||||
"max_objects": 65,
|
||||
"topology": {"nodes": 4, "drives_per_node": 1},
|
||||
"scope": "Target process restart, exact unversioned S3 bodies and replacement-disk shards; not power loss or EC8+4."
|
||||
},
|
||||
"background-target-crash": {
|
||||
"gate": "G14",
|
||||
"task": "W21",
|
||||
"lane": "e2e-nightly",
|
||||
"suite": "e2e_test",
|
||||
"name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_remote_shards_after_background_target_crash",
|
||||
"oracle": "background-target-crash.json",
|
||||
"evidence": "process-crash-restart",
|
||||
"unclean_shutdown_marker": true,
|
||||
"min_objects": 9,
|
||||
"max_objects": 65,
|
||||
"topology": {"nodes": 4, "drives_per_node": 1},
|
||||
"scope": "Target process killed during partial background rebuild, real unclean-shutdown marker, exact unversioned S3 bodies and replacement-disk shards; not power loss or EC8+4."
|
||||
}
|
||||
},
|
||||
"release_pending": {
|
||||
"G01": "W02/W04 complete root and quota authority coverage",
|
||||
"G02": "W03 bounded checkpoint progress and independent version inventory",
|
||||
"G03": "W17/W18 exact scoped ACK with durable publication and mixed peers",
|
||||
"G04": "W03/W15/W16 crash at every cache/root/floor/intent boundary",
|
||||
"G05": "W06/W07 per-object outcomes and bounded terminal retention",
|
||||
"G06": "W06/W08/W23 concurrent status, legacy clients and truncation",
|
||||
"G07": "W12/W13/W14 durable MRF responsibility at every commit boundary",
|
||||
"G08": "W12/W13/W14 MRF capacity, disk-full and replica-loss matrix",
|
||||
"G09": "W13/W18/W23 actual mixed-version reader/writer and rollback payloads",
|
||||
"G10": "W05/W09/W10/W11 bounded scheduling and pressure recovery",
|
||||
"G11": "W04/W19/W24 maintenance and complete producer coverage",
|
||||
"G12": "W02/W15/W16 both quota paths during reset and settlement",
|
||||
"G13": "W07/W14 quorum-minus-one, unknown disks, remount, Object Lock, dry-run, grace and commit tail",
|
||||
"G14": "W20/W21 same-window field evidence; 3x4 EC8+4 and multi-set/pool coverage",
|
||||
"P1": "W20 measured cold-walk share and foreground latency/throughput",
|
||||
"P2": "W20/W24 measured post-stop convergence and cold segment reuse",
|
||||
"P3": "W20 measured two-hour pressure/heal capacity and recovery window",
|
||||
"P4": "W20 measured MRF scale and replay cost with retained responsibility",
|
||||
"R-E": "W03/W05 fixed-budget real process restart through enumeration and classification",
|
||||
"R-D": "W07/W14 manager-to-event-to-ledger exact disposition, including grace",
|
||||
"R-L": "W13/W14 legacy source conflicts, migration gaps and crash-safe source retirement"
|
||||
}
|
||||
}
|
||||
@@ -4,11 +4,6 @@
|
||||
{ "workflow": ".github/workflows/ci.yml", "max_age_hours": 192 },
|
||||
{ "workflow": ".github/workflows/coverage.yml", "max_age_hours": 192 },
|
||||
{ "workflow": ".github/workflows/e2e-replication-nightly.yml", "max_age_hours": 36 },
|
||||
{
|
||||
"workflow": ".github/workflows/e2e-distributed.yml",
|
||||
"max_age_hours": 36,
|
||||
"never_ran_grace_until": "2026-09-18T00:00:00Z"
|
||||
},
|
||||
{ "workflow": ".github/workflows/e2e-s3tests.yml", "max_age_hours": 192 },
|
||||
{ "workflow": ".github/workflows/fuzz.yml", "max_age_hours": 36 },
|
||||
{ "workflow": ".github/workflows/mint.yml", "max_age_hours": 192 },
|
||||
|
||||
@@ -850,17 +850,6 @@ jobs:
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Install network fault-injection tools
|
||||
run: |
|
||||
sudo apt-get install -y iptables
|
||||
sudo -n iptables --version
|
||||
# The endpoint-blackhole heal scenario needs CAP_NET_ADMIN. Containerised
|
||||
# runners can run iptables but not touch the rule set; the test then logs
|
||||
# a skip instead of failing, so surface that here where it is visible.
|
||||
if ! sudo -n iptables -w 5 -S OUTPUT >/dev/null 2>&1; then
|
||||
echo "::warning::iptables cannot read the OUTPUT chain on this runner (no CAP_NET_ADMIN); the endpoint-blackhole heal scenario will be skipped"
|
||||
fi
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
|
||||
with:
|
||||
|
||||
@@ -121,22 +121,6 @@ jobs:
|
||||
create_latest=false
|
||||
source_ref="$GITHUB_SHA"
|
||||
|
||||
# Pre-GA policy: until the first stable (vX.Y.Z) tag exists, every
|
||||
# prerelease (alpha/beta/rc) also moves `latest`, so users pulling
|
||||
# `latest` get the newest test build. Once a stable tag is published
|
||||
# this returns false and `latest` follows stable releases only.
|
||||
prerelease_moves_latest() {
|
||||
local stable_tags
|
||||
stable_tags=$(git ls-remote --tags --refs origin 2>/dev/null \
|
||||
| awk '{print $2}' \
|
||||
| grep -E '^refs/tags/v?[0-9]+\.[0-9]+\.[0-9]+$' || true)
|
||||
if [[ -z "$stable_tags" ]]; then
|
||||
return 0
|
||||
fi
|
||||
echo "ℹ️ Stable release tag(s) already exist; prereleases no longer update latest"
|
||||
return 1
|
||||
}
|
||||
|
||||
if [[ "${{ github.event_name }}" == "workflow_run" ]]; then
|
||||
# Triggered by build workflow completion
|
||||
echo "🔗 Triggered by build workflow completion"
|
||||
@@ -200,8 +184,8 @@ jobs:
|
||||
if [[ "$version" == *"alpha"* ]] || [[ "$version" == *"beta"* ]] || [[ "$version" == *"rc"* ]]; then
|
||||
build_type="prerelease"
|
||||
is_prerelease=true
|
||||
# Pre-GA policy: prereleases update latest until the first stable tag exists.
|
||||
if prerelease_moves_latest; then
|
||||
# Current policy: create latest tags for stable releases and selected prereleases (alpha/beta).
|
||||
if [[ "$version" == *"alpha"* ]] || [[ "$version" == *"beta"* ]]; then
|
||||
create_latest=true
|
||||
echo "🧪 Building Docker image for prerelease: $version (creating latest tag)"
|
||||
else
|
||||
@@ -259,8 +243,8 @@ jobs:
|
||||
v*alpha*|v*beta*|v*rc*|*alpha*|*beta*|*rc*)
|
||||
build_type="prerelease"
|
||||
is_prerelease=true
|
||||
# Pre-GA policy: prereleases update latest until the first stable tag exists.
|
||||
if prerelease_moves_latest; then
|
||||
# Current policy: create latest tags for stable releases and selected prereleases (alpha/beta).
|
||||
if [[ "$version" == *"alpha"* ]] || [[ "$version" == *"beta"* ]]; then
|
||||
create_latest=true
|
||||
echo "🧪 Building with prerelease version: $input_version (creating latest tag)"
|
||||
else
|
||||
@@ -410,13 +394,11 @@ jobs:
|
||||
TAG_BASE="${VERSION}${VARIANT_SUFFIX}"
|
||||
TAGS="${{ env.REGISTRY_DOCKERHUB }}:$TAG_BASE,${{ env.REGISTRY_GHCR }}:$TAG_BASE,${{ env.REGISTRY_QUAY }}:$TAG_BASE"
|
||||
|
||||
# Add latest when requested (stable releases, and prereleases before GA)
|
||||
# Add channel tags for prereleases and latest for stable
|
||||
if [[ "$CREATE_LATEST" == "true" ]]; then
|
||||
# Create latest tags for stable releases and selected prereleases when CREATE_LATEST=true.
|
||||
TAGS="$TAGS,${{ env.REGISTRY_DOCKERHUB }}:latest${VARIANT_SUFFIX},${{ env.REGISTRY_GHCR }}:latest${VARIANT_SUFFIX},${{ env.REGISTRY_QUAY }}:latest${VARIANT_SUFFIX}"
|
||||
fi
|
||||
|
||||
# Always add the channel tag for prereleases, independent of latest
|
||||
if [[ "$BUILD_TYPE" == "prerelease" ]]; then
|
||||
elif [[ "$BUILD_TYPE" == "prerelease" ]]; then
|
||||
# Prerelease channel tags (alpha, beta, rc)
|
||||
if [[ "$VERSION" == *"alpha"* ]]; then
|
||||
CHANNEL="alpha"
|
||||
@@ -573,7 +555,7 @@ jobs:
|
||||
"prerelease")
|
||||
echo "🧪 Prerelease Docker image has been built with ${VERSION} tags"
|
||||
echo "⚠️ This is a prerelease image - use with caution"
|
||||
# Prereleases move latest until the first stable tag exists (pre-GA policy).
|
||||
# Create latest tags for stable releases and selected prereleases when CREATE_LATEST=true.
|
||||
if [[ "$CREATE_LATEST" == "true" ]]; then
|
||||
echo "🏷️ Latest tag has been created for prerelease: $VERSION"
|
||||
else
|
||||
|
||||
@@ -1,216 +0,0 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# 4-node 4-disk distributed e2e lane.
|
||||
#
|
||||
# Each selected test starts a real localhost cluster via
|
||||
# `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
|
||||
# case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
|
||||
# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
|
||||
# nightly runs, and manual dispatches all execute the same fail-closed suite.
|
||||
# Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
|
||||
#
|
||||
# Isolated pool filesystems: expand/decommission/rebalance cases require
|
||||
# independent `statfs` capacity. This job runs on GitHub-hosted
|
||||
# `ubuntu-latest` because the self-hosted `sm-standard-4` ARC pods cannot
|
||||
# create filesystems: `mount -o loop` fails with ENOENT (no
|
||||
# `/dev/loop-control`), and `mount -t tmpfs` fails with "cannot mount tmpfs
|
||||
# read-only" (no `CAP_SYS_ADMIN` in the initial namespace). The same reason
|
||||
# `uring-integration` and `e2e-s3tests.yml` left that label. The prepare
|
||||
# step mounts four 1 GiB tmpfs instances and exports `RUSTFS_E2E_POOL_ROOTS`.
|
||||
|
||||
name: e2e-distributed
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- "Cargo.lock"
|
||||
- "Cargo.toml"
|
||||
- ".config/nextest.toml"
|
||||
- ".github/workflows/e2e-distributed.yml"
|
||||
- "crates/audit/**"
|
||||
- "crates/common/**"
|
||||
- "crates/config/**"
|
||||
- "crates/e2e_test/**"
|
||||
- "crates/ecstore/**"
|
||||
- "crates/filemeta/**"
|
||||
- "crates/heal/**"
|
||||
- "crates/iam/**"
|
||||
- "crates/lock/**"
|
||||
- "crates/madmin/**"
|
||||
- "crates/notify/**"
|
||||
- "crates/replication/**"
|
||||
- "crates/s3-client/**"
|
||||
- "crates/s3-ops/**"
|
||||
- "crates/s3-types/**"
|
||||
- "crates/scanner/**"
|
||||
- "crates/storage-api/**"
|
||||
- "crates/utils/**"
|
||||
- "rustfs/**"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
filter:
|
||||
description: "Optional nextest -E filter (default: the whole e2e-distributed profile)"
|
||||
required: false
|
||||
default: ""
|
||||
schedule:
|
||||
# 05:53 UTC nightly — clear of e2e-nightly (04:29) and ODM interop (05:23).
|
||||
- cron: "53 5 * * *"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name != 'schedule' }}
|
||||
|
||||
jobs:
|
||||
distributed:
|
||||
name: Distributed 4-node 4-disk e2e
|
||||
# GitHub-hosted VM: loop and tmpfs mounts work here. sm-standard-4 is an
|
||||
# ARC pod and rejects both (`mount -o loop` ENOENT, tmpfs "read-only").
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 180
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
NO_PROXY: 127.0.0.1,localhost
|
||||
HTTP_PROXY: ""
|
||||
HTTPS_PROXY: ""
|
||||
# Pinned previous release used by distributed::upgrade_test (same pin as e2e-upgrade.yml).
|
||||
UPGRADE_SOURCE_VERSION: 1.0.0-rc.2
|
||||
UPGRADE_SOURCE_ASSET: rustfs-linux-x86_64-gnu-v1.0.0-rc.2.zip
|
||||
UPGRADE_SOURCE_SHA256: 7c789386bf85278f865b8e0d359bf4edb84d5aa408cc3fa54a18c25ca74cd6e7
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
# Dedicated key: ubuntu-latest and sm-standard-4 share runner.os, so
|
||||
# a shared key would mix VM and ARC pod target/ artifacts.
|
||||
cache-shared-key: ci-e2e-distributed-hosted
|
||||
cache-save-if: ${{ github.ref == 'refs/heads/main' }}
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Prepare isolated filesystems for pool movement
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mount_base="${RUNNER_TEMP}/rustfs-e2e-pools"
|
||||
mkdir -p "${mount_base}"
|
||||
roots=()
|
||||
for pool in 0 1 2 3; do
|
||||
mountpoint="${mount_base}/pool-${pool}"
|
||||
mkdir -p "${mountpoint}"
|
||||
# Sized tmpfs reports a distinct st_dev and independent 1G
|
||||
# statfs capacity. Requires a VM runner (ubuntu-latest).
|
||||
if ! sudo mount -t tmpfs -o size=1G,nosuid,nodev,mode=1777 tmpfs "${mountpoint}"; then
|
||||
echo "tmpfs mount failed on $(uname -a)" >&2
|
||||
findmnt || true
|
||||
grep Cap /proc/self/status || true
|
||||
exit 1
|
||||
fi
|
||||
sudo chmod 1777 "${mountpoint}"
|
||||
roots+=("${mountpoint}")
|
||||
done
|
||||
printf -v joined_roots '%s:' "${roots[@]}"
|
||||
echo "RUSTFS_E2E_POOL_ROOTS=${joined_roots%:}" >> "${GITHUB_ENV}"
|
||||
findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[0]}"
|
||||
findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[1]}"
|
||||
findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[2]}"
|
||||
findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[3]}"
|
||||
|
||||
- name: Download pinned previous release
|
||||
env:
|
||||
SOURCE_DIR: ${{ runner.temp }}/rustfs-upgrade-source
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p "$SOURCE_DIR"
|
||||
archive="$SOURCE_DIR/$UPGRADE_SOURCE_ASSET"
|
||||
curl --fail --location --retry 3 --output "$archive" \
|
||||
"https://github.com/${GITHUB_REPOSITORY}/releases/download/${UPGRADE_SOURCE_VERSION}/${UPGRADE_SOURCE_ASSET}"
|
||||
echo "$UPGRADE_SOURCE_SHA256 $archive" | sha256sum --check --strict
|
||||
unzip -q "$archive" -d "$SOURCE_DIR"
|
||||
chmod +x "$SOURCE_DIR/rustfs"
|
||||
test -x "$SOURCE_DIR/rustfs"
|
||||
echo "RUSTFS_UPGRADE_SOURCE_BINARY=$SOURCE_DIR/rustfs" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build rustfs binary
|
||||
run: |
|
||||
cargo build -p rustfs --bins
|
||||
: > target/debug/rustfs.features
|
||||
|
||||
- name: Verify distributed e2e membership
|
||||
env:
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-distributed-list.json
|
||||
run: |
|
||||
cargo nextest list --profile e2e-distributed -p e2e_test --message-format json > "${NEXTEST_LISTING}"
|
||||
python3 ./scripts/check_test_wiring.py --check-profile e2e-distributed "${NEXTEST_LISTING}"
|
||||
|
||||
- name: Run distributed 4-node e2e suite
|
||||
env:
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-distributed-logs
|
||||
FILTER: ${{ inputs.filter }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -n "${FILTER}" ]; then
|
||||
cargo nextest run --profile e2e-distributed -p e2e_test -E "${FILTER}"
|
||||
else
|
||||
cargo nextest run --profile e2e-distributed -p e2e_test --no-tests=fail
|
||||
fi
|
||||
|
||||
- name: Upload distributed e2e diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: e2e-distributed-${{ github.run_number }}
|
||||
path: |
|
||||
target/nextest/e2e-distributed/junit.xml
|
||||
${{ runner.temp }}/rustfs-e2e-distributed-list.json
|
||||
${{ runner.temp }}/rustfs-e2e-distributed-logs/
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Unmount isolated pool filesystems
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mount_base="${RUNNER_TEMP}/rustfs-e2e-pools"
|
||||
for pool in 0 1 2 3; do
|
||||
mountpoint="${mount_base}/pool-${pool}"
|
||||
if mountpoint --quiet "${mountpoint}"; then
|
||||
sudo umount "${mountpoint}"
|
||||
fi
|
||||
done
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [distributed]
|
||||
if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -22,7 +22,6 @@ on:
|
||||
- "Continuous Integration"
|
||||
- "coverage"
|
||||
- "e2e-nightly"
|
||||
- "e2e-distributed"
|
||||
- "e2e-s3tests"
|
||||
- "Fuzz"
|
||||
- "mint"
|
||||
|
||||
Generated
+27
-29
@@ -2527,18 +2527,18 @@ checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b"
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-channel"
|
||||
version = "0.5.17"
|
||||
version = "0.5.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1"
|
||||
checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e"
|
||||
dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-deque"
|
||||
version = "0.8.8"
|
||||
version = "0.8.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a"
|
||||
checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb"
|
||||
dependencies = [
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-utils",
|
||||
@@ -2546,27 +2546,27 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-epoch"
|
||||
version = "0.9.21"
|
||||
version = "0.9.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d"
|
||||
checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
|
||||
dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-queue"
|
||||
version = "0.3.14"
|
||||
version = "0.3.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae"
|
||||
checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26"
|
||||
dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.8.23"
|
||||
version = "0.8.22"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6"
|
||||
checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17"
|
||||
|
||||
[[package]]
|
||||
name = "crunchy"
|
||||
@@ -3673,9 +3673,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "der"
|
||||
version = "0.8.2"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a878c850e9e421b20262e9b41f9c860e4785fa07541c266b62ff9d1ef998a80a"
|
||||
checksum = "a69dedd701da44b0536442edf09c81a64b0ab97a7a4a5e3d1971f00027cbc63d"
|
||||
dependencies = [
|
||||
"const-oid 0.10.2",
|
||||
"pem-rfc7468 1.0.0",
|
||||
@@ -4090,7 +4090,7 @@ version = "0.17.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c0681a4fc24c767085329728d8dfba959af91228aa4610cca4f8ce317ba46ae0"
|
||||
dependencies = [
|
||||
"der 0.8.2",
|
||||
"der 0.8.1",
|
||||
"digest 0.11.3",
|
||||
"elliptic-curve 0.14.1",
|
||||
"rfc6979 0.6.0",
|
||||
@@ -5734,9 +5734,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ipnet"
|
||||
version = "2.12.2"
|
||||
version = "2.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0"
|
||||
checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
@@ -6139,9 +6139,9 @@ checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
|
||||
|
||||
[[package]]
|
||||
name = "libflate"
|
||||
version = "2.3.2"
|
||||
version = "2.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "561a8da1a50e1428d3c51321dafeca849df992a5bb67720c386131234caba82e"
|
||||
checksum = "a4da9b700e758e57152a1fd1c52cbdc5727c1aa6d8743dc1acda917398f1d76c"
|
||||
dependencies = [
|
||||
"adler32",
|
||||
"crc32fast",
|
||||
@@ -7933,7 +7933,7 @@ version = "0.8.0-rc.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "986d2e952779af96ea048f160fd9194e1751b4faea78bcf3ceb456efe008088e"
|
||||
dependencies = [
|
||||
"der 0.8.2",
|
||||
"der 0.8.1",
|
||||
"spki 0.8.0",
|
||||
]
|
||||
|
||||
@@ -7976,7 +7976,7 @@ dependencies = [
|
||||
"aes 0.9.3",
|
||||
"aes-gcm",
|
||||
"cbc 0.2.1",
|
||||
"der 0.8.2",
|
||||
"der 0.8.1",
|
||||
"pbkdf2 0.13.0",
|
||||
"rand_core 0.10.1",
|
||||
"scrypt 0.12.0",
|
||||
@@ -8000,7 +8000,7 @@ version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7"
|
||||
dependencies = [
|
||||
"der 0.8.2",
|
||||
"der 0.8.1",
|
||||
"pkcs5 0.8.1",
|
||||
"rand_core 0.10.1",
|
||||
"spki 0.8.0",
|
||||
@@ -8942,9 +8942,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "redis"
|
||||
version = "1.7.0"
|
||||
version = "1.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2acbc41a996f7652b2ddd9dfd98cc4ff602cfd742ae35382f07f608405ab50ed"
|
||||
checksum = "e37a4ca5c6ca42aa3e6df2fd32b987a65d32a4c2159a6f3fe0fd1df306a2658f"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arcstr",
|
||||
@@ -9326,7 +9326,7 @@ dependencies = [
|
||||
"curve25519-dalek 5.0.0",
|
||||
"data-encoding",
|
||||
"delegate",
|
||||
"der 0.8.2",
|
||||
"der 0.8.1",
|
||||
"digest 0.11.3",
|
||||
"ecdsa 0.17.0",
|
||||
"ed25519-dalek 3.0.0",
|
||||
@@ -9999,7 +9999,6 @@ dependencies = [
|
||||
"rustls-pki-types",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_with",
|
||||
"serial_test",
|
||||
"sha1 0.11.0",
|
||||
"sha2 0.11.0",
|
||||
@@ -10746,7 +10745,6 @@ dependencies = [
|
||||
"rustfs-data-usage",
|
||||
"rustfs-ecstore",
|
||||
"rustfs-filemeta",
|
||||
"rustfs-heal",
|
||||
"rustfs-heal-contracts",
|
||||
"rustfs-lifecycle",
|
||||
"rustfs-lock",
|
||||
@@ -10944,9 +10942,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-uring"
|
||||
version = "0.2.2"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b29bc57b4bd62a73f4fae408b536adf578332e50e464797d09dc2382c7cb68c2"
|
||||
checksum = "0486e62d0efe25db95c00aeacb2da84368adcba299216cda99fcb11328061c84"
|
||||
dependencies = [
|
||||
"io-uring",
|
||||
"libc",
|
||||
@@ -11421,7 +11419,7 @@ checksum = "d56d437c2f19203ce5f7122e507831de96f3d2d4d3be5af44a0b0a09d8a80e4d"
|
||||
dependencies = [
|
||||
"base16ct 1.0.0",
|
||||
"ctutils",
|
||||
"der 0.8.2",
|
||||
"der 0.8.1",
|
||||
"hybrid-array",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
@@ -11995,7 +11993,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d9efca8738c78ee9484207732f728b1ef517bbb1833d6fc0879ca898a522f6f"
|
||||
dependencies = [
|
||||
"base64ct",
|
||||
"der 0.8.2",
|
||||
"der 0.8.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
+5
-6
@@ -191,7 +191,6 @@ rmp = { version = "0.8.15" }
|
||||
rmp-serde = { version = "1.3.1" }
|
||||
serde = { version = "1.0.229" }
|
||||
serde_ignored = { version = "0.1" }
|
||||
serde_with = { version = "3", default-features = false, features = ["macros", "std"] }
|
||||
serde_json = { version = "1.0.151" }
|
||||
serde_urlencoded = "0.7.1"
|
||||
|
||||
@@ -257,10 +256,10 @@ clap = { version = "4.6.6" }
|
||||
const-str = { version = "1.1.0" }
|
||||
convert_case = "0.12.0"
|
||||
criterion = { version = "0.8" }
|
||||
crossbeam-queue = "0.3.14"
|
||||
crossbeam-channel = "0.5.17"
|
||||
crossbeam-deque = "0.8.8"
|
||||
crossbeam-utils = "0.8.23"
|
||||
crossbeam-queue = "0.3.13"
|
||||
crossbeam-channel = "0.5.16"
|
||||
crossbeam-deque = "0.8.7"
|
||||
crossbeam-utils = "0.8.22"
|
||||
datafusion = { default-features = false, version = "55.0.0" }
|
||||
derive_builder = "0.20.2"
|
||||
enumset = "1.1.14"
|
||||
@@ -307,7 +306,7 @@ rustfs-erasure-codec = { version = "8.0.2" }
|
||||
reed-solomon-simd = "3.1.0"
|
||||
regex = { version = "1.13.1" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.34.0" }
|
||||
redis = { version = "1.7.0" }
|
||||
redis = { version = "1.6.0" }
|
||||
rustify = { version = "0.7", default-features = false }
|
||||
rustix = { version = "1.1.4" }
|
||||
rust-embed = { version = "8.12.0" }
|
||||
|
||||
@@ -109,15 +109,6 @@ Star RustFS on GitHub and be instantly notified of new releases.
|
||||
|
||||
## Quickstart
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **Pool expansion notice:**
|
||||
>
|
||||
> - A single-node single-drive (SNSD) deployment is supported only as a standalone local path. It cannot expand in place or be added as a Pool. To move to a multi-drive topology, create a new deployment and migrate data through S3.
|
||||
> - Keep an existing multi-drive Pool's endpoints and Erasure Set width unchanged; expand by appending a new Pool. With ellipsis-based expansion, every Pool argument must contain an ellipsis expression and expand to at least two drive endpoints.
|
||||
> - Single-node multi-drive Pools and multi-node Pools with one drive per node are allowed, subject to valid Erasure Set geometry and EC settings; acceptance does not guarantee host-failure tolerance.
|
||||
>
|
||||
> These topology rules follow MinIO, but automatic parity selection differs between the projects. See the [Pool layout compatibility and regression tests](docs/testing/pool-layout-compatibility.md) before expanding a deployment.
|
||||
|
||||
To get started with RustFS, follow these steps:
|
||||
|
||||
### 1. One-click Installation (Option 1)
|
||||
|
||||
@@ -89,15 +89,6 @@ RustFS 是一个基于 Rust 构建的高性能分布式对象存储系统。Rust
|
||||
|
||||
## 快速开始
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **Pool 扩容 Notice:**
|
||||
>
|
||||
> - 单节点单盘(SNSD)部署仅支持使用本地路径独立运行,不支持原地扩容,也不能作为 Pool 加入集群。如需改为多盘拓扑,请创建新部署并通过 S3 迁移数据。
|
||||
> - 已有多盘 Pool 的端点和 Erasure Set 宽度应保持不变,扩容应追加新的 Pool。使用省略号表达式扩容时,每个 Pool 参数都必须包含省略号表达式,并展开为至少两个磁盘端点。
|
||||
> - 允许单节点多盘 Pool,也允许多节点、每节点一盘的 Pool,但必须满足 Erasure Set 布局和 EC 配置要求;配置合法不代表能够容忍整台主机故障。
|
||||
>
|
||||
> 这些拓扑规则与 MinIO 一致,但两者的默认 parity 选择方式存在差异。扩容前请阅读 [Pool 布局兼容性与回归测试说明](docs/testing/pool-layout-compatibility.md)。
|
||||
|
||||
请按照以下步骤快速上手 RustFS:
|
||||
|
||||
### 1. 一键安装脚本 (选项 1)
|
||||
|
||||
@@ -422,9 +422,9 @@ fn unix_now_ms() -> u64 {
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
/// Legacy, unverified repair notice. Its identity lacks kind, set scope,
|
||||
/// bucket incarnation and responsibility generation. Consumers must not use
|
||||
/// it to discharge persisted repair responsibility.
|
||||
/// A repair the MRF consumer landed, fanned out so retry ledgers can drop
|
||||
/// entries the journal no longer tracks (backlog#1894 axis B). The payload
|
||||
/// mirrors the intent identity so consumers match without re-parsing.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct MrfRepairedEvent {
|
||||
pub bucket: Arc<str>,
|
||||
@@ -439,8 +439,8 @@ const MRF_REPAIRED_EVENT_CAP: usize = 4096;
|
||||
|
||||
static MRF_REPAIRED_EVENTS: OnceLock<std::sync::Mutex<std::collections::VecDeque<MrfRepairedEvent>>> = OnceLock::new();
|
||||
|
||||
/// Record a legacy notification for compatibility. This is not an
|
||||
/// acknowledgement of storage verification or durable repair completion.
|
||||
/// Record that the MRF consumer landed a repair. Never blocks: the critical
|
||||
/// section is a deque push under a std mutex.
|
||||
pub fn note_mrf_repaired(bucket: &str, object: &str, version_id: Option<[u8; 16]>) {
|
||||
let registry = MRF_REPAIRED_EVENTS.get_or_init(|| std::sync::Mutex::new(std::collections::VecDeque::new()));
|
||||
let Ok(mut events) = registry.lock() else {
|
||||
@@ -515,9 +515,6 @@ mod tests {
|
||||
}
|
||||
coalescer_release(&key, Some(lease));
|
||||
let retry_lease = coalescer_admit(key.clone()).expect("released identity must admit a retry");
|
||||
assert_ne!(lease, retry_lease);
|
||||
coalescer_release(&key, Some(lease));
|
||||
assert_eq!(coalescer_admit(key.clone()), Err(MrfIngressResult::Coalesced));
|
||||
coalescer_release(&key, Some(retry_lease));
|
||||
}
|
||||
|
||||
|
||||
@@ -130,37 +130,6 @@ Scanner cycle budget controls:
|
||||
- timeout returns S3 `SlowDown`, so clients should use normal SDK retry handling.
|
||||
- this is not a fdatasync or group-commit switch. Track fdatasync batching separately with `rustfs_s3_put_object_rename_fdatasync_batch_files`.
|
||||
|
||||
## Foreground write admission environment variables
|
||||
|
||||
Large direct `PutObject` requests and multipart `UploadPart` requests share one
|
||||
per-process permit pool that bounds how many bodies are ingested and written
|
||||
concurrently. Small direct PUTs stay on the legacy path.
|
||||
|
||||
- `RUSTFS_PUT_LARGE_FOREGROUND_ADMISSION_ENABLE`
|
||||
- enables the default-on pool; `false` keeps only the soft request counter.
|
||||
- default is `true`.
|
||||
- `RUSTFS_PUT_LARGE_FOREGROUND_ADMISSION_LIMIT`
|
||||
- permits in the pool; `0` derives half of `RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS`, clamped to `32`.
|
||||
- default is `0` (32 permits at stock settings).
|
||||
- `RUSTFS_PUT_LARGE_FOREGROUND_ADMISSION_MIN_SIZE_BYTES`
|
||||
- smallest direct `PutObject` that takes a permit; unknown-size requests always do.
|
||||
- default is `33554432` (32 MiB).
|
||||
- `RUSTFS_PUT_LARGE_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS`
|
||||
- how long a direct `PutObject` waits for a permit before returning S3 `SlowDown`.
|
||||
- default is `250`.
|
||||
- `RUSTFS_PUT_MULTIPART_FOREGROUND_ADMISSION_MIN_SIZE_BYTES`
|
||||
- smallest `UploadPart` that takes a permit; `0` gates every part.
|
||||
- default is `0`.
|
||||
- `RUSTFS_PUT_MULTIPART_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS`
|
||||
- how long an `UploadPart` waits in the bounded queue for a permit before returning S3 `SlowDown`; `0` rejects immediately when the pool is full.
|
||||
- default is `30000`. Parts wait before body ingest, so SDK-default clients that send every part of an upload concurrently drain through the pool instead of failing.
|
||||
- `RUSTFS_PUT_MULTIPART_FOREGROUND_ADMISSION_MAX_PENDING`
|
||||
- maximum `UploadPart` requests waiting for a permit at once; parts beyond it return `SlowDown` without waiting.
|
||||
- default is `0`, which derives 16 times the permit limit (512 at stock settings).
|
||||
- `RUSTFS_PUT_FOREGROUND_ADMISSION_ENABLE`, `RUSTFS_PUT_FOREGROUND_ADMISSION_LIMIT`, `RUSTFS_PUT_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS`
|
||||
- experimental strict gate that applies to every foreground write regardless of size and replaces the pool above when enabled.
|
||||
- default is disabled; enabling it with limit `0` disables foreground write admission entirely.
|
||||
|
||||
## Remote tier timeout environment variables
|
||||
|
||||
- `RUSTFS_TIER_REMOTE_CONNECT_TIMEOUT_SECS`
|
||||
@@ -203,14 +172,6 @@ Drive timeout profile preset:
|
||||
- Then `RUSTFS_DRIVE_MAX_TIMEOUT_DURATION` legacy fallback.
|
||||
- Then the profile-derived default (`default` or `high_latency`).
|
||||
|
||||
## Admin peer probe timeout
|
||||
|
||||
- `RUSTFS_ADMIN_PEER_PROBE_TIMEOUT_SECS`
|
||||
- total per-peer budget for the `server_info`/`storage_info` admin probe round; `server_info` may reconnect once and `storage_info` remains a single attempt.
|
||||
- default is `10` seconds, preserving the previous two-attempt worst-case budget.
|
||||
- values must be positive; `0` or an invalid value falls back to the default, and values above `60` are clamped to `60`.
|
||||
- the setting is read by the aggregating node only; it does not change the internode RPC wire contract. Any retry shares one round deadline rather than receiving a fresh timeout.
|
||||
|
||||
## Startup filesystem boundary policy
|
||||
|
||||
- `RUSTFS_UNSUPPORTED_FS_POLICY` controls startup behavior when RustFS detects local endpoint filesystems that are outside the supported production boundary.
|
||||
|
||||
@@ -39,15 +39,6 @@ pub const DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS: u64 = 20;
|
||||
pub const ENV_INTERNODE_RPC_TIMEOUT_SECS: &str = "RUSTFS_INTERNODE_RPC_TIMEOUT_SECS";
|
||||
pub const DEFAULT_INTERNODE_RPC_TIMEOUT_SECS: u64 = 30;
|
||||
|
||||
/// Total budget for one admin peer probe round, including any reconnect retry.
|
||||
///
|
||||
/// This is intentionally separate from the transport-level RPC timeout: admin
|
||||
/// probes may retry once, but the retry must consume the same round budget.
|
||||
pub const ENV_ADMIN_PEER_PROBE_TIMEOUT_SECS: &str = "RUSTFS_ADMIN_PEER_PROBE_TIMEOUT_SECS";
|
||||
pub const DEFAULT_ADMIN_PEER_PROBE_TIMEOUT_SECS: u64 = 10;
|
||||
pub const MAX_ADMIN_PEER_PROBE_TIMEOUT_SECS: u64 = 60;
|
||||
const _: () = assert!(DEFAULT_ADMIN_PEER_PROBE_TIMEOUT_SECS <= MAX_ADMIN_PEER_PROBE_TIMEOUT_SECS);
|
||||
|
||||
// ── Client-side internode gRPC channel tuning (P0) ──
|
||||
// These mirror the server-side HTTP/2 transport tuning in `rustfs/src/server/http.rs`
|
||||
// on the *client* `tonic` `Endpoint` used for internode control-plane RPCs. Prior to
|
||||
@@ -321,7 +312,6 @@ mod tests {
|
||||
assert_eq!(DEFAULT_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS, 5);
|
||||
assert_eq!(DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS, 20);
|
||||
assert_eq!(DEFAULT_INTERNODE_RPC_TIMEOUT_SECS, 30);
|
||||
assert_eq!(DEFAULT_ADMIN_PEER_PROBE_TIMEOUT_SECS, 10);
|
||||
assert_eq!(DEFAULT_INTERNODE_HTTP_TUNING_PROFILE, "legacy");
|
||||
}
|
||||
|
||||
@@ -422,7 +412,6 @@ mod tests {
|
||||
"RUSTFS_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS"
|
||||
);
|
||||
assert_eq!(ENV_INTERNODE_RPC_TIMEOUT_SECS, "RUSTFS_INTERNODE_RPC_TIMEOUT_SECS");
|
||||
assert_eq!(ENV_ADMIN_PEER_PROBE_TIMEOUT_SECS, "RUSTFS_ADMIN_PEER_PROBE_TIMEOUT_SECS");
|
||||
assert_eq!(ENV_INTERNODE_HTTP_TUNING_PROFILE, "RUSTFS_INTERNODE_HTTP_TUNING_PROFILE");
|
||||
assert_eq!(ENV_INTERNODE_HTTP_POOL_MAX_IDLE_PER_HOST, "RUSTFS_INTERNODE_HTTP_POOL_MAX_IDLE_PER_HOST");
|
||||
assert_eq!(ENV_INTERNODE_HTTP_POOL_IDLE_TIMEOUT_SECS, "RUSTFS_INTERNODE_HTTP_POOL_IDLE_TIMEOUT_SECS");
|
||||
|
||||
@@ -365,36 +365,13 @@ pub const ENV_PUT_MULTIPART_FOREGROUND_ADMISSION_MIN_SIZE_BYTES: &str =
|
||||
"RUSTFS_PUT_MULTIPART_FOREGROUND_ADMISSION_MIN_SIZE_BYTES";
|
||||
pub const DEFAULT_PUT_MULTIPART_FOREGROUND_ADMISSION_MIN_SIZE_BYTES: usize = 0;
|
||||
|
||||
/// Time in milliseconds an automatic foreground direct PutObject waits for a permit.
|
||||
/// Time in milliseconds an automatic foreground write waits for a permit.
|
||||
///
|
||||
/// A short wait smooths transient bursts while still returning S3
|
||||
/// `SlowDown`/503 before body ingest when the node is already saturated.
|
||||
pub const ENV_PUT_LARGE_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS: &str = "RUSTFS_PUT_LARGE_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS";
|
||||
pub const DEFAULT_PUT_LARGE_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS: u64 = 250;
|
||||
|
||||
/// Time in milliseconds a multipart UploadPart waits for a foreground write permit.
|
||||
///
|
||||
/// SDK-default multipart clients send every part of an upload concurrently, so
|
||||
/// a single node routinely sees several times more parts in flight than the
|
||||
/// permit pool allows. Those parts have not ingested a body yet, so queueing
|
||||
/// them costs a connection rather than memory or internode streams; the pool
|
||||
/// still bounds the number of parts being written. The wait is long enough for
|
||||
/// an ordinary queue to drain on modest hardware, and a part that cannot get a
|
||||
/// permit within it fails with S3 `SlowDown`/503 for the client to retry.
|
||||
/// `0` rejects immediately when the pool is full.
|
||||
pub const ENV_PUT_MULTIPART_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS: &str =
|
||||
"RUSTFS_PUT_MULTIPART_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS";
|
||||
pub const DEFAULT_PUT_MULTIPART_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS: u64 = 30_000;
|
||||
|
||||
/// Maximum multipart UploadPart requests waiting for a foreground write permit per process.
|
||||
///
|
||||
/// Parts beyond this queue depth are rejected with S3 `SlowDown`/503 without
|
||||
/// waiting, so a genuinely saturated node still fails fast instead of holding
|
||||
/// an unbounded set of connections open for the whole wait timeout.
|
||||
/// `0` derives the depth from the permit limit.
|
||||
pub const ENV_PUT_MULTIPART_FOREGROUND_ADMISSION_MAX_PENDING: &str = "RUSTFS_PUT_MULTIPART_FOREGROUND_ADMISSION_MAX_PENDING";
|
||||
pub const DEFAULT_PUT_MULTIPART_FOREGROUND_ADMISSION_MAX_PENDING: usize = 0;
|
||||
|
||||
const _: () = assert!(DEFAULT_PUT_LARGE_FOREGROUND_ADMISSION_ENABLE);
|
||||
|
||||
/// Environment variable for minimum GetObject timeout in seconds.
|
||||
|
||||
@@ -26,7 +26,6 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
|
||||
| **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
|
||||
| **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
|
||||
| **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
|
||||
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
|
||||
| **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
|
||||
| **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
|
||||
|
||||
@@ -172,7 +171,6 @@ the same profile for membership and execution with one nightly worker.
|
||||
| KMS suite | `e2e-full` job, merge queue + main | **Active** |
|
||||
| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
|
||||
| Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
|
||||
| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) |
|
||||
| Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
|
||||
| Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
|
||||
| Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |
|
||||
@@ -184,8 +182,6 @@ the wiring source of truth. Committed test-ID digests under
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**Endpoint blackhole scenario skipped** — `heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_after_target_endpoint_blackhole` installs a loopback `iptables` DROP rule and therefore needs `CAP_NET_ADMIN` (root or passwordless `sudo -n iptables`). A host where `iptables` is missing or cannot read the OUTPUT chain (typical inside an unprivileged container, where the nf_tables backend reports "Permission denied" even under `sudo`) logs a `heal_interruption_skipped` warning and returns without exercising heal. Set `RUSTFS_E2E_REQUIRE_NET_FAULT_INJECTION=1` on lanes that do provision the capability so a broken runner fails instead of skipping.
|
||||
|
||||
**Reproduce a CI failure locally** — run the exact profile/lane:
|
||||
|
||||
```bash
|
||||
@@ -195,9 +191,6 @@ cargo nextest run --profile e2e-smoke -p e2e_test
|
||||
cargo nextest run --profile e2e-full -p e2e_test
|
||||
# Cluster fault nightly lane
|
||||
cargo nextest run --profile e2e-nightly -p e2e_test
|
||||
# 4-node 4-disk distributed lane (S3 / lock / versioning / replication / decommission / chaos / upgrade)
|
||||
# Upgrade cases need RUSTFS_UPGRADE_SOURCE_BINARY; without it they fail closed.
|
||||
cargo nextest run --profile e2e-distributed -p e2e_test
|
||||
# Replication nightly lane; awscurl is required for STS paths
|
||||
cargo nextest run --profile e2e-repl-nightly -p e2e_test
|
||||
# Fixed-port protocol nightly lane
|
||||
|
||||
@@ -1,74 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use std::path::Path;
|
||||
use std::process::Command;
|
||||
|
||||
fn git(root: &Path, args: &[&str]) -> Option<String> {
|
||||
let output = Command::new("git").args(args).current_dir(root).output().ok()?;
|
||||
output
|
||||
.status
|
||||
.success()
|
||||
.then(|| String::from_utf8_lossy(&output.stdout).trim().to_owned())
|
||||
}
|
||||
|
||||
fn emit(name: &str, value: &str) {
|
||||
let value = if value.contains(['\n', '\r']) { "unknown" } else { value };
|
||||
println!("cargo:rustc-env=RUSTFS_E2E_BUILD_{name}={value}");
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let manifest = std::env::var_os("CARGO_MANIFEST_DIR").unwrap_or_default();
|
||||
let root = Path::new(&manifest).join("../..");
|
||||
// Cover dependency/common sources as well as this crate. HEAD/ref/index
|
||||
// changes must refresh identity even when no Rust source mtime changes.
|
||||
for path in [
|
||||
"crates",
|
||||
"rustfs",
|
||||
"Cargo.toml",
|
||||
"Cargo.lock",
|
||||
"rust-toolchain.toml",
|
||||
".cargo",
|
||||
".config",
|
||||
] {
|
||||
println!("cargo:rerun-if-changed={}", root.join(path).display());
|
||||
}
|
||||
let mut git_paths = vec!["HEAD".to_owned(), "index".to_owned(), "packed-refs".to_owned()];
|
||||
if let Some(reference) = git(&root, &["symbolic-ref", "-q", "HEAD"]) {
|
||||
git_paths.push(reference);
|
||||
}
|
||||
for path in git_paths {
|
||||
if let Some(path) = git(&root, &["rev-parse", "--git-path", &path]) {
|
||||
let path = Path::new(&path);
|
||||
let path = if path.is_absolute() {
|
||||
path.to_owned()
|
||||
} else {
|
||||
root.join(path)
|
||||
};
|
||||
if path.exists() {
|
||||
println!("cargo:rerun-if-changed={}", path.display());
|
||||
}
|
||||
}
|
||||
}
|
||||
let revision = git(&root, &["rev-parse", "HEAD"]).unwrap_or_else(|| "unknown".to_owned());
|
||||
let dirty = git(&root, &["status", "--porcelain", "--untracked-files=normal"]).is_none_or(|status| !status.is_empty());
|
||||
let lock = git(&root, &["hash-object", "Cargo.lock"]).unwrap_or_else(|| "unknown".to_owned());
|
||||
let mut features = std::env::vars()
|
||||
.filter_map(|(key, _)| {
|
||||
key.strip_prefix("CARGO_FEATURE_")
|
||||
.map(|name| name.to_ascii_lowercase().replace('_', "-"))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
features.sort();
|
||||
emit("COMMIT", &revision);
|
||||
emit("DIRTY", if dirty { "true" } else { "false" });
|
||||
emit("LOCK", &lock);
|
||||
emit("FEATURES", &features.join(","));
|
||||
for name in ["TARGET", "PROFILE"] {
|
||||
emit(name, &std::env::var(name).unwrap_or_else(|_| "unknown".to_owned()));
|
||||
}
|
||||
println!("cargo:rerun-if-env-changed=CARGO_ENCODED_RUSTFLAGS");
|
||||
let flags = std::env::var("CARGO_ENCODED_RUSTFLAGS").unwrap_or_default();
|
||||
let flags: String = flags.as_bytes().iter().map(|byte| format!("{byte:02x}")).collect();
|
||||
emit("RUSTFLAGS_HEX", &flags);
|
||||
}
|
||||
@@ -55,20 +55,18 @@ type ChaosResult<T> = Result<T, Box<dyn Error + Send + Sync>>;
|
||||
/// A successful S3 GET only proves that a quorum can serve an object. Replacement
|
||||
/// tests need this lower-level record to prove that the rebuilt target holds the
|
||||
/// `xl.meta` selected for a specific version and every `part.N` it declares.
|
||||
#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub(crate) struct VersionShardCensus {
|
||||
pub version_id: Option<String>,
|
||||
pub has_xl_meta: bool,
|
||||
pub data_dir: Option<String>,
|
||||
pub erasure_index: Option<usize>,
|
||||
pub data_blocks: Option<usize>,
|
||||
pub parity_blocks: Option<usize>,
|
||||
pub expected_part_numbers: BTreeSet<usize>,
|
||||
pub present_part_fingerprints: BTreeMap<usize, PartShardFingerprint>,
|
||||
pub inline_data_fingerprint: Option<PartShardFingerprint>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub(crate) struct PartShardFingerprint {
|
||||
pub size: u64,
|
||||
pub sha256: String,
|
||||
@@ -90,15 +88,13 @@ impl VersionShardCensus {
|
||||
&& manifest.is_complete()
|
||||
&& self.data_dir == manifest.data_dir
|
||||
&& self.erasure_index == manifest.erasure_index
|
||||
&& self.data_blocks == manifest.data_blocks
|
||||
&& self.parity_blocks == manifest.parity_blocks
|
||||
&& self.expected_part_numbers == manifest.expected_part_numbers
|
||||
&& self.present_part_fingerprints == manifest.present_part_fingerprints
|
||||
&& self.inline_data_fingerprint == manifest.inline_data_fingerprint
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn sha256_hex(data: &[u8]) -> String {
|
||||
fn sha256_hex(data: &[u8]) -> String {
|
||||
let digest = Sha256::digest(data);
|
||||
digest.iter().map(|byte| format!("{byte:02x}")).collect()
|
||||
}
|
||||
@@ -317,8 +313,6 @@ pub(crate) fn census_object_version_on_disk(
|
||||
has_xl_meta: false,
|
||||
data_dir: None,
|
||||
erasure_index: None,
|
||||
data_blocks: None,
|
||||
parity_blocks: None,
|
||||
expected_part_numbers: BTreeSet::new(),
|
||||
present_part_fingerprints: BTreeMap::new(),
|
||||
inline_data_fingerprint: None,
|
||||
@@ -366,8 +360,6 @@ pub(crate) fn census_object_version_on_disk(
|
||||
has_xl_meta: true,
|
||||
data_dir,
|
||||
erasure_index,
|
||||
data_blocks: Some(file_info.erasure.data_blocks),
|
||||
parity_blocks: Some(file_info.erasure.parity_blocks),
|
||||
expected_part_numbers,
|
||||
present_part_fingerprints,
|
||||
inline_data_fingerprint,
|
||||
@@ -421,8 +413,6 @@ mod tests {
|
||||
has_xl_meta: true,
|
||||
data_dir: Some("data-dir".to_string()),
|
||||
erasure_index: Some(3),
|
||||
data_blocks: Some(2),
|
||||
parity_blocks: Some(2),
|
||||
expected_part_numbers: BTreeSet::from([1]),
|
||||
present_part_fingerprints: BTreeMap::from([(1, shard_fingerprint(b"part").unwrap())]),
|
||||
inline_data_fingerprint: None,
|
||||
|
||||
+12
-159
@@ -57,8 +57,6 @@ const RUSTFS_FULL_FEATURE: &str = "full";
|
||||
const TEST_PORT_MIN: u16 = 20_000;
|
||||
// Keep allocator ports below the ephemeral range used by bind(..., 0) test helpers.
|
||||
const TEST_PORT_RANGE: u16 = 10_000;
|
||||
const TEST_PORT_MIN_ENV: &str = "RUSTFS_E2E_TEST_PORT_MIN";
|
||||
const TEST_PORT_RANGE_ENV: &str = "RUSTFS_E2E_TEST_PORT_RANGE";
|
||||
const TEST_PORT_COUNTER_PATH: &str = "/tmp/rustfs_e2e_next_port";
|
||||
const TEST_PORT_LOCK_DIR: &str = "/tmp/rustfs_e2e_port_allocator.lock";
|
||||
const TEST_PORT_LOCK_STALE_AFTER: Duration = Duration::from_secs(30);
|
||||
@@ -101,74 +99,22 @@ impl Drop for PortAllocatorGuard {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
struct TestPortAllocatorConfig {
|
||||
min: u16,
|
||||
range: u16,
|
||||
fn advance_test_port(port: u16) -> u16 {
|
||||
let offset = (port - TEST_PORT_MIN + 1) % TEST_PORT_RANGE;
|
||||
TEST_PORT_MIN + offset
|
||||
}
|
||||
|
||||
impl TestPortAllocatorConfig {
|
||||
fn max_exclusive(self) -> u32 {
|
||||
u32::from(self.min) + u32::from(self.range)
|
||||
}
|
||||
|
||||
fn contains(self, port: &u16) -> bool {
|
||||
(u32::from(self.min)..self.max_exclusive()).contains(&u32::from(*port))
|
||||
}
|
||||
fn seeded_test_port() -> u16 {
|
||||
let offset = (Uuid::new_v4().as_u128() % u128::from(TEST_PORT_RANGE)) as u16;
|
||||
TEST_PORT_MIN + offset
|
||||
}
|
||||
|
||||
fn parse_test_port_allocator_config(
|
||||
min_override: Option<&str>,
|
||||
range_override: Option<&str>,
|
||||
) -> Result<TestPortAllocatorConfig, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let min = match min_override {
|
||||
Some(value) => value
|
||||
.parse::<u16>()
|
||||
.map_err(|err| format!("{TEST_PORT_MIN_ENV} must be a valid u16: {err}"))?,
|
||||
None => TEST_PORT_MIN,
|
||||
};
|
||||
let range = match range_override {
|
||||
Some(value) => value
|
||||
.parse::<u16>()
|
||||
.map_err(|err| format!("{TEST_PORT_RANGE_ENV} must be a valid u16: {err}"))?,
|
||||
None => TEST_PORT_RANGE,
|
||||
};
|
||||
if range == 0 {
|
||||
return Err(format!("{TEST_PORT_RANGE_ENV} must be greater than zero").into());
|
||||
}
|
||||
if min < 1024 {
|
||||
return Err(format!("{TEST_PORT_MIN_ENV} must be at least 1024").into());
|
||||
}
|
||||
let max_exclusive = u32::from(min) + u32::from(range);
|
||||
if max_exclusive > u32::from(u16::MAX) + 1 {
|
||||
return Err(format!("{TEST_PORT_MIN_ENV} + {TEST_PORT_RANGE_ENV} exceeds u16 port space").into());
|
||||
}
|
||||
Ok(TestPortAllocatorConfig { min, range })
|
||||
}
|
||||
|
||||
fn test_port_allocator_config() -> Result<TestPortAllocatorConfig, Box<dyn std::error::Error + Send + Sync>> {
|
||||
parse_test_port_allocator_config(
|
||||
std::env::var(TEST_PORT_MIN_ENV).ok().as_deref(),
|
||||
std::env::var(TEST_PORT_RANGE_ENV).ok().as_deref(),
|
||||
)
|
||||
}
|
||||
|
||||
fn advance_test_port(port: u16, config: TestPortAllocatorConfig) -> u16 {
|
||||
let offset = (port - config.min + 1) % config.range;
|
||||
config.min + offset
|
||||
}
|
||||
|
||||
fn seeded_test_port(config: TestPortAllocatorConfig) -> u16 {
|
||||
let offset = (Uuid::new_v4().as_u128() % u128::from(config.range)) as u16;
|
||||
config.min + offset
|
||||
}
|
||||
|
||||
fn read_next_test_port(config: TestPortAllocatorConfig) -> u16 {
|
||||
fn read_next_test_port() -> u16 {
|
||||
stdfs::read_to_string(TEST_PORT_COUNTER_PATH)
|
||||
.ok()
|
||||
.and_then(|value| value.trim().parse::<u16>().ok())
|
||||
.filter(|port| config.contains(port))
|
||||
.unwrap_or_else(|| seeded_test_port(config))
|
||||
.filter(|port| (TEST_PORT_MIN..TEST_PORT_MIN + TEST_PORT_RANGE).contains(port))
|
||||
.unwrap_or_else(seeded_test_port)
|
||||
}
|
||||
|
||||
fn remove_stale_port_allocator_lock() {
|
||||
@@ -683,12 +629,11 @@ impl RustFSTestEnvironment {
|
||||
pub async fn find_available_port() -> Result<u16, Box<dyn std::error::Error + Send + Sync>> {
|
||||
use std::net::TcpListener;
|
||||
let _guard = PortAllocatorGuard::acquire().await?;
|
||||
let config = test_port_allocator_config()?;
|
||||
let mut next_port = read_next_test_port(config);
|
||||
let mut next_port = read_next_test_port();
|
||||
|
||||
for _ in 0..config.range {
|
||||
for _ in 0..TEST_PORT_RANGE {
|
||||
let port = next_port;
|
||||
next_port = advance_test_port(next_port, config);
|
||||
next_port = advance_test_port(next_port);
|
||||
write_next_test_port(next_port)?;
|
||||
|
||||
if let Ok(listener) = TcpListener::bind(("127.0.0.1", port)) {
|
||||
@@ -1755,69 +1700,6 @@ impl RustFSTestClusterEnvironment {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Append a new single-node erasure pool to a stopped multi-pool cluster.
|
||||
///
|
||||
/// Used to simulate pool expansion on localhost: every pool already owns
|
||||
/// exactly one node with `drives_per_node >= 2` (the only multi-pool layout
|
||||
/// the single-host `RUSTFS_VOLUMES` syntax can express). The new node is
|
||||
/// allocated a fresh port and empty drive directories; callers must
|
||||
/// [`Self::start`] afterwards so every process picks up the extended
|
||||
/// volumes argument. Existing data directories are left untouched.
|
||||
pub async fn append_single_node_pool(&mut self) -> Result<usize, Box<dyn std::error::Error + Send + Sync>> {
|
||||
if self.nodes.iter().any(|node| node.process.is_some()) {
|
||||
return Err("stop the cluster before appending a pool".into());
|
||||
}
|
||||
if self.topology.drives_per_node < 2 {
|
||||
return Err(
|
||||
"append_single_node_pool requires drives_per_node >= 2 (the server parser rejects a single-drive ellipses pool)"
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
|
||||
let mut pools = self.topology.normalized_pools();
|
||||
for (pool_idx, nodes) in pools.iter().enumerate() {
|
||||
if nodes.len() != 1 {
|
||||
return Err(format!(
|
||||
"pool {pool_idx} spans {} nodes; append_single_node_pool requires one node per pool",
|
||||
nodes.len()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
}
|
||||
|
||||
let new_idx = self.nodes.len();
|
||||
let port = RustFSTestEnvironment::find_available_port().await?;
|
||||
let address = format!("127.0.0.1:{port}");
|
||||
let data_dirs: Vec<String> = (0..self.topology.drives_per_node)
|
||||
.map(|drive| format!("{}/node{}/drive{}", self.temp_dir, new_idx, drive))
|
||||
.collect();
|
||||
for dir in &data_dirs {
|
||||
fs::create_dir_all(dir).await?;
|
||||
}
|
||||
|
||||
self.nodes.push(ClusterNode {
|
||||
url: format!("http://{address}"),
|
||||
address,
|
||||
data_dir: data_dirs[0].clone(),
|
||||
data_dirs,
|
||||
pool_idx: pools.len(),
|
||||
process: None,
|
||||
});
|
||||
pools.push(vec![new_idx]);
|
||||
self.topology.node_count = self.nodes.len();
|
||||
self.topology.pools = pools;
|
||||
self.node_extra_env.push(Vec::new());
|
||||
self.node_capture_log_paths.push(None);
|
||||
self.volume_proxy_addresses.push(None);
|
||||
|
||||
if !self.extra_env.iter().any(|(key, _)| key == "RUSTFS_UNSAFE_BYPASS_DISK_CHECK") {
|
||||
self.extra_env
|
||||
.push(("RUSTFS_UNSAFE_BYPASS_DISK_CHECK".to_string(), "true".to_string()));
|
||||
}
|
||||
|
||||
Ok(new_idx)
|
||||
}
|
||||
|
||||
/// Gracefully stop one cluster node and wait for its process to exit.
|
||||
///
|
||||
/// This is intentionally separate from [`Self::stop_node`]: the latter is
|
||||
@@ -2163,35 +2045,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn e2e_port_allocator_uses_default_range() {
|
||||
assert_eq!(
|
||||
parse_test_port_allocator_config(None, None).expect("default port allocator config"),
|
||||
TestPortAllocatorConfig {
|
||||
min: TEST_PORT_MIN,
|
||||
range: TEST_PORT_RANGE
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn e2e_port_allocator_accepts_explicit_test_range() {
|
||||
let config = parse_test_port_allocator_config(Some("31000"), Some("128")).expect("explicit port range");
|
||||
|
||||
assert_eq!(advance_test_port(31127, config), 31000);
|
||||
assert!(config.contains(&31000));
|
||||
assert!(config.contains(&31127));
|
||||
assert!(!config.contains(&31128));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn e2e_port_allocator_rejects_invalid_override() {
|
||||
assert!(parse_test_port_allocator_config(Some("1023"), Some("1")).is_err());
|
||||
assert!(parse_test_port_allocator_config(Some("65000"), Some("1000")).is_err());
|
||||
assert!(parse_test_port_allocator_config(Some("31000"), Some("0")).is_err());
|
||||
assert!(parse_test_port_allocator_config(Some("not-a-port"), Some("128")).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolves_rustfs_binary_in_configured_cargo_target_directory() {
|
||||
let workspace = Path::new("workspace");
|
||||
|
||||
@@ -35,15 +35,11 @@ where
|
||||
{
|
||||
let mut last_usage = DataUsageInfo::default();
|
||||
let mut last_query_error = None;
|
||||
for _ in 0..90 {
|
||||
for _ in 0..45 {
|
||||
match get_data_usage_info(env).await {
|
||||
Ok(usage) => {
|
||||
last_query_error = None;
|
||||
if usage.is_complete_bucket_usage_snapshot()
|
||||
&& usage.usage_snapshot_converged != Some(false)
|
||||
&& usage.buckets_usage.contains_key(bucket)
|
||||
&& predicate(&usage)
|
||||
{
|
||||
if usage.buckets_usage.contains_key(bucket) && predicate(&usage) {
|
||||
return Ok(usage);
|
||||
}
|
||||
last_usage = usage;
|
||||
|
||||
@@ -1,222 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{
|
||||
DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, retrying_get_equals, unique_bucket,
|
||||
wait_for_ready, wait_until,
|
||||
};
|
||||
use crate::chaos::{census_object_version_on_disk, signed_admin_post};
|
||||
use crate::common::{build_test_s3_config, init_logging};
|
||||
use crate::fault_proxy::FaultMode;
|
||||
use aws_sdk_s3::Client;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::{Barrier, mpsc};
|
||||
use tokio::time::timeout;
|
||||
|
||||
#[tokio::test]
|
||||
async fn kill_and_restart_node_preserves_objects() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("killnode");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let body = vec![0x11u8; 128 * 1024];
|
||||
put_object(&dist.client(0)?, &bucket, "keep.bin", body.clone()).await?;
|
||||
|
||||
dist.cluster.stop_node(3)?;
|
||||
retrying_get_equals(&dist.client(0)?, &bucket, "keep.bin", &body, Duration::from_secs(20)).await?;
|
||||
|
||||
dist.cluster.start_node(3).await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
assert_object_bytes(&dist.client(3)?, &bucket, "keep.bin", &body).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn full_cluster_restart_preserves_objects() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("pwr");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let body = vec![0x44u8; 64 * 1024];
|
||||
put_object(&dist.client(1)?, &bucket, "survive.bin", body.clone()).await?;
|
||||
|
||||
dist.cluster.stop();
|
||||
dist.cluster.start().await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
for node_idx in 0..dist.cluster.nodes.len() {
|
||||
assert_object_bytes(&dist.client(node_idx)?, &bucket, "survive.bin", &body).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fresh_drive_replacement_is_physically_healed_without_data_change() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start_with_env(DistLayout::FourByFour, &[("RUSTFS_HEAL_ENABLED", "true")]).await?;
|
||||
let bucket = unique_bucket("baddrive");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let body = payload_for("fresh-drive/durable.bin", 8 * 1024 * 1024);
|
||||
put_object(&dist.client(1)?, &bucket, "durable.bin", body.clone()).await?;
|
||||
|
||||
let replaced_drive = PathBuf::from(&dist.cluster.nodes[0].data_dirs[0]);
|
||||
let baseline = census_object_version_on_disk(&replaced_drive, &bucket, "durable.bin", None)?;
|
||||
assert!(
|
||||
baseline.is_complete(),
|
||||
"replacement target did not hold a complete baseline shard: {baseline:?}"
|
||||
);
|
||||
assert!(
|
||||
!baseline.expected_part_numbers.is_empty(),
|
||||
"replacement witness must use physical part shards: {baseline:?}"
|
||||
);
|
||||
|
||||
dist.cluster.stop_node(0)?;
|
||||
let format_path = replaced_drive.join(".rustfs.sys/format.json");
|
||||
let format = std::fs::read(&format_path)?;
|
||||
let retired_drive = PathBuf::from(format!("{}.retired", replaced_drive.display()));
|
||||
std::fs::rename(&replaced_drive, &retired_drive)?;
|
||||
std::fs::create_dir_all(format_path.parent().ok_or("replacement format path omitted parent")?)?;
|
||||
std::fs::write(&format_path, format)?;
|
||||
let empty = census_object_version_on_disk(&replaced_drive, &bucket, "durable.bin", None)?;
|
||||
assert!(!empty.has_xl_meta, "fresh replacement unexpectedly retained object metadata: {empty:?}");
|
||||
|
||||
dist.cluster.start_node(0).await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
let heal_body =
|
||||
r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
|
||||
let heal_url = format!("{}/rustfs/admin/v3/heal/{bucket}?forceStart=true", dist.cluster.nodes[1].url);
|
||||
signed_admin_post(&heal_url, Some(heal_body), &dist.cluster.access_key, &dist.cluster.secret_key).await?;
|
||||
wait_until(
|
||||
Duration::from_secs(90),
|
||||
|| async {
|
||||
let healed = census_object_version_on_disk(&replaced_drive, &bucket, "durable.bin", None)?;
|
||||
Ok(healed.matches_manifest(&baseline))
|
||||
},
|
||||
"fresh replacement contains the original complete shard manifest",
|
||||
)
|
||||
.await?;
|
||||
|
||||
for node_idx in 0..dist.cluster.nodes.len() {
|
||||
assert_object_bytes(&dist.client(node_idx)?, &bucket, "durable.bin", &body).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_gets_survive_peer_node_kill() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("getkill");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let body = payload_for("inflight/steady.bin", 8 * 1024 * 1024);
|
||||
put_object(&dist.client(0)?, &bucket, "steady.bin", body.clone()).await?;
|
||||
|
||||
let live: Vec<_> = (0..3).map(|idx| dist.client(idx)).collect::<Result<Vec<_>, _>>()?;
|
||||
let worker_count = 12;
|
||||
let release = Arc::new(Barrier::new(worker_count + 1));
|
||||
let (started_tx, mut started_rx) = mpsc::unbounded_channel();
|
||||
let mut handles = Vec::new();
|
||||
for idx in 0..worker_count {
|
||||
let client = live[idx % live.len()].clone();
|
||||
let bucket = bucket.clone();
|
||||
let body = body.clone();
|
||||
let release = release.clone();
|
||||
let started_tx = started_tx.clone();
|
||||
handles.push(tokio::spawn(async move {
|
||||
let response = client.get_object().bucket(&bucket).key("steady.bin").send().await?;
|
||||
if response.content_length() != Some(body.len() as i64) {
|
||||
return Err::<(), Box<dyn std::error::Error + Send + Sync>>(
|
||||
format!("worker {idx} received a wrong content length").into(),
|
||||
);
|
||||
}
|
||||
started_tx.send(idx)?;
|
||||
release.wait().await;
|
||||
let actual = response.body.collect().await?.into_bytes();
|
||||
if actual.as_ref() != body.as_slice() {
|
||||
return Err(format!("worker {idx} received corrupted bytes after peer kill").into());
|
||||
}
|
||||
Ok(())
|
||||
}));
|
||||
}
|
||||
drop(started_tx);
|
||||
for _ in 0..worker_count {
|
||||
timeout(Duration::from_secs(30), started_rx.recv())
|
||||
.await?
|
||||
.ok_or("a streaming GET exited before reaching the kill barrier")?;
|
||||
}
|
||||
|
||||
dist.cluster.stop_node(3)?;
|
||||
release.wait().await;
|
||||
for handle in handles {
|
||||
handle.await??;
|
||||
}
|
||||
|
||||
dist.cluster.start_node(3).await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
assert_object_bytes(&dist.client(3)?, &bucket, "steady.bin", &body).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn blackholed_node_client_network_preserves_cluster_availability_and_recovers() -> TestResult {
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let proxy = crate::fault_proxy::FaultProxy::start(dist.cluster.nodes[3].address.parse()?).await?;
|
||||
let proxied_url = format!("http://{}", proxy.local_addr());
|
||||
let proxied_client = Client::from_conf(build_test_s3_config(
|
||||
&proxied_url,
|
||||
&dist.cluster.access_key,
|
||||
&dist.cluster.secret_key,
|
||||
None,
|
||||
"distributed-network-chaos",
|
||||
));
|
||||
|
||||
let result: TestResult = async {
|
||||
let bucket = unique_bucket("netfault");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let baseline = payload_for("network/baseline.bin", 1024 * 1024);
|
||||
put_object(&dist.client(0)?, &bucket, "baseline.bin", baseline.clone()).await?;
|
||||
assert_object_bytes(&proxied_client, &bucket, "baseline.bin", &baseline).await?;
|
||||
|
||||
proxy.set_mode(FaultMode::Blackhole);
|
||||
assert_eq!(proxy.mode(), FaultMode::Blackhole);
|
||||
if let Ok(Ok(_)) = timeout(
|
||||
Duration::from_secs(5),
|
||||
proxied_client.get_object().bucket(&bucket).key("baseline.bin").send(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
return Err("blackholed node endpoint unexpectedly completed a GET".into());
|
||||
}
|
||||
|
||||
let during = payload_for("network/during.bin", 1024 * 1024);
|
||||
timeout(Duration::from_secs(30), async {
|
||||
put_object(&dist.client(1)?, &bucket, "during-blackhole.bin", during.clone()).await?;
|
||||
assert_object_bytes(&dist.client(2)?, &bucket, "baseline.bin", &baseline).await?;
|
||||
assert_object_bytes(&dist.client(0)?, &bucket, "during-blackhole.bin", &during).await?;
|
||||
Ok::<_, Box<dyn std::error::Error + Send + Sync>>(())
|
||||
})
|
||||
.await??;
|
||||
|
||||
proxy.set_mode(FaultMode::Pass);
|
||||
retrying_get_equals(&proxied_client, &bucket, "during-blackhole.bin", &during, Duration::from_secs(30)).await?;
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
|
||||
proxy.set_mode(FaultMode::Pass);
|
||||
proxy.shutdown().await;
|
||||
result
|
||||
}
|
||||
@@ -1,98 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
|
||||
use crate::common::init_logging;
|
||||
use std::collections::BTreeSet;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::Barrier;
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult {
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("conc");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let clients = Arc::new(dist.clients()?);
|
||||
let worker_count = 24;
|
||||
let rounds = 4;
|
||||
let barrier = Arc::new(Barrier::new(worker_count));
|
||||
|
||||
let mut handles = Vec::new();
|
||||
for idx in 0..worker_count {
|
||||
let clients = clients.clone();
|
||||
let barrier = barrier.clone();
|
||||
let bucket = bucket.clone();
|
||||
handles.push(tokio::spawn(async move {
|
||||
barrier.wait().await;
|
||||
let writer = &clients[idx % clients.len()];
|
||||
let reader = &clients[(idx + 1) % clients.len()];
|
||||
let copier = &clients[(idx + 2) % clients.len()];
|
||||
let mut retained = Vec::with_capacity(rounds);
|
||||
for round in 0..rounds {
|
||||
let key = format!("source/worker-{idx:02}-round-{round}.bin");
|
||||
let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin");
|
||||
let body = payload_for(&key, 64 * 1024);
|
||||
put_object(writer, &bucket, &key, body.clone()).await?;
|
||||
|
||||
let head = reader.head_object().bucket(&bucket).key(&key).send().await?;
|
||||
if head.content_length() != Some(body.len() as i64) {
|
||||
return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into());
|
||||
}
|
||||
assert_object_bytes(reader, &bucket, &key, &body).await?;
|
||||
|
||||
copier
|
||||
.copy_object()
|
||||
.bucket(&bucket)
|
||||
.key(©_key)
|
||||
.copy_source(format!("{bucket}/{key}"))
|
||||
.send()
|
||||
.await?;
|
||||
assert_object_bytes(writer, &bucket, ©_key, &body).await?;
|
||||
|
||||
writer.delete_object().bucket(&bucket).key(&key).send().await?;
|
||||
let missing = reader
|
||||
.head_object()
|
||||
.bucket(&bucket)
|
||||
.key(&key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("deleted source key must not remain visible");
|
||||
if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) {
|
||||
return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into());
|
||||
}
|
||||
retained.push((copy_key, body));
|
||||
}
|
||||
Ok::<_, Box<dyn std::error::Error + Send + Sync>>(retained)
|
||||
}));
|
||||
}
|
||||
|
||||
let mut inventory = Vec::new();
|
||||
for handle in handles {
|
||||
inventory.extend(handle.await??);
|
||||
}
|
||||
|
||||
let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect();
|
||||
for (node_idx, client) in clients.iter().enumerate() {
|
||||
let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?;
|
||||
let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
|
||||
assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing");
|
||||
for (key, body) in &inventory {
|
||||
assert_object_bytes(client, &bucket, key, body)
|
||||
.await
|
||||
.map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{
|
||||
DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, decommission_running_with_progress,
|
||||
decommission_status_json, payload_for, put_inventory_retrying, retrying_get_equals, retrying_put, start_decommission,
|
||||
unique_bucket, wait_for_decommission_complete, wait_for_decommission_running_with_progress,
|
||||
};
|
||||
use crate::common::init_logging;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::Barrier;
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_puts_during_decommission_do_not_lose_baseline_or_new_objects() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
|
||||
let bucket = unique_bucket("concdecom");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let baseline_client = dist.client(0)?;
|
||||
let inventory = put_inventory_retrying(&baseline_client, &bucket, 96, 256 * 1024, Duration::from_secs(30)).await?;
|
||||
dist.expand_to_four_pools().await?;
|
||||
|
||||
start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
|
||||
|
||||
let clients = Arc::new(dist.clients()?);
|
||||
let barrier = Arc::new(Barrier::new(17));
|
||||
let mut handles = Vec::new();
|
||||
for idx in 0..16 {
|
||||
let clients = clients.clone();
|
||||
let barrier = barrier.clone();
|
||||
let bucket = bucket.clone();
|
||||
handles.push(tokio::spawn(async move {
|
||||
barrier.wait().await;
|
||||
let client = &clients[idx % clients.len()];
|
||||
let key = format!("live/{idx:02}.bin");
|
||||
let body = payload_for(&key, 8 * 1024);
|
||||
retrying_put(client, &bucket, &key, body.clone(), Duration::from_secs(45)).await?;
|
||||
Ok::<_, Box<dyn std::error::Error + Send + Sync>>((key, body))
|
||||
}));
|
||||
}
|
||||
|
||||
wait_for_decommission_running_with_progress(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
|
||||
barrier.wait().await;
|
||||
|
||||
let mut live_objects = Vec::new();
|
||||
for handle in handles {
|
||||
live_objects.push(handle.await??);
|
||||
}
|
||||
let status = decommission_status_json(&dist.cluster).await?;
|
||||
if !decommission_running_with_progress(&status, DECOMMISSION_POOL_ID)? {
|
||||
return Err(format!("decommission did not remain active across concurrent PUTs: {status}").into());
|
||||
}
|
||||
|
||||
wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
|
||||
|
||||
let checker = dist.client(2)?;
|
||||
assert_inventory(&checker, &bucket, &inventory).await?;
|
||||
for (key, body) in live_objects {
|
||||
retrying_get_equals(&checker, &bucket, &key, &body, Duration::from_secs(30)).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,156 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{
|
||||
DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, enable_versioning, put_inventory_retrying,
|
||||
sha256_hex, start_decommission, unique_bucket, wait_for_decommission_active, wait_for_decommission_complete,
|
||||
};
|
||||
use crate::common::init_logging;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
use std::time::Duration;
|
||||
|
||||
#[tokio::test]
|
||||
async fn decommission_does_not_alter_object_sha256_across_pools() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
|
||||
let bucket = unique_bucket("integrity");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let client = dist.client(0)?;
|
||||
enable_versioning(&client, &bucket).await?;
|
||||
let inventory = put_inventory_retrying(&client, &bucket, 96, 256 * 1024, Duration::from_secs(30)).await?;
|
||||
let before: Vec<(String, String)> = inventory.iter().map(|(key, body)| (key.clone(), sha256_hex(body))).collect();
|
||||
|
||||
let versioned_key = "history/versioned.bin";
|
||||
let version_one = b"historical bytes before data movement".to_vec();
|
||||
let version_two = b"current bytes before data movement".to_vec();
|
||||
let version_one_id = client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(versioned_key)
|
||||
.body(ByteStream::from(version_one.clone()))
|
||||
.send()
|
||||
.await?
|
||||
.version_id()
|
||||
.ok_or("historical PUT omitted version ID")?
|
||||
.to_string();
|
||||
let version_two_id = client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(versioned_key)
|
||||
.body(ByteStream::from(version_two.clone()))
|
||||
.send()
|
||||
.await?
|
||||
.version_id()
|
||||
.ok_or("current PUT omitted version ID")?
|
||||
.to_string();
|
||||
|
||||
let multipart_key = "multipart/moved.bin";
|
||||
let first_part = vec![0x31; 5 * 1024 * 1024];
|
||||
let second_part = vec![0x72; 1024 * 1024];
|
||||
let upload = client
|
||||
.create_multipart_upload()
|
||||
.bucket(&bucket)
|
||||
.key(multipart_key)
|
||||
.send()
|
||||
.await?;
|
||||
let upload_id = upload.upload_id().ok_or("movement multipart upload omitted upload ID")?;
|
||||
let uploaded_one = client
|
||||
.upload_part()
|
||||
.bucket(&bucket)
|
||||
.key(multipart_key)
|
||||
.upload_id(upload_id)
|
||||
.part_number(1)
|
||||
.body(ByteStream::from(first_part.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
let uploaded_two = client
|
||||
.upload_part()
|
||||
.bucket(&bucket)
|
||||
.key(multipart_key)
|
||||
.upload_id(upload_id)
|
||||
.part_number(2)
|
||||
.body(ByteStream::from(second_part.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
client
|
||||
.complete_multipart_upload()
|
||||
.bucket(&bucket)
|
||||
.key(multipart_key)
|
||||
.upload_id(upload_id)
|
||||
.multipart_upload(
|
||||
CompletedMultipartUpload::builder()
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(1)
|
||||
.e_tag(uploaded_one.e_tag().ok_or("movement part 1 omitted ETag")?)
|
||||
.build(),
|
||||
)
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(2)
|
||||
.e_tag(uploaded_two.e_tag().ok_or("movement part 2 omitted ETag")?)
|
||||
.build(),
|
||||
)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
dist.expand_to_four_pools().await?;
|
||||
|
||||
start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
|
||||
wait_for_decommission_active(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
|
||||
wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
|
||||
|
||||
let after_client = dist.client(2)?;
|
||||
assert_inventory(&after_client, &bucket, &inventory).await?;
|
||||
for (key, expected_hash) in before {
|
||||
let got = after_client.get_object().bucket(&bucket).key(&key).send().await?;
|
||||
let body = got.body.collect().await?.into_bytes();
|
||||
assert_eq!(sha256_hex(body.as_ref()), expected_hash, "checksum changed for {key} after decommission");
|
||||
}
|
||||
for (version_id, expected) in [(&version_one_id, &version_one), (&version_two_id, &version_two)] {
|
||||
let got = after_client
|
||||
.get_object()
|
||||
.bucket(&bucket)
|
||||
.key(versioned_key)
|
||||
.version_id(version_id)
|
||||
.send()
|
||||
.await?
|
||||
.body
|
||||
.collect()
|
||||
.await?
|
||||
.into_bytes();
|
||||
assert_eq!(got.as_ref(), expected.as_slice(), "version {version_id} changed after decommission");
|
||||
}
|
||||
let mut expected_multipart = first_part;
|
||||
expected_multipart.extend_from_slice(&second_part);
|
||||
let got_multipart = after_client
|
||||
.get_object()
|
||||
.bucket(&bucket)
|
||||
.key(multipart_key)
|
||||
.send()
|
||||
.await?
|
||||
.body
|
||||
.collect()
|
||||
.await?
|
||||
.into_bytes();
|
||||
assert_eq!(
|
||||
sha256_hex(got_multipart.as_ref()),
|
||||
sha256_hex(&expected_multipart),
|
||||
"multipart checksum changed after decommission"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,81 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{
|
||||
DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, list_pools_json, put_inventory,
|
||||
put_inventory_retrying, start_decommission, start_rebalance, unique_bucket, wait_for_decommission_active,
|
||||
wait_for_decommission_complete, wait_for_rebalance_active, wait_for_rebalance_complete,
|
||||
};
|
||||
use crate::common::init_logging;
|
||||
use std::time::Duration;
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_pool_expand_preserves_objects_then_rebalance() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
|
||||
let bucket = unique_bucket("expand");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let client = dist.client(0)?;
|
||||
let inventory = put_inventory(&client, &bucket, 64, 256 * 1024).await?;
|
||||
assert_inventory(&client, &bucket, &inventory).await?;
|
||||
|
||||
for expected_nodes in 2..=4 {
|
||||
let new_node = dist.append_pool_and_restart().await?;
|
||||
assert_eq!(new_node + 1, expected_nodes);
|
||||
assert_inventory(&dist.client(new_node)?, &bucket, &inventory).await?;
|
||||
}
|
||||
assert_eq!(dist.cluster.nodes.len(), 4);
|
||||
|
||||
// Prove that the expanded pool map is durable, and clear any recovery
|
||||
// latch raised while the newly-added pool replicas converged.
|
||||
dist.restart_current_binary_gracefully().await?;
|
||||
|
||||
let after_expand = dist.client(0)?;
|
||||
assert_inventory(&after_expand, &bucket, &inventory).await?;
|
||||
let peer = dist.client(3)?;
|
||||
assert_inventory(&peer, &bucket, &inventory).await?;
|
||||
|
||||
let rebalance_id = start_rebalance(&dist.cluster).await?;
|
||||
wait_for_rebalance_active(&dist.cluster, &rebalance_id, Duration::from_secs(30)).await?;
|
||||
wait_for_rebalance_complete(&dist.cluster, &rebalance_id, Duration::from_secs(180)).await?;
|
||||
assert_inventory(&peer, &bucket, &inventory).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_pool_decommission_moves_objects_without_loss() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
|
||||
let bucket = unique_bucket("decom");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let client = dist.client(0)?;
|
||||
let inventory = put_inventory_retrying(&client, &bucket, 96, 128 * 1024, Duration::from_secs(30)).await?;
|
||||
dist.expand_to_four_pools().await?;
|
||||
|
||||
let pools_before = list_pools_json(&dist.cluster).await?;
|
||||
let pool_count = pools_before
|
||||
.as_array()
|
||||
.map(Vec::len)
|
||||
.or_else(|| pools_before.get("pools").and_then(serde_json::Value::as_array).map(Vec::len))
|
||||
.ok_or_else(|| format!("pool list omitted an array: {pools_before}"))?;
|
||||
assert_eq!(pool_count, 4, "expected exactly four pools before decommission: {pools_before}");
|
||||
|
||||
start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
|
||||
wait_for_decommission_active(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
|
||||
wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
|
||||
|
||||
let after = dist.client(2)?;
|
||||
assert_inventory(&after, &bucket, &inventory).await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,149 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{
|
||||
DistCluster, DistLayout, TestResult, assert_object_bytes, get_object_bytes, put_object, unique_bucket, wait_until,
|
||||
};
|
||||
use crate::common::init_logging;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
use std::time::Duration;
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_four_drive_multipart_and_cross_node_listing_agree() -> TestResult {
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("extra");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let client = dist.client(0)?;
|
||||
|
||||
let key = "multipart.bin";
|
||||
let part1 = vec![0x41u8; 5 * 1024 * 1024];
|
||||
let part2 = vec![0x42u8; 5 * 1024 * 1024];
|
||||
let upload = client.create_multipart_upload().bucket(&bucket).key(key).send().await?;
|
||||
let upload_id = upload.upload_id().ok_or("missing upload id")?.to_string();
|
||||
|
||||
let uploaded1 = client
|
||||
.upload_part()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(1)
|
||||
.body(ByteStream::from(part1.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
let uploaded2 = client
|
||||
.upload_part()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(2)
|
||||
.body(ByteStream::from(part2.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
client
|
||||
.complete_multipart_upload()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.upload_id(&upload_id)
|
||||
.multipart_upload(
|
||||
CompletedMultipartUpload::builder()
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(1)
|
||||
.e_tag(uploaded1.e_tag().unwrap_or_default())
|
||||
.build(),
|
||||
)
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(2)
|
||||
.e_tag(uploaded2.e_tag().unwrap_or_default())
|
||||
.build(),
|
||||
)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let mut expected = part1;
|
||||
expected.extend_from_slice(&part2);
|
||||
for node_idx in 0..dist.cluster.nodes.len() {
|
||||
assert_object_bytes(&dist.client(node_idx)?, &bucket, key, &expected).await?;
|
||||
}
|
||||
|
||||
put_object(&client, &bucket, "list/a", b"a".to_vec()).await?;
|
||||
put_object(&dist.client(2)?, &bucket, "list/b", b"b".to_vec()).await?;
|
||||
let mut seen = Vec::new();
|
||||
for node_idx in 0..dist.cluster.nodes.len() {
|
||||
let listed = dist
|
||||
.client(node_idx)?
|
||||
.list_objects_v2()
|
||||
.bucket(&bucket)
|
||||
.prefix("list/")
|
||||
.send()
|
||||
.await?;
|
||||
let keys: Vec<String> = listed
|
||||
.contents()
|
||||
.iter()
|
||||
.filter_map(|object| object.key().map(str::to_string))
|
||||
.collect();
|
||||
seen.push(keys);
|
||||
}
|
||||
for keys in &seen[1..] {
|
||||
assert_eq!(&seen[0], keys, "list results diverged across nodes: {seen:?}");
|
||||
}
|
||||
|
||||
let got = get_object_bytes(&dist.client(3)?, &bucket, "list/a").await?;
|
||||
assert_eq!(got, b"a");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_list_buckets_agree_across_all_nodes() -> TestResult {
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("listed");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
put_object(&dist.client(0)?, &bucket, "seed.bin", b"seed".to_vec()).await?;
|
||||
|
||||
for node_idx in 0..dist.cluster.nodes.len() {
|
||||
let client = dist.client(node_idx)?;
|
||||
let name = bucket.clone();
|
||||
wait_until(
|
||||
Duration::from_secs(20),
|
||||
|| {
|
||||
let client = client.clone();
|
||||
let name = name.clone();
|
||||
async move {
|
||||
let listed = client.list_buckets().send().await?;
|
||||
Ok(listed.buckets().iter().any(|entry| entry.name() == Some(name.as_str())))
|
||||
}
|
||||
},
|
||||
&format!("node {node_idx} lists {bucket}"),
|
||||
)
|
||||
.await?;
|
||||
wait_until(
|
||||
Duration::from_secs(20),
|
||||
|| {
|
||||
let client = dist.client(node_idx).expect("client");
|
||||
let name = bucket.clone();
|
||||
async move { Ok(get_object_bytes(&client, &name, "seed.bin").await.ok() == Some(b"seed".to_vec())) }
|
||||
},
|
||||
&format!("node {node_idx} reads seed.bin"),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,35 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! 4-node 4-drive distributed e2e coverage.
|
||||
//!
|
||||
//! Selected by `[profile.e2e-distributed]` and run from
|
||||
//! `.github/workflows/e2e-distributed.yml`. Excluded from `e2e-full` because
|
||||
//! each case starts four real `rustfs` processes.
|
||||
|
||||
mod chaos_test;
|
||||
mod concurrency_stability_test;
|
||||
mod concurrent_data_movement_test;
|
||||
mod data_integrity_movement_test;
|
||||
mod expand_decommission_rebalance_test;
|
||||
mod extra_test;
|
||||
mod harness;
|
||||
mod object_lock_test;
|
||||
mod observability_test;
|
||||
mod replication_quota_test;
|
||||
mod s3_basic_test;
|
||||
mod s3_during_data_movement_test;
|
||||
mod site_replication_test;
|
||||
mod upgrade_test;
|
||||
mod versioning_test;
|
||||
@@ -1,219 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, unique_bucket};
|
||||
use crate::common::init_logging;
|
||||
use crate::object_lock::common::{
|
||||
delete_object_with_bypass, put_object_lock_configuration, put_object_with_legal_hold, put_object_with_retention,
|
||||
};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::error::SdkError;
|
||||
use aws_sdk_s3::operation::delete_object::DeleteObjectError;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
DefaultRetention, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
|
||||
ObjectLockRule,
|
||||
};
|
||||
use chrono::{Duration as ChronoDuration, Utc};
|
||||
|
||||
fn delete_denied(error: &SdkError<DeleteObjectError>, context: &str) -> TestResult {
|
||||
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
|
||||
if code == Some("AccessDenied") {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(format!("{context}: expected AccessDenied, got {error:?}").into())
|
||||
}
|
||||
}
|
||||
|
||||
async fn expect_versioned_delete_denied(
|
||||
client: &Client,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
version_id: &str,
|
||||
bypass: bool,
|
||||
context: &str,
|
||||
) -> TestResult {
|
||||
match delete_object_with_bypass(client, bucket, key, Some(version_id), bypass).await {
|
||||
Ok(_) => Err(format!("{context}: DeleteObject of retained version must be denied").into()),
|
||||
Err(error) => delete_denied(error.as_ref(), context),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_four_drive_object_lock_worm_blocks_delete() -> TestResult {
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let client = dist.client(0)?;
|
||||
let peer = dist.client(2)?;
|
||||
let bucket = unique_bucket("objlock");
|
||||
|
||||
client
|
||||
.create_bucket()
|
||||
.bucket(&bucket)
|
||||
.object_lock_enabled_for_bucket(true)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let retain_until = Utc::now() + ChronoDuration::days(1);
|
||||
|
||||
let compliance_key = "compliance.bin";
|
||||
let compliance_version = put_object_with_retention(
|
||||
&client,
|
||||
&bucket,
|
||||
compliance_key,
|
||||
b"locked-compliance",
|
||||
ObjectLockRetentionMode::Compliance,
|
||||
retain_until,
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Unversioned DELETE is allowed: it only creates a delete marker. WORM
|
||||
// applies to a specific version id.
|
||||
let marker = peer.delete_object().bucket(&bucket).key(compliance_key).send().await?;
|
||||
assert_eq!(
|
||||
marker.delete_marker(),
|
||||
Some(true),
|
||||
"unversioned DELETE on a locked object must create a delete marker"
|
||||
);
|
||||
|
||||
expect_versioned_delete_denied(&peer, &bucket, compliance_key, &compliance_version, false, "COMPLIANCE without bypass")
|
||||
.await?;
|
||||
expect_versioned_delete_denied(&peer, &bucket, compliance_key, &compliance_version, true, "COMPLIANCE with bypass").await?;
|
||||
|
||||
let governance_key = "governance.bin";
|
||||
let governance_version = put_object_with_retention(
|
||||
&client,
|
||||
&bucket,
|
||||
governance_key,
|
||||
b"locked-governance",
|
||||
ObjectLockRetentionMode::Governance,
|
||||
retain_until,
|
||||
)
|
||||
.await?;
|
||||
|
||||
expect_versioned_delete_denied(&peer, &bucket, governance_key, &governance_version, false, "GOVERNANCE without bypass")
|
||||
.await?;
|
||||
delete_object_with_bypass(&peer, &bucket, governance_key, Some(&governance_version), true).await?;
|
||||
let deleted_governance = peer
|
||||
.head_object()
|
||||
.bucket(&bucket)
|
||||
.key(governance_key)
|
||||
.version_id(&governance_version)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("GOVERNANCE bypass must remove the retained version");
|
||||
assert_eq!(
|
||||
deleted_governance.raw_response().map(|response| response.status().as_u16()),
|
||||
Some(404),
|
||||
"deleted GOVERNANCE version returned an unexpected HEAD result: {deleted_governance:?}"
|
||||
);
|
||||
|
||||
let hold_key = "legal-hold.bin";
|
||||
let hold_version =
|
||||
put_object_with_legal_hold(&client, &bucket, hold_key, b"legal-hold", ObjectLockLegalHoldStatus::On).await?;
|
||||
expect_versioned_delete_denied(&peer, &bucket, hold_key, &hold_version, false, "legal hold without bypass").await?;
|
||||
expect_versioned_delete_denied(&peer, &bucket, hold_key, &hold_version, true, "legal hold with bypass").await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_default_retention_is_visible_and_non_lock_bucket_rejects_configuration() -> TestResult {
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let writer = dist.client(0)?;
|
||||
let reader = dist.client(3)?;
|
||||
let bucket = unique_bucket("default-lock");
|
||||
|
||||
writer
|
||||
.create_bucket()
|
||||
.bucket(&bucket)
|
||||
.object_lock_enabled_for_bucket(true)
|
||||
.send()
|
||||
.await?;
|
||||
put_object_lock_configuration(&writer, &bucket, ObjectLockRetentionMode::Governance, Some(1), None).await?;
|
||||
|
||||
let key = "default-governance.bin";
|
||||
let put = writer
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from_static(b"default retention payload"))
|
||||
.send()
|
||||
.await?;
|
||||
let version_id = put.version_id().ok_or("default-retained PUT omitted version ID")?;
|
||||
|
||||
let config = reader.get_object_lock_configuration().bucket(&bucket).send().await?;
|
||||
let default_retention = config
|
||||
.object_lock_configuration()
|
||||
.and_then(|configuration| configuration.rule())
|
||||
.and_then(|rule| rule.default_retention())
|
||||
.ok_or("GetObjectLockConfiguration omitted default retention")?;
|
||||
assert_eq!(default_retention.mode().map(|mode| mode.as_str()), Some("GOVERNANCE"));
|
||||
assert_eq!(default_retention.days(), Some(1));
|
||||
|
||||
let retention = reader
|
||||
.get_object_retention()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.version_id(version_id)
|
||||
.send()
|
||||
.await?;
|
||||
let retention = retention.retention().ok_or("GetObjectRetention omitted applied retention")?;
|
||||
assert_eq!(retention.mode().map(|mode| mode.as_str()), Some("GOVERNANCE"));
|
||||
let retain_until = retention
|
||||
.retain_until_date()
|
||||
.ok_or("default retention omitted retain-until date")?;
|
||||
assert!(retain_until.secs() > Utc::now().timestamp(), "default retention is not in the future");
|
||||
|
||||
let versioning = reader.get_bucket_versioning().bucket(&bucket).send().await?;
|
||||
assert_eq!(versioning.status().map(|status| status.as_str()), Some("Enabled"));
|
||||
expect_versioned_delete_denied(&reader, &bucket, key, version_id, false, "default GOVERNANCE retention without bypass")
|
||||
.await?;
|
||||
|
||||
let plain_bucket = unique_bucket("no-lock");
|
||||
dist.create_bucket(&plain_bucket).await?;
|
||||
let configuration = ObjectLockConfiguration::builder()
|
||||
.object_lock_enabled(ObjectLockEnabled::Enabled)
|
||||
.rule(
|
||||
ObjectLockRule::builder()
|
||||
.default_retention(
|
||||
DefaultRetention::builder()
|
||||
.mode(ObjectLockRetentionMode::Governance)
|
||||
.days(1)
|
||||
.build(),
|
||||
)
|
||||
.build(),
|
||||
)
|
||||
.build();
|
||||
let error = writer
|
||||
.put_object_lock_configuration()
|
||||
.bucket(&plain_bucket)
|
||||
.object_lock_configuration(configuration)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("an unversioned bucket must reject Object Lock enablement");
|
||||
let service_error = error
|
||||
.as_service_error()
|
||||
.ok_or("non-lock bucket rejection was not an S3 service error")?;
|
||||
assert_eq!(service_error.code(), Some("InvalidBucketState"), "unexpected error: {error:?}");
|
||||
assert_eq!(
|
||||
service_error.message(),
|
||||
Some("Object Lock configuration cannot be enabled on existing buckets"),
|
||||
"unexpected error: {error:?}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,236 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, cluster_admin_ok, unique_bucket, wait_for_ready};
|
||||
use crate::common::{admin_request, init_logging, local_http_client};
|
||||
use aws_sdk_s3::operation::RequestId;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use bytes::Bytes;
|
||||
use http::Method;
|
||||
use http_body_util::{BodyExt, Empty};
|
||||
use hyper::body::Incoming;
|
||||
use hyper::service::service_fn;
|
||||
use hyper::{Request, Response};
|
||||
use hyper_util::rt::TokioIo;
|
||||
use local_ip_address::local_ip;
|
||||
use rustfs_madmin::metrics::RealtimeMetrics;
|
||||
use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
|
||||
use serde_json::Value;
|
||||
use std::convert::Infallible;
|
||||
use std::time::Duration;
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::task::JoinHandle;
|
||||
use tokio::time::{Instant, timeout};
|
||||
|
||||
async fn spawn_audit_collector() -> TestResult<(String, mpsc::UnboundedReceiver<Value>, JoinHandle<()>)> {
|
||||
let listener = TcpListener::bind("0.0.0.0:0").await?;
|
||||
let endpoint = format!("http://{}/audit", std::net::SocketAddr::new(local_ip()?, listener.local_addr()?.port()));
|
||||
let (tx, rx) = mpsc::unbounded_channel();
|
||||
let handle = tokio::spawn(async move {
|
||||
loop {
|
||||
let Ok((stream, _)) = listener.accept().await else {
|
||||
return;
|
||||
};
|
||||
let tx = tx.clone();
|
||||
tokio::spawn(async move {
|
||||
let service = service_fn(move |request: Request<Incoming>| {
|
||||
let tx = tx.clone();
|
||||
async move {
|
||||
let method = request.method().clone();
|
||||
if let Ok(body) = request.into_body().collect().await
|
||||
&& method == Method::POST
|
||||
&& let Ok(payload) = serde_json::from_slice::<Value>(&body.to_bytes())
|
||||
{
|
||||
if let Some(records) = payload["Records"].as_array() {
|
||||
for entry in records {
|
||||
let _ = tx.send(entry.clone());
|
||||
}
|
||||
} else {
|
||||
let _ = tx.send(payload);
|
||||
}
|
||||
}
|
||||
Ok::<_, Infallible>(Response::new(Empty::<Bytes>::new()))
|
||||
}
|
||||
});
|
||||
let _ = hyper::server::conn::http1::Builder::new()
|
||||
.serve_connection(TokioIo::new(stream), service)
|
||||
.await;
|
||||
});
|
||||
}
|
||||
});
|
||||
Ok((endpoint, rx, handle))
|
||||
}
|
||||
|
||||
async fn wait_for_audit_entry(
|
||||
rx: &mut mpsc::UnboundedReceiver<Value>,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
request_id: &str,
|
||||
) -> TestResult<Value> {
|
||||
let deadline = Instant::now() + Duration::from_secs(30);
|
||||
let mut seen = Vec::new();
|
||||
loop {
|
||||
let remaining = deadline.saturating_duration_since(Instant::now());
|
||||
if remaining.is_zero() {
|
||||
return Err(format!(
|
||||
"audit webhook did not receive PutObject for {bucket}/{key}; received {} other records: {seen:?}",
|
||||
seen.len()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
let entry = match timeout(remaining, rx.recv()).await {
|
||||
Ok(Some(entry)) => entry,
|
||||
Ok(None) => return Err("audit collector stopped before the expected entry arrived".into()),
|
||||
Err(_) => {
|
||||
return Err(format!(
|
||||
"audit webhook did not receive PutObject for {bucket}/{key}; received {} other records: {seen:?}",
|
||||
seen.len()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
};
|
||||
if entry["api"]["name"].as_str() == Some("s3:PutObject")
|
||||
&& entry["api"]["bucket"].as_str() == Some(bucket)
|
||||
&& entry["api"]["object"].as_str() == Some(key)
|
||||
&& entry["requestID"].as_str() == Some(request_id)
|
||||
{
|
||||
return Ok(entry);
|
||||
}
|
||||
if seen.len() < 8 {
|
||||
seen.push(format!(
|
||||
"api={:?} bucket={:?} object={:?} requestID={:?}",
|
||||
entry["api"]["name"].as_str(),
|
||||
entry["api"]["bucket"].as_str(),
|
||||
entry["api"]["object"].as_str(),
|
||||
entry["requestID"].as_str()
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_health_inventory_metrics_and_audit_delivery_are_consistent() -> TestResult {
|
||||
init_logging();
|
||||
let (audit_endpoint, mut audit_entries, collector) = spawn_audit_collector().await?;
|
||||
let audit_origin = reqwest::Url::parse(&audit_endpoint)?.origin().ascii_serialization();
|
||||
let audit_env = [
|
||||
("RUSTFS_AUDIT_ENABLE", "true"),
|
||||
("RUSTFS_AUDIT_WEBHOOK_ENABLE_DISTRIBUTED", "on"),
|
||||
("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_DISTRIBUTED", audit_endpoint.as_str()),
|
||||
(ENV_OUTBOUND_ALLOW_ORIGINS, audit_origin.as_str()),
|
||||
];
|
||||
let mut dist = DistCluster::new_stopped_with_env(DistLayout::FourByFour, &audit_env).await?;
|
||||
for node_idx in 0..dist.cluster.nodes.len() {
|
||||
let queue_dir = format!("{}/audit-queue-node-{node_idx}", dist.cluster.temp_dir);
|
||||
tokio::fs::create_dir_all(&queue_dir).await?;
|
||||
dist.cluster
|
||||
.set_node_env(node_idx, "RUSTFS_AUDIT_WEBHOOK_QUEUE_DIR_DISTRIBUTED", queue_dir)?;
|
||||
}
|
||||
dist.cluster.start().await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
|
||||
let http = local_http_client();
|
||||
for node in &dist.cluster.nodes {
|
||||
for probe in ["ready", "live"] {
|
||||
let response = http.get(format!("{}/health/{probe}", node.url)).send().await?;
|
||||
assert!(
|
||||
response.status().is_success(),
|
||||
"node {} {probe} probe failed: {}",
|
||||
node.address,
|
||||
response.status()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let info_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/info", None).await?;
|
||||
let info: Value = serde_json::from_str(&info_body)?;
|
||||
let servers = info["info"]["servers"]
|
||||
.as_array()
|
||||
.ok_or_else(|| format!("admin info omitted servers: {info}"))?;
|
||||
assert_eq!(servers.len(), 4, "admin info did not report all four nodes: {info}");
|
||||
|
||||
let storage_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/storageinfo", None).await?;
|
||||
let storage: Value = serde_json::from_str(&storage_body)?;
|
||||
let disks = storage["info"]["disks"]
|
||||
.as_array()
|
||||
.ok_or_else(|| format!("storageinfo omitted disks: {storage}"))?;
|
||||
assert_eq!(disks.len(), 16, "storageinfo did not report all sixteen drives: {storage}");
|
||||
assert!(
|
||||
disks.iter().all(|disk| {
|
||||
disk["state"].as_str().is_some_and(|state| state.eq_ignore_ascii_case("ok"))
|
||||
&& disk["runtimeState"]
|
||||
.as_str()
|
||||
.is_some_and(|state| state.eq_ignore_ascii_case("online"))
|
||||
}),
|
||||
"storageinfo reported a drive that was not healthy and online: {storage}"
|
||||
);
|
||||
|
||||
for (node_idx, node) in dist.cluster.nodes.iter().enumerate() {
|
||||
let (status, metrics_body) = admin_request(
|
||||
&node.url,
|
||||
Method::GET,
|
||||
"/rustfs/admin/v3/metrics?n=1&by-host=true&by-disk=true",
|
||||
None,
|
||||
&dist.cluster.access_key,
|
||||
&dist.cluster.secret_key,
|
||||
)
|
||||
.await?;
|
||||
assert!(status.is_success(), "node {node_idx} metrics failed: {status} {metrics_body}");
|
||||
let sample: RealtimeMetrics = serde_json::from_str(
|
||||
metrics_body
|
||||
.lines()
|
||||
.next()
|
||||
.ok_or_else(|| format!("node {node_idx} returned empty metrics"))?,
|
||||
)?;
|
||||
assert!(sample.finally, "node {node_idx} metrics sample was not terminal");
|
||||
assert!(sample.errors.is_empty(), "node {node_idx} metrics reported errors: {:?}", sample.errors);
|
||||
assert!(!sample.hosts.is_empty(), "node {node_idx} metrics omitted hosts");
|
||||
}
|
||||
|
||||
let targets_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/audit/target/list", None).await?;
|
||||
let targets: Value = serde_json::from_str(&targets_body)?;
|
||||
let configured = targets["audit_endpoints"]
|
||||
.as_array()
|
||||
.ok_or_else(|| format!("audit target list omitted audit_endpoints: {targets}"))?
|
||||
.iter()
|
||||
.any(|target| target["account_id"].as_str() == Some("distributed") && target["service"].as_str() == Some("webhook"));
|
||||
assert!(configured, "configured audit webhook was missing: {targets}");
|
||||
|
||||
let bucket = unique_bucket("audit");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let key = "correlated/audit-object.bin";
|
||||
let put = dist
|
||||
.client(2)?
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from_static(b"distributed audit payload"))
|
||||
.send()
|
||||
.await?;
|
||||
let request_id = put.request_id().ok_or("PutObject response omitted request ID")?;
|
||||
let audit = wait_for_audit_entry(&mut audit_entries, &bucket, key, request_id).await?;
|
||||
assert_eq!(
|
||||
audit["api"]["status_code"].as_i64(),
|
||||
Some(200),
|
||||
"audit entry did not report success: {audit}"
|
||||
);
|
||||
assert!(
|
||||
!audit.to_string().contains(&dist.cluster.secret_key),
|
||||
"audit entry leaked the root secret key"
|
||||
);
|
||||
|
||||
collector.abort();
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,191 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{
|
||||
DistCluster, DistLayout, TestResult, enable_versioning, put_bucket_replication, put_object, retrying_put, set_bucket_quota,
|
||||
set_remote_target, unique_bucket, wait_for_ready, wait_for_replicated_bytes, wait_until,
|
||||
};
|
||||
use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, init_logging};
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use http::Method;
|
||||
use std::time::Duration;
|
||||
|
||||
async fn wait_for_replication_status(
|
||||
client: &aws_sdk_s3::Client,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
expected: &[&str],
|
||||
timeout: Duration,
|
||||
) -> TestResult {
|
||||
wait_until(
|
||||
timeout,
|
||||
|| async {
|
||||
let head = client.head_object().bucket(bucket).key(key).send().await?;
|
||||
Ok(head
|
||||
.replication_status()
|
||||
.is_some_and(|status| expected.contains(&status.as_str())))
|
||||
},
|
||||
&format!("replication status for {bucket}/{key} in {expected:?}"),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_bucket_replication_converges_to_peer_cluster() -> TestResult {
|
||||
init_logging();
|
||||
let (source, mut target) = DistCluster::start_replication_pair().await?;
|
||||
let source_bucket = unique_bucket("replsrc");
|
||||
let target_bucket = unique_bucket("repldst");
|
||||
source.create_bucket(&source_bucket).await?;
|
||||
target.create_bucket(&target_bucket).await?;
|
||||
|
||||
let source_client = source.client(0)?;
|
||||
let target_client = target.client(0)?;
|
||||
enable_versioning(&source_client, &source_bucket).await?;
|
||||
enable_versioning(&target_client, &target_bucket).await?;
|
||||
|
||||
let arn = set_remote_target(&source.cluster, &source_bucket, &target.cluster, &target_bucket).await?;
|
||||
put_bucket_replication(&source.cluster, &source_bucket, &arn).await?;
|
||||
|
||||
let key = "replicated/metadata-and-tags.bin";
|
||||
let body = b"distributed-bucket-replication".to_vec();
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(&source_bucket)
|
||||
.key(key)
|
||||
.metadata("origin", "four-node-source")
|
||||
.tagging("suite=distributed&shape=metadata")
|
||||
.body(ByteStream::from(body.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
wait_for_replicated_bytes(&target_client, &target_bucket, key, &body, Duration::from_secs(45)).await?;
|
||||
wait_for_replication_status(&source_client, &source_bucket, key, &["COMPLETED"], Duration::from_secs(30)).await?;
|
||||
|
||||
let peer_read = target.client(3)?;
|
||||
wait_for_replicated_bytes(&peer_read, &target_bucket, key, &body, Duration::from_secs(15)).await?;
|
||||
let replica_head = peer_read.head_object().bucket(&target_bucket).key(key).send().await?;
|
||||
assert_eq!(
|
||||
replica_head
|
||||
.metadata()
|
||||
.and_then(|metadata| metadata.get("origin"))
|
||||
.map(String::as_str),
|
||||
Some("four-node-source")
|
||||
);
|
||||
assert_eq!(replica_head.replication_status().map(|status| status.as_str()), Some("REPLICA"));
|
||||
let replica_tags = peer_read.get_object_tagging().bucket(&target_bucket).key(key).send().await?;
|
||||
let tags: std::collections::BTreeMap<_, _> = replica_tags.tag_set().iter().map(|tag| (tag.key(), tag.value())).collect();
|
||||
assert_eq!(tags.get("suite"), Some(&"distributed"));
|
||||
assert_eq!(tags.get("shape"), Some(&"metadata"));
|
||||
|
||||
target.cluster.stop();
|
||||
let outage_key = "replicated/queued-during-target-outage.bin";
|
||||
let outage_body = b"retry-after-target-restart".to_vec();
|
||||
put_object(&source_client, &source_bucket, outage_key, outage_body.clone()).await?;
|
||||
wait_for_replication_status(
|
||||
&source_client,
|
||||
&source_bucket,
|
||||
outage_key,
|
||||
&["PENDING", "FAILED"],
|
||||
Duration::from_secs(30),
|
||||
)
|
||||
.await?;
|
||||
|
||||
target.cluster.start().await?;
|
||||
wait_for_ready(&target.cluster).await?;
|
||||
wait_for_replicated_bytes(&target.client(2)?, &target_bucket, outage_key, &outage_body, Duration::from_secs(90)).await?;
|
||||
wait_for_replication_status(&source_client, &source_bucket, outage_key, &["COMPLETED"], Duration::from_secs(45)).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_four_drive_hard_quota_rejects_over_limit_put() -> TestResult {
|
||||
init_logging();
|
||||
let dist = DistCluster::start_with_env(DistLayout::FourByFour, FAST_DATA_USAGE_SCANNER_ENV).await?;
|
||||
let bucket = unique_bucket("quota");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
set_bucket_quota(&dist.cluster, &bucket, 8 * 1024).await?;
|
||||
|
||||
let client = dist.client(1)?;
|
||||
retrying_put(&client, &bucket, "small.bin", vec![0u8; 1024], Duration::from_secs(30)).await?;
|
||||
wait_until(
|
||||
Duration::from_secs(30),
|
||||
|| async {
|
||||
let (status, body) = super::harness::cluster_admin(
|
||||
&dist.cluster,
|
||||
Method::GET,
|
||||
&format!("/rustfs/admin/v3/quota-stats/{bucket}"),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
if !status.is_success() {
|
||||
return Ok(false);
|
||||
}
|
||||
let stats: serde_json::Value =
|
||||
serde_json::from_str(&body).map_err(|error| format!("quota stats returned invalid JSON: {error}: {body}"))?;
|
||||
let usage = stats
|
||||
.get("current_usage")
|
||||
.and_then(serde_json::Value::as_u64)
|
||||
.ok_or_else(|| format!("quota stats omitted current_usage: {stats}"))?;
|
||||
Ok(usage >= 1024)
|
||||
},
|
||||
"quota stats observe small object",
|
||||
)
|
||||
.await?;
|
||||
|
||||
let oversized_key = "too-big.bin";
|
||||
let error = client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(oversized_key)
|
||||
.body(vec![0u8; 16 * 1024].into())
|
||||
.send()
|
||||
.await
|
||||
.expect_err("hard quota must reject the oversized PUT");
|
||||
let service_error = error
|
||||
.as_service_error()
|
||||
.ok_or("quota rejection was not an S3 service error")?;
|
||||
assert_eq!(
|
||||
error.raw_response().map(|response| response.status().as_u16()),
|
||||
Some(400),
|
||||
"quota rejection must be HTTP 400: {error:?}"
|
||||
);
|
||||
assert_eq!(service_error.code(), Some("InvalidRequest"), "unexpected quota error: {error:?}");
|
||||
assert!(
|
||||
service_error
|
||||
.message()
|
||||
.is_some_and(|message| message.starts_with("Bucket quota exceeded")),
|
||||
"PUT must fail specifically at quota admission: {error:?}"
|
||||
);
|
||||
|
||||
let missing = client
|
||||
.head_object()
|
||||
.bucket(&bucket)
|
||||
.key(oversized_key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("an object rejected by quota must not become visible");
|
||||
assert_eq!(
|
||||
missing.raw_response().map(|response| response.status().as_u16()),
|
||||
Some(404),
|
||||
"quota-rejected object returned an unexpected HEAD result: {missing:?}"
|
||||
);
|
||||
|
||||
let listed = client.list_objects_v2().bucket(&bucket).send().await?;
|
||||
assert!(
|
||||
listed.contents().iter().all(|object| object.key() != Some(oversized_key)),
|
||||
"quota-rejected key leaked into ListObjectsV2"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,258 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, get_object_bytes, put_object, unique_bucket};
|
||||
use crate::common::{init_logging, local_http_client};
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::presigning::PresigningConfig;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{Delete, MetadataDirective, ObjectIdentifier};
|
||||
use std::time::Duration;
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_four_drive_s3_put_get_head_list_copy_rename_delete_and_presign() -> TestResult {
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("s3basic");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
|
||||
let writer = dist.client(0)?;
|
||||
let reader = dist.client(3)?;
|
||||
let key = "dir/object.bin";
|
||||
let body = vec![0xA5u8; 256 * 1024];
|
||||
put_object(&writer, &bucket, key, body.clone()).await?;
|
||||
|
||||
let head = reader.head_object().bucket(&bucket).key(key).send().await?;
|
||||
assert_eq!(head.content_length(), Some(body.len() as i64));
|
||||
assert_object_bytes(&reader, &bucket, key, &body).await?;
|
||||
|
||||
let ranged = reader
|
||||
.get_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.range("bytes=0-15")
|
||||
.send()
|
||||
.await?;
|
||||
let ranged_body = ranged.body.collect().await?.into_bytes();
|
||||
assert_eq!(ranged_body.as_ref(), &body[..16]);
|
||||
|
||||
let listed = reader.list_objects_v2().bucket(&bucket).prefix("dir/").send().await?;
|
||||
let keys: Vec<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
|
||||
assert_eq!(keys, vec![key]);
|
||||
|
||||
let copy_key = "dir/object-copy.bin";
|
||||
reader
|
||||
.copy_object()
|
||||
.bucket(&bucket)
|
||||
.key(copy_key)
|
||||
.copy_source(format!("{bucket}/{key}"))
|
||||
.metadata_directive(MetadataDirective::Copy)
|
||||
.send()
|
||||
.await?;
|
||||
assert_object_bytes(&writer, &bucket, copy_key, &body).await?;
|
||||
|
||||
let moved_key = "dir/object-moved.bin";
|
||||
writer
|
||||
.copy_object()
|
||||
.bucket(&bucket)
|
||||
.key(moved_key)
|
||||
.copy_source(format!("{bucket}/{copy_key}"))
|
||||
.send()
|
||||
.await?;
|
||||
writer.delete_object().bucket(&bucket).key(copy_key).send().await?;
|
||||
match writer.head_object().bucket(&bucket).key(copy_key).send().await {
|
||||
Ok(_) => return Err("copied source still present after rename delete".into()),
|
||||
Err(error) if error.as_service_error().is_some_and(|err| err.is_not_found()) => {}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
assert_object_bytes(&reader, &bucket, moved_key, &body).await?;
|
||||
|
||||
let presigned = writer
|
||||
.get_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.presigned(PresigningConfig::expires_in(Duration::from_secs(120))?)
|
||||
.await?;
|
||||
let response = local_http_client().get(presigned.uri().to_string()).send().await?;
|
||||
assert!(response.status().is_success(), "presigned GET failed: {}", response.status());
|
||||
let presigned_body = response.bytes().await?;
|
||||
assert_eq!(presigned_body.as_ref(), body.as_slice());
|
||||
|
||||
let empty_key = "empty";
|
||||
put_object(&writer, &bucket, empty_key, Vec::new()).await?;
|
||||
let empty = get_object_bytes(&reader, &bucket, empty_key).await?;
|
||||
assert!(empty.is_empty());
|
||||
|
||||
let deleted = writer
|
||||
.delete_objects()
|
||||
.bucket(&bucket)
|
||||
.delete(
|
||||
Delete::builder()
|
||||
.objects(ObjectIdentifier::builder().key(key).build()?)
|
||||
.objects(ObjectIdentifier::builder().key(moved_key).build()?)
|
||||
.objects(ObjectIdentifier::builder().key(empty_key).build()?)
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
assert!(deleted.errors().is_empty(), "DeleteObjects reported failures: {deleted:?}");
|
||||
assert_eq!(deleted.deleted().len(), 3, "DeleteObjects did not acknowledge every key");
|
||||
|
||||
let remaining = reader.list_objects_v2().bucket(&bucket).send().await?;
|
||||
assert!(remaining.contents().is_empty(), "bucket still has objects after delete");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_s3_metadata_tags_special_keys_pagination_and_multipart_abort() -> TestResult {
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("s3matrix");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let writer = dist.client(0)?;
|
||||
let reader = dist.client(3)?;
|
||||
|
||||
let special_key = "unicode/测试 space+percent%25.txt";
|
||||
let special_body = b"metadata and tagging survive distributed routing".to_vec();
|
||||
let put = writer
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(special_key)
|
||||
.metadata("test-meta", "distributed")
|
||||
.tagging("purpose=compatibility&scope=four-by-four")
|
||||
.body(ByteStream::from(special_body.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
let etag = put.e_tag().ok_or("PutObject omitted ETag")?.to_string();
|
||||
|
||||
let head = reader.head_object().bucket(&bucket).key(special_key).send().await?;
|
||||
assert_eq!(
|
||||
head.metadata()
|
||||
.and_then(|metadata| metadata.get("test-meta"))
|
||||
.map(String::as_str),
|
||||
Some("distributed")
|
||||
);
|
||||
assert_eq!(head.e_tag(), Some(etag.as_str()));
|
||||
let tags = reader.get_object_tagging().bucket(&bucket).key(special_key).send().await?;
|
||||
let actual_tags: std::collections::BTreeMap<_, _> = tags
|
||||
.tag_set()
|
||||
.iter()
|
||||
.map(|tag| (tag.key().to_string(), tag.value().to_string()))
|
||||
.collect();
|
||||
assert_eq!(actual_tags.get("purpose").map(String::as_str), Some("compatibility"));
|
||||
assert_eq!(actual_tags.get("scope").map(String::as_str), Some("four-by-four"));
|
||||
|
||||
let conditional = reader
|
||||
.get_object()
|
||||
.bucket(&bucket)
|
||||
.key(special_key)
|
||||
.if_match(&etag)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(conditional.body.collect().await?.into_bytes().as_ref(), special_body.as_slice());
|
||||
let invalid_range = reader
|
||||
.get_object()
|
||||
.bucket(&bucket)
|
||||
.key(special_key)
|
||||
.range("bytes=999999-1000000")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("an unsatisfiable range must fail");
|
||||
assert_eq!(
|
||||
invalid_range.as_service_error().and_then(ProvideErrorMetadata::code),
|
||||
Some("InvalidRange"),
|
||||
"unexpected invalid-range error: {invalid_range:?}"
|
||||
);
|
||||
|
||||
let upload_key = "multipart/aborted.bin";
|
||||
let upload = writer
|
||||
.create_multipart_upload()
|
||||
.bucket(&bucket)
|
||||
.key(upload_key)
|
||||
.send()
|
||||
.await?;
|
||||
let upload_id = upload.upload_id().ok_or("CreateMultipartUpload omitted upload ID")?;
|
||||
writer
|
||||
.upload_part()
|
||||
.bucket(&bucket)
|
||||
.key(upload_key)
|
||||
.upload_id(upload_id)
|
||||
.part_number(1)
|
||||
.body(ByteStream::from(vec![0x5Au8; 5 * 1024 * 1024]))
|
||||
.send()
|
||||
.await?;
|
||||
let pending = reader
|
||||
.list_multipart_uploads()
|
||||
.bucket(&bucket)
|
||||
.prefix("multipart/")
|
||||
.send()
|
||||
.await?;
|
||||
assert!(pending.uploads().iter().any(|entry| entry.upload_id() == Some(upload_id)));
|
||||
writer
|
||||
.abort_multipart_upload()
|
||||
.bucket(&bucket)
|
||||
.key(upload_key)
|
||||
.upload_id(upload_id)
|
||||
.send()
|
||||
.await?;
|
||||
let after_abort = reader
|
||||
.list_multipart_uploads()
|
||||
.bucket(&bucket)
|
||||
.prefix("multipart/")
|
||||
.send()
|
||||
.await?;
|
||||
assert!(after_abort.uploads().iter().all(|entry| entry.upload_id() != Some(upload_id)));
|
||||
let aborted_head = reader
|
||||
.head_object()
|
||||
.bucket(&bucket)
|
||||
.key(upload_key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("aborted multipart upload must not create an object");
|
||||
assert_eq!(
|
||||
aborted_head.raw_response().map(|response| response.status().as_u16()),
|
||||
Some(404),
|
||||
"aborted multipart object returned an unexpected HEAD result: {aborted_head:?}"
|
||||
);
|
||||
|
||||
for index in 0..113 {
|
||||
let key = format!("page/{index:04}.txt");
|
||||
put_object(&writer, &bucket, &key, format!("page-{index}").into_bytes()).await?;
|
||||
}
|
||||
let mut token = None;
|
||||
let mut paged_keys = Vec::new();
|
||||
loop {
|
||||
let page = reader
|
||||
.list_objects_v2()
|
||||
.bucket(&bucket)
|
||||
.prefix("page/")
|
||||
.max_keys(37)
|
||||
.set_continuation_token(token.take())
|
||||
.send()
|
||||
.await?;
|
||||
paged_keys.extend(page.contents().iter().filter_map(|object| object.key().map(str::to_string)));
|
||||
if page.is_truncated() != Some(true) {
|
||||
break;
|
||||
}
|
||||
token = Some(
|
||||
page.next_continuation_token()
|
||||
.ok_or("truncated ListObjectsV2 page omitted next continuation token")?
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
assert_eq!(paged_keys.len(), 113);
|
||||
let expected: Vec<_> = (0..113).map(|index| format!("page/{index:04}.txt")).collect();
|
||||
assert_eq!(paged_keys, expected, "pagination lost, duplicated, or reordered keys");
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,94 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{
|
||||
DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, decommission_running_with_progress,
|
||||
decommission_status_json, put_inventory_retrying, rebalance_running_with_progress, rebalance_status_json,
|
||||
retrying_get_equals, retrying_put, start_decommission, start_rebalance, unique_bucket, wait_for_decommission_complete,
|
||||
wait_for_decommission_running_with_progress, wait_for_rebalance_complete, wait_for_rebalance_running_with_progress,
|
||||
};
|
||||
use crate::common::init_logging;
|
||||
use std::time::Duration;
|
||||
|
||||
#[tokio::test]
|
||||
async fn s3_put_get_list_succeed_during_decommission_and_rebalance() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
|
||||
let bucket = unique_bucket("s3move");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let client = dist.client(0)?;
|
||||
let inventory = put_inventory_retrying(&client, &bucket, 96, 256 * 1024, Duration::from_secs(30)).await?;
|
||||
dist.expand_to_four_pools().await?;
|
||||
|
||||
start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
|
||||
wait_for_decommission_running_with_progress(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
|
||||
let live = dist.client(2)?;
|
||||
retrying_put(
|
||||
&live,
|
||||
&bucket,
|
||||
"during-decommission.bin",
|
||||
b"written-while-decommissioning".to_vec(),
|
||||
Duration::from_secs(30),
|
||||
)
|
||||
.await?;
|
||||
retrying_get_equals(
|
||||
&live,
|
||||
&bucket,
|
||||
"during-decommission.bin",
|
||||
b"written-while-decommissioning",
|
||||
Duration::from_secs(30),
|
||||
)
|
||||
.await?;
|
||||
let listed = live.list_objects_v2().bucket(&bucket).send().await?;
|
||||
assert!(
|
||||
listed
|
||||
.contents()
|
||||
.iter()
|
||||
.any(|object| object.key() == Some("during-decommission.bin")),
|
||||
"list during decommission missed the newly written key"
|
||||
);
|
||||
let status = decommission_status_json(&dist.cluster).await?;
|
||||
if !decommission_running_with_progress(&status, DECOMMISSION_POOL_ID)? {
|
||||
return Err(format!("decommission did not remain active across the S3 operations: {status}").into());
|
||||
}
|
||||
|
||||
wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
|
||||
assert_inventory(&live, &bucket, &inventory).await?;
|
||||
|
||||
let rebalance_id = start_rebalance(&dist.cluster).await?;
|
||||
wait_for_rebalance_running_with_progress(&dist.cluster, &rebalance_id, Duration::from_secs(30)).await?;
|
||||
retrying_put(
|
||||
&live,
|
||||
&bucket,
|
||||
"during-rebalance.bin",
|
||||
b"written-while-rebalancing".to_vec(),
|
||||
Duration::from_secs(30),
|
||||
)
|
||||
.await?;
|
||||
retrying_get_equals(
|
||||
&live,
|
||||
&bucket,
|
||||
"during-rebalance.bin",
|
||||
b"written-while-rebalancing",
|
||||
Duration::from_secs(30),
|
||||
)
|
||||
.await?;
|
||||
let status = rebalance_status_json(&dist.cluster).await?;
|
||||
if !rebalance_running_with_progress(&status, &rebalance_id)? {
|
||||
return Err(format!("rebalance did not remain active across the S3 operations: {status}").into());
|
||||
}
|
||||
wait_for_rebalance_complete(&dist.cluster, &rebalance_id, Duration::from_secs(180)).await?;
|
||||
assert_inventory(&dist.client(1)?, &bucket, &inventory).await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,128 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{
|
||||
DistCluster, TestResult, cluster_admin_ok, enable_versioning, put_object, unique_bucket, wait_for_replicated_bytes,
|
||||
wait_until,
|
||||
};
|
||||
use crate::common::{init_logging, signed_request};
|
||||
use http::{Method, StatusCode};
|
||||
use rustfs_madmin::{PeerSite, ReplicateAddStatus, SiteReplicationInfo, SyncStatus};
|
||||
use std::time::Duration;
|
||||
|
||||
async fn site_replication_add(
|
||||
cluster: &crate::common::RustFSTestClusterEnvironment,
|
||||
sites: &[PeerSite],
|
||||
) -> TestResult<ReplicateAddStatus> {
|
||||
let url = format!("{}/rustfs/admin/v3/site-replication/add?replicateILMExpiry=false", cluster.nodes[0].url);
|
||||
let response = signed_request(
|
||||
Method::PUT,
|
||||
&url,
|
||||
&cluster.access_key,
|
||||
&cluster.secret_key,
|
||||
Some(serde_json::to_vec(sites)?),
|
||||
Some("application/json"),
|
||||
)
|
||||
.await?;
|
||||
if response.status() != StatusCode::OK {
|
||||
let status = response.status();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
return Err(format!("site replication add failed: {status} {body}").into());
|
||||
}
|
||||
Ok(serde_json::from_slice(&response.bytes().await?)?)
|
||||
}
|
||||
|
||||
async fn site_replication_info(cluster: &crate::common::RustFSTestClusterEnvironment) -> TestResult<SiteReplicationInfo> {
|
||||
let body = cluster_admin_ok(cluster, Method::GET, "/rustfs/admin/v3/site-replication/info", None).await?;
|
||||
Ok(serde_json::from_str(&body)?)
|
||||
}
|
||||
|
||||
async fn wait_for_site_replication_enabled(cluster: &crate::common::RustFSTestClusterEnvironment) -> TestResult {
|
||||
wait_until(
|
||||
Duration::from_secs(30),
|
||||
|| async {
|
||||
let info = site_replication_info(cluster).await?;
|
||||
Ok(info.enabled && info.sites.len() == 2 && info.sites.iter().all(|site| site.sync_state == SyncStatus::Enable))
|
||||
},
|
||||
"site replication enabled with two synchronized sites",
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_site_replication_replicates_object_to_peer_site() -> TestResult {
|
||||
init_logging();
|
||||
let (site_a, site_b) = DistCluster::start_replication_pair().await?;
|
||||
let bucket = unique_bucket("siterepl");
|
||||
site_a.create_bucket(&bucket).await?;
|
||||
site_b.create_bucket(&bucket).await?;
|
||||
|
||||
let client_a = site_a.client(0)?;
|
||||
let client_b = site_b.client(0)?;
|
||||
enable_versioning(&client_a, &bucket).await?;
|
||||
enable_versioning(&client_b, &bucket).await?;
|
||||
|
||||
let sites = vec![
|
||||
PeerSite {
|
||||
name: "site-a".to_string(),
|
||||
endpoint: site_a.cluster.nodes[0].url.clone(),
|
||||
access_key: site_a.cluster.access_key.clone(),
|
||||
secret_key: site_a.cluster.secret_key.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
PeerSite {
|
||||
name: "site-b".to_string(),
|
||||
endpoint: site_b.cluster.nodes[0].url.clone(),
|
||||
access_key: site_b.cluster.access_key.clone(),
|
||||
secret_key: site_b.cluster.secret_key.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
];
|
||||
let add_status = site_replication_add(&site_a.cluster, &sites).await?;
|
||||
assert!(
|
||||
add_status.success && add_status.err_detail.is_empty() && add_status.initial_sync_error_message.is_empty(),
|
||||
"site replication add reported failure: {add_status:?}"
|
||||
);
|
||||
wait_for_site_replication_enabled(&site_a.cluster).await?;
|
||||
wait_for_site_replication_enabled(&site_b.cluster).await?;
|
||||
|
||||
let info_a = site_replication_info(&site_a.cluster).await?;
|
||||
let remote = info_a
|
||||
.sites
|
||||
.iter()
|
||||
.find(|site| site.name == "site-b")
|
||||
.ok_or_else(|| format!("site A info omitted the configured site-b peer: {info_a:?}"))?;
|
||||
assert_eq!(remote.endpoint, site_b.cluster.nodes[0].url);
|
||||
let deployment_ids: std::collections::BTreeSet<_> = info_a.sites.iter().map(|site| site.deployment_id.as_str()).collect();
|
||||
assert!(
|
||||
deployment_ids.iter().all(|deployment_id| !deployment_id.is_empty()) && deployment_ids.len() == 2,
|
||||
"site peers must have two distinct non-empty deployment IDs: {info_a:?}"
|
||||
);
|
||||
assert!(info_a.retry_stats.is_none(), "site A has pending replication retries: {info_a:?}");
|
||||
assert!(info_a.pending_operation.is_none(), "site A has a pending operation: {info_a:?}");
|
||||
|
||||
let key = "site-object.bin";
|
||||
let body = b"four-node-site-replication".to_vec();
|
||||
put_object(&client_a, &bucket, key, body.clone()).await?;
|
||||
wait_for_replicated_bytes(&client_b, &bucket, key, &body, Duration::from_secs(60)).await?;
|
||||
|
||||
let peer_b = site_b.client(3)?;
|
||||
wait_for_replicated_bytes(&peer_b, &bucket, key, &body, Duration::from_secs(20)).await?;
|
||||
|
||||
let reverse_key = "reverse/site-object.bin";
|
||||
let reverse_body = b"site-b-to-site-a".to_vec();
|
||||
put_object(&site_b.client(2)?, &bucket, reverse_key, reverse_body.clone()).await?;
|
||||
wait_for_replicated_bytes(&site_a.client(3)?, &bucket, reverse_key, &reverse_body, Duration::from_secs(60)).await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,345 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! 4-node upgrade coverage for historical objects and IAM AK/SK.
|
||||
//!
|
||||
//! Complements `upgrade_compatibility_test` (single-node SSE/multipart and
|
||||
//! mixed-version listing). This module pins the distributed contract the
|
||||
//! hardware upgrade chain is meant to catch: after a 4-node upgrade, objects
|
||||
//! written on the previous release still read back, and IAM user credentials
|
||||
//! created before the upgrade still authenticate.
|
||||
//!
|
||||
//! Requires `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous
|
||||
//! release. The `e2e-distributed` workflow downloads that binary; a local run
|
||||
//! without it fails closed rather than skipping.
|
||||
|
||||
use super::harness::{
|
||||
DistCluster, DistLayout, TestResult, assert_object_bytes, cluster_admin_ok, enable_versioning, get_object_bytes, put_object,
|
||||
unique_bucket, wait_until,
|
||||
};
|
||||
use crate::common::{
|
||||
AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, init_logging,
|
||||
};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
use uuid::Uuid;
|
||||
|
||||
const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
|
||||
const IAM_SECRET: &str = "UpgradeTestSecretKey1";
|
||||
const WRONG_SECRET: &str = "WrongSecretKey000000";
|
||||
const CREDENTIAL_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
struct UpgradeSeed {
|
||||
history_bucket: String,
|
||||
history_key: &'static str,
|
||||
history_body: Vec<u8>,
|
||||
versioned_bucket: String,
|
||||
versioned_key: &'static str,
|
||||
version1: String,
|
||||
version1_body: Vec<u8>,
|
||||
version2: String,
|
||||
version2_body: Vec<u8>,
|
||||
iam_bucket: String,
|
||||
iam_key: &'static str,
|
||||
iam_body: Vec<u8>,
|
||||
iam_user: String,
|
||||
iam_secret: &'static str,
|
||||
}
|
||||
|
||||
fn source_binary() -> TestResult<PathBuf> {
|
||||
let path = std::env::var_os(SOURCE_BINARY_ENV).map(PathBuf::from).ok_or_else(|| {
|
||||
format!(
|
||||
"{SOURCE_BINARY_ENV} must point to the pinned previous release binary (the e2e-distributed workflow downloads it)"
|
||||
)
|
||||
})?;
|
||||
if !path.is_file() {
|
||||
return Err(format!("upgrade source binary does not exist: {}", path.display()).into());
|
||||
}
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
fn capture_upgrade_logs(cluster: &mut DistCluster, label: &str) -> TestResult {
|
||||
let Some(log_dir) = std::env::var_os("RUSTFS_E2E_LOG_DIR") else {
|
||||
return Ok(());
|
||||
};
|
||||
std::fs::create_dir_all(&log_dir)?;
|
||||
for node_idx in 0..cluster.cluster.nodes.len() {
|
||||
let path = Path::new(&log_dir).join(format!("{label}-node-{node_idx}.log"));
|
||||
cluster
|
||||
.cluster
|
||||
.set_node_capture_log_path(node_idx, path.to_string_lossy().into_owned())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn iam_rw_policy(bucket: &str) -> String {
|
||||
serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:*"],
|
||||
"Resource": [
|
||||
format!("arn:aws:s3:::{bucket}"),
|
||||
format!("arn:aws:s3:::{bucket}/*")
|
||||
]
|
||||
}]
|
||||
})
|
||||
.to_string()
|
||||
}
|
||||
|
||||
async fn create_iam_user(dist: &DistCluster, user: &str, secret: &str, policy_name: &str, bucket: &str) -> TestResult {
|
||||
let url = &dist.cluster.nodes[0].url;
|
||||
let access = &dist.cluster.access_key;
|
||||
let admin_secret = &dist.cluster.secret_key;
|
||||
admin_create_user_via(AdminTransport::Signed, url, access, admin_secret, user, secret).await?;
|
||||
admin_add_canned_policy_via(AdminTransport::Signed, url, access, admin_secret, policy_name, &iam_rw_policy(bucket)).await?;
|
||||
admin_attach_user_policy_via(AdminTransport::Signed, url, access, admin_secret, policy_name, user).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn wait_for_put(client: &Client, bucket: &str, key: &str, body: Vec<u8>, label: &str) -> TestResult {
|
||||
wait_until(
|
||||
CREDENTIAL_TIMEOUT,
|
||||
|| {
|
||||
let client = client.clone();
|
||||
let bucket = bucket.to_string();
|
||||
let key = key.to_string();
|
||||
let body = body.clone();
|
||||
async move {
|
||||
put_object(&client, &bucket, &key, body).await?;
|
||||
Ok(true)
|
||||
}
|
||||
},
|
||||
label,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn wait_for_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8], label: &str) -> TestResult {
|
||||
wait_until(
|
||||
CREDENTIAL_TIMEOUT,
|
||||
|| {
|
||||
let client = client.clone();
|
||||
let bucket = bucket.to_string();
|
||||
let key = key.to_string();
|
||||
let expected = expected.to_vec();
|
||||
async move {
|
||||
let got = get_object_bytes(&client, &bucket, &key).await?;
|
||||
Ok(got == expected)
|
||||
}
|
||||
},
|
||||
label,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn seed_history_and_iam(dist: &DistCluster) -> TestResult<UpgradeSeed> {
|
||||
let history_bucket = unique_bucket("upg-hist");
|
||||
let versioned_bucket = unique_bucket("upg-ver");
|
||||
let iam_bucket = unique_bucket("upg-iam");
|
||||
dist.create_bucket(&history_bucket).await?;
|
||||
dist.create_bucket(&versioned_bucket).await?;
|
||||
dist.create_bucket(&iam_bucket).await?;
|
||||
|
||||
let root = dist.client(0)?;
|
||||
enable_versioning(&root, &versioned_bucket).await?;
|
||||
|
||||
let history_key = "plain-history.bin";
|
||||
let history_body = b"written by the previous 4-node release".to_vec();
|
||||
put_object(&root, &history_bucket, history_key, history_body.clone()).await?;
|
||||
|
||||
let versioned_key = "versioned-history.txt";
|
||||
let version1_body = b"version-one-before-upgrade".to_vec();
|
||||
let version1 = root
|
||||
.put_object()
|
||||
.bucket(&versioned_bucket)
|
||||
.key(versioned_key)
|
||||
.body(aws_sdk_s3::primitives::ByteStream::from(version1_body.clone()))
|
||||
.send()
|
||||
.await?
|
||||
.version_id()
|
||||
.ok_or("first versioned PUT omitted version ID")?
|
||||
.to_string();
|
||||
let version2_body = b"version-two-before-upgrade".to_vec();
|
||||
let version2 = root
|
||||
.put_object()
|
||||
.bucket(&versioned_bucket)
|
||||
.key(versioned_key)
|
||||
.body(aws_sdk_s3::primitives::ByteStream::from(version2_body.clone()))
|
||||
.send()
|
||||
.await?
|
||||
.version_id()
|
||||
.ok_or("second versioned PUT omitted version ID")?
|
||||
.to_string();
|
||||
|
||||
let iam_user = format!("upg{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let policy_name = format!("upgpol{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
create_iam_user(dist, &iam_user, IAM_SECRET, &policy_name, &iam_bucket).await?;
|
||||
|
||||
let iam_key = "iam-history.bin";
|
||||
let iam_body = b"written with pre-upgrade IAM AK/SK".to_vec();
|
||||
let iam_client = dist.client_with_credentials(1, &iam_user, IAM_SECRET)?;
|
||||
wait_for_put(&iam_client, &iam_bucket, iam_key, iam_body.clone(), "IAM user PUT before upgrade").await?;
|
||||
|
||||
Ok(UpgradeSeed {
|
||||
history_bucket,
|
||||
history_key,
|
||||
history_body,
|
||||
versioned_bucket,
|
||||
versioned_key,
|
||||
version1,
|
||||
version1_body,
|
||||
version2,
|
||||
version2_body,
|
||||
iam_bucket,
|
||||
iam_key,
|
||||
iam_body,
|
||||
iam_user,
|
||||
iam_secret: IAM_SECRET,
|
||||
})
|
||||
}
|
||||
|
||||
async fn assert_history_and_iam(dist: &DistCluster, seed: &UpgradeSeed, context: &str) -> TestResult {
|
||||
let root_a = dist.client(0)?;
|
||||
let root_b = dist.client(3)?;
|
||||
wait_for_bytes(
|
||||
&root_b,
|
||||
&seed.history_bucket,
|
||||
seed.history_key,
|
||||
&seed.history_body,
|
||||
&format!("{context}: root GET historical object"),
|
||||
)
|
||||
.await?;
|
||||
assert_object_bytes(&root_a, &seed.history_bucket, seed.history_key, &seed.history_body).await?;
|
||||
|
||||
let v1 = root_b
|
||||
.get_object()
|
||||
.bucket(&seed.versioned_bucket)
|
||||
.key(seed.versioned_key)
|
||||
.version_id(&seed.version1)
|
||||
.send()
|
||||
.await?;
|
||||
let v1_body = v1.body.collect().await?.into_bytes();
|
||||
if v1_body.as_ref() != seed.version1_body.as_slice() {
|
||||
return Err(format!("{context}: version 1 bytes changed after upgrade").into());
|
||||
}
|
||||
let v2 = root_a
|
||||
.get_object()
|
||||
.bucket(&seed.versioned_bucket)
|
||||
.key(seed.versioned_key)
|
||||
.version_id(&seed.version2)
|
||||
.send()
|
||||
.await?;
|
||||
let v2_body = v2.body.collect().await?.into_bytes();
|
||||
if v2_body.as_ref() != seed.version2_body.as_slice() {
|
||||
return Err(format!("{context}: version 2 bytes changed after upgrade").into());
|
||||
}
|
||||
|
||||
let users = cluster_admin_ok(&dist.cluster, http::Method::GET, "/rustfs/admin/v3/list-users", None).await?;
|
||||
if !users.contains(&seed.iam_user) {
|
||||
return Err(format!("{context}: list-users lost IAM user {}: {users}", seed.iam_user).into());
|
||||
}
|
||||
|
||||
let iam_on_upgraded = dist.client_with_credentials(0, &seed.iam_user, seed.iam_secret)?;
|
||||
let iam_on_peer = dist.client_with_credentials(3, &seed.iam_user, seed.iam_secret)?;
|
||||
wait_for_bytes(
|
||||
&iam_on_upgraded,
|
||||
&seed.iam_bucket,
|
||||
seed.iam_key,
|
||||
&seed.iam_body,
|
||||
&format!("{context}: IAM GET historical object on node 0"),
|
||||
)
|
||||
.await?;
|
||||
wait_for_bytes(
|
||||
&iam_on_peer,
|
||||
&seed.iam_bucket,
|
||||
seed.iam_key,
|
||||
&seed.iam_body,
|
||||
&format!("{context}: IAM GET historical object on node 3"),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let post_key = format!("after-upgrade-{context}.txt");
|
||||
let post_body = format!("{context}: written with the same IAM AK/SK after upgrade").into_bytes();
|
||||
wait_for_put(
|
||||
&iam_on_peer,
|
||||
&seed.iam_bucket,
|
||||
&post_key,
|
||||
post_body.clone(),
|
||||
&format!("{context}: IAM PUT after upgrade"),
|
||||
)
|
||||
.await?;
|
||||
assert_object_bytes(&iam_on_upgraded, &seed.iam_bucket, &post_key, &post_body).await?;
|
||||
|
||||
let bad = dist.client_with_credentials(1, &seed.iam_user, WRONG_SECRET)?;
|
||||
match bad.get_object().bucket(&seed.iam_bucket).key(seed.iam_key).send().await {
|
||||
Ok(_) => return Err(format!("{context}: wrong secret must not read the IAM object").into()),
|
||||
Err(error) => {
|
||||
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
|
||||
let rejected = code == Some("SignatureDoesNotMatch")
|
||||
|| code == Some("InvalidAccessKeyId")
|
||||
|| code == Some("AccessDenied")
|
||||
|| code == Some("InvalidArgument")
|
||||
|| error.raw_response().is_some_and(|response| response.status().as_u16() == 403);
|
||||
if !rejected {
|
||||
return Err(format!("{context}: wrong secret failed with unexpected error {error:?}").into());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let post_root_key = format!("root-after-{context}.bin");
|
||||
let post_root_body = format!("{context}: root write after upgrade").into_bytes();
|
||||
put_object(&root_a, &seed.history_bucket, &post_root_key, post_root_body.clone()).await?;
|
||||
assert_object_bytes(&root_b, &seed.history_bucket, &post_root_key, &post_root_body).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_direct_upgrade_preserves_history_and_iam_credentials() -> TestResult {
|
||||
init_logging();
|
||||
let previous = source_binary()?;
|
||||
let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
|
||||
capture_upgrade_logs(&mut dist, "direct-upgrade")?;
|
||||
dist.start_from_binary(&previous).await?;
|
||||
|
||||
let seed = seed_history_and_iam(&dist).await?;
|
||||
dist.restart_with_current_binary().await?;
|
||||
assert_history_and_iam(&dist, &seed, "direct").await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> TestResult {
|
||||
init_logging();
|
||||
let previous = source_binary()?;
|
||||
let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
|
||||
capture_upgrade_logs(&mut dist, "rolling-upgrade")?;
|
||||
dist.start_from_binary(&previous).await?;
|
||||
|
||||
let seed = seed_history_and_iam(&dist).await?;
|
||||
|
||||
dist.replace_node_with_current_binary(0).await?;
|
||||
assert_history_and_iam(&dist, &seed, "one-current-node").await?;
|
||||
|
||||
for node_idx in [1, 2] {
|
||||
dist.replace_node_with_current_binary(node_idx).await?;
|
||||
}
|
||||
assert_history_and_iam(&dist, &seed, "one-previous-node").await?;
|
||||
|
||||
dist.replace_node_with_current_binary(3).await?;
|
||||
assert_history_and_iam(&dist, &seed, "homogeneous-current").await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,188 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, enable_versioning, get_object_bytes, put_object, unique_bucket};
|
||||
use crate::common::init_logging;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_four_drive_versioning_put_list_get_delete_marker() -> TestResult {
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("version");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let writer = dist.client(0)?;
|
||||
let reader = dist.client(3)?;
|
||||
enable_versioning(&writer, &bucket).await?;
|
||||
|
||||
let key = "versioned.txt";
|
||||
let v1_id = writer
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(b"v1".to_vec().into())
|
||||
.send()
|
||||
.await?
|
||||
.version_id()
|
||||
.ok_or("v1 PUT omitted version ID")?
|
||||
.to_string();
|
||||
let v2_id = writer
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(b"v2".to_vec().into())
|
||||
.send()
|
||||
.await?
|
||||
.version_id()
|
||||
.ok_or("v2 PUT omitted version ID")?
|
||||
.to_string();
|
||||
|
||||
let versions = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?;
|
||||
let matching_versions: Vec<_> = versions
|
||||
.versions()
|
||||
.iter()
|
||||
.filter(|version| version.key() == Some(key))
|
||||
.collect();
|
||||
assert_eq!(matching_versions.len(), 2, "fresh key must have exactly two versions: {versions:?}");
|
||||
assert!(versions.delete_markers().is_empty(), "fresh key unexpectedly has a delete marker");
|
||||
assert!(
|
||||
matching_versions
|
||||
.iter()
|
||||
.any(|version| version.version_id() == Some(v1_id.as_str()) && version.is_latest() != Some(true)),
|
||||
"v1 was not the historical version: {versions:?}"
|
||||
);
|
||||
assert!(
|
||||
matching_versions
|
||||
.iter()
|
||||
.any(|version| version.version_id() == Some(v2_id.as_str()) && version.is_latest() == Some(true)),
|
||||
"v2 was not the latest version: {versions:?}"
|
||||
);
|
||||
|
||||
let latest = get_object_bytes(&reader, &bucket, key).await?;
|
||||
assert_eq!(latest, b"v2");
|
||||
|
||||
let older = reader.get_object().bucket(&bucket).key(key).version_id(&v1_id).send().await?;
|
||||
let older_body = older.body.collect().await?.into_bytes();
|
||||
assert_eq!(older_body.as_ref(), b"v1");
|
||||
|
||||
let deleted = writer.delete_object().bucket(&bucket).key(key).send().await?;
|
||||
assert_eq!(deleted.delete_marker(), Some(true));
|
||||
let marker_id = deleted.version_id().ok_or("DeleteObject omitted delete-marker version ID")?;
|
||||
let after_delete = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?;
|
||||
let matching_markers: Vec<_> = after_delete
|
||||
.delete_markers()
|
||||
.iter()
|
||||
.filter(|marker| marker.key() == Some(key))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
matching_markers.len(),
|
||||
1,
|
||||
"delete marker missing or duplicated after current-version delete: {after_delete:?}"
|
||||
);
|
||||
assert!(
|
||||
matching_markers[0].version_id() == Some(marker_id) && matching_markers[0].is_latest() == Some(true),
|
||||
"DeleteObject response and ListObjectVersions disagree about the marker: {after_delete:?}"
|
||||
);
|
||||
|
||||
let latest_after_delete = reader.get_object().bucket(&bucket).key(key).send().await;
|
||||
match latest_after_delete {
|
||||
Ok(_) => return Err("current version should be a delete marker".into()),
|
||||
Err(error)
|
||||
if error
|
||||
.as_service_error()
|
||||
.and_then(ProvideErrorMetadata::code)
|
||||
.is_some_and(|code| code == "NoSuchKey" || code == "NotFound") => {}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
|
||||
let restored = reader.get_object().bucket(&bucket).key(key).version_id(&v1_id).send().await?;
|
||||
let restored_body = restored.body.collect().await?.into_bytes();
|
||||
assert_eq!(restored_body.as_ref(), b"v1");
|
||||
|
||||
writer
|
||||
.delete_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.version_id(marker_id)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(get_object_bytes(&reader, &bucket, key).await?, b"v2");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_versioning_suspension_keeps_one_null_version_and_history() -> TestResult {
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("suspend");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let writer = dist.client(0)?;
|
||||
let reader = dist.client(3)?;
|
||||
enable_versioning(&writer, &bucket).await?;
|
||||
|
||||
let key = "suspended.txt";
|
||||
let original = writer
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(b"enabled-history".to_vec().into())
|
||||
.send()
|
||||
.await?
|
||||
.version_id()
|
||||
.ok_or("enabled PUT omitted version ID")?
|
||||
.to_string();
|
||||
writer
|
||||
.put_bucket_versioning()
|
||||
.bucket(&bucket)
|
||||
.versioning_configuration(
|
||||
VersioningConfiguration::builder()
|
||||
.status(BucketVersioningStatus::Suspended)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
put_object(&writer, &bucket, key, b"null-one".to_vec()).await?;
|
||||
put_object(&writer, &bucket, key, b"null-two".to_vec()).await?;
|
||||
assert_eq!(get_object_bytes(&reader, &bucket, key).await?, b"null-two");
|
||||
|
||||
let versions = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?;
|
||||
let matching: Vec<_> = versions
|
||||
.versions()
|
||||
.iter()
|
||||
.filter(|version| version.key() == Some(key))
|
||||
.collect();
|
||||
assert!(matching.iter().any(|version| version.version_id() == Some(original.as_str())));
|
||||
let null_version_count = matching
|
||||
.iter()
|
||||
.filter(|version| {
|
||||
matches!(
|
||||
version.version_id(),
|
||||
None | Some("") | Some("null") | Some("00000000-0000-0000-0000-000000000000")
|
||||
)
|
||||
})
|
||||
.count();
|
||||
assert_eq!(null_version_count, 1, "suspended overwrites must keep one null version: {versions:?}");
|
||||
|
||||
let historical = reader
|
||||
.get_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.version_id(&original)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(historical.body.collect().await?.into_bytes().as_ref(), b"enabled-history");
|
||||
Ok(())
|
||||
}
|
||||
@@ -34,14 +34,12 @@ use s3s::dto::{
|
||||
AbortMultipartUploadInput, AbortMultipartUploadOutput, CommonPrefix, CompleteMultipartUploadInput,
|
||||
CompleteMultipartUploadOutput, CreateMultipartUploadInput, CreateMultipartUploadOutput, DeleteMarkerEntry, DeleteObjectInput,
|
||||
DeleteObjectOutput, DeleteObjectTaggingInput, DeleteObjectTaggingOutput, ETag, GetBucketVersioningInput,
|
||||
GetBucketVersioningOutput, GetObjectInput, GetObjectLegalHoldInput, GetObjectLegalHoldOutput,
|
||||
GetObjectLockConfigurationInput, GetObjectLockConfigurationOutput, GetObjectOutput, GetObjectRetentionInput,
|
||||
GetObjectRetentionOutput, GetObjectTaggingInput, GetObjectTaggingOutput, HeadBucketInput, HeadBucketOutput, HeadObjectInput,
|
||||
GetBucketVersioningOutput, GetObjectInput, GetObjectLockConfigurationInput, GetObjectLockConfigurationOutput,
|
||||
GetObjectOutput, GetObjectTaggingInput, GetObjectTaggingOutput, HeadBucketInput, HeadBucketOutput, HeadObjectInput,
|
||||
HeadObjectOutput, ListObjectVersionsInput, ListObjectVersionsOutput, ListObjectsV2Input, ListObjectsV2Output, Object,
|
||||
ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockMode,
|
||||
ObjectLockRetention, ObjectLockRetentionMode, ObjectStorageClass, ObjectVersionId, PutObjectInput, PutObjectLegalHoldInput,
|
||||
PutObjectLegalHoldOutput, PutObjectOutput, PutObjectRetentionInput, PutObjectRetentionOutput, PutObjectTaggingInput,
|
||||
PutObjectTaggingOutput, Range, StreamingBlob, Tag, TagSet, Timestamp, TimestampFormat, UploadPartInput, UploadPartOutput,
|
||||
ObjectLockConfiguration, ObjectLockEnabled, ObjectStorageClass, ObjectVersionId, PutObjectInput, PutObjectOutput,
|
||||
PutObjectTaggingInput, PutObjectTaggingOutput, Range, StreamingBlob, Tag, TagSet, Timestamp, TimestampFormat,
|
||||
UploadPartInput, UploadPartOutput,
|
||||
};
|
||||
use s3s::service::{S3Service, S3ServiceBuilder};
|
||||
use s3s::validation::{AwsNameValidation, NameValidation};
|
||||
@@ -129,10 +127,6 @@ pub enum Operation {
|
||||
GetObjectTagging,
|
||||
PutObjectTagging,
|
||||
DeleteObjectTagging,
|
||||
GetObjectRetention,
|
||||
PutObjectRetention,
|
||||
GetObjectLegalHold,
|
||||
PutObjectLegalHold,
|
||||
ListObjectVersions,
|
||||
ListObjectsV2,
|
||||
CreateMultipartUpload,
|
||||
@@ -457,40 +451,10 @@ impl JournaledHeaders {
|
||||
struct ControlState {
|
||||
scripts: HashMap<Operation, VecDeque<FaultAction>>,
|
||||
keyed_scripts: HashMap<(Operation, String), VecDeque<FaultAction>>,
|
||||
held_get: Option<HeldGetObject>,
|
||||
requests: VecDeque<RequestRecord>,
|
||||
next_sequence: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct HeldGetObject {
|
||||
bucket: String,
|
||||
key: String,
|
||||
entered: watch::Sender<usize>,
|
||||
released: watch::Receiver<bool>,
|
||||
}
|
||||
|
||||
/// Holds every GET of one object, including retries, until this guard is dropped.
|
||||
#[must_use = "dropping the guard releases the held GET requests"]
|
||||
pub struct GetObjectGate {
|
||||
control: Arc<Mutex<ControlState>>,
|
||||
entered: watch::Receiver<usize>,
|
||||
released: watch::Sender<bool>,
|
||||
}
|
||||
|
||||
impl GetObjectGate {
|
||||
pub async fn wait_until_entered(&mut self) -> Result<(), watch::error::RecvError> {
|
||||
self.entered.wait_for(|count| *count > 0).await.map(|_| ())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for GetObjectGate {
|
||||
fn drop(&mut self) {
|
||||
lock(&self.control).held_get = None;
|
||||
self.released.send_replace(true);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct StoreState {
|
||||
assign_own_version_ids: bool,
|
||||
@@ -507,10 +471,6 @@ struct StoreState {
|
||||
/// PutObject carrying any `x-amz-object-lock-*` header must also carry
|
||||
/// `Content-MD5` or an `x-amz-checksum-*` header.
|
||||
require_checksum_for_object_lock: bool,
|
||||
/// Models Wasabi (rustfs/backlog#2340): a version-addressed DELETE of a
|
||||
/// version id the target never had answers 404 `NoSuchVersion` instead of
|
||||
/// the idempotent 204 RustFS/MinIO give.
|
||||
reject_unknown_version_deletes: bool,
|
||||
limits: StoreLimits,
|
||||
buckets: HashMap<String, BucketState>,
|
||||
uploads: HashMap<String, MultipartState>,
|
||||
@@ -575,41 +535,6 @@ struct ObjectVersion {
|
||||
/// SSE-C passthrough transport headers stored with the version (RustFS
|
||||
/// target behavior); empty when the drop mode discarded them.
|
||||
replication_sse_headers: Vec<(String, String)>,
|
||||
/// Object Lock state of the version: retention (mode, retain-until) from
|
||||
/// the PUT / CreateMultipartUpload headers or PutObjectRetention, and the
|
||||
/// legal hold flag; replayed on HEAD.
|
||||
lock: VersionLock,
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct VersionLock {
|
||||
retention: Option<(String, Timestamp)>,
|
||||
/// `None` until a legal hold status was ever set; like S3, HEAD then
|
||||
/// reports nothing, while an explicit OFF is reported as `OFF`.
|
||||
legal_hold: Option<bool>,
|
||||
}
|
||||
|
||||
impl VersionLock {
|
||||
fn from_headers(
|
||||
mode: Option<ObjectLockMode>,
|
||||
retain_until: Option<Timestamp>,
|
||||
legal_hold: Option<ObjectLockLegalHoldStatus>,
|
||||
) -> Self {
|
||||
Self {
|
||||
retention: mode.zip(retain_until).map(|(mode, until)| (mode.as_str().to_string(), until)),
|
||||
legal_hold: legal_hold.map(|status| status.as_str().eq_ignore_ascii_case("ON")),
|
||||
}
|
||||
}
|
||||
|
||||
fn legal_hold_status(&self) -> Option<ObjectLockLegalHoldStatus> {
|
||||
self.legal_hold.map(|on| {
|
||||
ObjectLockLegalHoldStatus::from_static(if on {
|
||||
ObjectLockLegalHoldStatus::ON
|
||||
} else {
|
||||
ObjectLockLegalHoldStatus::OFF
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -621,7 +546,6 @@ struct MultipartState {
|
||||
metadata: Option<HashMap<String, String>>,
|
||||
standard_headers: StandardHeaders,
|
||||
replication_sse_headers: Vec<(String, String)>,
|
||||
lock: VersionLock,
|
||||
parts: BTreeMap<i32, MultipartPart>,
|
||||
}
|
||||
|
||||
@@ -630,10 +554,6 @@ struct MultipartPart {
|
||||
body: Bytes,
|
||||
e_tag: String,
|
||||
digest: [u8; 16],
|
||||
/// Plaintext length declared by an SSE-C passthrough sender
|
||||
/// (`x-rustfs-replication-part-actual-size`); RustFS validates the 5 MiB
|
||||
/// minimum against it rather than against the stored bytes.
|
||||
actual_size: Option<usize>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -895,7 +815,6 @@ impl FakeS3Target {
|
||||
standard_headers: seed.standard_headers.clone(),
|
||||
tags: Vec::new(),
|
||||
replication_sse_headers: Vec::new(),
|
||||
lock: VersionLock::default(),
|
||||
};
|
||||
upsert_version(&mut state, bucket, key.into(), version).expect("seed object must fit the storage budget");
|
||||
e_tag
|
||||
@@ -969,12 +888,6 @@ impl FakeS3Target {
|
||||
/// PutObject that carries Object Lock parameters (AWS S3 / MinIO rule,
|
||||
/// rustfs#7082). `Content-MD5`, when present, is always verified against
|
||||
/// the body regardless of this mode.
|
||||
/// Wasabi-like mode: DELETE of an unknown version id answers 404
|
||||
/// `NoSuchVersion` (the default 204 models RustFS/MinIO).
|
||||
pub fn reject_unknown_version_deletes(&self, enabled: bool) {
|
||||
lock(&self.backend.store).reject_unknown_version_deletes = enabled;
|
||||
}
|
||||
|
||||
pub fn require_checksum_for_object_lock(&self, enabled: bool) {
|
||||
lock(&self.backend.store).require_checksum_for_object_lock = enabled;
|
||||
}
|
||||
@@ -1023,30 +936,6 @@ impl FakeS3Target {
|
||||
.extend(std::iter::repeat_n(action, times));
|
||||
}
|
||||
|
||||
/// Hold one exact bucket/key before any GET response can reach the client.
|
||||
/// The fixture supports one live gate; request and connection deadlines still apply.
|
||||
pub fn hold_get_object(&self, bucket: &str, key: &str) -> GetObjectGate {
|
||||
assert!(
|
||||
bucket.len() <= MAX_RETAINED_IDENTIFIER_BYTES && key.len() <= MAX_RETAINED_IDENTIFIER_BYTES,
|
||||
"held GET identifiers exceed the fixture limit"
|
||||
);
|
||||
let mut state = lock(&self.control);
|
||||
assert!(state.held_get.is_none(), "fake target already holds a GET gate");
|
||||
let (entered, entered_rx) = watch::channel(0);
|
||||
let (released, released_rx) = watch::channel(false);
|
||||
state.held_get = Some(HeldGetObject {
|
||||
bucket: bucket.to_string(),
|
||||
key: key.to_string(),
|
||||
entered,
|
||||
released: released_rx,
|
||||
});
|
||||
GetObjectGate {
|
||||
control: Arc::clone(&self.control),
|
||||
entered: entered_rx,
|
||||
released,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn clear_faults(&self) {
|
||||
let mut state = lock(&self.control);
|
||||
state.scripts.clear();
|
||||
@@ -1209,10 +1098,6 @@ fn operation_from_s3_name(name: &str) -> Operation {
|
||||
"GetObjectTagging" => Operation::GetObjectTagging,
|
||||
"PutObjectTagging" => Operation::PutObjectTagging,
|
||||
"DeleteObjectTagging" => Operation::DeleteObjectTagging,
|
||||
"GetObjectRetention" => Operation::GetObjectRetention,
|
||||
"PutObjectRetention" => Operation::PutObjectRetention,
|
||||
"GetObjectLegalHold" => Operation::GetObjectLegalHold,
|
||||
"PutObjectLegalHold" => Operation::PutObjectLegalHold,
|
||||
"ListObjectsV2" => Operation::ListObjectsV2,
|
||||
"CreateMultipartUpload" => Operation::CreateMultipartUpload,
|
||||
"UploadPart" => Operation::UploadPart,
|
||||
@@ -1350,18 +1235,6 @@ fn parse_request(method: &Method, uri: &Uri) -> ParsedRequest {
|
||||
(&Method::DELETE, true) if query.contains_key("tagging") && only_query_keys(&["tagging", "versionId"]) => {
|
||||
Operation::DeleteObjectTagging
|
||||
}
|
||||
(&Method::GET, true) if query.contains_key("retention") && only_query_keys(&["retention", "versionId"]) => {
|
||||
Operation::GetObjectRetention
|
||||
}
|
||||
(&Method::PUT, true) if query.contains_key("retention") && only_query_keys(&["retention", "versionId"]) => {
|
||||
Operation::PutObjectRetention
|
||||
}
|
||||
(&Method::GET, true) if query.contains_key("legal-hold") && only_query_keys(&["legal-hold", "versionId"]) => {
|
||||
Operation::GetObjectLegalHold
|
||||
}
|
||||
(&Method::PUT, true) if query.contains_key("legal-hold") && only_query_keys(&["legal-hold", "versionId"]) => {
|
||||
Operation::PutObjectLegalHold
|
||||
}
|
||||
// A replication PUT addresses the source version via `?versionId=`.
|
||||
(&Method::PUT, true) if only_query_keys(&["versionId"]) => Operation::PutObject,
|
||||
(&Method::GET, true) if only_query_keys(&["versionId"]) => Operation::GetObject,
|
||||
@@ -1915,28 +1788,6 @@ fn set_version_tags(
|
||||
Ok(resolved)
|
||||
}
|
||||
|
||||
fn update_version_lock(
|
||||
state: &mut StoreState,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
version_id: Option<&str>,
|
||||
update: impl FnOnce(&mut VersionLock),
|
||||
) -> S3Result<String> {
|
||||
let resolved = find_version(state, bucket, key, version_id)?.version_id;
|
||||
let version = state
|
||||
.buckets
|
||||
.get_mut(bucket)
|
||||
.expect("bucket existence checked by find_version")
|
||||
.objects
|
||||
.get_mut(key)
|
||||
.expect("key existence checked by find_version")
|
||||
.iter_mut()
|
||||
.find(|version| version.version_id == resolved)
|
||||
.expect("version existence checked by find_version");
|
||||
update(&mut version.lock);
|
||||
Ok(resolved)
|
||||
}
|
||||
|
||||
/// Whether version ids are surfaced for this bucket. Unknown buckets report
|
||||
/// `true`; the caller's lookup raises `NoSuchBucket` first.
|
||||
fn bucket_versioned(state: &StoreState, bucket: &str) -> bool {
|
||||
@@ -2376,11 +2227,6 @@ impl S3 for FakeBackend {
|
||||
standard_headers,
|
||||
tags: Vec::new(),
|
||||
replication_sse_headers: captured_replication_sse_headers(&headers, drop_unlisted),
|
||||
lock: VersionLock::from_headers(
|
||||
input.object_lock_mode,
|
||||
input.object_lock_retain_until_date,
|
||||
input.object_lock_legal_hold_status,
|
||||
),
|
||||
};
|
||||
upsert_version(&mut lock(&self.store), &input.bucket, input.key, version)?;
|
||||
Ok(apply_response_fault(
|
||||
@@ -2397,17 +2243,6 @@ impl S3 for FakeBackend {
|
||||
let fault = request_fault(&req);
|
||||
apply_non_body_fault(fault.as_ref(), &self.control).await?;
|
||||
let input = req.input;
|
||||
let held_get = lock(&self.control)
|
||||
.held_get
|
||||
.as_ref()
|
||||
.filter(|held| held.bucket == input.bucket && held.key == input.key)
|
||||
.cloned();
|
||||
if let Some(mut held) = held_get {
|
||||
held.entered.send_modify(|count| *count += 1);
|
||||
// Keep the gate installed when a request is cancelled or times out:
|
||||
// a retry must cross the same boundary before returning any bytes.
|
||||
let _ = held.released.wait_for(|released| *released).await;
|
||||
}
|
||||
let (version, versioned) = {
|
||||
let state = lock(&self.store);
|
||||
(
|
||||
@@ -2439,13 +2274,6 @@ impl S3 for FakeBackend {
|
||||
last_modified: Some(version.last_modified.clone()),
|
||||
version_id: versioned.then_some(version.version_id),
|
||||
sse_customer_algorithm,
|
||||
object_lock_mode: version
|
||||
.lock
|
||||
.retention
|
||||
.as_ref()
|
||||
.map(|(mode, _)| ObjectLockMode::from(mode.clone())),
|
||||
object_lock_retain_until_date: version.lock.retention.as_ref().map(|(_, until)| until.clone()),
|
||||
object_lock_legal_hold_status: version.lock.legal_hold_status(),
|
||||
..Default::default()
|
||||
});
|
||||
response.status = served.status;
|
||||
@@ -2480,13 +2308,6 @@ impl S3 for FakeBackend {
|
||||
last_modified: Some(version.last_modified.clone()),
|
||||
version_id: versioned.then_some(version.version_id),
|
||||
sse_customer_algorithm,
|
||||
object_lock_mode: version
|
||||
.lock
|
||||
.retention
|
||||
.as_ref()
|
||||
.map(|(mode, _)| ObjectLockMode::from(mode.clone())),
|
||||
object_lock_retain_until_date: version.lock.retention.as_ref().map(|(_, until)| until.clone()),
|
||||
object_lock_legal_hold_status: version.lock.legal_hold_status(),
|
||||
..Default::default()
|
||||
});
|
||||
response.status = served.status;
|
||||
@@ -2546,82 +2367,6 @@ impl S3 for FakeBackend {
|
||||
))
|
||||
}
|
||||
|
||||
async fn get_object_retention(
|
||||
&self,
|
||||
req: S3Request<GetObjectRetentionInput>,
|
||||
) -> S3Result<S3Response<GetObjectRetentionOutput>> {
|
||||
let fault = request_fault(&req);
|
||||
apply_non_body_fault(fault.as_ref(), &self.control).await?;
|
||||
let input = req.input;
|
||||
let version = find_version(&lock(&self.store), &input.bucket, &input.key, input.version_id.as_deref())?;
|
||||
Ok(apply_response_fault(
|
||||
S3Response::new(GetObjectRetentionOutput {
|
||||
retention: version.lock.retention.map(|(mode, until)| ObjectLockRetention {
|
||||
mode: Some(ObjectLockRetentionMode::from(mode)),
|
||||
retain_until_date: Some(until),
|
||||
}),
|
||||
}),
|
||||
fault.as_ref(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn put_object_retention(
|
||||
&self,
|
||||
req: S3Request<PutObjectRetentionInput>,
|
||||
) -> S3Result<S3Response<PutObjectRetentionOutput>> {
|
||||
let fault = request_fault(&req);
|
||||
apply_non_body_fault(fault.as_ref(), &self.control).await?;
|
||||
let input = req.input;
|
||||
let retention = input
|
||||
.retention
|
||||
.and_then(|retention| retention.mode.zip(retention.retain_until_date))
|
||||
.map(|(mode, until)| (mode.as_str().to_string(), until));
|
||||
update_version_lock(&mut lock(&self.store), &input.bucket, &input.key, input.version_id.as_deref(), |lock| {
|
||||
lock.retention = retention;
|
||||
})?;
|
||||
Ok(apply_response_fault(S3Response::new(PutObjectRetentionOutput::default()), fault.as_ref()))
|
||||
}
|
||||
|
||||
async fn get_object_legal_hold(
|
||||
&self,
|
||||
req: S3Request<GetObjectLegalHoldInput>,
|
||||
) -> S3Result<S3Response<GetObjectLegalHoldOutput>> {
|
||||
let fault = request_fault(&req);
|
||||
apply_non_body_fault(fault.as_ref(), &self.control).await?;
|
||||
let input = req.input;
|
||||
let version = find_version(&lock(&self.store), &input.bucket, &input.key, input.version_id.as_deref())?;
|
||||
Ok(apply_response_fault(
|
||||
S3Response::new(GetObjectLegalHoldOutput {
|
||||
legal_hold: Some(ObjectLockLegalHold {
|
||||
status: Some(
|
||||
version
|
||||
.lock
|
||||
.legal_hold_status()
|
||||
.unwrap_or_else(|| ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::OFF)),
|
||||
),
|
||||
}),
|
||||
}),
|
||||
fault.as_ref(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn put_object_legal_hold(
|
||||
&self,
|
||||
req: S3Request<PutObjectLegalHoldInput>,
|
||||
) -> S3Result<S3Response<PutObjectLegalHoldOutput>> {
|
||||
let fault = request_fault(&req);
|
||||
apply_non_body_fault(fault.as_ref(), &self.control).await?;
|
||||
let input = req.input;
|
||||
let legal_hold_on = input
|
||||
.legal_hold
|
||||
.and_then(|hold| hold.status)
|
||||
.is_some_and(|status| status.as_str().eq_ignore_ascii_case("ON"));
|
||||
update_version_lock(&mut lock(&self.store), &input.bucket, &input.key, input.version_id.as_deref(), |lock| {
|
||||
lock.legal_hold = Some(legal_hold_on);
|
||||
})?;
|
||||
Ok(apply_response_fault(S3Response::new(PutObjectLegalHoldOutput::default()), fault.as_ref()))
|
||||
}
|
||||
|
||||
async fn delete_object_tagging(
|
||||
&self,
|
||||
req: S3Request<DeleteObjectTaggingInput>,
|
||||
@@ -2675,7 +2420,6 @@ impl S3 for FakeBackend {
|
||||
return Ok(apply_response_fault(S3Response::new(DeleteObjectOutput::default()), fault.as_ref()));
|
||||
}
|
||||
if let Some(version_id) = input.version_id {
|
||||
let reject_unknown = state.reject_unknown_version_deletes;
|
||||
let (removed_bytes, removed_versions, delete_marker, remove_key) = {
|
||||
let Some(versions) = state
|
||||
.buckets
|
||||
@@ -2684,9 +2428,6 @@ impl S3 for FakeBackend {
|
||||
.objects
|
||||
.get_mut(&input.key)
|
||||
else {
|
||||
if reject_unknown {
|
||||
return Err(s3s::s3_error!(NoSuchVersion, "The specified version does not exist."));
|
||||
}
|
||||
return Ok(apply_response_fault(
|
||||
S3Response::new(DeleteObjectOutput {
|
||||
version_id: Some(version_id),
|
||||
@@ -2695,9 +2436,6 @@ impl S3 for FakeBackend {
|
||||
fault.as_ref(),
|
||||
));
|
||||
};
|
||||
if reject_unknown && !versions.iter().any(|version| version.version_id == version_id) {
|
||||
return Err(s3s::s3_error!(NoSuchVersion, "The specified version does not exist."));
|
||||
}
|
||||
let mut removed_bytes = 0usize;
|
||||
let mut removed_versions = 0usize;
|
||||
let mut delete_marker = None;
|
||||
@@ -2751,7 +2489,6 @@ impl S3 for FakeBackend {
|
||||
standard_headers: StandardHeaders::default(),
|
||||
tags: Vec::new(),
|
||||
replication_sse_headers: Vec::new(),
|
||||
lock: VersionLock::default(),
|
||||
},
|
||||
)?;
|
||||
Ok(apply_response_fault(
|
||||
@@ -2806,11 +2543,6 @@ impl S3 for FakeBackend {
|
||||
metadata: input.metadata,
|
||||
standard_headers,
|
||||
replication_sse_headers: captured_replication_sse_headers(&headers, drop_unlisted),
|
||||
lock: VersionLock::from_headers(
|
||||
input.object_lock_mode,
|
||||
input.object_lock_retain_until_date,
|
||||
input.object_lock_legal_hold_status,
|
||||
),
|
||||
parts: BTreeMap::new(),
|
||||
},
|
||||
);
|
||||
@@ -2827,11 +2559,6 @@ impl S3 for FakeBackend {
|
||||
|
||||
async fn upload_part(&self, req: S3Request<UploadPartInput>) -> S3Result<S3Response<UploadPartOutput>> {
|
||||
let fault = request_fault(&req);
|
||||
let declared_actual_size = req
|
||||
.headers
|
||||
.get("x-rustfs-replication-part-actual-size")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| value.parse::<usize>().ok());
|
||||
let _body_permit = timeout(MAX_FAULT_DURATION, Arc::clone(&self.body_limit).acquire_owned())
|
||||
.await
|
||||
.map_err(|_| s3s::s3_error!(RequestTimeout, "fake target body limiter wait exceeded 30 seconds"))?
|
||||
@@ -2873,7 +2600,6 @@ impl S3 for FakeBackend {
|
||||
body,
|
||||
e_tag: e_tag.clone(),
|
||||
digest,
|
||||
actual_size: declared_actual_size,
|
||||
},
|
||||
);
|
||||
Ok(apply_response_fault(
|
||||
@@ -2943,9 +2669,7 @@ impl S3 for FakeBackend {
|
||||
if requested_etag != &stored.e_tag {
|
||||
return Err(s3s::s3_error!(InvalidPart, "part ETag does not match"));
|
||||
}
|
||||
if index + 1 != requested_parts.len()
|
||||
&& stored.actual_size.unwrap_or(stored.body.len()) < MIN_MULTIPART_PART_BYTES
|
||||
{
|
||||
if index + 1 != requested_parts.len() && stored.body.len() < MIN_MULTIPART_PART_BYTES {
|
||||
return Err(s3s::s3_error!(EntityTooSmall, "non-final multipart part is smaller than 5 MiB"));
|
||||
}
|
||||
selected.push((*number, stored.clone()));
|
||||
@@ -2959,7 +2683,6 @@ impl S3 for FakeBackend {
|
||||
metadata: upload.metadata.clone(),
|
||||
standard_headers: upload.standard_headers.clone(),
|
||||
replication_sse_headers: upload.replication_sse_headers.clone(),
|
||||
lock: upload.lock.clone(),
|
||||
parts: BTreeMap::new(),
|
||||
},
|
||||
selected,
|
||||
@@ -2990,7 +2713,6 @@ impl S3 for FakeBackend {
|
||||
standard_headers: upload.standard_headers,
|
||||
tags: Vec::new(),
|
||||
replication_sse_headers: upload.replication_sse_headers,
|
||||
lock: upload.lock,
|
||||
};
|
||||
let mut state = lock(&self.store);
|
||||
let versioned = bucket_versioned(&state, &input.bucket);
|
||||
@@ -3172,81 +2894,6 @@ mod tests {
|
||||
aws_sdk_s3::primitives::DateTime::from_secs(4_102_444_800)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_object_gate_holds_retries_and_releases_on_drop() -> Result<(), BoxError> {
|
||||
let target = FakeS3Target::start().await?;
|
||||
let bucket = "gated-target";
|
||||
target.create_bucket(bucket);
|
||||
for key in ["held", "unrelated"] {
|
||||
target.put_seed_object(bucket, key, Bytes::from_static(b"payload"), &SeedMetadata::default());
|
||||
}
|
||||
{
|
||||
let gate = target.hold_get_object(bucket, "held");
|
||||
let request = || S3Request {
|
||||
input: GetObjectInput {
|
||||
bucket: bucket.to_string(),
|
||||
key: "held".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
method: Method::GET,
|
||||
uri: Uri::from_static("/gated-target/held"),
|
||||
headers: HeaderMap::new(),
|
||||
extensions: http::Extensions::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
};
|
||||
// Without a fault, only the gate can suspend this backend method.
|
||||
let mut first = target.backend.get_object(request());
|
||||
assert!(futures::poll!(first.as_mut()).is_pending(), "the first GET must wait at the gate");
|
||||
drop(first);
|
||||
let mut retry = target.backend.get_object(request());
|
||||
assert!(futures::poll!(retry.as_mut()).is_pending(), "a cancelled GET must not consume the gate");
|
||||
drop(gate);
|
||||
let std::task::Poll::Ready(response) = futures::poll!(retry.as_mut()) else {
|
||||
panic!("dropping the gate must release the waiting GET");
|
||||
};
|
||||
let mut body = response?.output.body.expect("released GET body");
|
||||
assert_eq!(body.next().await.transpose()?, Some(Bytes::from_static(b"payload")));
|
||||
assert!(body.next().await.is_none(), "released GET body must be complete");
|
||||
}
|
||||
let client = client(&target);
|
||||
let mut gate = target.hold_get_object(bucket, "held");
|
||||
let mut requests = tokio::task::JoinSet::new();
|
||||
let first = client.clone();
|
||||
requests.spawn(async move { get_bytes(&first, bucket, "held", None).await });
|
||||
timeout(Duration::from_secs(2), gate.wait_until_entered()).await??;
|
||||
requests.abort_all();
|
||||
assert!(
|
||||
requests
|
||||
.join_next()
|
||||
.await
|
||||
.expect("first GET task")
|
||||
.expect_err("cancel the first GET attempt")
|
||||
.is_cancelled()
|
||||
);
|
||||
|
||||
let retry = client.clone();
|
||||
requests.spawn(async move { get_bytes(&retry, bucket, "held", None).await });
|
||||
timeout(Duration::from_secs(2), gate.entered.wait_for(|count| *count == 2)).await??;
|
||||
assert_eq!(
|
||||
timeout(Duration::from_secs(2), get_bytes(&client, bucket, "unrelated", None)).await??,
|
||||
Bytes::from_static(b"payload")
|
||||
);
|
||||
assert!(requests.try_join_next().is_none(), "the retry must remain behind the gate");
|
||||
drop(gate);
|
||||
assert_eq!(
|
||||
timeout(Duration::from_secs(2), requests.join_next())
|
||||
.await?
|
||||
.expect("retried GET task")??,
|
||||
Bytes::from_static(b"payload")
|
||||
);
|
||||
assert_eq!(get_bytes(&client, bucket, "held", None).await?, Bytes::from_static(b"payload"));
|
||||
assert_eq!(target.count_requests(Operation::GetObject, "held"), 3);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_lock_target_requires_a_checksum_on_locked_puts() -> Result<(), BoxError> {
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
@@ -4822,7 +4469,6 @@ mod tests {
|
||||
metadata: None,
|
||||
standard_headers: StandardHeaders::default(),
|
||||
replication_sse_headers: Vec::new(),
|
||||
lock: VersionLock::default(),
|
||||
parts: BTreeMap::new(),
|
||||
},
|
||||
);
|
||||
|
||||
@@ -16,141 +16,24 @@
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::chaos::{VersionShardCensus, census_object_version_on_disk, sha256_hex, signed_admin_post};
|
||||
use crate::chaos::{VersionShardCensus, census_object_version_on_disk, signed_admin_post};
|
||||
use crate::common::{
|
||||
FAST_DATA_USAGE_SCANNER_ENV, RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging,
|
||||
rustfs_binary_path,
|
||||
};
|
||||
use crate::storage_api::RUSTFS_META_BUCKET;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use http::Method;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashSet;
|
||||
use std::error::Error;
|
||||
use std::io::{Read, Write};
|
||||
use std::net::SocketAddr;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
use tokio::net::TcpStream;
|
||||
use tokio::time::{Duration, Instant, sleep, timeout};
|
||||
use tracing::info;
|
||||
#[cfg(target_os = "linux")]
|
||||
use tracing::warn;
|
||||
|
||||
const POOL_METADATA_OBJECT: &str = "pool.bin";
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct EvidenceBuild {
|
||||
sha256: String,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct RestartEvidenceRun {
|
||||
schema: u32,
|
||||
run_id: String,
|
||||
source_revision: String,
|
||||
test_build: serde_json::Value,
|
||||
binary: EvidenceBuild,
|
||||
test_binary: EvidenceBuild,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct ScannerHealEvidenceCase {
|
||||
id: &'static str,
|
||||
oracle: &'static str,
|
||||
evidence: &'static str,
|
||||
unclean_shutdown_marker: bool,
|
||||
}
|
||||
|
||||
const BACKGROUND_TARGET_RESTART_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase {
|
||||
id: "background-target-restart",
|
||||
oracle: "background-target-restart.json",
|
||||
evidence: "process-restart",
|
||||
unclean_shutdown_marker: false,
|
||||
};
|
||||
|
||||
const BACKGROUND_TARGET_CRASH_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase {
|
||||
id: "background-target-crash",
|
||||
oracle: "background-target-crash.json",
|
||||
evidence: "process-crash-restart",
|
||||
unclean_shutdown_marker: true,
|
||||
};
|
||||
|
||||
struct RestartEvidenceContext {
|
||||
directory: PathBuf,
|
||||
run: RestartEvidenceRun,
|
||||
case: ScannerHealEvidenceCase,
|
||||
}
|
||||
|
||||
fn file_sha256(path: &Path) -> Result<String, Box<dyn Error + Send + Sync>> {
|
||||
let mut file = std::fs::File::open(path)?;
|
||||
let mut digest = Sha256::new();
|
||||
let mut buffer = [0_u8; 64 * 1024];
|
||||
loop {
|
||||
let read = file.read(&mut buffer)?;
|
||||
if read == 0 {
|
||||
break;
|
||||
}
|
||||
digest.update(&buffer[..read]);
|
||||
}
|
||||
Ok(digest.finalize().iter().map(|byte| format!("{byte:02x}")).collect())
|
||||
}
|
||||
|
||||
fn restart_evidence_run(
|
||||
binary: &Path,
|
||||
case: ScannerHealEvidenceCase,
|
||||
) -> Result<Option<RestartEvidenceContext>, Box<dyn Error + Send + Sync>> {
|
||||
let Some(directory) = std::env::var_os("RUSTFS_SCANNER_HEAL_RUN_DIR") else {
|
||||
return Ok(None);
|
||||
};
|
||||
if case.id.is_empty()
|
||||
|| case.oracle.is_empty()
|
||||
|| !case.oracle.ends_with(".json")
|
||||
|| case.oracle.contains('/')
|
||||
|| case.oracle.contains('\\')
|
||||
|| case.oracle.contains("..")
|
||||
|| !matches!(case.evidence, "process-restart" | "process-crash-restart")
|
||||
|| (case.evidence == "process-crash-restart") != case.unclean_shutdown_marker
|
||||
{
|
||||
return Err("invalid scanner/heal evidence case".into());
|
||||
}
|
||||
let directory = PathBuf::from(directory);
|
||||
let receipt = directory.join("run.json");
|
||||
if receipt.metadata()?.len() > 1024 * 1024 {
|
||||
return Err("oversized scanner/heal execution receipt".into());
|
||||
}
|
||||
let run: RestartEvidenceRun = serde_json::from_slice(&std::fs::read(receipt)?)?;
|
||||
if run.schema != 1 || run.run_id.len() != 32 || run.source_revision.len() != 40 {
|
||||
return Err("invalid scanner/heal execution identity".into());
|
||||
}
|
||||
let built = compiled_test_identity();
|
||||
for key in ["source_revision", "dirty", "lock_blob", "features"] {
|
||||
assert_eq!(built[key], run.test_build[key], "compiled test identity differs for {key}");
|
||||
}
|
||||
assert_eq!(file_sha256(binary)?, run.binary.sha256, "server binary must match the run receipt");
|
||||
assert_eq!(
|
||||
file_sha256(&std::env::current_exe()?)?,
|
||||
run.test_binary.sha256,
|
||||
"test executable must match the run receipt"
|
||||
);
|
||||
if directory.join(case.oracle).exists() {
|
||||
return Err("scanner/heal oracle already exists; create a new execution receipt".into());
|
||||
}
|
||||
Ok(Some(RestartEvidenceContext { directory, run, case }))
|
||||
}
|
||||
|
||||
fn compiled_test_identity() -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"source_revision": env!("RUSTFS_E2E_BUILD_COMMIT"),
|
||||
"dirty": env!("RUSTFS_E2E_BUILD_DIRTY") != "false",
|
||||
"lock_blob": env!("RUSTFS_E2E_BUILD_LOCK"),
|
||||
"features": env!("RUSTFS_E2E_BUILD_FEATURES"),
|
||||
"target": env!("RUSTFS_E2E_BUILD_TARGET"),
|
||||
"profile": env!("RUSTFS_E2E_BUILD_PROFILE"),
|
||||
"rustflags_hex": env!("RUSTFS_E2E_BUILD_RUSTFLAGS_HEX"),
|
||||
})
|
||||
}
|
||||
|
||||
struct TcpPortBlackhole {
|
||||
port: u16,
|
||||
comment: String,
|
||||
@@ -159,49 +42,6 @@ mod tests {
|
||||
}
|
||||
|
||||
impl TcpPortBlackhole {
|
||||
/// Environment flag that turns an unusable fault-injection host into a
|
||||
/// hard failure instead of a logged skip. Lanes that provision
|
||||
/// `CAP_NET_ADMIN` set it so a broken runner cannot pass silently.
|
||||
#[cfg(target_os = "linux")]
|
||||
const REQUIRE_ENV: &str = "RUSTFS_E2E_REQUIRE_NET_FAULT_INJECTION";
|
||||
|
||||
/// Probe whether this host can manipulate the OUTPUT chain at all.
|
||||
///
|
||||
/// Returns `Ok(Some(reason))` when `iptables` is missing or lacks
|
||||
/// `CAP_NET_ADMIN` (the nf_tables backend reports "Permission denied"
|
||||
/// even under `sudo` inside an unprivileged container) and the lane did
|
||||
/// not demand fault injection; returns an error when the lane demands
|
||||
/// it; returns `Ok(None)` when the blackhole can be installed.
|
||||
#[cfg(target_os = "linux")]
|
||||
fn unavailable_reason() -> Result<Option<String>, Box<dyn Error + Send + Sync>> {
|
||||
let id = Command::new("id").arg("-u").output()?;
|
||||
if !id.status.success() {
|
||||
return Err(format!("failed to determine the test process uid: {}", String::from_utf8_lossy(&id.stderr)).into());
|
||||
}
|
||||
let use_sudo = String::from_utf8_lossy(&id.stdout).trim() != "0";
|
||||
let mut command = if use_sudo {
|
||||
let mut command = Command::new("sudo");
|
||||
command.args(["-n", "iptables"]);
|
||||
command
|
||||
} else {
|
||||
Command::new("iptables")
|
||||
};
|
||||
let probe = command.args(["-w", "5", "-S", "OUTPUT"]).output();
|
||||
let reason = match probe {
|
||||
Ok(output) if output.status.success() => return Ok(None),
|
||||
Ok(output) => format!(
|
||||
"iptables cannot read the OUTPUT chain (status {}): {}",
|
||||
output.status,
|
||||
String::from_utf8_lossy(&output.stderr).trim()
|
||||
),
|
||||
Err(err) => format!("iptables is not runnable: {err}"),
|
||||
};
|
||||
if std::env::var_os(Self::REQUIRE_ENV).is_some() {
|
||||
return Err(format!("{} is set but network fault injection is unavailable: {reason}", Self::REQUIRE_ENV).into());
|
||||
}
|
||||
Ok(Some(reason))
|
||||
}
|
||||
|
||||
fn install(address: &str) -> Result<Self, Box<dyn Error + Send + Sync>> {
|
||||
let address = address.parse::<SocketAddr>()?;
|
||||
if !address.ip().is_loopback() {
|
||||
@@ -286,27 +126,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// Remove a disk directory underneath a running server. Background writers
|
||||
/// (scanner, usage cache, heal markers) can recreate entries between the
|
||||
/// recursive listing and the final `rmdir`, which surfaces as
|
||||
/// `DirectoryNotEmpty` on macOS; retry briefly so the wipe reflects the
|
||||
/// operator action rather than a listing race.
|
||||
fn wipe_directory_while_server_runs(disk: &Path) -> std::io::Result<()> {
|
||||
let mut last_err = None;
|
||||
for _ in 0..20 {
|
||||
match std::fs::remove_dir_all(disk) {
|
||||
Ok(()) => return Ok(()),
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(()),
|
||||
Err(err) if err.kind() == std::io::ErrorKind::DirectoryNotEmpty => {
|
||||
last_err = Some(err);
|
||||
std::thread::sleep(std::time::Duration::from_millis(100));
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
Err(last_err.expect("retry loop only exits without success after recording an error"))
|
||||
}
|
||||
|
||||
fn has_file_under(path: &Path) -> bool {
|
||||
let Ok(entries) = std::fs::read_dir(path) else {
|
||||
return false;
|
||||
@@ -376,9 +195,8 @@ mod tests {
|
||||
clients: &[aws_sdk_s3::Client],
|
||||
bucket: &str,
|
||||
expected_keys: &HashSet<String>,
|
||||
) -> Result<Vec<Vec<String>>, Box<dyn Error + Send + Sync>> {
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
const PAGE_SIZE: i32 = 10;
|
||||
let mut node_listings = Vec::with_capacity(clients.len());
|
||||
for (node_index, client) in clients.iter().enumerate() {
|
||||
let mut listed_keys = Vec::new();
|
||||
let mut continuation_token = None;
|
||||
@@ -425,10 +243,8 @@ mod tests {
|
||||
&listed_key_set, expected_keys,
|
||||
"node {node_index} did not expose the complete recovered namespace"
|
||||
);
|
||||
listed_keys.sort();
|
||||
node_listings.push(listed_keys);
|
||||
}
|
||||
Ok(node_listings)
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn heal_task_status_diagnostic(body: &str) -> String {
|
||||
@@ -589,7 +405,7 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
wipe_directory_while_server_runs(&disk0).expect("disk0 wipe should succeed while server is running");
|
||||
std::fs::remove_dir_all(&disk0).expect("disk0 wipe should succeed while server is running");
|
||||
std::fs::create_dir_all(&disk0).expect("disk0 should be recreated empty while server is running");
|
||||
assert!(!has_file_under(&disk0), "disk0 must be empty immediately after runtime wipe");
|
||||
|
||||
@@ -963,16 +779,6 @@ mod tests {
|
||||
.await?
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_cluster_root_heal_recovers_remote_shards_after_background_target_crash()
|
||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
timeout(
|
||||
Duration::from_secs(420),
|
||||
run_cluster_root_heal_interruption(InterruptionScenario::BackgroundTargetCrash),
|
||||
)
|
||||
.await?
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_cluster_root_heal_recovers_remote_shards_after_coordinator_restart() -> Result<(), Box<dyn Error + Send + Sync>>
|
||||
{
|
||||
@@ -986,18 +792,6 @@ mod tests {
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_cluster_root_heal_recovers_after_target_endpoint_blackhole() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
if let Some(reason) = TcpPortBlackhole::unavailable_reason()? {
|
||||
init_logging();
|
||||
warn!(
|
||||
event = "heal_interruption_skipped",
|
||||
component = "e2e_test",
|
||||
subsystem = "heal",
|
||||
interruption_kind = "target_endpoint_blackhole",
|
||||
reason,
|
||||
"Skipping endpoint blackhole scenario: network fault injection is unavailable on this host"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
timeout(
|
||||
Duration::from_secs(420),
|
||||
run_cluster_root_heal_interruption(InterruptionScenario::TargetEndpointBlackhole),
|
||||
@@ -1009,27 +803,14 @@ mod tests {
|
||||
enum InterruptionScenario {
|
||||
IsolatedTargetRestart,
|
||||
BackgroundTargetRestart,
|
||||
BackgroundTargetCrash,
|
||||
BackgroundCoordinatorRestart,
|
||||
TargetEndpointBlackhole,
|
||||
}
|
||||
|
||||
async fn run_cluster_root_heal_interruption(scenario: InterruptionScenario) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let server_binary = rustfs_binary_path();
|
||||
let evidence_run = match scenario {
|
||||
InterruptionScenario::BackgroundTargetRestart => {
|
||||
restart_evidence_run(&server_binary, BACKGROUND_TARGET_RESTART_EVIDENCE)?
|
||||
}
|
||||
InterruptionScenario::BackgroundTargetCrash => {
|
||||
restart_evidence_run(&server_binary, BACKGROUND_TARGET_CRASH_EVIDENCE)?
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let mut evidence_objects = Vec::new();
|
||||
let (background_enabled, interruption_node, interruption_kind) = match scenario {
|
||||
InterruptionScenario::IsolatedTargetRestart => (false, 1, "target_restart"),
|
||||
InterruptionScenario::BackgroundTargetRestart => (true, 1, "background_target_restart"),
|
||||
InterruptionScenario::BackgroundTargetCrash => (true, 1, "background_target_crash"),
|
||||
InterruptionScenario::BackgroundCoordinatorRestart => (true, 0, "coordinator_restart"),
|
||||
InterruptionScenario::TargetEndpointBlackhole => (false, 1, "target_endpoint_blackhole"),
|
||||
};
|
||||
@@ -1074,7 +855,7 @@ mod tests {
|
||||
for node_index in 0..cluster.nodes.len() {
|
||||
cluster.set_node_capture_log_path(node_index, format!("{log_dir}/node{node_index}.log"))?;
|
||||
}
|
||||
cluster.start_with_binary(&server_binary).await?;
|
||||
cluster.start().await?;
|
||||
let clients = cluster.create_all_clients()?;
|
||||
|
||||
let bucket = "heal-restart-during-rebuild";
|
||||
@@ -1096,7 +877,6 @@ mod tests {
|
||||
.unwrap_or(4 * 1024 * 1024)
|
||||
.clamp(1024 * 1024, 16 * 1024 * 1024);
|
||||
let mut expected_manifests = Vec::with_capacity(online_object_count);
|
||||
let mut unclean_shutdown_marker_observed = None;
|
||||
for index in 0..online_object_count {
|
||||
let key = format!("cluster/online/object-{index:04}.bin");
|
||||
let payload_seed = u8::try_from(index + 1).expect("clamped object count must fit in u8");
|
||||
@@ -1216,7 +996,7 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
cluster.start_node_from_binary(1, &server_binary).await?;
|
||||
cluster.start_node(1).await?;
|
||||
|
||||
let status_url = format!("{}/rustfs/admin/v3/background-heal/status", cluster.nodes[0].url);
|
||||
let recovery_deadline = Instant::now() + Duration::from_secs(60);
|
||||
@@ -1463,11 +1243,7 @@ mod tests {
|
||||
"Restored target endpoint forwarding"
|
||||
);
|
||||
} else {
|
||||
if scenario == InterruptionScenario::BackgroundTargetRestart {
|
||||
cluster.stop_node_gracefully(interruption_node).await?;
|
||||
} else {
|
||||
cluster.stop_node(interruption_node)?;
|
||||
}
|
||||
cluster.stop_node(interruption_node)?;
|
||||
let stopped_count = metadata_count(&replaced_disk, bucket, &expected_manifests);
|
||||
assert!(
|
||||
stopped_count > 0 && stopped_count < expected_manifests.len(),
|
||||
@@ -1483,12 +1259,9 @@ mod tests {
|
||||
.join(".rustfs.sys")
|
||||
.join("unclean-shutdown");
|
||||
if background_enabled {
|
||||
let marker_exists = unclean_shutdown_marker.is_file();
|
||||
unclean_shutdown_marker_observed = Some(marker_exists);
|
||||
let expected_marker = !matches!(scenario, InterruptionScenario::BackgroundTargetRestart);
|
||||
assert!(
|
||||
marker_exists == expected_marker,
|
||||
"background restart/crash lane observed unexpected unclean-shutdown marker state"
|
||||
unclean_shutdown_marker.is_file(),
|
||||
"background restart must retain the real unclean-shutdown marker"
|
||||
);
|
||||
} else {
|
||||
match std::fs::remove_file(&unclean_shutdown_marker) {
|
||||
@@ -1501,7 +1274,7 @@ mod tests {
|
||||
}
|
||||
}
|
||||
}
|
||||
cluster.start_node_from_binary(interruption_node, &server_binary).await?;
|
||||
cluster.start_node(interruption_node).await?;
|
||||
if interruption_node == 0 {
|
||||
let target = cluster.nodes[1]
|
||||
.process
|
||||
@@ -1600,7 +1373,7 @@ mod tests {
|
||||
.map(|manifest| manifest.key.clone())
|
||||
.collect::<HashSet<_>>();
|
||||
assert!(expected_keys.insert(outage_key.to_string()));
|
||||
let node_listings = assert_all_nodes_list_exact_keys(&clients, bucket, &expected_keys).await?;
|
||||
assert_all_nodes_list_exact_keys(&clients, bucket, &expected_keys).await?;
|
||||
|
||||
let target_client = cluster.create_s3_client(1)?;
|
||||
for expected in &expected_manifests {
|
||||
@@ -1608,31 +1381,11 @@ mod tests {
|
||||
let actual = response.body.collect().await?.into_bytes();
|
||||
let expected_body = deterministic_object_body(object_size_bytes, expected.payload_seed);
|
||||
assert_eq!(actual.as_ref(), expected_body.as_slice(), "object body changed for {}", expected.key);
|
||||
if evidence_run.is_some() {
|
||||
evidence_objects.push(serde_json::json!({
|
||||
"key": expected.key, "version_id": expected.shard_census.version_id,
|
||||
"expected_bytes": expected_body.len(), "actual_bytes": actual.len(),
|
||||
"expected_sha256": sha256_hex(&expected_body),
|
||||
"actual_sha256": sha256_hex(&actual),
|
||||
"expected_physical": expected.shard_census,
|
||||
"physical": census_object_version_on_disk(&replaced_disk, bucket, &expected.key, None)?,
|
||||
}));
|
||||
}
|
||||
}
|
||||
let response = target_client.get_object().bucket(bucket).key(outage_key).send().await?;
|
||||
let actual = response.body.collect().await?.into_bytes();
|
||||
let expected_outage_body = deterministic_object_body(object_size_bytes, outage_payload_seed);
|
||||
assert_eq!(actual.as_ref(), expected_outage_body.as_slice(), "object body changed for {outage_key}");
|
||||
if evidence_run.is_some() {
|
||||
evidence_objects.push(serde_json::json!({
|
||||
"key": outage_key, "version_id": null,
|
||||
"expected_bytes": expected_outage_body.len(), "actual_bytes": actual.len(),
|
||||
"expected_sha256": sha256_hex(&expected_outage_body),
|
||||
"actual_sha256": sha256_hex(&actual),
|
||||
"expected_physical": null,
|
||||
"physical": census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?,
|
||||
}));
|
||||
}
|
||||
|
||||
let terminal_deadline = Instant::now() + Duration::from_secs(30);
|
||||
loop {
|
||||
@@ -1679,37 +1432,6 @@ mod tests {
|
||||
return Err(format!("heal data rebuilt but task did not finish successfully: {task_status}").into());
|
||||
}
|
||||
|
||||
if let Some(evidence_context) = evidence_run {
|
||||
let restarted_pid = cluster.nodes[1].process.as_ref().ok_or("restarted target is absent")?.id();
|
||||
assert_ne!(target_pid, restarted_pid, "target must be a new process");
|
||||
assert_eq!(
|
||||
file_sha256(&server_binary)?,
|
||||
evidence_context.run.binary.sha256,
|
||||
"server build changed during restart"
|
||||
);
|
||||
let evidence = serde_json::json!({
|
||||
"schema": 1, "case": evidence_context.case.id, "evidence": evidence_context.case.evidence,
|
||||
"run_id": evidence_context.run.run_id, "source_revision": evidence_context.run.source_revision,
|
||||
"test_build": compiled_test_identity(),
|
||||
"binary_sha256": evidence_context.run.binary.sha256,
|
||||
"test_binary_sha256": evidence_context.run.test_binary.sha256,
|
||||
"topology": {"nodes": cluster.nodes.len(), "drives_per_node": cluster.nodes[0].data_dirs.len()},
|
||||
"pid_before": target_pid, "pid_after": restarted_pid,
|
||||
"unclean_shutdown_marker": unclean_shutdown_marker_observed.unwrap_or(false),
|
||||
"objects": evidence_objects, "node_listings": node_listings,
|
||||
});
|
||||
let data = serde_json::to_vec(&evidence)?;
|
||||
if data.len() > 1024 * 1024 {
|
||||
return Err("scanner/heal oracle exceeds the 1 MiB artifact budget".into());
|
||||
}
|
||||
let mut output = std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(evidence_context.directory.join(evidence_context.case.oracle))?;
|
||||
output.write_all(&data)?;
|
||||
output.sync_all()?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -378,11 +378,6 @@ mod bucket_stats_regression_test;
|
||||
#[cfg(test)]
|
||||
mod distributed_startup_regression_test;
|
||||
|
||||
// 4-node / 4-disk distributed Actions suite (S3, lock, versioning, replication,
|
||||
// quota, observability, expand/decommission/rebalance, site replication, chaos).
|
||||
#[cfg(test)]
|
||||
mod distributed;
|
||||
|
||||
// P1 regression: tier/ILM transition (rustfs#5218, #5130, #5011, #4826, #5024)
|
||||
#[cfg(test)]
|
||||
mod tier_transition_regression_test;
|
||||
|
||||
@@ -21,7 +21,7 @@
|
||||
use super::common::{BoxError, OdmSourceSpec, OdmTestEnv, SeedObject};
|
||||
use crate::fake_s3_target::{BucketMode, Operation};
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, ObjectAttributes, VersioningConfiguration};
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
use bytes::Bytes;
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -103,19 +103,11 @@ async fn get_miss_pulls_inline_and_serves_locally_afterwards() -> TestResult {
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_large_object_streams_through_and_backfills_in_background() -> TestResult {
|
||||
const PART_SIZE: usize = 5 * 1024 * 1024;
|
||||
let bucket = "odm-get-large";
|
||||
let env = configured_env(bucket, |spec| {
|
||||
spec.policy.inline_max_bytes = 4096;
|
||||
spec.policy.multipart_part_size_bytes = u64::try_from(PART_SIZE).expect("part size fits in u64");
|
||||
})
|
||||
.await?;
|
||||
let env = configured_env(bucket, |spec| spec.policy.inline_max_bytes = 4096).await?;
|
||||
let key = "large/archive.bin";
|
||||
let body = payload(PART_SIZE + 4096);
|
||||
let etag = env
|
||||
.seed_source(SOURCE_BUCKET, &[SeedObject::new(key, body.clone())])
|
||||
.remove(0);
|
||||
assert_eq!(etag.len(), 32, "the source fixture has a plain MD5 ETag");
|
||||
let body = payload(512 * 1024);
|
||||
env.seed_source(SOURCE_BUCKET, &[SeedObject::new(key, body.clone())]);
|
||||
|
||||
let response = env.raw_get(bucket, key).await?;
|
||||
assert_eq!(response.status, 200, "{}", String::from_utf8_lossy(&response.body));
|
||||
@@ -133,68 +125,6 @@ async fn get_large_object_streams_through_and_backfills_in_background() -> TestR
|
||||
vec![None, None],
|
||||
"one passthrough GET plus one background pull, both unranged"
|
||||
);
|
||||
|
||||
let source_requests = env.source.requests().len();
|
||||
let second_part = env.client.get_object().bucket(bucket).key(key).part_number(2).send().await?;
|
||||
assert_eq!(second_part.content_length(), Some(4096), "the completed second part is the tail");
|
||||
assert_eq!(
|
||||
second_part.content_range(),
|
||||
Some(format!("bytes {PART_SIZE}-{}/{}", body.len() - 1, body.len()).as_str()),
|
||||
"partNumber reads the stored multipart boundary"
|
||||
);
|
||||
assert_eq!(
|
||||
second_part.body.collect().await?.into_bytes(),
|
||||
body.slice(PART_SIZE..),
|
||||
"the local second part contains the exact source tail"
|
||||
);
|
||||
let third_part = env
|
||||
.client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.part_number(3)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("the completed object has exactly two parts");
|
||||
assert_eq!(third_part.code(), Some("InvalidPart"));
|
||||
|
||||
let mut part_marker = None;
|
||||
for (part_number, part_size) in [(1, PART_SIZE), (2, 4096)] {
|
||||
let attributes = env
|
||||
.client
|
||||
.get_object_attributes()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.object_attributes(ObjectAttributes::ObjectParts)
|
||||
.object_attributes(ObjectAttributes::Etag)
|
||||
.max_parts(1)
|
||||
.set_part_number_marker(part_marker.clone())
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
attributes.e_tag().map(|value| value.trim_matches('"')),
|
||||
Some(etag.as_str()),
|
||||
"multipart write-back preserves the source MD5 ETag"
|
||||
);
|
||||
let parts = attributes
|
||||
.object_parts()
|
||||
.expect("RustFS must expose the stored multipart layout");
|
||||
assert_eq!(parts.total_parts_count(), Some(2));
|
||||
assert_eq!(parts.max_parts(), Some(1));
|
||||
assert_eq!(parts.is_truncated(), Some(part_number == 1));
|
||||
assert_eq!(parts.parts().len(), 1, "RustFS returns one stored part per requested page");
|
||||
assert_eq!(parts.parts()[0].part_number(), Some(part_number));
|
||||
assert_eq!(parts.parts()[0].size(), Some(i64::try_from(part_size).expect("part size fits in i64")));
|
||||
part_marker = parts.next_part_number_marker().map(str::to_owned);
|
||||
if part_number == 1 {
|
||||
assert_eq!(part_marker.as_deref(), Some("1"), "the next request continues after the first part");
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
env.source.requests().len(),
|
||||
source_requests,
|
||||
"local part reads must not consult the source"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -22,21 +22,17 @@
|
||||
//! local object and what the source was asked for.
|
||||
|
||||
use super::common::{
|
||||
ALLOW_LOOPBACK_SOURCE_ENV, AdminResponse, BackfillOp, BackfillRequest, BoxError, ODM_MODULE_SWITCH_ENV, ODM_SERVER_ENV,
|
||||
OdmEnvOptions, OdmSourceSpec, OdmTestEnv, SeedObject, start_configured_env, start_configured_env_with, start_source_rustfs,
|
||||
AdminResponse, BoxError, OdmEnvOptions, OdmSourceSpec, OdmTestEnv, SeedObject, start_configured_env,
|
||||
start_configured_env_with,
|
||||
};
|
||||
use crate::common::{RustFSTestEnvironment, replication_fast_env, signed_request};
|
||||
use crate::fake_s3_target::{BucketMode, FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, Operation};
|
||||
use crate::object_lock::common::put_object_lock_configuration;
|
||||
use crate::replication_extension_test::{
|
||||
ReplicationTargetOptions, enable_bucket_versioning, set_replication_target_with_options,
|
||||
};
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::types::{
|
||||
BucketVersioningStatus, Event, FilterRule, FilterRuleName, NotificationConfiguration, NotificationConfigurationFilter,
|
||||
ObjectAttributes, ObjectLockRetentionMode, QueueConfiguration, S3KeyFilter, ServerSideEncryption,
|
||||
ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, Tagging,
|
||||
VersioningConfiguration,
|
||||
ObjectLockRetentionMode, QueueConfiguration, S3KeyFilter, ServerSideEncryption, ServerSideEncryptionByDefault,
|
||||
ServerSideEncryptionConfiguration, ServerSideEncryptionRule, VersioningConfiguration,
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use local_ip_address::local_ip;
|
||||
@@ -584,130 +580,6 @@ async fn test_odm_pulled_object_replicates_and_target_as_source_is_rejected() ->
|
||||
"a bucket may not migrate from its own replication target: {}",
|
||||
rejected.body
|
||||
);
|
||||
Box::pin(assert_odm_multipart_replicates_to_rustfs(&env, bucket)).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_odm_multipart_replicates_to_rustfs(env: &OdmTestEnv, bucket: &str) -> TestResult {
|
||||
const PART_SIZE: usize = 5 * 1024 * 1024;
|
||||
let replica = start_source_rustfs().await?;
|
||||
let replica_bucket = "odm-real-replica";
|
||||
replica.create_test_bucket(replica_bucket).await?;
|
||||
enable_bucket_versioning(&replica, replica_bucket).await?;
|
||||
let arn = set_replication_target_with_options(
|
||||
&env.rustfs,
|
||||
bucket,
|
||||
ReplicationTargetOptions {
|
||||
endpoint: &replica.address,
|
||||
access_key: &replica.access_key,
|
||||
secret_key: &replica.secret_key,
|
||||
target_bucket: replica_bucket,
|
||||
secure: false,
|
||||
skip_tls_verify: false,
|
||||
ca_cert_pem: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
put_bucket_replication(&env.rustfs, bucket, &arn).await?;
|
||||
let mut spec = env.fake_source_spec(SOURCE_BUCKET);
|
||||
// Below the 16 MiB inline default the pull is one tee'd PUT with a single
|
||||
// part; force the passthrough + background multipart write-back instead.
|
||||
spec.policy.inline_max_bytes = 4096;
|
||||
spec.policy.multipart_part_size_bytes = PART_SIZE as u64;
|
||||
spec.policy.preserve_etag = true;
|
||||
env.configure_and_wait(bucket, &spec).await?;
|
||||
|
||||
let key = "replicated/preserved-md5-multipart.bin";
|
||||
let body = payload(PART_SIZE + 4096);
|
||||
let source_put = env
|
||||
.source_client()
|
||||
.put_object()
|
||||
.bucket(SOURCE_BUCKET)
|
||||
.key(key)
|
||||
.body(aws_sdk_s3::primitives::ByteStream::from(body.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
let etag = source_put.e_tag().ok_or("source PUT omitted its ETag")?.trim_matches('"');
|
||||
assert_eq!(etag.len(), 32, "the source must retain a single-PUT MD5 ETag");
|
||||
assert!(etag.bytes().all(|byte| byte.is_ascii_hexdigit()));
|
||||
let pulled = env.raw_get(bucket, key).await?;
|
||||
assert_eq!(pulled.status, 200, "{}", String::from_utf8_lossy(&pulled.body));
|
||||
assert_eq!(pulled.body, body);
|
||||
assert!(env.wait_local_listed(bucket, key, SETTLE).await?, "the multipart pull must persist");
|
||||
|
||||
let deadline = Instant::now() + SETTLE;
|
||||
let source_head = loop {
|
||||
let head = env.client.head_object().bucket(bucket).key(key).send().await?;
|
||||
match head.replication_status().map(|status| status.as_str()) {
|
||||
Some("COMPLETED") => break head,
|
||||
Some("FAILED") => return Err("the ODM multipart copy failed replication to RustFS".into()),
|
||||
_ => {
|
||||
assert!(Instant::now() < deadline, "the ODM multipart copy never completed replication to RustFS");
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
}
|
||||
}
|
||||
};
|
||||
let version = source_head
|
||||
.version_id()
|
||||
.ok_or("the versioned ODM copy omitted its version id")?;
|
||||
assert_ne!(version, "null");
|
||||
let replica_client = replica.create_s3_client();
|
||||
for (client, object_bucket) in [(&env.client, bucket), (&replica_client, replica_bucket)] {
|
||||
let attributes = client
|
||||
.get_object_attributes()
|
||||
.bucket(object_bucket)
|
||||
.key(key)
|
||||
.version_id(version)
|
||||
.object_attributes(ObjectAttributes::Etag)
|
||||
.object_attributes(ObjectAttributes::ObjectParts)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(attributes.e_tag().map(|value| value.trim_matches('"')), Some(etag));
|
||||
let parts = attributes
|
||||
.object_parts()
|
||||
.ok_or("the local copy and replica must both expose two parts")?;
|
||||
assert_eq!(parts.total_parts_count(), Some(2));
|
||||
assert_eq!(
|
||||
parts
|
||||
.parts()
|
||||
.iter()
|
||||
.map(|part| (part.part_number(), part.size()))
|
||||
.collect::<Vec<_>>(),
|
||||
[(Some(1), Some(PART_SIZE as i64)), (Some(2), Some(4096))]
|
||||
);
|
||||
}
|
||||
// REPLICA status surfaces on HEAD, like the other inbound-replica checks.
|
||||
let replica_head = replica_client
|
||||
.head_object()
|
||||
.bucket(replica_bucket)
|
||||
.key(key)
|
||||
.version_id(version)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(replica_head.replication_status().map(|status| status.as_str()), Some("REPLICA"));
|
||||
let replica_get = replica_client
|
||||
.get_object()
|
||||
.bucket(replica_bucket)
|
||||
.key(key)
|
||||
.version_id(version)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(replica_get.version_id(), Some(version));
|
||||
assert_eq!(replica_get.body.collect().await?.into_bytes(), body);
|
||||
let boundary = replica_client
|
||||
.get_object()
|
||||
.bucket(replica_bucket)
|
||||
.key(key)
|
||||
.version_id(version)
|
||||
.range(format!("bytes={}-{}", PART_SIZE - 32, PART_SIZE + 31))
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(boundary.body.collect().await?.into_bytes(), body.slice(PART_SIZE - 32..PART_SIZE + 32));
|
||||
assert_eq!(
|
||||
env.source.count_requests(Operation::GetObject, key),
|
||||
2,
|
||||
"one passthrough GET plus one background pull; replication and local reads must not fetch the migration source again"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -861,242 +733,6 @@ async fn test_odm_disable_keeps_pulled_objects_and_stops_source_traffic() -> Tes
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The process switch preserves configured buckets and unfinished jobs while
|
||||
/// restoring local-only S3 behavior, including after an ordinary metadata write.
|
||||
#[tokio::test]
|
||||
async fn test_odm_global_disable_preserves_data_config_and_backfill_across_restarts() -> TestResult {
|
||||
let bucket = "odm-global-disable";
|
||||
let mut env = start_configured_env(bucket, SOURCE_BUCKET, |spec| spec.policy.list_through = true).await?;
|
||||
let pulled_key = "migrated/pulled.bin";
|
||||
let remote_key = "remote/untouched.bin";
|
||||
let pending_key = "backfill/pending.bin";
|
||||
let local_key = "local/kept.bin";
|
||||
let source_body = Bytes::from_static(b"source payload");
|
||||
let local_body = Bytes::from_static(b"client payload");
|
||||
env.seed_source(
|
||||
SOURCE_BUCKET,
|
||||
&[
|
||||
SeedObject::new(pulled_key, source_body.clone()),
|
||||
SeedObject::new(remote_key, source_body.clone()),
|
||||
SeedObject::new(pending_key, source_body.clone()),
|
||||
],
|
||||
);
|
||||
env.client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(local_key)
|
||||
.body(local_body.clone().into())
|
||||
.send()
|
||||
.await?;
|
||||
let pulled = env.raw_get(bucket, pulled_key).await?;
|
||||
assert_eq!(pulled.status, 200);
|
||||
assert_eq!(pulled.header(ODM_RESPONSE_HEADER), Some("source"));
|
||||
assert_eq!(pulled.body, source_body);
|
||||
let stored = env.raw_get(bucket, pulled_key).await?;
|
||||
assert_eq!(stored.status, 200);
|
||||
assert_eq!(stored.header(ODM_RESPONSE_HEADER), None, "the inline pull has committed locally");
|
||||
assert_eq!(stored.body, source_body);
|
||||
let config = env.get_config(bucket).await?;
|
||||
assert_eq!(config.status, 200, "{}", config.body);
|
||||
let config = config.json()?;
|
||||
|
||||
// Hold every attempt until the process has exited, so retries cannot commit
|
||||
// the only backfill object before the crash. The start checkpoint exists.
|
||||
let mut pending_get = env.source.hold_get_object(SOURCE_BUCKET, pending_key);
|
||||
let started = env
|
||||
.start_backfill(
|
||||
bucket,
|
||||
BackfillRequest {
|
||||
prefix: Some("backfill/".to_string()),
|
||||
..BackfillRequest::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(started.status, 200, "{}", started.body);
|
||||
let job_id = started.json()?["job"]["job_id"].as_str().ok_or("missing job ID")?.to_string();
|
||||
tokio::time::timeout(Duration::from_secs(10), pending_get.wait_until_entered())
|
||||
.await
|
||||
.expect("backfill never reached the held source GET")?;
|
||||
let process = env.rustfs.process.as_mut().ok_or("missing RustFS process before crash")?;
|
||||
assert!(process.try_wait()?.is_none(), "RustFS exited before the controlled crash");
|
||||
process.kill()?;
|
||||
let stopped = process.wait()?;
|
||||
assert!(!stopped.success(), "the interrupted process must exit after being killed");
|
||||
drop(env.rustfs.process.take());
|
||||
drop(pending_get);
|
||||
env.source.take_requests();
|
||||
env.rustfs
|
||||
.restart_server_preserving_data(vec![], &[(ODM_MODULE_SWITCH_ENV, "false"), (ALLOW_LOOPBACK_SOURCE_ENV, "true")])
|
||||
.await?;
|
||||
|
||||
let off_config = env.get_config(bucket).await?;
|
||||
assert_eq!(off_config.status, 200, "{}", off_config.body);
|
||||
assert_eq!(off_config.json()?, config, "the saved configuration and timestamp survive disabling");
|
||||
let status = env.status_json(bucket).await?;
|
||||
assert_eq!(status["configured"], true, "{status}");
|
||||
assert_eq!(status["enabled"], true, "the bucket remains configured as enabled: {status}");
|
||||
assert_eq!(status["module_enabled"], false, "{status}");
|
||||
assert_eq!(status["counters"], Value::Null, "no bucket runtime is installed: {status}");
|
||||
let checkpoint = env.backfill_job(bucket).await?.ok_or("disabled module lost the checkpoint")?;
|
||||
assert_eq!(checkpoint["job_id"], job_id);
|
||||
assert_eq!(checkpoint["state"], "running", "the interrupted job is retained: {checkpoint}");
|
||||
|
||||
for (key, body) in [(local_key, &local_body), (pulled_key, &source_body)] {
|
||||
let get = env.raw_get(bucket, key).await?;
|
||||
assert_eq!(get.status, 200);
|
||||
assert_eq!(&get.body, body);
|
||||
assert_eq!(get.header(ODM_RESPONSE_HEADER), None);
|
||||
let head = env.client.head_object().bucket(bucket).key(key).send().await?;
|
||||
assert_eq!(head.content_length(), Some(i64::try_from(body.len())?));
|
||||
}
|
||||
for key in [remote_key, pending_key] {
|
||||
let get = env.raw_get(bucket, key).await?;
|
||||
assert_eq!(get.status, 404, "disabled source GET {key}: {}", String::from_utf8_lossy(&get.body));
|
||||
let head = env.client.head_object().bucket(bucket).key(key).send().await;
|
||||
let err = head.expect_err("a source-only object must remain absent locally");
|
||||
assert_eq!(err.raw_response().map(|response| response.status().as_u16()), Some(404));
|
||||
}
|
||||
|
||||
let replacement = Bytes::from_static(b"written while the module is off");
|
||||
for key in [local_key, "local/deleted.bin"] {
|
||||
env.client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(replacement.clone().into())
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
env.client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key("local/deleted.bin")
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(env.raw_get(bucket, "local/deleted.bin").await?.status, 404);
|
||||
assert_eq!(env.raw_get(bucket, local_key).await?.body, replacement);
|
||||
|
||||
// Both wire protocols must finish their local pages even though the saved
|
||||
// configuration still requests list-through.
|
||||
for use_v2 in [false, true] {
|
||||
let mut cursor = None;
|
||||
let mut listed = Vec::new();
|
||||
for page_number in 0..2 {
|
||||
let (keys, truncated, next) = if use_v2 {
|
||||
let page = env
|
||||
.client
|
||||
.list_objects_v2()
|
||||
.bucket(bucket)
|
||||
.max_keys(1)
|
||||
.set_continuation_token(cursor)
|
||||
.send()
|
||||
.await?;
|
||||
(
|
||||
page.contents()
|
||||
.iter()
|
||||
.map(|object| object.key().expect("listed key").to_string())
|
||||
.collect::<Vec<_>>(),
|
||||
page.is_truncated(),
|
||||
page.next_continuation_token().map(str::to_string),
|
||||
)
|
||||
} else {
|
||||
let page = env
|
||||
.client
|
||||
.list_objects()
|
||||
.bucket(bucket)
|
||||
.max_keys(1)
|
||||
.set_marker(cursor)
|
||||
.send()
|
||||
.await?;
|
||||
// V1 may omit NextMarker without a delimiter; clients then
|
||||
// continue from the last returned key.
|
||||
let next = page.next_marker().or_else(|| {
|
||||
if page.is_truncated() == Some(true) {
|
||||
page.contents().last().and_then(|object| object.key())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
});
|
||||
(
|
||||
page.contents()
|
||||
.iter()
|
||||
.map(|object| object.key().expect("listed key").to_string())
|
||||
.collect::<Vec<_>>(),
|
||||
page.is_truncated(),
|
||||
next.map(str::to_string),
|
||||
)
|
||||
};
|
||||
assert_eq!(keys.len(), 1, "one local key per page, V2={use_v2}");
|
||||
assert_eq!(truncated, Some(page_number == 0), "local pagination must terminate, V2={use_v2}");
|
||||
if page_number == 0 {
|
||||
assert!(next.as_ref().is_some_and(|value| !value.is_empty()), "missing local cursor, V2={use_v2}");
|
||||
}
|
||||
cursor = next;
|
||||
listed.extend(keys);
|
||||
}
|
||||
assert_eq!(listed, [local_key, pulled_key], "source-only keys must stay absent, V2={use_v2}");
|
||||
}
|
||||
|
||||
let spec = env.fake_source_spec(SOURCE_BUCKET);
|
||||
for response in [
|
||||
env.configure_source(bucket, &spec).await?,
|
||||
env.validate_source(bucket, &spec).await?,
|
||||
env.backfill(bucket, BackfillOp::Start(BackfillRequest::default())).await?,
|
||||
] {
|
||||
assert_eq!(response.status, 400, "{}", response.body);
|
||||
assert!(response.body.contains("OnDemandMigrationDisabled"), "{}", response.body);
|
||||
}
|
||||
let tagging = Tagging::builder()
|
||||
.tag_set(Tag::builder().key("module").value("disabled").build()?)
|
||||
.build()?;
|
||||
env.client
|
||||
.put_bucket_tagging()
|
||||
.bucket(bucket)
|
||||
.tagging(tagging.clone())
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(env.get_config(bucket).await?.json()?, config, "an unrelated metadata write preserves ODM");
|
||||
assert_eq!(
|
||||
env.backfill_job(bucket).await?,
|
||||
Some(checkpoint),
|
||||
"no recovery or checkpoint update while disabled"
|
||||
);
|
||||
assert!(
|
||||
env.source.requests().is_empty(),
|
||||
"disabled startup and all requests must leave the source untouched"
|
||||
);
|
||||
|
||||
env.rustfs.restart_server_preserving_data(vec![], ODM_SERVER_ENV).await?;
|
||||
env.wait_until_source_consulted(bucket).await?;
|
||||
assert_eq!(
|
||||
env.get_config(bucket).await?.json()?,
|
||||
config,
|
||||
"reenabling uses the persisted configuration"
|
||||
);
|
||||
let tags = env.client.get_bucket_tagging().bucket(bucket).send().await?;
|
||||
assert_eq!(tags.tag_set(), tagging.tag_set(), "the ordinary metadata write also persists");
|
||||
let resumed = env.raw_get(bucket, remote_key).await?;
|
||||
assert_eq!(resumed.status, 200);
|
||||
assert_eq!(resumed.header(ODM_RESPONSE_HEADER), Some("source"));
|
||||
assert_eq!(resumed.body, source_body, "stored credentials still authenticate without reconfiguration");
|
||||
let completed = env
|
||||
.wait_for_backfill(bucket, SETTLE, |job| job["state"] == "completed")
|
||||
.await?;
|
||||
assert_eq!(completed["job_id"], job_id, "the interrupted job resumes without a new start");
|
||||
assert_eq!(completed["failed"], 0, "{completed}");
|
||||
for (key, body) in [
|
||||
(local_key, &replacement),
|
||||
(pulled_key, &source_body),
|
||||
(pending_key, &source_body),
|
||||
] {
|
||||
let get = env.raw_get(bucket, key).await?;
|
||||
assert_eq!(get.status, 200);
|
||||
assert_eq!(&get.body, body);
|
||||
assert_eq!(get.header(ODM_RESPONSE_HEADER), None, "{key} remains stored locally");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Case 19: the admin surface an operator sees — the configuration read back
|
||||
/// without its secret, and a status document whose counters match the source
|
||||
/// journal exactly.
|
||||
|
||||
@@ -368,23 +368,23 @@ impl Drop for SlowReplicationTargetGuard {
|
||||
// Mirrors madmin-go `ResyncTargetsInfo`/`ResyncTarget` json tags — the same
|
||||
// shape `mc replicate resync status` decodes.
|
||||
#[derive(Debug, Clone, serde::Deserialize)]
|
||||
pub(crate) struct ReplicationResetStatusResponse {
|
||||
struct ReplicationResetStatusResponse {
|
||||
#[serde(rename = "target", default)]
|
||||
pub(crate) targets: Vec<ReplicationResetStatusTarget>,
|
||||
targets: Vec<ReplicationResetStatusTarget>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, serde::Deserialize)]
|
||||
pub(crate) struct ReplicationResetStatusTarget {
|
||||
struct ReplicationResetStatusTarget {
|
||||
#[serde(rename = "arn", default)]
|
||||
pub(crate) arn: String,
|
||||
arn: String,
|
||||
#[serde(rename = "resetid", default)]
|
||||
pub(crate) reset_id: String,
|
||||
reset_id: String,
|
||||
#[serde(rename = "resyncStatus", default)]
|
||||
pub(crate) status: String,
|
||||
status: String,
|
||||
#[serde(rename = "replicationCount", default)]
|
||||
pub(crate) replicated_count: i64,
|
||||
replicated_count: i64,
|
||||
#[serde(rename = "object", default)]
|
||||
pub(crate) object: String,
|
||||
object: String,
|
||||
}
|
||||
|
||||
fn extract_xml_tag(xml: &str, tag: &str) -> Option<String> {
|
||||
@@ -512,7 +512,7 @@ pub(crate) async fn put_bucket_replication(
|
||||
put_bucket_replication_with_delete_statuses(env, bucket, target_arn, "Enabled", None).await
|
||||
}
|
||||
|
||||
pub(crate) async fn put_bucket_replication_with_delete_statuses(
|
||||
async fn put_bucket_replication_with_delete_statuses(
|
||||
env: &RustFSTestEnvironment,
|
||||
bucket: &str,
|
||||
target_arn: &str,
|
||||
@@ -627,7 +627,7 @@ async fn put_bucket_replication_rules(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_bucket_replication(
|
||||
async fn delete_bucket_replication(
|
||||
env: &RustFSTestEnvironment,
|
||||
bucket: &str,
|
||||
) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
|
||||
@@ -2294,7 +2294,7 @@ async fn site_replication_state_edit(
|
||||
/// return the target `(arn, reset_id)`, asserting the response carries the
|
||||
/// madmin `ResyncTargetsInfo` shape (`target[0].arn` / `target[0].resetid`)
|
||||
/// that `mc replicate resync start` decodes.
|
||||
pub(crate) async fn start_bucket_replication_reset(
|
||||
async fn start_bucket_replication_reset(
|
||||
env: &RustFSTestEnvironment,
|
||||
bucket: &str,
|
||||
) -> Result<(String, String), Box<dyn Error + Send + Sync>> {
|
||||
@@ -2314,7 +2314,7 @@ pub(crate) async fn start_bucket_replication_reset(
|
||||
Ok((arn, reset_id))
|
||||
}
|
||||
|
||||
pub(crate) async fn get_replication_reset_status(
|
||||
async fn get_replication_reset_status(
|
||||
env: &RustFSTestEnvironment,
|
||||
bucket: &str,
|
||||
arn: &str,
|
||||
@@ -3837,244 +3837,6 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Regression for rustfs/backlog#2340 (not Wasabi specific): a directory
|
||||
/// marker (`prefix/` with a body) in a versioned bucket is stored as the null
|
||||
/// version, like MinIO (`putOpts`: "for directory objects skip creating new
|
||||
/// versions"), and must still replicate to completion instead of staying
|
||||
/// `PENDING`.
|
||||
#[tokio::test]
|
||||
async fn test_bucket_replication_replicates_directory_marker_in_versioned_bucket() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let mut source_env_vars = replication_fast_env();
|
||||
source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
|
||||
|
||||
let mut target_env = RustFSTestEnvironment::new().await?;
|
||||
target_env.start_rustfs_server_without_cleanup(vec![]).await?;
|
||||
|
||||
let source_bucket = "replication-dir-marker-src";
|
||||
let target_bucket = "replication-dir-marker-dst";
|
||||
let source_client = source_env.create_s3_client();
|
||||
let target_client = target_env.create_s3_client();
|
||||
|
||||
source_client.create_bucket().bucket(source_bucket).send().await?;
|
||||
target_client.create_bucket().bucket(target_bucket).send().await?;
|
||||
enable_bucket_versioning(&source_env, source_bucket).await?;
|
||||
enable_bucket_versioning(&target_env, target_bucket).await?;
|
||||
let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
|
||||
put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
|
||||
|
||||
let marker_key = "dir/trailing/";
|
||||
let body = b"directory marker body";
|
||||
let put = source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(marker_key)
|
||||
.body(ByteStream::from_static(body))
|
||||
.send()
|
||||
.await?;
|
||||
assert!(
|
||||
put.version_id()
|
||||
.is_none_or(|id| id == "null" || id == uuid::Uuid::nil().to_string()),
|
||||
"a directory marker is the null version even in a versioned bucket: {:?}",
|
||||
put.version_id()
|
||||
);
|
||||
|
||||
wait_for_source_replication_status(&source_client, source_bucket, marker_key, "COMPLETED", false).await?;
|
||||
|
||||
let replica = target_client
|
||||
.get_object()
|
||||
.bucket(target_bucket)
|
||||
.key(marker_key)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(replica.body.collect().await?.into_bytes().as_ref(), body);
|
||||
let listed = target_client
|
||||
.list_object_versions()
|
||||
.bucket(target_bucket)
|
||||
.prefix(marker_key)
|
||||
.send()
|
||||
.await?;
|
||||
let marker_versions: Vec<_> = listed.versions().iter().filter(|v| v.key() == Some(marker_key)).collect();
|
||||
assert_eq!(marker_versions.len(), 1, "the marker must land exactly once: {marker_versions:?}");
|
||||
assert_eq!(
|
||||
marker_versions[0].version_id(),
|
||||
Some("null"),
|
||||
"the replica keeps the null version identity"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Regression for rustfs/backlog#2340 (not Wasabi specific): permanently
|
||||
/// deleting a version whose payload lives in a data dir must leave the source
|
||||
/// clean once the purge replicates. Managed-SSE objects are never inlined and a
|
||||
/// plain object above the inline threshold takes the same layout. The version
|
||||
/// retained with a pending purge used to lose its data dir, so the purge state
|
||||
/// could never be applied (`VersionNotFound` on every retry) and the bucket
|
||||
/// stayed `BucketNotEmpty` while `ListObjectVersions` was already empty.
|
||||
#[tokio::test]
|
||||
async fn test_bucket_replication_version_purge_of_non_inline_object_releases_source_bucket() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let (source_env, target_env, source_bucket, target_bucket) = build_sse_replication_pair("purge-datadir", true, true).await?;
|
||||
let target_arn = wait_for_remote_target_arn(&source_env, &source_bucket).await?;
|
||||
put_bucket_replication_with_delete_statuses(&source_env, &source_bucket, &target_arn, "Enabled", Some("Enabled")).await?;
|
||||
let source_client = source_env.create_s3_client();
|
||||
let target_client = target_env.create_s3_client();
|
||||
|
||||
let sse_key = "sse-object.bin";
|
||||
let large_key = "large-object.bin";
|
||||
let sse_put = source_client
|
||||
.put_object()
|
||||
.bucket(&source_bucket)
|
||||
.key(sse_key)
|
||||
.body(ByteStream::from_static(b"encrypted source payload"))
|
||||
.server_side_encryption(ServerSideEncryption::Aes256)
|
||||
.send()
|
||||
.await?;
|
||||
let large_put = source_client
|
||||
.put_object()
|
||||
.bucket(&source_bucket)
|
||||
.key(large_key)
|
||||
.body(ByteStream::from(vec![0x5a; 2 * 1024 * 1024]))
|
||||
.send()
|
||||
.await?;
|
||||
let purged = [
|
||||
(sse_key, sse_put.version_id().ok_or("SSE PUT omitted version ID")?.to_string()),
|
||||
(large_key, large_put.version_id().ok_or("large PUT omitted version ID")?.to_string()),
|
||||
];
|
||||
assert_replication_converged(&source_client, &source_bucket, &target_client, &target_bucket).await?;
|
||||
|
||||
for (key, version_id) in &purged {
|
||||
source_client
|
||||
.delete_object()
|
||||
.bucket(&source_bucket)
|
||||
.key(*key)
|
||||
.version_id(version_id)
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
assert_replication_converged(&source_client, &source_bucket, &target_client, &target_bucket).await?;
|
||||
let target_state = list_replication_state(&target_client, &target_bucket).await?;
|
||||
assert!(target_state.is_empty(), "target retained an explicitly purged version: {target_state:?}");
|
||||
|
||||
// The purge state is applied on the source asynchronously after the target
|
||||
// acknowledges the delete; only then does the retained version go away and
|
||||
// the bucket become deletable. A listing that is empty while DeleteBucket
|
||||
// keeps answering BucketNotEmpty is exactly the regression.
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(60);
|
||||
loop {
|
||||
let listing = source_client.list_object_versions().bucket(&source_bucket).send().await?;
|
||||
let listed = listing.versions().len() + listing.delete_markers().len();
|
||||
match source_client.delete_bucket().bucket(&source_bucket).send().await {
|
||||
Ok(_) => break,
|
||||
Err(err) if err.code() == Some("BucketNotEmpty") => {
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Err(format!(
|
||||
"source bucket stayed BucketNotEmpty after the version purge replicated; \
|
||||
ListObjectVersions shows {listed} entries"
|
||||
)
|
||||
.into());
|
||||
}
|
||||
sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
Err(err) => return Err(err.into()),
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Regression for rustfs/backlog#2340 (not Wasabi specific): a single-part
|
||||
/// object uploaded with `x-amz-checksum-*` must reach the target with the same
|
||||
/// checksum. The outbound options keyed the stored record by algorithm name,
|
||||
/// which the target client sent as `x-amz-meta-*` user metadata, so a replica
|
||||
/// never carried a checksum although the source HEAD returned one.
|
||||
#[tokio::test]
|
||||
async fn test_bucket_replication_forwards_single_part_object_checksums() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let mut source_env_vars = replication_fast_env();
|
||||
source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
|
||||
|
||||
let mut target_env = RustFSTestEnvironment::new().await?;
|
||||
target_env.start_rustfs_server_without_cleanup(vec![]).await?;
|
||||
|
||||
let source_bucket = "replication-checksum-src";
|
||||
let target_bucket = "replication-checksum-dst";
|
||||
let source_client = source_env.create_s3_client();
|
||||
let target_client = target_env.create_s3_client();
|
||||
|
||||
source_client.create_bucket().bucket(source_bucket).send().await?;
|
||||
target_client.create_bucket().bucket(target_bucket).send().await?;
|
||||
enable_bucket_versioning(&source_env, source_bucket).await?;
|
||||
enable_bucket_versioning(&target_env, target_bucket).await?;
|
||||
let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
|
||||
put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
|
||||
|
||||
let body = b"123456789";
|
||||
let crc32_key = "checksum-crc32.txt";
|
||||
let sha256_key = "checksum-sha256.txt";
|
||||
let crc32_put = source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(crc32_key)
|
||||
.body(ByteStream::from_static(body))
|
||||
.checksum_algorithm(aws_sdk_s3::types::ChecksumAlgorithm::Crc32)
|
||||
.send()
|
||||
.await?;
|
||||
let expected_crc32 = crc32_put.checksum_crc32().ok_or("source PUT omitted CRC32")?.to_string();
|
||||
let sha256_put = source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(sha256_key)
|
||||
.body(ByteStream::from_static(body))
|
||||
.checksum_algorithm(aws_sdk_s3::types::ChecksumAlgorithm::Sha256)
|
||||
.send()
|
||||
.await?;
|
||||
let expected_sha256 = sha256_put.checksum_sha256().ok_or("source PUT omitted SHA256")?.to_string();
|
||||
|
||||
for key in [crc32_key, sha256_key] {
|
||||
wait_for_source_replication_status(&source_client, source_bucket, key, "COMPLETED", false).await?;
|
||||
}
|
||||
|
||||
let replica = target_client
|
||||
.head_object()
|
||||
.bucket(target_bucket)
|
||||
.key(crc32_key)
|
||||
.checksum_mode(aws_sdk_s3::types::ChecksumMode::Enabled)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(replica.checksum_crc32(), Some(expected_crc32.as_str()), "replica lost the CRC32 checksum");
|
||||
let replica = target_client
|
||||
.head_object()
|
||||
.bucket(target_bucket)
|
||||
.key(sha256_key)
|
||||
.checksum_mode(aws_sdk_s3::types::ChecksumMode::Enabled)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
replica.checksum_sha256(),
|
||||
Some(expected_sha256.as_str()),
|
||||
"replica lost the SHA256 checksum"
|
||||
);
|
||||
// The bare algorithm name must not leak as user metadata either.
|
||||
assert!(
|
||||
replica
|
||||
.metadata()
|
||||
.is_none_or(|meta| !meta.keys().any(|k| k.eq_ignore_ascii_case("sha256"))),
|
||||
"replica carries the checksum as user metadata: {:?}",
|
||||
replica.metadata()
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_bucket_replication_disabled_delete_marker_does_not_propagate() -> TestResult {
|
||||
init_logging();
|
||||
@@ -9055,11 +8817,9 @@ async fn test_replication_check_flags_multipart_only_version_minting_target() ->
|
||||
.is_some_and(|error| error.contains("CreateMultipartUpload")),
|
||||
"the failure must name the multipart path: {payload}"
|
||||
);
|
||||
// The PutObject leg mirrored, so it is the multipart probe that failed;
|
||||
// the mutation phases address the id the PUT reported and still run.
|
||||
// The PutObject leg mirrored, so it is the multipart probe that failed.
|
||||
assert_eq!(target_report["Phases"]["Put"]["Status"], "OK", "{payload}");
|
||||
assert_eq!(target_report["Phases"]["DeleteMarker"]["Status"], "OK", "{payload}");
|
||||
assert_eq!(target_report["Phases"]["VersionDelete"]["Status"], "OK", "{payload}");
|
||||
assert_eq!(target_report["Phases"]["DeleteMarker"]["Status"], "SKIPPED", "{payload}");
|
||||
assert_eq!(target_report["Phases"]["Cleanup"]["Status"], "OK", "{payload}");
|
||||
|
||||
let probe_key = target
|
||||
@@ -9247,9 +9007,6 @@ async fn test_replication_check_flags_version_minting_target() -> TestResult {
|
||||
let target_bucket = "version-fidelity-dst";
|
||||
target.create_bucket(target_bucket);
|
||||
target.assign_own_version_ids(true);
|
||||
// Wasabi shape: the probe version the VersionDelete phase removed answers
|
||||
// NoSuchVersion to cleanup's second DELETE, which must count as clean.
|
||||
target.reject_unknown_version_deletes(true);
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let mut env_vars = replication_fast_env();
|
||||
@@ -9294,13 +9051,11 @@ async fn test_replication_check_flags_version_minting_target() -> TestResult {
|
||||
fidelity["Code"], "BucketRemoteTargetVersionMismatch",
|
||||
"the failure must carry a machine-readable code: {payload}"
|
||||
);
|
||||
// The probe PUT itself succeeded (fidelity is judged from its response).
|
||||
// The mutation phases address the id the target assigned — the ledger
|
||||
// the worker records per object (rustfs/backlog#2340) — so they run and
|
||||
// pass on a drifting target, and cleanup uses the same id.
|
||||
// The probe PUT itself succeeded (fidelity is judged from its response);
|
||||
// the later mutation phases are pointless against a drifting target and
|
||||
// must be skipped, but cleanup still runs.
|
||||
assert_eq!(target_report["Phases"]["Put"]["Status"], "OK", "{payload}");
|
||||
assert_eq!(target_report["Phases"]["DeleteMarker"]["Status"], "OK", "{payload}");
|
||||
assert_eq!(target_report["Phases"]["VersionDelete"]["Status"], "OK", "{payload}");
|
||||
assert_eq!(target_report["Phases"]["DeleteMarker"]["Status"], "SKIPPED", "{payload}");
|
||||
assert_eq!(target_report["Phases"]["Cleanup"]["Status"], "OK", "{payload}");
|
||||
|
||||
// The probe PUT must carry the source version as `?versionId=` — the
|
||||
@@ -10190,199 +9945,3 @@ async fn test_get_object_tagging_proxies_unreplicated_object_to_replication_targ
|
||||
target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// backlog#2363
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Wait until the source reports a terminal replication status for `key`.
|
||||
async fn wait_terminal_replication_status(
|
||||
client: &Client,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
ssec: bool,
|
||||
timeout: Duration,
|
||||
) -> Result<String, Box<dyn Error + Send + Sync>> {
|
||||
let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY);
|
||||
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
loop {
|
||||
let request = client.head_object().bucket(bucket).key(key);
|
||||
let head = if ssec {
|
||||
request
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&customer_key)
|
||||
.sse_customer_key_md5(&customer_key_md5)
|
||||
.send()
|
||||
.await?
|
||||
} else {
|
||||
request.send().await?
|
||||
};
|
||||
let status = head.replication_status().map(|status| status.as_str().to_string());
|
||||
if matches!(status.as_deref(), Some("COMPLETED") | Some("FAILED")) {
|
||||
return Ok(status.unwrap_or_default());
|
||||
}
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Err(format!("{bucket}/{key}: replication never reached a terminal status; last {status:?}").into());
|
||||
}
|
||||
sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// backlog#2363: SSE-C ciphertext passthrough of objects the source stored
|
||||
/// compressed. The replica on a RustFS target must decrypt to the original
|
||||
/// bytes for a single PUT and for a multipart upload.
|
||||
#[tokio::test]
|
||||
async fn test_bucket_replication_sse_c_compressed_passthrough() -> TestResult {
|
||||
init_logging();
|
||||
const PART_SIZE: usize = 5 * 1024 * 1024;
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let mut target_env = RustFSTestEnvironment::new().await?;
|
||||
let mut source_process_env = replication_fast_env();
|
||||
source_process_env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
source_process_env.extend_from_slice(FAST_SCANNER_ENV);
|
||||
source_process_env.extend_from_slice(&[
|
||||
("NO_PROXY", "127.0.0.1,localhost"),
|
||||
("HTTP_PROXY", ""),
|
||||
("HTTPS_PROXY", ""),
|
||||
("RUSTFS_COMPRESSION_ENABLED", "true"),
|
||||
("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true"),
|
||||
]);
|
||||
source_env.start_rustfs_server_with_env(vec![], &source_process_env).await?;
|
||||
target_env
|
||||
.start_rustfs_server_without_cleanup_with_env(&[
|
||||
("NO_PROXY", "127.0.0.1,localhost"),
|
||||
("HTTP_PROXY", ""),
|
||||
("HTTPS_PROXY", ""),
|
||||
])
|
||||
.await?;
|
||||
|
||||
let source_bucket = "ssec-compressed-src";
|
||||
let target_bucket = "ssec-compressed-dst";
|
||||
let source_client = source_env.create_s3_client();
|
||||
let target_client = target_env.create_s3_client();
|
||||
source_client.create_bucket().bucket(source_bucket).send().await?;
|
||||
target_client.create_bucket().bucket(target_bucket).send().await?;
|
||||
enable_bucket_versioning(&source_env, source_bucket).await?;
|
||||
enable_bucket_versioning(&target_env, target_bucket).await?;
|
||||
let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
|
||||
put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
|
||||
|
||||
let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY);
|
||||
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
|
||||
let text = |len: usize, seed: u32| -> Vec<u8> {
|
||||
let mut out = Vec::with_capacity(len + 64);
|
||||
let mut line = 0u64;
|
||||
while out.len() < len {
|
||||
out.extend_from_slice(format!("ssec compressed passthrough seed={seed} line={line} lorem ipsum dolor\n").as_bytes());
|
||||
line += 1;
|
||||
}
|
||||
out.truncate(len);
|
||||
out
|
||||
};
|
||||
|
||||
let single_key = "ssec-compressed-single.txt";
|
||||
let single_body = text(1024 * 1024 + 17, 1);
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(single_key)
|
||||
.content_type("text/plain")
|
||||
.body(ByteStream::from(single_body.clone()))
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&customer_key)
|
||||
.sse_customer_key_md5(&customer_key_md5)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let multipart_key = "ssec-compressed-multipart.txt";
|
||||
let multipart_parts = [text(PART_SIZE, 2), text(1024 * 1024 + 4096, 3)];
|
||||
let multipart_body: Vec<u8> = multipart_parts.concat();
|
||||
let created = source_client
|
||||
.create_multipart_upload()
|
||||
.bucket(source_bucket)
|
||||
.key(multipart_key)
|
||||
.content_type("text/plain")
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&customer_key)
|
||||
.sse_customer_key_md5(&customer_key_md5)
|
||||
.send()
|
||||
.await?;
|
||||
let upload_id = created.upload_id().ok_or("missing multipart upload id")?.to_string();
|
||||
let mut completed = Vec::new();
|
||||
for (index, part) in multipart_parts.iter().enumerate() {
|
||||
let part_number = i32::try_from(index + 1)?;
|
||||
let uploaded = source_client
|
||||
.upload_part()
|
||||
.bucket(source_bucket)
|
||||
.key(multipart_key)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(part_number)
|
||||
.body(ByteStream::from(part.clone()))
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&customer_key)
|
||||
.sse_customer_key_md5(&customer_key_md5)
|
||||
.send()
|
||||
.await?;
|
||||
completed.push(
|
||||
CompletedPart::builder()
|
||||
.part_number(part_number)
|
||||
.set_e_tag(uploaded.e_tag().map(str::to_string))
|
||||
.build(),
|
||||
);
|
||||
}
|
||||
source_client
|
||||
.complete_multipart_upload()
|
||||
.bucket(source_bucket)
|
||||
.key(multipart_key)
|
||||
.upload_id(&upload_id)
|
||||
.multipart_upload(CompletedMultipartUpload::builder().set_parts(Some(completed)).build())
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&customer_key)
|
||||
.sse_customer_key_md5(&customer_key_md5)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let mut failures = Vec::new();
|
||||
for (key, body) in [(single_key, &single_body), (multipart_key, &multipart_body)] {
|
||||
let status = wait_terminal_replication_status(&source_client, source_bucket, key, true, Duration::from_secs(120)).await?;
|
||||
if status != "COMPLETED" {
|
||||
failures.push(format!("{key}: source reports {status}"));
|
||||
continue;
|
||||
}
|
||||
let replica = target_client
|
||||
.get_object()
|
||||
.bucket(target_bucket)
|
||||
.key(key)
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&customer_key)
|
||||
.sse_customer_key_md5(&customer_key_md5)
|
||||
.send()
|
||||
.await;
|
||||
match replica {
|
||||
Ok(replica) => {
|
||||
let content_length = replica.content_length();
|
||||
match replica.body.collect().await {
|
||||
Ok(collected) => {
|
||||
let bytes = collected.into_bytes();
|
||||
if bytes.as_ref() != body.as_slice() {
|
||||
failures.push(format!(
|
||||
"{key}: replica bytes differ (content_length={content_length:?}, got {} bytes, want {})",
|
||||
bytes.len(),
|
||||
body.len()
|
||||
));
|
||||
}
|
||||
}
|
||||
Err(err) => failures.push(format!("{key}: replica body read failed: {err}")),
|
||||
}
|
||||
}
|
||||
Err(err) => failures.push(format!("{key}: replica GET failed: {err}")),
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
failures.is_empty(),
|
||||
"SSE-C compressed passthrough replicas must decrypt to the source bytes: {failures:?}"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -31,22 +31,17 @@
|
||||
//! Adding a target behavior the fleet has shown: add the mode to the fake
|
||||
//! target, add a row here, and record any cell that is red before the fix.
|
||||
|
||||
use crate::common::{init_logging, replication_fast_env};
|
||||
use crate::fake_s3_target::{BucketMode, FAKE_ACCESS_KEY, FAKE_SECRET_KEY};
|
||||
use crate::fake_s3_target::{FakeS3Target, FaultAction as FakeTargetFault, Operation as FakeTargetOperation, RequestRecord};
|
||||
use crate::on_demand_migration::common::{OdmEnvOptions, OdmTestEnv, fake_source_client};
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, replication_fast_env};
|
||||
use crate::fake_s3_target::{FAKE_ACCESS_KEY, FAKE_SECRET_KEY};
|
||||
use crate::fake_s3_target::{FakeS3Target, Operation as FakeTargetOperation, RequestRecord};
|
||||
use crate::on_demand_migration::common::fake_source_client;
|
||||
use crate::replication_extension_test::{
|
||||
LOOPBACK_REPLICATION_TARGET_ENV, ReplicationTargetOptions, delete_bucket_replication, enable_bucket_versioning,
|
||||
get_replication_reset_status, put_bucket_replication, put_bucket_replication_with_delete_statuses,
|
||||
set_replication_target_with_options, start_bucket_replication_reset,
|
||||
LOOPBACK_REPLICATION_TARGET_ENV, ReplicationTargetOptions, enable_bucket_versioning, put_bucket_replication,
|
||||
set_replication_target_with_options,
|
||||
};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::primitives::{ByteStream, DateTime};
|
||||
use aws_sdk_s3::types::{
|
||||
Checksum, ChecksumAlgorithm, CompletedMultipartUpload, CompletedPart, ObjectAttributes, ObjectLockLegalHold,
|
||||
ObjectLockLegalHoldStatus, ObjectLockMode, ObjectLockRetention, ObjectLockRetentionMode, Tag, Tagging,
|
||||
};
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart, ObjectLockLegalHoldStatus, ObjectLockMode};
|
||||
use bytes::Bytes;
|
||||
use std::error::Error;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
@@ -68,10 +63,7 @@ enum TargetMode {
|
||||
/// Object Lock parameters must carry `Content-MD5` or `x-amz-checksum-*`.
|
||||
RequireChecksumWithObjectLock,
|
||||
/// AWS S3 / Wasabi / Impossible Cloud: mints its own version ids
|
||||
/// (rustfs/backlog#2085) and, like Wasabi, answers NoSuchVersion to a
|
||||
/// DELETE of an id it never had (rustfs/backlog#2340). Data must still
|
||||
/// land, and every version-addressed mutation must resolve the replica
|
||||
/// through the target-version ledger.
|
||||
/// (rustfs/backlog#2085). Data must still land.
|
||||
MintOwnVersionIds,
|
||||
}
|
||||
|
||||
@@ -88,10 +80,7 @@ impl TargetMode {
|
||||
TargetMode::Baseline => {}
|
||||
TargetMode::RejectAwsChunked => target.reject_aws_chunked_uploads(true),
|
||||
TargetMode::RequireChecksumWithObjectLock => target.require_checksum_for_object_lock(true),
|
||||
TargetMode::MintOwnVersionIds => {
|
||||
target.assign_own_version_ids(true);
|
||||
target.reject_unknown_version_deletes(true);
|
||||
}
|
||||
TargetMode::MintOwnVersionIds => target.assign_own_version_ids(true),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -121,23 +110,16 @@ enum ObjectShape {
|
||||
/// Two-part multipart upload with a GOVERNANCE retention period; the
|
||||
/// lock headers travel on CreateMultipartUpload, which has no body.
|
||||
LockedMultipart,
|
||||
/// ODM stores two local parts while preserving a single-PUT source's MD5 ETag.
|
||||
OdmPreservedMd5Multipart,
|
||||
/// Single-part object uploaded with `x-amz-checksum-sha256`; the replica
|
||||
/// must carry the same header (rustfs/backlog#2340).
|
||||
Checksummed,
|
||||
}
|
||||
|
||||
impl ObjectShape {
|
||||
const ALL: [ObjectShape; 8] = [
|
||||
const ALL: [ObjectShape; 6] = [
|
||||
ObjectShape::Empty,
|
||||
ObjectShape::Plain,
|
||||
ObjectShape::Retention,
|
||||
ObjectShape::LegalHold,
|
||||
ObjectShape::Multipart,
|
||||
ObjectShape::LockedMultipart,
|
||||
ObjectShape::OdmPreservedMd5Multipart,
|
||||
ObjectShape::Checksummed,
|
||||
];
|
||||
|
||||
fn key(self) -> &'static str {
|
||||
@@ -148,17 +130,6 @@ impl ObjectShape {
|
||||
ObjectShape::LegalHold => "matrix/legal-hold.bin",
|
||||
ObjectShape::Multipart => "matrix/multipart.bin",
|
||||
ObjectShape::LockedMultipart => "matrix/locked-multipart.bin",
|
||||
ObjectShape::OdmPreservedMd5Multipart => "matrix/odm-preserved-md5.bin",
|
||||
ObjectShape::Checksummed => "matrix/checksummed.bin",
|
||||
}
|
||||
}
|
||||
|
||||
/// The `x-amz-checksum-*` header the source stored and every upload of
|
||||
/// the replica must repeat.
|
||||
fn forwarded_checksum_header(self) -> Option<&'static str> {
|
||||
match self {
|
||||
ObjectShape::Checksummed => Some("x-amz-checksum-sha256"),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -168,8 +139,7 @@ impl ObjectShape {
|
||||
|
||||
/// Upload the shape to the source and return the bytes the target must
|
||||
/// end up holding.
|
||||
async fn put(self, env: &OdmTestEnv, bucket: &str) -> Result<Bytes, Box<dyn Error + Send + Sync>> {
|
||||
let client = &env.client;
|
||||
async fn put(self, client: &Client, bucket: &str) -> Result<Bytes, Box<dyn Error + Send + Sync>> {
|
||||
let key = self.key();
|
||||
match self {
|
||||
ObjectShape::Empty => {
|
||||
@@ -220,19 +190,6 @@ impl ObjectShape {
|
||||
}
|
||||
ObjectShape::Multipart => multipart_put(client, bucket, key, 0x44, false).await,
|
||||
ObjectShape::LockedMultipart => multipart_put(client, bucket, key, 0x55, true).await,
|
||||
ObjectShape::OdmPreservedMd5Multipart => odm_preserved_md5_multipart(env, bucket, key).await,
|
||||
ObjectShape::Checksummed => {
|
||||
let body = payload(40 * 1024, 0x66);
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(body.clone()))
|
||||
.checksum_algorithm(ChecksumAlgorithm::Sha256)
|
||||
.send()
|
||||
.await?;
|
||||
Ok(body)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -262,595 +219,6 @@ fn expectation(mode: TargetMode, shape: ObjectShape) -> Expectation {
|
||||
.unwrap_or(Expectation::Completed)
|
||||
}
|
||||
|
||||
/// rustfs/backlog#2340: a target that mints its own version ids (Wasabi,
|
||||
/// AWS S3) answers 404 to a HEAD by the source uuid, which the worker used to
|
||||
/// read as "replica missing" and re-drive the PUT — one more target version
|
||||
/// per heal, MRF retry or resync. Two re-drive shapes, both must converge on
|
||||
/// the single version the first PUT created:
|
||||
/// - the first PUT lands but its response is lost, so the object is FAILED
|
||||
/// and the scanner heal pass re-drives it;
|
||||
/// - an existing-object resync re-drives a COMPLETED object unconditionally.
|
||||
#[tokio::test]
|
||||
async fn matrix_mint_own_version_ids_redrive_does_not_duplicate() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let target = FakeS3Target::start().await?;
|
||||
let target_bucket = "matrix-mint-own-redrive-dst".to_string();
|
||||
target.create_bucket_with_object_lock(target_bucket.clone());
|
||||
target.assign_own_version_ids(true);
|
||||
|
||||
let mut env_vars = replication_fast_env();
|
||||
env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
env_vars.extend_from_slice(&[
|
||||
("NO_PROXY", "127.0.0.1,localhost"),
|
||||
("HTTP_PROXY", ""),
|
||||
("HTTPS_PROXY", ""),
|
||||
// The scanner heal pass is what re-drives a FAILED object.
|
||||
("RUSTFS_SCANNER_CYCLE", "1"),
|
||||
("RUSTFS_SCANNER_START_DELAY_SECS", "1"),
|
||||
]);
|
||||
let env = OdmTestEnv::start_with(OdmEnvOptions {
|
||||
env: env_vars,
|
||||
..OdmEnvOptions::default()
|
||||
})
|
||||
.await?;
|
||||
let source_env = &env.rustfs;
|
||||
|
||||
let source_bucket = "matrix-mint-own-redrive-src";
|
||||
let source_client = source_env.create_s3_client();
|
||||
source_client
|
||||
.create_bucket()
|
||||
.bucket(source_bucket)
|
||||
.object_lock_enabled_for_bucket(true)
|
||||
.send()
|
||||
.await?;
|
||||
enable_bucket_versioning(source_env, source_bucket).await?;
|
||||
let target_arn = set_replication_target_with_options(
|
||||
source_env,
|
||||
source_bucket,
|
||||
ReplicationTargetOptions {
|
||||
endpoint: &target.address(),
|
||||
access_key: FAKE_ACCESS_KEY,
|
||||
secret_key: FAKE_SECRET_KEY,
|
||||
target_bucket: &target_bucket,
|
||||
secure: false,
|
||||
skip_tls_verify: false,
|
||||
ca_cert_pem: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
put_bucket_replication(source_env, source_bucket, &target_arn).await?;
|
||||
|
||||
// Teach the worker the target's identity contract with one ordinary
|
||||
// write, exactly as production learns it (the PUT response carries the
|
||||
// minted id).
|
||||
let probe_key = "redrive/identity-probe.bin";
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(probe_key)
|
||||
.body(ByteStream::from(payload(4 * 1024, 0x01)))
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
wait_for_terminal_replication_status(&source_client, source_bucket, probe_key).await?,
|
||||
"COMPLETED"
|
||||
);
|
||||
|
||||
// Shape 1: the PUT is stored, its response never arrives, heal re-drives.
|
||||
let heal_key = "redrive/heal.bin";
|
||||
target.inject_for_key(FakeTargetOperation::PutObject, heal_key, FakeTargetFault::DisconnectAfterResponse, 1);
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(heal_key)
|
||||
.body(ByteStream::from(payload(8 * 1024, 0x02)))
|
||||
.send()
|
||||
.await?;
|
||||
wait_for_replication_status_and_single_version(&source_client, source_bucket, &target, &target_bucket, heal_key).await?;
|
||||
|
||||
// Shape 2: an existing-object resync re-drives a COMPLETED object.
|
||||
let resync_key = "redrive/resync.bin";
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(resync_key)
|
||||
.body(ByteStream::from(payload(8 * 1024, 0x03)))
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
wait_for_terminal_replication_status(&source_client, source_bucket, resync_key).await?,
|
||||
"COMPLETED"
|
||||
);
|
||||
let (reset_arn, _reset_id) = start_bucket_replication_reset(source_env, source_bucket).await?;
|
||||
assert_eq!(reset_arn, target_arn);
|
||||
let resync = async {
|
||||
loop {
|
||||
let status = get_replication_reset_status(source_env, source_bucket, &target_arn).await?;
|
||||
if let Some(entry) = status.targets.iter().find(|entry| entry.arn == target_arn)
|
||||
&& entry.status == "Completed"
|
||||
{
|
||||
return Ok::<_, Box<dyn Error + Send + Sync>>(entry.replicated_count);
|
||||
}
|
||||
sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
};
|
||||
let replicated = timeout(Duration::from_secs(90), resync)
|
||||
.await
|
||||
.map_err(|_| "existing-object resync did not complete within 90 seconds")??;
|
||||
assert!(replicated >= 3, "resync must count the located replicas as replicated, got {replicated}");
|
||||
for key in [probe_key, heal_key, resync_key] {
|
||||
let versions = target.stored_versions(&target_bucket, key);
|
||||
assert_eq!(
|
||||
versions.len(),
|
||||
1,
|
||||
"{key}: a re-drive against a target that mints its own version ids must not mint another one: {versions:?}"
|
||||
);
|
||||
}
|
||||
|
||||
target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// rustfs/backlog#2340 (target-version ledger): on a target that mints its own
|
||||
/// version ids and answers NoSuchVersion to an unknown id (the Wasabi shape),
|
||||
/// every version-addressed mutation must land on the version the target
|
||||
/// assigned, which the replication PUT recorded on the source:
|
||||
/// - a tag update changes the existing target version, no new version;
|
||||
/// - a retention extension and legal hold ON/OFF change that version too;
|
||||
/// - a permanent delete of the older of two same-content generations removes
|
||||
/// exactly that replica and keeps the live one (content identity alone
|
||||
/// could not tell them apart).
|
||||
#[tokio::test]
|
||||
async fn matrix_mint_own_version_ids_addresses_mutations_through_the_ledger() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let target = FakeS3Target::start().await?;
|
||||
let target_bucket = "matrix-mint-own-ledger-dst".to_string();
|
||||
target.create_bucket_with_object_lock(target_bucket.clone());
|
||||
TargetMode::MintOwnVersionIds.apply(&target);
|
||||
|
||||
let mut env_vars = replication_fast_env();
|
||||
env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
env_vars.extend_from_slice(&[
|
||||
("NO_PROXY", "127.0.0.1,localhost"),
|
||||
("HTTP_PROXY", ""),
|
||||
("HTTPS_PROXY", ""),
|
||||
// The scanner heal pass retries a purge the first attempt lost.
|
||||
("RUSTFS_SCANNER_CYCLE", "1"),
|
||||
("RUSTFS_SCANNER_START_DELAY_SECS", "1"),
|
||||
]);
|
||||
let env = OdmTestEnv::start_with(OdmEnvOptions {
|
||||
env: env_vars,
|
||||
..OdmEnvOptions::default()
|
||||
})
|
||||
.await?;
|
||||
let source_env = &env.rustfs;
|
||||
|
||||
let source_bucket = "matrix-mint-own-ledger-src";
|
||||
let source_client = source_env.create_s3_client();
|
||||
source_client
|
||||
.create_bucket()
|
||||
.bucket(source_bucket)
|
||||
.object_lock_enabled_for_bucket(true)
|
||||
.send()
|
||||
.await?;
|
||||
enable_bucket_versioning(source_env, source_bucket).await?;
|
||||
let target_arn = set_replication_target_with_options(
|
||||
source_env,
|
||||
source_bucket,
|
||||
ReplicationTargetOptions {
|
||||
endpoint: &target.address(),
|
||||
access_key: FAKE_ACCESS_KEY,
|
||||
secret_key: FAKE_SECRET_KEY,
|
||||
target_bucket: &target_bucket,
|
||||
secure: false,
|
||||
skip_tls_verify: false,
|
||||
ca_cert_pem: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
put_bucket_replication_with_delete_statuses(source_env, source_bucket, &target_arn, "Enabled", Some("Enabled")).await?;
|
||||
let target_client = fake_source_client(&target);
|
||||
|
||||
// Tag update on an existing version.
|
||||
let tag_key = "ledger/tags.bin";
|
||||
let tagged = source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(tag_key)
|
||||
.body(ByteStream::from(payload(4 * 1024, 0x01)))
|
||||
.send()
|
||||
.await?;
|
||||
let tag_source_version = tagged.version_id().ok_or("source PUT returned no version id")?.to_string();
|
||||
assert_eq!(
|
||||
wait_for_terminal_replication_status(&source_client, source_bucket, tag_key).await?,
|
||||
"COMPLETED"
|
||||
);
|
||||
let tag_target_version = single_target_version(&target, &target_bucket, tag_key)?;
|
||||
source_client
|
||||
.put_object_tagging()
|
||||
.bucket(source_bucket)
|
||||
.key(tag_key)
|
||||
.version_id(&tag_source_version)
|
||||
.tagging(
|
||||
Tagging::builder()
|
||||
.tag_set(Tag::builder().key("phase").value("after").build()?)
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
wait_until("tag update on the existing target version", || async {
|
||||
let tags = target_client
|
||||
.get_object_tagging()
|
||||
.bucket(&target_bucket)
|
||||
.key(tag_key)
|
||||
.version_id(&tag_target_version)
|
||||
.send()
|
||||
.await?;
|
||||
Ok(tags
|
||||
.tag_set()
|
||||
.iter()
|
||||
.any(|tag| tag.key() == "phase" && tag.value() == "after"))
|
||||
})
|
||||
.await?;
|
||||
assert_stable_single_version(&target, &target_bucket, tag_key, &tag_target_version).await?;
|
||||
|
||||
// Retention extension and legal hold on an existing version.
|
||||
let lock_key = "ledger/lock.bin";
|
||||
let locked = source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(lock_key)
|
||||
.body(ByteStream::from(payload(4 * 1024, 0x02)))
|
||||
.object_lock_mode(ObjectLockMode::Governance)
|
||||
.object_lock_retain_until_date(retain_until())
|
||||
.send()
|
||||
.await?;
|
||||
let lock_source_version = locked.version_id().ok_or("source PUT returned no version id")?.to_string();
|
||||
assert_eq!(
|
||||
wait_for_terminal_replication_status(&source_client, source_bucket, lock_key).await?,
|
||||
"COMPLETED"
|
||||
);
|
||||
let lock_target_version = single_target_version(&target, &target_bucket, lock_key)?;
|
||||
let extended = DateTime::from_secs(retain_until().secs() + 86_400);
|
||||
source_client
|
||||
.put_object_retention()
|
||||
.bucket(source_bucket)
|
||||
.key(lock_key)
|
||||
.version_id(&lock_source_version)
|
||||
.retention(
|
||||
ObjectLockRetention::builder()
|
||||
.mode(ObjectLockRetentionMode::Governance)
|
||||
.retain_until_date(extended)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
source_client
|
||||
.put_object_legal_hold()
|
||||
.bucket(source_bucket)
|
||||
.key(lock_key)
|
||||
.version_id(&lock_source_version)
|
||||
.legal_hold(ObjectLockLegalHold::builder().status(ObjectLockLegalHoldStatus::On).build())
|
||||
.send()
|
||||
.await?;
|
||||
wait_until("retention extension and legal hold on the existing target version", || async {
|
||||
let head = target_client
|
||||
.head_object()
|
||||
.bucket(&target_bucket)
|
||||
.key(lock_key)
|
||||
.version_id(&lock_target_version)
|
||||
.send()
|
||||
.await?;
|
||||
Ok(head.object_lock_retain_until_date().map(|date| date.secs()) == Some(extended.secs())
|
||||
&& head.object_lock_legal_hold_status() == Some(&ObjectLockLegalHoldStatus::On))
|
||||
})
|
||||
.await?;
|
||||
source_client
|
||||
.put_object_legal_hold()
|
||||
.bucket(source_bucket)
|
||||
.key(lock_key)
|
||||
.version_id(&lock_source_version)
|
||||
.legal_hold(ObjectLockLegalHold::builder().status(ObjectLockLegalHoldStatus::Off).build())
|
||||
.send()
|
||||
.await?;
|
||||
wait_until("legal hold removal on the existing target version", || async {
|
||||
let head = target_client
|
||||
.head_object()
|
||||
.bucket(&target_bucket)
|
||||
.key(lock_key)
|
||||
.version_id(&lock_target_version)
|
||||
.send()
|
||||
.await?;
|
||||
Ok(head.object_lock_legal_hold_status() == Some(&ObjectLockLegalHoldStatus::Off))
|
||||
})
|
||||
.await?;
|
||||
assert_stable_single_version(&target, &target_bucket, lock_key, &lock_target_version).await?;
|
||||
|
||||
// Permanent delete of the older of two same-content generations.
|
||||
let generations_key = "ledger/generations.bin";
|
||||
let body = payload(4 * 1024, 0x03);
|
||||
let older = source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(generations_key)
|
||||
.body(ByteStream::from(body.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
let older_version = older.version_id().ok_or("source PUT returned no version id")?.to_string();
|
||||
assert_eq!(
|
||||
wait_for_terminal_replication_status(&source_client, source_bucket, generations_key).await?,
|
||||
"COMPLETED"
|
||||
);
|
||||
let older_replica = single_target_version(&target, &target_bucket, generations_key)?;
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(generations_key)
|
||||
.body(ByteStream::from(body))
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
wait_for_terminal_replication_status(&source_client, source_bucket, generations_key).await?,
|
||||
"COMPLETED"
|
||||
);
|
||||
wait_until("both generations replicated", || async {
|
||||
Ok(target.stored_versions(&target_bucket, generations_key).len() == 2)
|
||||
})
|
||||
.await?;
|
||||
let newer_replica = target
|
||||
.stored_versions(&target_bucket, generations_key)
|
||||
.into_iter()
|
||||
.map(|(version_id, _)| version_id)
|
||||
.find(|version_id| version_id != &older_replica)
|
||||
.ok_or("the second generation must have its own target version")?;
|
||||
|
||||
source_client
|
||||
.delete_object()
|
||||
.bucket(source_bucket)
|
||||
.key(generations_key)
|
||||
.version_id(&older_version)
|
||||
.send()
|
||||
.await?;
|
||||
wait_until("permanent delete of the older generation's replica", || async {
|
||||
let versions: Vec<String> = target
|
||||
.stored_versions(&target_bucket, generations_key)
|
||||
.into_iter()
|
||||
.map(|(version_id, _)| version_id)
|
||||
.collect();
|
||||
Ok(versions == [newer_replica.clone()])
|
||||
})
|
||||
.await?;
|
||||
assert_stable_single_version(&target, &target_bucket, generations_key, &newer_replica).await?;
|
||||
|
||||
// No mutation above may have gone out as a re-PUT: one upload per key.
|
||||
for key in [tag_key, lock_key] {
|
||||
let puts = target
|
||||
.requests()
|
||||
.iter()
|
||||
.filter(|record| record.key.as_deref() == Some(key) && record.operation == FakeTargetOperation::PutObject)
|
||||
.count();
|
||||
assert_eq!(
|
||||
puts, 1,
|
||||
"{key}: a metadata update must not re-PUT the object on a target that mints its own ids"
|
||||
);
|
||||
}
|
||||
|
||||
target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// rustfs/backlog#2340 (pending purge lifecycle): a permanent delete whose
|
||||
/// replication keeps failing leaves the version in xl.meta as a PENDING purge,
|
||||
/// hidden from listings. Once the bucket's replication configuration is
|
||||
/// removed nothing can ever confirm that purge remotely, so the delete worker
|
||||
/// must settle it locally (abandoned, with the replica left on the former
|
||||
/// target) — otherwise the bucket stays `BucketNotEmpty` forever with a
|
||||
/// residue the client cannot see.
|
||||
#[tokio::test]
|
||||
async fn matrix_removed_replication_config_abandons_pending_purge() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let target = FakeS3Target::start().await?;
|
||||
let target_bucket = "matrix-abandoned-purge-dst".to_string();
|
||||
target.create_bucket_with_object_lock(target_bucket.clone());
|
||||
TargetMode::MintOwnVersionIds.apply(&target);
|
||||
|
||||
let mut env_vars = replication_fast_env();
|
||||
env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
env_vars.extend_from_slice(&[
|
||||
("NO_PROXY", "127.0.0.1,localhost"),
|
||||
("HTTP_PROXY", ""),
|
||||
("HTTPS_PROXY", ""),
|
||||
// The scanner heal pass is what revisits a pending purge.
|
||||
("RUSTFS_SCANNER_CYCLE", "1"),
|
||||
("RUSTFS_SCANNER_START_DELAY_SECS", "1"),
|
||||
]);
|
||||
let env = OdmTestEnv::start_with(OdmEnvOptions {
|
||||
env: env_vars,
|
||||
..OdmEnvOptions::default()
|
||||
})
|
||||
.await?;
|
||||
let source_env = &env.rustfs;
|
||||
|
||||
let source_bucket = "matrix-abandoned-purge-src";
|
||||
let source_client = source_env.create_s3_client();
|
||||
source_client
|
||||
.create_bucket()
|
||||
.bucket(source_bucket)
|
||||
.object_lock_enabled_for_bucket(true)
|
||||
.send()
|
||||
.await?;
|
||||
enable_bucket_versioning(source_env, source_bucket).await?;
|
||||
let target_arn = set_replication_target_with_options(
|
||||
source_env,
|
||||
source_bucket,
|
||||
ReplicationTargetOptions {
|
||||
endpoint: &target.address(),
|
||||
access_key: FAKE_ACCESS_KEY,
|
||||
secret_key: FAKE_SECRET_KEY,
|
||||
target_bucket: &target_bucket,
|
||||
secure: false,
|
||||
skip_tls_verify: false,
|
||||
ca_cert_pem: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
put_bucket_replication_with_delete_statuses(source_env, source_bucket, &target_arn, "Enabled", Some("Enabled")).await?;
|
||||
|
||||
let key = "purge/orphaned.bin";
|
||||
let put = source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(payload(4 * 1024, 0x07)))
|
||||
.send()
|
||||
.await?;
|
||||
let source_version = put.version_id().ok_or("source PUT returned no version id")?.to_string();
|
||||
assert_eq!(
|
||||
wait_for_terminal_replication_status(&source_client, source_bucket, key).await?,
|
||||
"COMPLETED"
|
||||
);
|
||||
let replica = single_target_version(&target, &target_bucket, key)?;
|
||||
|
||||
// The target refuses every purge: the version stays a pending purge.
|
||||
// More refusals than any scanner cycle can consume within the test.
|
||||
target.inject_for_key(FakeTargetOperation::DeleteObject, key, FakeTargetFault::ResponseStatus(503), 4_000);
|
||||
source_client
|
||||
.delete_object()
|
||||
.bucket(source_bucket)
|
||||
.key(key)
|
||||
.version_id(&source_version)
|
||||
.send()
|
||||
.await?;
|
||||
wait_until("the refused purge to reach the target at least once", || async {
|
||||
Ok(target.count_requests(FakeTargetOperation::DeleteObject, key) >= 1)
|
||||
})
|
||||
.await?;
|
||||
let listed = source_client.list_object_versions().bucket(source_bucket).send().await?;
|
||||
assert!(
|
||||
listed.versions().is_empty() && listed.delete_markers().is_empty(),
|
||||
"a pending purge is hidden from listings: {listed:?}"
|
||||
);
|
||||
let blocked = source_client.delete_bucket().bucket(source_bucket).send().await;
|
||||
assert!(
|
||||
blocked
|
||||
.as_ref()
|
||||
.err()
|
||||
.and_then(|err| err.as_service_error())
|
||||
.is_some_and(|err| err.code() == Some("BucketNotEmpty")),
|
||||
"the hidden pending purge must block DeleteBucket while the target is still configured: {blocked:?}"
|
||||
);
|
||||
|
||||
// Removing the replication configuration orphans the purge; the scanner
|
||||
// heal pass must settle it locally so the bucket becomes deletable.
|
||||
let response = delete_bucket_replication(source_env, source_bucket).await?;
|
||||
assert!(response.status().is_success(), "DeleteBucketReplication: {}", response.status());
|
||||
wait_until("DeleteBucket to succeed once the orphaned purge is abandoned", || async {
|
||||
match source_client.delete_bucket().bucket(source_bucket).send().await {
|
||||
Ok(_) => Ok(true),
|
||||
Err(err) if err.as_service_error().is_some_and(|err| err.code() == Some("BucketNotEmpty")) => Ok(false),
|
||||
Err(err) => Err(err.into()),
|
||||
}
|
||||
})
|
||||
.await?;
|
||||
// Abandoned means abandoned: the replica stays on the former target and,
|
||||
// once the attempts in flight at removal time have drained, no further
|
||||
// purge attempts are sent to it.
|
||||
assert_eq!(
|
||||
single_target_version(&target, &target_bucket, key)?,
|
||||
replica,
|
||||
"an abandoned purge must not touch the replica on the former target"
|
||||
);
|
||||
sleep(Duration::from_secs(3)).await;
|
||||
let settled = target.count_requests(FakeTargetOperation::DeleteObject, key);
|
||||
sleep(Duration::from_secs(3)).await;
|
||||
assert_eq!(
|
||||
target.count_requests(FakeTargetOperation::DeleteObject, key),
|
||||
settled,
|
||||
"purge attempts must stop once the target is no longer configured"
|
||||
);
|
||||
|
||||
target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn single_target_version(target: &FakeS3Target, target_bucket: &str, key: &str) -> Result<String, Box<dyn Error + Send + Sync>> {
|
||||
let versions = target.stored_versions(target_bucket, key);
|
||||
match versions.as_slice() {
|
||||
[(version_id, false)] => Ok(version_id.clone()),
|
||||
other => Err(format!("{key}: expected exactly one live target version, got {other:?}").into()),
|
||||
}
|
||||
}
|
||||
|
||||
/// The target keeps holding exactly `version_id` for a few scanner cycles: a
|
||||
/// re-driven PUT or a wrong delete would show up here.
|
||||
async fn assert_stable_single_version(target: &FakeS3Target, target_bucket: &str, key: &str, version_id: &str) -> TestResult {
|
||||
for _ in 0..8 {
|
||||
let versions = target.stored_versions(target_bucket, key);
|
||||
if versions.len() != 1 || versions[0].0 != version_id {
|
||||
return Err(
|
||||
format!("{key}: target versions drifted from the single expected replica {version_id}: {versions:?}").into(),
|
||||
);
|
||||
}
|
||||
sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn wait_until<F, Fut>(what: &str, mut probe: F) -> TestResult
|
||||
where
|
||||
F: FnMut() -> Fut,
|
||||
Fut: std::future::Future<Output = Result<bool, Box<dyn Error + Send + Sync>>>,
|
||||
{
|
||||
let wait = async {
|
||||
loop {
|
||||
if probe().await? {
|
||||
return Ok::<_, Box<dyn Error + Send + Sync>>(());
|
||||
}
|
||||
sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
};
|
||||
timeout(Duration::from_secs(90), wait)
|
||||
.await
|
||||
.map_err(|_| format!("{what} did not happen within 90 seconds"))?
|
||||
}
|
||||
|
||||
/// Wait until `key` is COMPLETED on the source and, for the observation
|
||||
/// window after that, the target still holds exactly one live version of it.
|
||||
async fn wait_for_replication_status_and_single_version(
|
||||
source_client: &Client,
|
||||
source_bucket: &str,
|
||||
target: &FakeS3Target,
|
||||
target_bucket: &str,
|
||||
key: &str,
|
||||
) -> TestResult {
|
||||
// The lost PUT response first settles the object FAILED; only the next
|
||||
// scanner heal pass can turn that into COMPLETED, so FAILED is transient
|
||||
// here and the wait is for COMPLETED alone.
|
||||
let converged = async {
|
||||
loop {
|
||||
let head = source_client.head_object().bucket(source_bucket).key(key).send().await?;
|
||||
if head.replication_status().is_some_and(|status| status.as_str() == "COMPLETED") {
|
||||
return Ok::<_, Box<dyn Error + Send + Sync>>(());
|
||||
}
|
||||
sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
};
|
||||
timeout(Duration::from_secs(90), converged)
|
||||
.await
|
||||
.map_err(|_| format!("{key}: heal re-drive did not converge to COMPLETED within 90 seconds"))??;
|
||||
// The heal pass keeps visiting the key for a few scanner cycles; a
|
||||
// duplicate would show up here as a second stored version.
|
||||
for _ in 0..12 {
|
||||
let versions = target.stored_versions(target_bucket, key);
|
||||
assert_eq!(versions.len(), 1, "{key}: target minted another version on re-drive: {versions:?}");
|
||||
sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn matrix_baseline_target() -> TestResult {
|
||||
run_row(TargetMode::Baseline).await
|
||||
@@ -902,15 +270,11 @@ async fn run_row(mode: TargetMode) -> TestResult {
|
||||
target.create_bucket_with_object_lock(target_bucket.clone());
|
||||
mode.apply(&target);
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let mut env_vars = replication_fast_env();
|
||||
env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
env_vars.extend_from_slice(&[("NO_PROXY", "127.0.0.1,localhost"), ("HTTP_PROXY", ""), ("HTTPS_PROXY", "")]);
|
||||
let env = OdmTestEnv::start_with(OdmEnvOptions {
|
||||
env: env_vars,
|
||||
..OdmEnvOptions::default()
|
||||
})
|
||||
.await?;
|
||||
let source_env = &env.rustfs;
|
||||
source_env.start_rustfs_server_with_env(vec![], &env_vars).await?;
|
||||
|
||||
let source_bucket = format!("matrix-{}-src", mode.slug());
|
||||
let source_client = source_env.create_s3_client();
|
||||
@@ -920,9 +284,9 @@ async fn run_row(mode: TargetMode) -> TestResult {
|
||||
.object_lock_enabled_for_bucket(true)
|
||||
.send()
|
||||
.await?;
|
||||
enable_bucket_versioning(source_env, &source_bucket).await?;
|
||||
enable_bucket_versioning(&source_env, &source_bucket).await?;
|
||||
let target_arn = set_replication_target_with_options(
|
||||
source_env,
|
||||
&source_env,
|
||||
&source_bucket,
|
||||
ReplicationTargetOptions {
|
||||
endpoint: &target.address(),
|
||||
@@ -935,21 +299,14 @@ async fn run_row(mode: TargetMode) -> TestResult {
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
put_bucket_replication(source_env, &source_bucket, &target_arn).await?;
|
||||
put_bucket_replication(&source_env, &source_bucket, &target_arn).await?;
|
||||
|
||||
let target_client = fake_source_client(&target);
|
||||
let mut failures = Vec::new();
|
||||
for shape in ObjectShape::ALL {
|
||||
let cell = format!("{}/{:?}", mode.slug(), shape);
|
||||
let expected_body = shape.put(&env, &source_bucket).await?;
|
||||
let expected_body = shape.put(&source_client, &source_bucket).await?;
|
||||
let status = wait_for_terminal_replication_status(&source_client, &source_bucket, shape.key()).await?;
|
||||
if shape == ObjectShape::OdmPreservedMd5Multipart {
|
||||
assert_eq!(
|
||||
env.source.count_requests(FakeTargetOperation::GetObject, shape.key()),
|
||||
2,
|
||||
"one passthrough GET plus one background pull; replication must read the persisted local parts"
|
||||
);
|
||||
}
|
||||
let journal = target.requests();
|
||||
let outcome = match expectation(mode, shape) {
|
||||
Expectation::Completed => {
|
||||
@@ -1022,36 +379,6 @@ async fn check_completed_cell(
|
||||
if uploads.is_empty() {
|
||||
return Err("no upload reached the target although the source reports COMPLETED".into());
|
||||
}
|
||||
if shape == ObjectShape::OdmPreservedMd5Multipart {
|
||||
let key_requests: Vec<_> = journal
|
||||
.iter()
|
||||
.filter(|record| record.key.as_deref() == Some(shape.key()))
|
||||
.collect();
|
||||
for operation in [
|
||||
FakeTargetOperation::CreateMultipartUpload,
|
||||
FakeTargetOperation::CompleteMultipartUpload,
|
||||
] {
|
||||
if !key_requests.iter().any(|record| record.operation == operation) {
|
||||
return Err(format!("preserved-MD5 multipart object did not use {operation:?}").into());
|
||||
}
|
||||
}
|
||||
if key_requests
|
||||
.iter()
|
||||
.any(|record| record.operation == FakeTargetOperation::PutObject)
|
||||
{
|
||||
return Err("preserved-MD5 multipart object used a single PutObject".into());
|
||||
}
|
||||
let mut part_numbers: Vec<_> = key_requests
|
||||
.iter()
|
||||
.filter(|record| record.operation == FakeTargetOperation::UploadPart)
|
||||
.map(|record| record.part_number)
|
||||
.collect();
|
||||
part_numbers.sort_unstable();
|
||||
part_numbers.dedup();
|
||||
if part_numbers != [Some(1), Some(2)] {
|
||||
return Err(format!("preserved-MD5 multipart object uploaded unexpected parts: {part_numbers:?}").into());
|
||||
}
|
||||
}
|
||||
if let Some(framed) = uploads.iter().find(|record| record.transport.aws_chunked) {
|
||||
return Err(format!("{cell}: an upload went out aws-chunked (rustfs#6853 framing): {framed:?}").into());
|
||||
}
|
||||
@@ -1074,19 +401,6 @@ async fn check_completed_cell(
|
||||
}) {
|
||||
return Err(format!("a locked PutObject went out without any integrity header (rustfs#7082): {bare:?}").into());
|
||||
}
|
||||
// rustfs/backlog#2340 contract: a source checksum reaches the target as
|
||||
// the `x-amz-checksum-*` header, not as user metadata; every PutObject of
|
||||
// the shape carries it.
|
||||
if let Some(header) = shape.forwarded_checksum_header()
|
||||
&& let Some(missing) = uploads.iter().find(|record| {
|
||||
record.operation == FakeTargetOperation::PutObject
|
||||
&& !record.transport.checksum_headers.iter().any(|name| name == header)
|
||||
})
|
||||
{
|
||||
return Err(
|
||||
format!("a PutObject went out without the source's {header} header (rustfs/backlog#2340): {missing:?}").into(),
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1141,71 +455,6 @@ async fn wait_for_terminal_replication_status(
|
||||
}
|
||||
}
|
||||
|
||||
async fn odm_preserved_md5_multipart(env: &OdmTestEnv, bucket: &str, key: &str) -> Result<Bytes, Box<dyn Error + Send + Sync>> {
|
||||
const PART_SIZE: usize = 5 * 1024 * 1024;
|
||||
let origin_bucket = format!("{bucket}-origin");
|
||||
env.source.create_bucket_with_mode(&origin_bucket, BucketMode::Unversioned);
|
||||
let mut spec = env.fake_source_spec(&origin_bucket);
|
||||
// Below the 16 MiB inline default the pull is one tee'd PUT with a single
|
||||
// part; force the passthrough + background multipart write-back instead.
|
||||
spec.policy.inline_max_bytes = 4096;
|
||||
spec.policy.multipart_part_size_bytes = PART_SIZE as u64;
|
||||
spec.policy.preserve_etag = true;
|
||||
env.configure_and_wait(bucket, &spec).await?;
|
||||
|
||||
// A normal source PUT produces the MD5 ETag; only ODM chooses the local parts.
|
||||
let body = payload(PART_SIZE + 4096, 0x66);
|
||||
let source_put = env
|
||||
.source_client()
|
||||
.put_object()
|
||||
.bucket(&origin_bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(body.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
let source_etag = source_put.e_tag().ok_or("source PUT omitted its ETag")?.trim_matches('"');
|
||||
assert_eq!(source_etag.len(), 32, "source fixture must have a single-PUT MD5 ETag");
|
||||
assert!(source_etag.bytes().all(|byte| byte.is_ascii_hexdigit()));
|
||||
|
||||
let pulled = env.raw_get(bucket, key).await?;
|
||||
assert_eq!(pulled.status, 200, "{}", String::from_utf8_lossy(&pulled.body));
|
||||
assert_eq!(pulled.body, body);
|
||||
assert!(
|
||||
env.wait_local_listed(bucket, key, Duration::from_secs(30)).await?,
|
||||
"ODM must persist the object"
|
||||
);
|
||||
let attributes = env
|
||||
.client
|
||||
.get_object_attributes()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.object_attributes(ObjectAttributes::Etag)
|
||||
.object_attributes(ObjectAttributes::ObjectParts)
|
||||
.object_attributes(ObjectAttributes::Checksum)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(attributes.e_tag().map(|etag| etag.trim_matches('"')), Some(source_etag));
|
||||
let parts = attributes
|
||||
.object_parts()
|
||||
.ok_or("the ODM copy must expose its two local parts")?;
|
||||
assert_eq!(parts.total_parts_count(), Some(2));
|
||||
assert_eq!(
|
||||
parts
|
||||
.parts()
|
||||
.iter()
|
||||
.map(|part| (part.part_number(), part.size()))
|
||||
.collect::<Vec<_>>(),
|
||||
[(Some(1), Some(PART_SIZE as i64)), (Some(2), Some(4096))]
|
||||
);
|
||||
assert!(
|
||||
attributes
|
||||
.checksum()
|
||||
.is_none_or(|checksum| checksum == &Checksum::builder().build()),
|
||||
"multipart routing must work without an object checksum record"
|
||||
);
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
async fn multipart_put(
|
||||
client: &Client,
|
||||
bucket: &str,
|
||||
|
||||
@@ -14,10 +14,9 @@
|
||||
|
||||
use crate::common::{
|
||||
RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging, replication_fast_env, rustfs_binary_path,
|
||||
signed_request,
|
||||
};
|
||||
use crate::fake_s3_target::{BucketMode, FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, Operation as FakeTargetOperation};
|
||||
use crate::on_demand_migration::common::{ODM_SERVER_ENV, OdmTestEnv, SeedObject, fake_source_client};
|
||||
use crate::fake_s3_target::{BucketMode, FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target};
|
||||
use crate::on_demand_migration::common::{ODM_SERVER_ENV, OdmTestEnv, SeedObject};
|
||||
use crate::replication_extension_test::{
|
||||
LOOPBACK_REPLICATION_TARGET_ENV, ReplicationTargetOptions, put_bucket_replication, set_replication_target_with_options,
|
||||
};
|
||||
@@ -26,10 +25,9 @@ use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
BucketLifecycleConfiguration, BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DefaultRetention,
|
||||
ExpirationStatus, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter, ObjectAttributes, ObjectLockConfiguration,
|
||||
ObjectLockEnabled, ObjectLockRetentionMode, ObjectLockRule, PublicAccessBlockConfiguration, ServerSideEncryption,
|
||||
ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, Tagging,
|
||||
VersioningConfiguration,
|
||||
ExpirationStatus, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter, ObjectLockConfiguration, ObjectLockEnabled,
|
||||
ObjectLockRetentionMode, ObjectLockRule, PublicAccessBlockConfiguration, ServerSideEncryption, ServerSideEncryptionByDefault,
|
||||
ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, Tagging, VersioningConfiguration,
|
||||
};
|
||||
use http::{Method, StatusCode};
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -1207,770 +1205,3 @@ async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestRes
|
||||
replication_target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// rc.5 multipart layouts under the current build (backlog#2147 follow-up to
|
||||
// rustfs#7305)
|
||||
// ---------------------------------------------------------------------------
|
||||
//
|
||||
// rustfs#7305 changed `ObjectInfo::is_multipart` to consult the stored part
|
||||
// list before the ETag shape. Every earlier check of that change used
|
||||
// synthetic metadata; this scenario writes the layouts with the published
|
||||
// rc.5 binary and then reads, describes, and replicates them with the
|
||||
// current build on the same data directory.
|
||||
|
||||
const LAYOUT_PLAIN_BUCKET: &str = "upgrade-layout-plain";
|
||||
const LAYOUT_ENCRYPTED_BUCKET: &str = "upgrade-layout-encrypted";
|
||||
const LAYOUT_REPLICA_BUCKET: &str = "upgrade-layout-replica";
|
||||
const LAYOUT_PART_SIZE: usize = 5 * 1024 * 1024;
|
||||
const LAYOUT_TAIL_SIZE: usize = 1024 * 1024 + 4096;
|
||||
const LAYOUT_SSEC_KEY: &str = "0123456789abcdef0123456789abcdef";
|
||||
const LAYOUT_REPLICATION_TIMEOUT: Duration = Duration::from_secs(180);
|
||||
|
||||
struct LayoutCase {
|
||||
bucket: &'static str,
|
||||
key: &'static str,
|
||||
/// Empty for a single PUT.
|
||||
part_sizes: Vec<usize>,
|
||||
body: Vec<u8>,
|
||||
ssec: bool,
|
||||
/// `false` for layouts whose replication is a known pre-existing failure;
|
||||
/// their outcome is logged, not asserted.
|
||||
assert_replication: bool,
|
||||
/// Recorded from the rc.5 writer.
|
||||
rc5_etag: String,
|
||||
/// Whether rc.5 reported `ObjectParts` for the object.
|
||||
rc5_reported_parts: Option<usize>,
|
||||
}
|
||||
|
||||
impl LayoutCase {
|
||||
fn is_multipart_layout(&self) -> bool {
|
||||
self.part_sizes.len() > 1
|
||||
}
|
||||
|
||||
fn label(&self) -> String {
|
||||
format!("{}/{}", self.bucket, self.key)
|
||||
}
|
||||
}
|
||||
|
||||
fn layout_noise(len: usize, seed: u64) -> Vec<u8> {
|
||||
let mut state = seed ^ 0x9E37_79B9_7F4A_7C15;
|
||||
(0..len)
|
||||
.map(|_| {
|
||||
state ^= state << 13;
|
||||
state ^= state >> 7;
|
||||
state ^= state << 17;
|
||||
(state >> 24) as u8
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn layout_text(len: usize, seed: u64) -> Vec<u8> {
|
||||
let mut out = Vec::with_capacity(len + 64);
|
||||
let mut line = 0u64;
|
||||
while out.len() < len {
|
||||
out.extend_from_slice(format!("rc5 legacy layout seed={seed} line={line} lorem ipsum dolor sit amet\n").as_bytes());
|
||||
line += 1;
|
||||
}
|
||||
out.truncate(len);
|
||||
out
|
||||
}
|
||||
|
||||
fn layout_ssec_key_md5() -> String {
|
||||
use md5::{Digest as _, Md5};
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(LAYOUT_SSEC_KEY.as_bytes());
|
||||
base64_simd::STANDARD.encode_to_string(hasher.finalize())
|
||||
}
|
||||
|
||||
fn layout_ssec_key() -> String {
|
||||
base64_simd::STANDARD.encode_to_string(LAYOUT_SSEC_KEY)
|
||||
}
|
||||
|
||||
async fn layout_head(
|
||||
client: &Client,
|
||||
case: &LayoutCase,
|
||||
) -> Result<aws_sdk_s3::operation::head_object::HeadObjectOutput, BoxError> {
|
||||
let request = client.head_object().bucket(case.bucket).key(case.key);
|
||||
let request = if case.ssec {
|
||||
request
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(layout_ssec_key())
|
||||
.sse_customer_key_md5(layout_ssec_key_md5())
|
||||
} else {
|
||||
request
|
||||
};
|
||||
Ok(request.send().await?)
|
||||
}
|
||||
|
||||
async fn layout_get(
|
||||
client: &Client,
|
||||
case: &LayoutCase,
|
||||
range: Option<String>,
|
||||
part_number: Option<i32>,
|
||||
) -> Result<aws_sdk_s3::operation::get_object::GetObjectOutput, BoxError> {
|
||||
let request = client.get_object().bucket(case.bucket).key(case.key);
|
||||
let request = if case.ssec {
|
||||
request
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(layout_ssec_key())
|
||||
.sse_customer_key_md5(layout_ssec_key_md5())
|
||||
} else {
|
||||
request
|
||||
};
|
||||
let request = request.set_range(range).set_part_number(part_number);
|
||||
Ok(request.send().await?)
|
||||
}
|
||||
|
||||
async fn layout_attributes(
|
||||
client: &Client,
|
||||
case: &LayoutCase,
|
||||
) -> Result<aws_sdk_s3::operation::get_object_attributes::GetObjectAttributesOutput, BoxError> {
|
||||
let request = client
|
||||
.get_object_attributes()
|
||||
.bucket(case.bucket)
|
||||
.key(case.key)
|
||||
.object_attributes(ObjectAttributes::Etag)
|
||||
.object_attributes(ObjectAttributes::ObjectParts)
|
||||
.object_attributes(ObjectAttributes::ObjectSize)
|
||||
.max_parts(100);
|
||||
let request = if case.ssec {
|
||||
request
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(layout_ssec_key())
|
||||
.sse_customer_key_md5(layout_ssec_key_md5())
|
||||
} else {
|
||||
request
|
||||
};
|
||||
Ok(request.send().await?)
|
||||
}
|
||||
|
||||
/// Write `case` with the rc.5 client; single PUT when `part_sizes` is empty.
|
||||
async fn layout_write(client: &Client, case: &LayoutCase) -> Result<(), BoxError> {
|
||||
let content_type = "text/plain";
|
||||
if case.part_sizes.is_empty() {
|
||||
let request = client
|
||||
.put_object()
|
||||
.bucket(case.bucket)
|
||||
.key(case.key)
|
||||
.content_type(content_type)
|
||||
.body(ByteStream::from(case.body.clone()));
|
||||
let request = if case.ssec {
|
||||
request
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(layout_ssec_key())
|
||||
.sse_customer_key_md5(layout_ssec_key_md5())
|
||||
} else {
|
||||
request
|
||||
};
|
||||
request.send().await?;
|
||||
return Ok(());
|
||||
}
|
||||
let create = client
|
||||
.create_multipart_upload()
|
||||
.bucket(case.bucket)
|
||||
.key(case.key)
|
||||
.content_type(content_type);
|
||||
let create = if case.ssec {
|
||||
create
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(layout_ssec_key())
|
||||
.sse_customer_key_md5(layout_ssec_key_md5())
|
||||
} else {
|
||||
create
|
||||
};
|
||||
let created = create.send().await?;
|
||||
let upload_id = created.upload_id().ok_or("CreateMultipartUpload omitted upload ID")?;
|
||||
let mut completed = Vec::with_capacity(case.part_sizes.len());
|
||||
let mut offset = 0usize;
|
||||
for (index, size) in case.part_sizes.iter().enumerate() {
|
||||
let part_number = i32::try_from(index + 1)?;
|
||||
let chunk = case.body[offset..offset + size].to_vec();
|
||||
offset += size;
|
||||
let upload = client
|
||||
.upload_part()
|
||||
.bucket(case.bucket)
|
||||
.key(case.key)
|
||||
.upload_id(upload_id)
|
||||
.part_number(part_number)
|
||||
.body(ByteStream::from(chunk));
|
||||
let upload = if case.ssec {
|
||||
upload
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(layout_ssec_key())
|
||||
.sse_customer_key_md5(layout_ssec_key_md5())
|
||||
} else {
|
||||
upload
|
||||
};
|
||||
let uploaded = upload.send().await?;
|
||||
completed.push(
|
||||
CompletedPart::builder()
|
||||
.part_number(part_number)
|
||||
.e_tag(uploaded.e_tag().ok_or("UploadPart omitted ETag")?)
|
||||
.build(),
|
||||
);
|
||||
}
|
||||
client
|
||||
.complete_multipart_upload()
|
||||
.bucket(case.bucket)
|
||||
.key(case.key)
|
||||
.upload_id(upload_id)
|
||||
.multipart_upload(CompletedMultipartUpload::builder().set_parts(Some(completed)).build())
|
||||
.send()
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn layout_cases() -> Vec<LayoutCase> {
|
||||
let two = vec![LAYOUT_PART_SIZE, LAYOUT_TAIL_SIZE];
|
||||
let three = vec![LAYOUT_PART_SIZE, LAYOUT_PART_SIZE, 4096];
|
||||
let total = |sizes: &[usize]| sizes.iter().sum::<usize>();
|
||||
let case = |bucket, key, part_sizes: Vec<usize>, body: Vec<u8>, ssec| LayoutCase {
|
||||
bucket,
|
||||
key,
|
||||
part_sizes,
|
||||
body,
|
||||
ssec,
|
||||
assert_replication: true,
|
||||
rc5_etag: String::new(),
|
||||
rc5_reported_parts: None,
|
||||
};
|
||||
vec![
|
||||
case(LAYOUT_PLAIN_BUCKET, "plain/single.bin", vec![], layout_noise(1024 * 1024 + 17, 1), false),
|
||||
case(
|
||||
LAYOUT_PLAIN_BUCKET,
|
||||
"plain/multipart-2.bin",
|
||||
two.clone(),
|
||||
layout_noise(total(&two), 2),
|
||||
false,
|
||||
),
|
||||
case(
|
||||
LAYOUT_PLAIN_BUCKET,
|
||||
"plain/multipart-3.bin",
|
||||
three.clone(),
|
||||
layout_noise(total(&three), 3),
|
||||
false,
|
||||
),
|
||||
case(
|
||||
LAYOUT_PLAIN_BUCKET,
|
||||
"plain/compressed-single.txt",
|
||||
vec![],
|
||||
layout_text(1024 * 1024 + 17, 4),
|
||||
false,
|
||||
),
|
||||
case(
|
||||
LAYOUT_PLAIN_BUCKET,
|
||||
"plain/compressed-multipart-2.txt",
|
||||
two.clone(),
|
||||
layout_text(total(&two), 5),
|
||||
false,
|
||||
),
|
||||
case(
|
||||
LAYOUT_PLAIN_BUCKET,
|
||||
"plain/ssec-multipart-2.bin",
|
||||
two.clone(),
|
||||
layout_noise(total(&two), 6),
|
||||
true,
|
||||
),
|
||||
// SSE-C passthrough replicates the stored ciphertext part by part; a
|
||||
// compressible first part is stored well below 5 MiB, so the sender
|
||||
// declares each part's plaintext length and the target validates the
|
||||
// 5 MiB minimum against it (rustfs/backlog#2363). rc.5 as the sender
|
||||
// still fails this layout (see `rc5_baseline_replicates_multipart_layouts`).
|
||||
case(
|
||||
LAYOUT_PLAIN_BUCKET,
|
||||
"plain/ssec-compressed-multipart-2.txt",
|
||||
two.clone(),
|
||||
layout_text(total(&two), 7),
|
||||
true,
|
||||
),
|
||||
case(
|
||||
LAYOUT_ENCRYPTED_BUCKET,
|
||||
"encrypted/single.bin",
|
||||
vec![],
|
||||
layout_noise(1024 * 1024 + 17, 8),
|
||||
false,
|
||||
),
|
||||
case(
|
||||
LAYOUT_ENCRYPTED_BUCKET,
|
||||
"encrypted/multipart-2.bin",
|
||||
two.clone(),
|
||||
layout_noise(total(&two), 9),
|
||||
false,
|
||||
),
|
||||
case(
|
||||
LAYOUT_ENCRYPTED_BUCKET,
|
||||
"encrypted/multipart-3.bin",
|
||||
three.clone(),
|
||||
layout_noise(total(&three), 10),
|
||||
false,
|
||||
),
|
||||
case(
|
||||
LAYOUT_ENCRYPTED_BUCKET,
|
||||
"encrypted/compressed-multipart-2.txt",
|
||||
two.clone(),
|
||||
layout_text(total(&two), 11),
|
||||
false,
|
||||
),
|
||||
]
|
||||
}
|
||||
|
||||
fn layout_server_env() -> Vec<(&'static str, &'static str)> {
|
||||
let mut env = bucket_config_server_env();
|
||||
env.push(("RUSTFS_COMPRESSION_ENABLED", "true"));
|
||||
env.push(("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true"));
|
||||
env
|
||||
}
|
||||
|
||||
fn layout_reported_parts(attributes: &aws_sdk_s3::operation::get_object_attributes::GetObjectAttributesOutput) -> Option<usize> {
|
||||
attributes.object_parts().map(|parts| parts.parts().len())
|
||||
}
|
||||
|
||||
async fn assert_layout_readable(client: &Client, case: &LayoutCase, context: &str) -> TestResult {
|
||||
let label = case.label();
|
||||
let head = layout_head(client, case).await?;
|
||||
assert_eq!(
|
||||
head.e_tag().map(|etag| etag.trim_matches('"')),
|
||||
Some(case.rc5_etag.as_str()),
|
||||
"{context}: {label}: the ETag written by rc.5 must be reported unchanged"
|
||||
);
|
||||
assert_eq!(
|
||||
head.content_length(),
|
||||
Some(i64::try_from(case.body.len())?),
|
||||
"{context}: {label}: HEAD content length"
|
||||
);
|
||||
|
||||
let full = layout_get(client, case, None, None).await?.body.collect().await?.into_bytes();
|
||||
assert_eq!(full.len(), case.body.len(), "{context}: {label}: full GET length");
|
||||
assert!(full == case.body, "{context}: {label}: full GET body must equal the rc.5 upload");
|
||||
|
||||
if case.is_multipart_layout() {
|
||||
let first = case.part_sizes[0];
|
||||
let range = format!("bytes={}-{}", first - 32, first + 31);
|
||||
let crossing = layout_get(client, case, Some(range), None)
|
||||
.await?
|
||||
.body
|
||||
.collect()
|
||||
.await?
|
||||
.into_bytes();
|
||||
assert!(
|
||||
crossing == case.body[first - 32..first + 32],
|
||||
"{context}: {label}: range across the first part boundary"
|
||||
);
|
||||
let tail_start: usize = case.part_sizes[..case.part_sizes.len() - 1].iter().sum();
|
||||
let last_number = i32::try_from(case.part_sizes.len())?;
|
||||
let last = layout_get(client, case, None, Some(last_number)).await?;
|
||||
assert_eq!(
|
||||
last.content_length(),
|
||||
Some(i64::try_from(case.part_sizes[case.part_sizes.len() - 1])?),
|
||||
"{context}: {label}: partNumber={last_number} length"
|
||||
);
|
||||
let last_body = last.body.collect().await?.into_bytes();
|
||||
assert!(
|
||||
last_body == case.body[tail_start..],
|
||||
"{context}: {label}: partNumber={last_number} body must be the stored last part"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_layout_attributes(client: &Client, case: &LayoutCase, context: &str) -> TestResult {
|
||||
let label = case.label();
|
||||
let attributes = layout_attributes(client, case).await?;
|
||||
assert_eq!(
|
||||
attributes.e_tag().map(|etag| etag.trim_matches('"')),
|
||||
Some(case.rc5_etag.as_str()),
|
||||
"{context}: {label}: attributes ETag"
|
||||
);
|
||||
assert_eq!(
|
||||
attributes.object_size(),
|
||||
Some(i64::try_from(case.body.len())?),
|
||||
"{context}: {label}: attributes ObjectSize"
|
||||
);
|
||||
if case.is_multipart_layout() {
|
||||
let parts = attributes
|
||||
.object_parts()
|
||||
.ok_or_else(|| format!("{context}: {label}: multipart layout must expose ObjectParts"))?;
|
||||
assert_eq!(
|
||||
parts.total_parts_count(),
|
||||
Some(i32::try_from(case.part_sizes.len())?),
|
||||
"{context}: {label}: TotalPartsCount"
|
||||
);
|
||||
let observed: Vec<(Option<i32>, Option<i64>)> =
|
||||
parts.parts().iter().map(|part| (part.part_number(), part.size())).collect();
|
||||
let expected: Vec<(Option<i32>, Option<i64>)> = case
|
||||
.part_sizes
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, size)| (Some(index as i32 + 1), Some(*size as i64)))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
observed, expected,
|
||||
"{context}: {label}: ObjectParts must report the plaintext part layout"
|
||||
);
|
||||
} else {
|
||||
assert!(
|
||||
attributes.object_parts().is_none_or(|parts| parts.parts().is_empty()),
|
||||
"{context}: {label}: a single PUT must not report stored parts"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn put_layout_replication_rule(env: &RustFSTestEnvironment, bucket: &str, arn: &str) -> TestResult {
|
||||
let body = format!(
|
||||
r#"<ReplicationConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Role></Role>
|
||||
<Rule>
|
||||
<ID>legacy-layouts</ID>
|
||||
<Priority>1</Priority>
|
||||
<Status>Enabled</Status>
|
||||
<Filter><Prefix></Prefix></Filter>
|
||||
<DeleteMarkerReplication><Status>Enabled</Status></DeleteMarkerReplication>
|
||||
<DeleteReplication><Status>Enabled</Status></DeleteReplication>
|
||||
<ExistingObjectReplication><Status>Enabled</Status></ExistingObjectReplication>
|
||||
<Destination><Bucket>{arn}</Bucket></Destination>
|
||||
</Rule>
|
||||
</ReplicationConfiguration>"#
|
||||
);
|
||||
let url = format!("{}/{bucket}?replication", env.url);
|
||||
let response = signed_request(
|
||||
Method::PUT,
|
||||
&url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
Some(body.into_bytes()),
|
||||
Some("application/xml"),
|
||||
)
|
||||
.await?;
|
||||
if response.status() != StatusCode::OK {
|
||||
let status = response.status();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
return Err(format!("put replication rule on {bucket} failed: {status} {body}").into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Wait for the existing-object replication of `case` to reach a terminal
|
||||
/// status and return it (`COMPLETED` or `FAILED`).
|
||||
async fn wait_layout_replication_terminal(client: &Client, case: &LayoutCase) -> Result<String, BoxError> {
|
||||
let deadline = Instant::now() + LAYOUT_REPLICATION_TIMEOUT;
|
||||
loop {
|
||||
let head = layout_head(client, case).await?;
|
||||
let status = head.replication_status().map(|status| status.as_str().to_string());
|
||||
if matches!(status.as_deref(), Some("COMPLETED") | Some("FAILED")) {
|
||||
return Ok(status.unwrap_or_default());
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
return Err(format!(
|
||||
"{}: existing-object replication never reached a terminal status; last {status:?}",
|
||||
case.label()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct LayoutTransport {
|
||||
status: String,
|
||||
uploaded_parts: Vec<i32>,
|
||||
single_puts: usize,
|
||||
completes: usize,
|
||||
/// Raw per-key journal in target order: (sequence, operation, part number,
|
||||
/// upload id), so duplicate drives can be told apart from retries.
|
||||
journal: Vec<(u64, String, Option<i32>, Option<String>)>,
|
||||
}
|
||||
|
||||
/// Configure every layout bucket to replicate its existing objects to a fresh
|
||||
/// fake target, wait for each case to settle, and report the transport the
|
||||
/// target observed per case.
|
||||
async fn replicate_layouts(
|
||||
env: &RustFSTestEnvironment,
|
||||
client: &Client,
|
||||
cases: &[LayoutCase],
|
||||
) -> Result<(FakeS3Target, Vec<LayoutTransport>), BoxError> {
|
||||
let target = FakeS3Target::start().await?;
|
||||
target.create_bucket(LAYOUT_REPLICA_BUCKET);
|
||||
for bucket in [LAYOUT_PLAIN_BUCKET, LAYOUT_ENCRYPTED_BUCKET] {
|
||||
let arn = set_replication_target_with_options(
|
||||
env,
|
||||
bucket,
|
||||
ReplicationTargetOptions {
|
||||
endpoint: &target.address(),
|
||||
access_key: FAKE_ACCESS_KEY,
|
||||
secret_key: FAKE_SECRET_KEY,
|
||||
target_bucket: LAYOUT_REPLICA_BUCKET,
|
||||
secure: false,
|
||||
skip_tls_verify: false,
|
||||
ca_cert_pem: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
put_layout_replication_rule(env, bucket, &arn).await?;
|
||||
}
|
||||
let mut statuses = Vec::with_capacity(cases.len());
|
||||
for case in cases {
|
||||
statuses.push(wait_layout_replication_terminal(client, case).await?);
|
||||
}
|
||||
let journal = target.requests();
|
||||
let mut transports = Vec::with_capacity(cases.len());
|
||||
for (case, status) in cases.iter().zip(statuses) {
|
||||
let key_requests: Vec<_> = journal
|
||||
.iter()
|
||||
.filter(|record| record.key.as_deref() == Some(case.key))
|
||||
.collect();
|
||||
let mut uploaded_parts: Vec<i32> = key_requests
|
||||
.iter()
|
||||
.filter(|record| record.operation == FakeTargetOperation::UploadPart)
|
||||
.filter_map(|record| record.part_number)
|
||||
.collect();
|
||||
uploaded_parts.sort_unstable();
|
||||
uploaded_parts.dedup();
|
||||
let transport = LayoutTransport {
|
||||
status,
|
||||
uploaded_parts,
|
||||
single_puts: key_requests
|
||||
.iter()
|
||||
.filter(|record| record.operation == FakeTargetOperation::PutObject)
|
||||
.count(),
|
||||
completes: key_requests
|
||||
.iter()
|
||||
.filter(|record| record.operation == FakeTargetOperation::CompleteMultipartUpload)
|
||||
.count(),
|
||||
journal: key_requests
|
||||
.iter()
|
||||
.map(|record| {
|
||||
(
|
||||
record.sequence,
|
||||
format!("{:?}", record.operation),
|
||||
record.part_number,
|
||||
record.upload_id.as_ref().map(|id| id.chars().take(12).collect()),
|
||||
)
|
||||
})
|
||||
.collect(),
|
||||
};
|
||||
tracing::info!(
|
||||
target: "e2e_test::upgrade_compatibility_test",
|
||||
object = %case.label(),
|
||||
?transport,
|
||||
"replication transport observed on the target"
|
||||
);
|
||||
transports.push(transport);
|
||||
}
|
||||
Ok((target, transports))
|
||||
}
|
||||
|
||||
/// rc.5 writes single-PUT, multipart, compressed, SSE-C and SSE-S3 layouts;
|
||||
/// the current build must read every byte, expose the stored part layout
|
||||
/// through GetObjectAttributes and partNumber reads, and replicate the objects
|
||||
/// with the transport that matches their stored parts.
|
||||
#[tokio::test]
|
||||
#[ignore = "requires the pinned 1.0.0-rc.5 release binary"]
|
||||
async fn direct_upgrade_from_rc5_preserves_multipart_layouts() -> TestResult {
|
||||
init_logging();
|
||||
let previous_binary = source_binary()?;
|
||||
let server_env = layout_server_env();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_from_binary(&previous_binary, vec![], &server_env)
|
||||
.await?;
|
||||
let old_client = env.create_s3_client();
|
||||
env.create_test_bucket(LAYOUT_PLAIN_BUCKET).await?;
|
||||
env.create_test_bucket(LAYOUT_ENCRYPTED_BUCKET).await?;
|
||||
enable_versioning(&old_client, LAYOUT_PLAIN_BUCKET).await?;
|
||||
enable_versioning(&old_client, LAYOUT_ENCRYPTED_BUCKET).await?;
|
||||
put_default_sse_s3_encryption(&old_client, LAYOUT_ENCRYPTED_BUCKET).await?;
|
||||
assert_default_sse_s3_encryption(&old_client, LAYOUT_ENCRYPTED_BUCKET, "rc.5").await?;
|
||||
|
||||
let mut cases = layout_cases();
|
||||
for case in cases.iter_mut() {
|
||||
layout_write(&old_client, case).await?;
|
||||
let head = layout_head(&old_client, case).await?;
|
||||
case.rc5_etag = head
|
||||
.e_tag()
|
||||
.ok_or_else(|| format!("{}: rc.5 HEAD omitted the ETag", case.label()))?
|
||||
.trim_matches('"')
|
||||
.to_string();
|
||||
case.rc5_reported_parts = layout_attributes(&old_client, case)
|
||||
.await
|
||||
.ok()
|
||||
.and_then(|a| layout_reported_parts(&a));
|
||||
tracing::info!(
|
||||
target: "e2e_test::upgrade_compatibility_test",
|
||||
object = %case.label(),
|
||||
parts = case.part_sizes.len(),
|
||||
etag = %case.rc5_etag,
|
||||
rc5_reported_parts = ?case.rc5_reported_parts,
|
||||
"rc.5 wrote a legacy layout"
|
||||
);
|
||||
}
|
||||
// The rc.5 writer must itself still read what it wrote, so a later
|
||||
// failure is attributable to the upgrade rather than to the fixture.
|
||||
for case in &cases {
|
||||
assert_layout_readable(&old_client, case, "rc.5").await?;
|
||||
}
|
||||
|
||||
// Upgrade in place.
|
||||
env.restart_server_preserving_data(vec![], &server_env).await?;
|
||||
let client = env.create_s3_client();
|
||||
for case in &cases {
|
||||
assert_layout_readable(&client, case, "upgraded").await?;
|
||||
assert_layout_attributes(&client, case, "upgraded").await?;
|
||||
}
|
||||
|
||||
// Replicate the pre-existing objects with the current build.
|
||||
let (target, transports) = replicate_layouts(&env, &client, &cases).await?;
|
||||
let replica_client = fake_source_client(&target);
|
||||
for (case, transport) in cases.iter().zip(&transports) {
|
||||
let label = case.label();
|
||||
if !case.assert_replication {
|
||||
continue;
|
||||
}
|
||||
assert_eq!(transport.status, "COMPLETED", "{label}: existing-object replication must complete");
|
||||
if case.is_multipart_layout() {
|
||||
let expected: Vec<i32> = (1..=i32::try_from(case.part_sizes.len())?).collect();
|
||||
assert_eq!(
|
||||
transport.uploaded_parts, expected,
|
||||
"{label}: stored parts must replicate as the same multipart layout"
|
||||
);
|
||||
// An object still PENDING when the next scanner cycle arrives is
|
||||
// not driven a second time (rustfs/backlog#2362); the journal is
|
||||
// logged so a duplicate round is visible if this ever regresses.
|
||||
assert_eq!(
|
||||
transport.completes, 1,
|
||||
"{label}: exactly one CompleteMultipartUpload; journal {:?}",
|
||||
transport.journal
|
||||
);
|
||||
assert_eq!(
|
||||
transport.single_puts, 0,
|
||||
"{label}: a multipart layout must not go out as a single PutObject"
|
||||
);
|
||||
} else {
|
||||
assert_eq!(transport.single_puts, 1, "{label}: a single PUT replicates as exactly one PutObject");
|
||||
assert!(transport.uploaded_parts.is_empty(), "{label}: a single PUT must not go out as multipart");
|
||||
}
|
||||
if !case.ssec {
|
||||
let replica = replica_client
|
||||
.get_object()
|
||||
.bucket(LAYOUT_REPLICA_BUCKET)
|
||||
.key(case.key)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| format!("{label}: replica missing on the target: {err}"))?
|
||||
.body
|
||||
.collect()
|
||||
.await?
|
||||
.into_bytes();
|
||||
assert_eq!(replica.len(), case.body.len(), "{label}: replica length");
|
||||
assert!(replica == case.body, "{label}: replica body must equal the rc.5 upload");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The same layouts replicated by rc.5 itself, without an upgrade. This is the
|
||||
/// baseline that tells a pre-existing transport failure apart from one the
|
||||
/// current build introduced; it records the outcome per layout and only fails
|
||||
/// when the fixture cannot run.
|
||||
#[tokio::test]
|
||||
#[ignore = "requires the pinned 1.0.0-rc.5 release binary"]
|
||||
async fn rc5_baseline_replicates_multipart_layouts() -> TestResult {
|
||||
init_logging();
|
||||
let previous_binary = source_binary()?;
|
||||
let server_env = layout_server_env();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_from_binary(&previous_binary, vec![], &server_env)
|
||||
.await?;
|
||||
let client = env.create_s3_client();
|
||||
env.create_test_bucket(LAYOUT_PLAIN_BUCKET).await?;
|
||||
env.create_test_bucket(LAYOUT_ENCRYPTED_BUCKET).await?;
|
||||
enable_versioning(&client, LAYOUT_PLAIN_BUCKET).await?;
|
||||
enable_versioning(&client, LAYOUT_ENCRYPTED_BUCKET).await?;
|
||||
put_default_sse_s3_encryption(&client, LAYOUT_ENCRYPTED_BUCKET).await?;
|
||||
|
||||
let mut cases = layout_cases();
|
||||
for case in cases.iter_mut() {
|
||||
layout_write(&client, case).await?;
|
||||
let head = layout_head(&client, case).await?;
|
||||
case.rc5_etag = head
|
||||
.e_tag()
|
||||
.ok_or_else(|| format!("{}: rc.5 HEAD omitted the ETag", case.label()))?
|
||||
.trim_matches('"')
|
||||
.to_string();
|
||||
}
|
||||
let (_target, transports) = replicate_layouts(&env, &client, &cases).await?;
|
||||
let summary: Vec<String> = cases
|
||||
.iter()
|
||||
.zip(&transports)
|
||||
.map(|(case, transport)| {
|
||||
format!(
|
||||
"{}: {} parts={:?} puts={}",
|
||||
case.label(),
|
||||
transport.status,
|
||||
transport.uploaded_parts,
|
||||
transport.single_puts
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
tracing::info!(target: "e2e_test::upgrade_compatibility_test", ?summary, "rc.5 baseline replication outcomes");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// backlog#2362 under the same conditions that reproduced it with the rc.5
|
||||
/// writer, but with the workspace build on both sides so it runs in the
|
||||
/// ordinary lane: every pre-existing layout is driven through exactly one
|
||||
/// upload round even though the scanner re-scans it every second while the
|
||||
/// first round is still in flight.
|
||||
#[tokio::test]
|
||||
async fn existing_object_replication_drives_each_layout_once() -> TestResult {
|
||||
init_logging();
|
||||
let server_env = layout_server_env();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(vec![], &server_env).await?;
|
||||
let writer = env.create_s3_client();
|
||||
env.create_test_bucket(LAYOUT_PLAIN_BUCKET).await?;
|
||||
env.create_test_bucket(LAYOUT_ENCRYPTED_BUCKET).await?;
|
||||
enable_versioning(&writer, LAYOUT_PLAIN_BUCKET).await?;
|
||||
enable_versioning(&writer, LAYOUT_ENCRYPTED_BUCKET).await?;
|
||||
put_default_sse_s3_encryption(&writer, LAYOUT_ENCRYPTED_BUCKET).await?;
|
||||
|
||||
let mut cases = layout_cases();
|
||||
for case in cases.iter_mut() {
|
||||
layout_write(&writer, case).await?;
|
||||
let head = layout_head(&writer, case).await?;
|
||||
case.rc5_etag = head
|
||||
.e_tag()
|
||||
.ok_or_else(|| format!("{}: HEAD omitted the ETag", case.label()))?
|
||||
.trim_matches('"')
|
||||
.to_string();
|
||||
}
|
||||
|
||||
// The objects come from an earlier process lifetime: the scanner starts
|
||||
// cold and every object is a candidate at once.
|
||||
env.restart_server_preserving_data(vec![], &server_env).await?;
|
||||
let client = env.create_s3_client();
|
||||
let (_target, transports) = replicate_layouts(&env, &client, &cases).await?;
|
||||
let mut duplicates = Vec::new();
|
||||
for (case, transport) in cases.iter().zip(&transports) {
|
||||
assert_eq!(
|
||||
transport.status,
|
||||
"COMPLETED",
|
||||
"{}: existing-object replication must complete",
|
||||
case.label()
|
||||
);
|
||||
let rounds = if case.is_multipart_layout() {
|
||||
transport.completes
|
||||
} else {
|
||||
transport.single_puts
|
||||
};
|
||||
if rounds != 1 {
|
||||
duplicates.push(format!("{}: {rounds} upload rounds; journal {:?}", case.label(), transport.journal));
|
||||
}
|
||||
}
|
||||
assert!(duplicates.is_empty(), "each existing object must be driven exactly once: {duplicates:?}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -226,7 +226,7 @@ metrics = { workspace = true }
|
||||
# crates.io. The guard scripts/check_no_tokio_io_uring.sh allows an explicit
|
||||
# io-uring integration; only the tokio "io-uring" runtime feature is banned.
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
rustfs-uring = "0.2.2"
|
||||
rustfs-uring = "0.2.1"
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
winapi-util.workspace = true
|
||||
|
||||
@@ -32,8 +32,7 @@ pub mod bucket {
|
||||
pub mod bucket_target_sys {
|
||||
pub use crate::bucket::bucket_target_sys::{
|
||||
AdvancedPutOptions, BucketTargetError, BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError,
|
||||
SsecPassthroughCapability, TargetClient, VersionIdentityCapability, append_version_id_query,
|
||||
resolve_delete_api_version_id,
|
||||
SsecPassthroughCapability, TargetClient, append_version_id_query,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -70,33 +69,11 @@ pub mod bucket {
|
||||
};
|
||||
}
|
||||
|
||||
pub mod recovery_control {
|
||||
pub use crate::bucket::lifecycle::recovery_control::{
|
||||
IlmRecoveryClassification, IlmRecoveryControlPage, IlmRecoveryControlView, IlmRecoveryProtocol,
|
||||
inspect_recovery_control, list_recovery_controls,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod recovery_disposition {
|
||||
pub use crate::bucket::lifecycle::recovery_disposition::{
|
||||
IlmRecoveryDispositionExecutionOutcome, IlmRecoveryDispositionReasonCode, IlmRecoveryDispositionState,
|
||||
dry_run_recovery_disposition, execute_recovery_disposition,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod recovery_export {
|
||||
pub use crate::bucket::lifecycle::recovery_export::{
|
||||
IlmRecoveryExportCreated, IlmRecoveryExportObservation, create_recovery_export,
|
||||
inspect_recovery_export_observation, load_recovery_export,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod transition_transaction {
|
||||
pub use crate::bucket::lifecycle::transition_transaction::{
|
||||
TransitionOperatorDeleteResult, TransitionOperatorError, TransitionOperatorProbe, TransitionOperatorStatus,
|
||||
TransitionRecoveryRetryResult, TransitionRecoveryRetryStatus, delete_transition_candidate_for_operator,
|
||||
finalize_missing_transition_transaction_for_operator, inspect_transition_recovery_retry_for_operator,
|
||||
inspect_transition_transaction_for_operator, retry_transition_recovery_for_operator,
|
||||
delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator,
|
||||
inspect_transition_transaction_for_operator,
|
||||
};
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::bucket::lifecycle::transition_transaction::{
|
||||
@@ -112,9 +89,8 @@ pub mod bucket {
|
||||
#[allow(clippy::module_inception)]
|
||||
pub mod lifecycle {
|
||||
pub use crate::bucket::lifecycle::lifecycle::{
|
||||
Event, ExpirationOptions, IlmAction, LIFECYCLE_MALFORMED_XML_ERROR_KIND, Lifecycle, LifecycleCalculate,
|
||||
ObjectOpts, RuleValidate, TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time,
|
||||
object_opts_from_object_info,
|
||||
Event, ExpirationOptions, IlmAction, Lifecycle, LifecycleCalculate, ObjectOpts, RuleValidate,
|
||||
TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time, object_opts_from_object_info,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -175,17 +151,15 @@ pub mod bucket {
|
||||
BUCKET_CONFIG_PUBLISH_HOOK, BucketConfigPublishHook, BucketMetadataMutationGuard, BucketMetadataSys,
|
||||
ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
|
||||
acquire_bucket_metadata_transaction_lock_for_incarnation, acquire_scanner_bucket_incarnation_fence,
|
||||
capture_bucket_metadata_incarnation, delete, delete_if_incarnation, delete_if_incarnation_at,
|
||||
delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy, get_bucket_policy_raw,
|
||||
get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
|
||||
get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
|
||||
get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config,
|
||||
capture_bucket_metadata_incarnation, delete, delete_if_incarnation, delete_under_transaction_lock, get,
|
||||
get_accelerate_config, get_bucket_policy, get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk,
|
||||
get_cors_config, get_durability_config, get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config,
|
||||
get_notification_config, get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config,
|
||||
get_on_demand_migration_config_in, get_public_access_block_config, get_quota_config, get_replication_config,
|
||||
get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config,
|
||||
init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata,
|
||||
update, update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation,
|
||||
update_if_incarnation_at, update_quota_if_incarnation, update_quota_if_incarnation_at, update_under_transaction_lock,
|
||||
update_under_transaction_lock_at,
|
||||
update_quota_if_incarnation, update_under_transaction_lock,
|
||||
};
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::bucket::metadata_sys::{ConfigWriteLockProbe, test_support};
|
||||
@@ -309,7 +283,7 @@ pub mod cache {
|
||||
pub mod capacity {
|
||||
pub use crate::core::pools::{
|
||||
DecommissionUnresolvedEntry, PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free,
|
||||
is_pool_activation_fleet_proof_error, path2_bucket_object, path2_bucket_object_with_base_path,
|
||||
path2_bucket_object, path2_bucket_object_with_base_path,
|
||||
};
|
||||
pub use crate::store::utils::is_reserved_or_invalid_bucket;
|
||||
}
|
||||
@@ -462,12 +436,9 @@ pub mod notification {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub use crate::services::notification_sys::rotate_cross_pool_fence_fleet_proof_for_test;
|
||||
pub use crate::services::notification_sys::{
|
||||
ClusterTierDailyStats, CrossPoolFenceFleetProofToken, IlmRecoveryExportFleetProofToken,
|
||||
LegacyTransitionStateReconcileFleetProofToken, NotificationPeerErr, NotificationSys, ScannerPublicationLeaseGrant,
|
||||
acquire_cross_pool_fence_fleet_proof, acquire_ilm_recovery_export_fleet_proof,
|
||||
ClusterTierDailyStats, CrossPoolFenceFleetProofToken, LegacyTransitionStateReconcileFleetProofToken, NotificationPeerErr,
|
||||
NotificationSys, ScannerPublicationLeaseGrant, acquire_cross_pool_fence_fleet_proof,
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof, cross_pool_fence_fleet_proof_matches, get_global_notification_sys,
|
||||
ilm_recovery_export_fleet_proof_matches, ilm_recovery_export_local_process_epoch,
|
||||
ilm_recovery_export_member_epochs_sha256, ilm_recovery_export_topology_generation,
|
||||
legacy_transition_state_reconcile_fleet_proof_matches, new_global_notification_sys,
|
||||
scanner_peer_transport_error_message_is_retryable, start_remote_version_state_fleet_probe,
|
||||
};
|
||||
@@ -520,8 +491,7 @@ pub mod rpc {
|
||||
pub use crate::cluster::rpc::{
|
||||
AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
|
||||
PeerRestClient, PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
|
||||
ScannerBucketListing, ScannerDirtyUsageAcknowledgement, ScannerPeerActivity, ScannerPeerDirtyUsageBucket,
|
||||
ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease, ScannerScopedDirtyUsageAckEntry, TONIC_RPC_PREFIX,
|
||||
ScannerBucketListing, ScannerPeerActivity, ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease, TONIC_RPC_PREFIX,
|
||||
TonicInterceptor, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, decode_heal_bucket_rpc_options,
|
||||
encode_heal_bucket_rpc_options, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
||||
gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
|
||||
|
||||
@@ -18,7 +18,7 @@ use crate::bucket::metadata_sys::get_replication_config;
|
||||
use crate::bucket::remote_s3_client::{
|
||||
PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, build_remote_s3_client,
|
||||
};
|
||||
use crate::bucket::replication::{ObjectLockIntegrity, object_lock_put_integrity, replication_etags_match};
|
||||
use crate::bucket::replication::{ObjectLockIntegrity, object_lock_put_integrity};
|
||||
use crate::bucket::replication::{ReplicationStatusType, ReplicationTargetConfigBridge};
|
||||
use crate::bucket::target::ARN;
|
||||
use crate::bucket::target::BucketTargetType;
|
||||
@@ -33,8 +33,6 @@ use aws_sdk_s3::operation::get_object::{GetObjectError, GetObjectOutput};
|
||||
use aws_sdk_s3::operation::get_object_tagging::{GetObjectTaggingError, GetObjectTaggingOutput};
|
||||
use aws_sdk_s3::operation::head_bucket::HeadBucketError;
|
||||
use aws_sdk_s3::operation::head_object::HeadObjectError;
|
||||
use aws_sdk_s3::operation::put_object_legal_hold::{PutObjectLegalHoldError, PutObjectLegalHoldOutput};
|
||||
use aws_sdk_s3::operation::put_object_retention::{PutObjectRetentionError, PutObjectRetentionOutput};
|
||||
use aws_sdk_s3::operation::put_object_tagging::{PutObjectTaggingError, PutObjectTaggingOutput};
|
||||
use aws_sdk_s3::operation::upload_part::UploadPartOutput;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
@@ -44,7 +42,6 @@ use aws_sdk_s3::types::{
|
||||
ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
|
||||
ServerSideEncryption,
|
||||
};
|
||||
use aws_sdk_s3::types::{ObjectLockLegalHold, ObjectLockRetention};
|
||||
use aws_sdk_s3::{Client as S3Client, operation::head_object::HeadObjectOutput};
|
||||
use aws_smithy_runtime_api::client::orchestrator::HttpRequest;
|
||||
use futures::{StreamExt, stream};
|
||||
@@ -129,25 +126,6 @@ impl From<&BucketTarget> for RemoteS3EndpointSpec {
|
||||
}
|
||||
|
||||
pub type HeadObjectSdkError = Box<SdkError<HeadObjectError>>;
|
||||
|
||||
/// Whether an edited bucket target still addresses the same remote service
|
||||
/// (endpoint, bucket, path style, TLS and identity), so a verdict learned
|
||||
/// about that service stays valid across the edit.
|
||||
fn same_replication_service(edited: &BucketTarget, previous: &BucketTarget) -> bool {
|
||||
let access_key = |target: &BucketTarget| target.credentials.as_ref().map(|credentials| credentials.access_key.clone());
|
||||
edited.endpoint == previous.endpoint
|
||||
&& edited.target_bucket == previous.target_bucket
|
||||
&& edited.secure == previous.secure
|
||||
&& edited.path == previous.path
|
||||
&& access_key(edited) == access_key(previous)
|
||||
}
|
||||
|
||||
/// Page size and page budget for [`TargetClient::locate_replica_by_etag`].
|
||||
const FIND_VERSION_BY_ETAG_PAGE_SIZE: i32 = 1000;
|
||||
const FIND_VERSION_BY_ETAG_MAX_PAGES: usize = 8;
|
||||
/// Candidate cap for [`TargetClient::replica_candidates_by_etag`]: more than
|
||||
/// this many same-content versions of one key is ambiguity by any measure.
|
||||
const FIND_VERSION_BY_ETAG_MAX_MATCHES: usize = 16;
|
||||
pub type GetObjectSdkError = Box<SdkError<GetObjectError>>;
|
||||
pub type GetObjectTaggingSdkError = Box<SdkError<GetObjectTaggingError>>;
|
||||
pub type PutObjectTaggingSdkError = Box<SdkError<PutObjectTaggingError>>;
|
||||
@@ -371,13 +349,6 @@ struct TargetClientBuildProbe {
|
||||
/// their import path while the verdict vocabulary lives with the
|
||||
/// replication decision logic.
|
||||
pub use crate::bucket::replication::SsecPassthroughCapability;
|
||||
/// Version-identity verdicts (see the enum's own docs in
|
||||
/// `rustfs-replication`) are cached here per target ARN and follow the same
|
||||
/// `arn_remotes_map` lifecycle. They carry no TTL: the verdict is refreshed
|
||||
/// by every replication write's response, so it can only go stale on a
|
||||
/// target that receives no writes — and a stale `MintsOwn` costs one extra
|
||||
/// content-identity lookup before a PUT, never a lost replica.
|
||||
pub use crate::bucket::replication::VersionIdentityCapability;
|
||||
|
||||
/// How long an audited SSE-C passthrough verdict stays authoritative.
|
||||
///
|
||||
@@ -404,11 +375,6 @@ pub struct BucketTargetSys {
|
||||
/// SSE-C passthrough capability verdicts keyed by target ARN. See
|
||||
/// [`SsecPassthroughCapability`]; reset alongside `arn_remotes_map`.
|
||||
ssec_passthrough_map: Arc<RwLock<HashMap<String, SsecPassthroughRecord>>>,
|
||||
/// Version-identity verdicts keyed by target ARN. See
|
||||
/// [`VersionIdentityCapability`]; reset alongside `arn_remotes_map`. A std
|
||||
/// lock (never held across an await) so the replication worker can record
|
||||
/// a verdict from inside its synchronous PUT-response audit.
|
||||
version_identity_map: Arc<std::sync::RwLock<HashMap<String, VersionIdentityCapability>>>,
|
||||
pub targets_map: Arc<RwLock<HashMap<String, Vec<BucketTarget>>>>,
|
||||
/// Buckets whose persisted `bucket-targets.json` exists but cannot be
|
||||
/// decoded (rustfs/backlog#2282). Written under the bucket's update mutex
|
||||
@@ -457,7 +423,6 @@ impl BucketTargetSys {
|
||||
Self {
|
||||
arn_remotes_map: Arc::new(RwLock::new(HashMap::new())),
|
||||
ssec_passthrough_map: Arc::new(RwLock::new(HashMap::new())),
|
||||
version_identity_map: Arc::new(std::sync::RwLock::new(HashMap::new())),
|
||||
targets_map: Arc::new(RwLock::new(HashMap::new())),
|
||||
unreadable_targets: Arc::new(RwLock::new(HashSet::new())),
|
||||
h_mutex: Arc::new(RwLock::new(HashMap::new())),
|
||||
@@ -781,40 +746,10 @@ impl BucketTargetSys {
|
||||
arn_remotes_map.remove(&target.arn);
|
||||
health_map.remove(&target.arn);
|
||||
ssec_map.remove(&target.arn);
|
||||
self.forget_version_identity_capability(&target.arn);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Cached version-identity verdict for a target ARN; `Unknown` until a
|
||||
/// replication write or a replication-check VersionFidelity probe judged
|
||||
/// it since the target was built.
|
||||
pub fn version_identity_capability(&self, arn: &str) -> VersionIdentityCapability {
|
||||
self.version_identity_map
|
||||
.read()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.get(arn)
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Record a version-identity verdict for a target ARN. Written by the
|
||||
/// replication worker after every PutObject / CompleteMultipartUpload
|
||||
/// response and by the replication-check VersionFidelity phase.
|
||||
pub fn record_version_identity_capability(&self, arn: &str, capability: VersionIdentityCapability) {
|
||||
self.version_identity_map
|
||||
.write()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.insert(arn.to_string(), capability);
|
||||
}
|
||||
|
||||
fn forget_version_identity_capability(&self, arn: &str) {
|
||||
self.version_identity_map
|
||||
.write()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.remove(arn);
|
||||
}
|
||||
|
||||
/// Cached SSE-C passthrough capability for a target ARN, plus whether the
|
||||
/// verdict is older than [`SSEC_PASSTHROUGH_CAPABILITY_TTL`]. `(Unknown,
|
||||
/// false)` when no verdict has been recorded since the target was built.
|
||||
@@ -859,24 +794,17 @@ impl BucketTargetSys {
|
||||
) -> Result<BucketTargets, BucketTargetError> {
|
||||
self.validate_target(bucket, target).await?;
|
||||
|
||||
let mut bucket_targets = self.targets_base_for_write(bucket).await?;
|
||||
let mut bucket_targets = match self.list_bucket_targets(bucket).await {
|
||||
Ok(targets) => targets,
|
||||
Err(BucketTargetError::BucketRemoteTargetNotFound { .. }) => BucketTargets::default(),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
|
||||
Self::upsert_target_entry(&mut bucket_targets.targets, target, update)?;
|
||||
|
||||
Ok(bucket_targets)
|
||||
}
|
||||
|
||||
/// Ordinary writes must not turn an unreadable cached snapshot into an
|
||||
/// empty configuration. Explicit repair belongs to the metadata transaction
|
||||
/// that can inspect the current persisted state.
|
||||
async fn targets_base_for_write(&self, bucket: &str) -> Result<BucketTargets, BucketTargetError> {
|
||||
match self.list_bucket_targets(bucket).await {
|
||||
Ok(targets) => Ok(targets),
|
||||
Err(BucketTargetError::BucketRemoteTargetNotFound { .. }) => Ok(BucketTargets::default()),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn validate_target(&self, bucket: &str, target: &BucketTarget) -> Result<(), BucketTargetError> {
|
||||
if !target.target_type.is_valid() {
|
||||
return Err(BucketTargetError::BucketRemoteArnTypeInvalid {
|
||||
@@ -935,9 +863,7 @@ impl BucketTargetSys {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Merge a validated target into a caller-owned snapshot. The caller must
|
||||
/// protect that snapshot through persistence.
|
||||
pub fn upsert_target_entry(
|
||||
fn upsert_target_entry(
|
||||
bucket_targets: &mut Vec<BucketTarget>,
|
||||
target: &BucketTarget,
|
||||
update: bool,
|
||||
@@ -1227,32 +1153,12 @@ impl BucketTargetSys {
|
||||
// Remove existing targets
|
||||
if let Some(existing_targets) = targets_map.remove(bucket) {
|
||||
let mut ssec_map = self.ssec_passthrough_map.write().await;
|
||||
let unchanged_service: HashMap<&str, &BucketTarget> = targets
|
||||
.map(|new_targets| {
|
||||
new_targets
|
||||
.targets
|
||||
.iter()
|
||||
.map(|target| (target.arn.as_str(), target))
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
for target in existing_targets {
|
||||
arn_remotes_map.remove(&target.arn);
|
||||
health_map.remove(&target.arn);
|
||||
// A rebuilt/edited target may point at a different service:
|
||||
// the SSE-C passthrough verdict must be re-audited from Unknown.
|
||||
ssec_map.remove(&target.arn);
|
||||
// The version-identity verdict survives an edit that keeps the
|
||||
// same remote service (a resync start or a bandwidth change
|
||||
// rewrites the entry in place): forgetting it there would make
|
||||
// the very resync that follows re-drive every object as a
|
||||
// duplicate on a target that mints its own version ids.
|
||||
if unchanged_service
|
||||
.get(target.arn.as_str())
|
||||
.is_none_or(|edited| !same_replication_service(edited, &target))
|
||||
{
|
||||
self.forget_version_identity_capability(&target.arn);
|
||||
}
|
||||
self.update_bandwidth_limit(bucket, &target.arn, 0);
|
||||
}
|
||||
}
|
||||
@@ -1321,27 +1227,25 @@ impl BucketTargetSys {
|
||||
return (String::new(), false);
|
||||
};
|
||||
|
||||
let targets_map = self.targets_map.read().await;
|
||||
let targets = targets_map.get(bucket).map(Vec::as_slice).unwrap_or_default();
|
||||
Self::remote_arn_for_targets(targets, target, depl_id)
|
||||
}
|
||||
|
||||
/// Resolve create idempotency against the snapshot the caller will persist.
|
||||
pub fn remote_arn_for_targets(targets: &[BucketTarget], target: &BucketTarget, depl_id: &str) -> (String, bool) {
|
||||
for tgt in targets {
|
||||
if tgt.target_type == target.target_type
|
||||
&& tgt.target_bucket == target.target_bucket
|
||||
&& target.endpoint == tgt.endpoint
|
||||
&& tgt
|
||||
.credentials
|
||||
.as_ref()
|
||||
.map(|c| {
|
||||
let default_creds = Credentials::default();
|
||||
c.access_key == target.credentials.as_ref().unwrap_or(&default_creds).access_key
|
||||
})
|
||||
.unwrap_or(false)
|
||||
{
|
||||
return (tgt.arn.clone(), true);
|
||||
{
|
||||
let targets_map = self.targets_map.read().await;
|
||||
if let Some(targets) = targets_map.get(bucket) {
|
||||
for tgt in targets {
|
||||
if tgt.target_type == target.target_type
|
||||
&& tgt.target_bucket == target.target_bucket
|
||||
&& target.endpoint == tgt.endpoint
|
||||
&& tgt
|
||||
.credentials
|
||||
.as_ref()
|
||||
.map(|c| {
|
||||
let default_creds = Credentials::default();
|
||||
c.access_key == target.credentials.as_ref().unwrap_or(&default_creds).access_key
|
||||
})
|
||||
.unwrap_or(false)
|
||||
{
|
||||
return (tgt.arn.clone(), true);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1369,7 +1273,6 @@ fn generate_arn(t: &BucketTarget, depl_id: &str) -> String {
|
||||
arn.to_string()
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RemoveObjectOptions {
|
||||
pub force_delete: bool,
|
||||
pub governance_bypass: bool,
|
||||
@@ -1425,12 +1328,7 @@ fn build_remove_object_headers(version_id: Option<&str>, opts: &RemoveObjectOpti
|
||||
/// and silently creates a delete marker instead of removing the version, while
|
||||
/// the source stamps `VersionPurgeStatus=Complete` (backlog#799 B8 / #857).
|
||||
/// Non-replication callers always pass the version through unchanged.
|
||||
/// The `versionId` a replicated DELETE puts on the wire: none for a
|
||||
/// delete-marker creation (the target mints the marker; the source version
|
||||
/// travels in the internal headers for RustFS peers), the addressed version
|
||||
/// otherwise. A generic S3 target given the version id on a marker-creation
|
||||
/// DELETE would permanently delete that version instead.
|
||||
pub fn resolve_delete_api_version_id(version_id: Option<String>, opts: &RemoveObjectOptions) -> Option<String> {
|
||||
fn resolve_delete_api_version_id(version_id: Option<String>, opts: &RemoveObjectOptions) -> Option<String> {
|
||||
if opts.replication_request && opts.replication_delete_marker {
|
||||
None
|
||||
} else {
|
||||
@@ -1983,125 +1881,6 @@ impl TargetClient {
|
||||
.map_err(Box::new)
|
||||
}
|
||||
|
||||
/// Candidate replicas by content identity on a target that mints its own
|
||||
/// version ids: page `ListObjectVersions` under the exact key and report
|
||||
/// the live versions whose ETag matches `source_etag`, newest first.
|
||||
/// Delete markers and prefix siblings never match. Bounded to
|
||||
/// [`FIND_VERSION_BY_ETAG_MAX_PAGES`] pages and
|
||||
/// [`FIND_VERSION_BY_ETAG_MAX_MATCHES`] candidates so a key with a very
|
||||
/// deep history cannot turn one convergence check into an unbounded scan;
|
||||
/// a replica beyond that window reads as missing, which only costs a
|
||||
/// re-PUT (today's behaviour), never a lost object.
|
||||
///
|
||||
/// Content identity is not version identity: two source generations with
|
||||
/// the same bytes have the same ETag. Callers drop the candidates other
|
||||
/// source versions already claim through their ledgers and refuse an
|
||||
/// [`ReplicaLocation::Ambiguous`] remainder before mutating or deleting.
|
||||
pub async fn replica_candidates_by_etag(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
source_etag: &str,
|
||||
) -> Result<Vec<String>, Box<SdkError<aws_sdk_s3::operation::list_object_versions::ListObjectVersionsError>>> {
|
||||
let mut key_marker: Option<String> = None;
|
||||
let mut version_id_marker: Option<String> = None;
|
||||
let mut matches: Vec<String> = Vec::new();
|
||||
for _ in 0..FIND_VERSION_BY_ETAG_MAX_PAGES {
|
||||
let page = self
|
||||
.client
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.prefix(object)
|
||||
.max_keys(FIND_VERSION_BY_ETAG_PAGE_SIZE)
|
||||
.set_key_marker(key_marker.take())
|
||||
.set_version_id_marker(version_id_marker.take())
|
||||
.send()
|
||||
.await
|
||||
.map_err(Box::new)?;
|
||||
matches.extend(
|
||||
page.versions()
|
||||
.iter()
|
||||
.filter(|version| {
|
||||
version.key() == Some(object)
|
||||
&& version.version_id().is_some_and(|id| !id.is_empty())
|
||||
&& replication_etags_match(Some(source_etag), version.e_tag())
|
||||
})
|
||||
.filter_map(|version| version.version_id().map(str::to_string)),
|
||||
);
|
||||
// A listing that moved past the exact key (every listed key is >=
|
||||
// the prefix), ended, or already filled the candidate cap decides.
|
||||
if matches.len() >= FIND_VERSION_BY_ETAG_MAX_MATCHES
|
||||
|| page
|
||||
.versions()
|
||||
.iter()
|
||||
.any(|version| version.key().is_some_and(|key| key > object))
|
||||
|| !page.is_truncated().unwrap_or(false)
|
||||
{
|
||||
break;
|
||||
}
|
||||
key_marker = page.next_key_marker().map(str::to_string);
|
||||
version_id_marker = page.next_version_id_marker().map(str::to_string);
|
||||
if key_marker.is_none() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
matches.truncate(FIND_VERSION_BY_ETAG_MAX_MATCHES);
|
||||
Ok(matches)
|
||||
}
|
||||
|
||||
/// PutObjectRetention against a replica version on a target that does not
|
||||
/// take retention through the replication PUT's own headers (it mints its
|
||||
/// own version ids, so a re-PUT would create another version instead of
|
||||
/// updating this one). Anti-loop marker always added.
|
||||
pub async fn put_object_retention(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
mode: ObjectLockRetentionMode,
|
||||
retain_until: aws_sdk_s3::primitives::DateTime,
|
||||
) -> Result<PutObjectRetentionOutput, Box<SdkError<PutObjectRetentionError>>> {
|
||||
let headers = proxy_outbound_headers(HeaderMap::new());
|
||||
self.client
|
||||
.put_object_retention()
|
||||
.bucket(bucket)
|
||||
.key(object)
|
||||
.set_version_id(resolve_read_api_version_id(version_id))
|
||||
.retention(
|
||||
ObjectLockRetention::builder()
|
||||
.mode(mode)
|
||||
.retain_until_date(retain_until)
|
||||
.build(),
|
||||
)
|
||||
.customize()
|
||||
.map_request(move |req| apply_extra_headers(req, &headers))
|
||||
.send()
|
||||
.await
|
||||
.map_err(Box::new)
|
||||
}
|
||||
|
||||
/// PutObjectLegalHold counterpart of [`Self::put_object_retention`].
|
||||
pub async fn put_object_legal_hold(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
status: ObjectLockLegalHoldStatus,
|
||||
) -> Result<PutObjectLegalHoldOutput, Box<SdkError<PutObjectLegalHoldError>>> {
|
||||
let headers = proxy_outbound_headers(HeaderMap::new());
|
||||
self.client
|
||||
.put_object_legal_hold()
|
||||
.bucket(bucket)
|
||||
.key(object)
|
||||
.set_version_id(resolve_read_api_version_id(version_id))
|
||||
.legal_hold(ObjectLockLegalHold::builder().status(status).build())
|
||||
.customize()
|
||||
.map_request(move |req| apply_extra_headers(req, &headers))
|
||||
.send()
|
||||
.await
|
||||
.map_err(Box::new)
|
||||
}
|
||||
|
||||
/// HEAD used by the read-proxy path (GET/HEAD of an object not yet
|
||||
/// replicated locally, MinIO `proxyHeadToRepTarget`).
|
||||
///
|
||||
@@ -2274,15 +2053,7 @@ impl TargetClient {
|
||||
}
|
||||
}
|
||||
|
||||
// A forwarded source checksum is this PUT's integrity header. In
|
||||
// streaming-checksum mode (`RUSTFS_REPLICATION_STREAMING_CHECKSUMS`)
|
||||
// the SDK would still add its default CRC32 trailer, and a target that
|
||||
// receives both keeps the trailer's algorithm: a forwarded SHA256
|
||||
// vanished from the replica while the source reported COMPLETED. Pin
|
||||
// this request to WhenRequired so nothing is sent beside the source's
|
||||
// own checksum.
|
||||
let forwards_source_checksum = headers.keys().any(|name| name.as_str().starts_with("x-amz-checksum-"));
|
||||
let mut operation = builder
|
||||
match builder
|
||||
.bucket(bucket)
|
||||
.key(object)
|
||||
.content_length(size)
|
||||
@@ -2302,14 +2073,10 @@ impl TargetClient {
|
||||
}
|
||||
|
||||
Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
|
||||
});
|
||||
if forwards_source_checksum {
|
||||
operation = operation.config_override(
|
||||
aws_sdk_s3::config::Builder::new()
|
||||
.request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired),
|
||||
);
|
||||
}
|
||||
match operation.send().await {
|
||||
})
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
Ok(output) => {
|
||||
// Under SSE-KMS/DSSE or SSE-C the target's ETag is not the MD5
|
||||
// of the stored plaintext, so it cannot be compared against the
|
||||
@@ -2553,45 +2320,6 @@ impl TargetClient {
|
||||
}
|
||||
}
|
||||
|
||||
/// Where a replica stands on a target that mints its own version ids, by
|
||||
/// content identity (exact key + ETag) after the candidates other source
|
||||
/// versions claim were removed. See
|
||||
/// [`TargetClient::replica_candidates_by_etag`].
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum ReplicaLocation {
|
||||
/// No live version under the key carries the source ETag.
|
||||
Missing,
|
||||
/// Exactly one live version carries it: safe to address.
|
||||
Unique(String),
|
||||
/// More than one live version carries it (same bytes replicated for
|
||||
/// several source generations). `newest` is the most recently listed
|
||||
/// one — good enough to prove the replica exists, never good enough to
|
||||
/// pick which one to mutate or delete.
|
||||
Ambiguous { newest: String },
|
||||
}
|
||||
|
||||
impl ReplicaLocation {
|
||||
/// `matches` newest first, as the target listed them.
|
||||
pub fn from_matches(mut matches: Vec<String>) -> Self {
|
||||
match matches.len() {
|
||||
0 => Self::Missing,
|
||||
1 => Self::Unique(matches.remove(0)),
|
||||
_ => Self::Ambiguous {
|
||||
newest: matches.remove(0),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// The version to read for existence/ETag checks, where an ambiguous
|
||||
/// match is still a located replica.
|
||||
pub fn any_version_id(&self) -> Option<&str> {
|
||||
match self {
|
||||
Self::Missing => None,
|
||||
Self::Unique(version_id) | Self::Ambiguous { newest: version_id } => Some(version_id),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum BucketTargetError {
|
||||
BucketRemoteTargetNotFound {
|
||||
@@ -2768,21 +2496,13 @@ mod tests {
|
||||
}
|
||||
|
||||
fn header_recording_target_client(response_headers: Vec<(String, String)>) -> (TargetClient, RecordedHeaders) {
|
||||
header_recording_target_client_with_checksums(response_headers, replication_request_checksum_calculation())
|
||||
}
|
||||
|
||||
fn header_recording_target_client_with_checksums(
|
||||
response_headers: Vec<(String, String)>,
|
||||
checksums: RequestChecksumCalculation,
|
||||
) -> (TargetClient, RecordedHeaders) {
|
||||
let request_headers: RecordedHeaders = Arc::new(std::sync::Mutex::new(Vec::new()));
|
||||
let connector = SharedHttpConnector::new(RecordingHeaderConnector {
|
||||
request_headers: Arc::clone(&request_headers),
|
||||
response_headers,
|
||||
});
|
||||
let http_client = http_client_fn(move |_settings, _components| connector.clone());
|
||||
let client =
|
||||
s3_client_for_endpoint_test_with_checksums("https://localhost:443".to_string(), Some(http_client), checksums);
|
||||
let client = s3_client_for_test(443, Some(http_client));
|
||||
(
|
||||
TargetClient {
|
||||
endpoint: "https://localhost:443".to_string(),
|
||||
@@ -2949,47 +2669,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// With streaming checksums enabled the SDK adds a CRC32 trailer to every
|
||||
/// upload. A PUT that forwards the source's checksum must not get that
|
||||
/// second algorithm: a target that receives both keeps the trailer's and
|
||||
/// the forwarded SHA256 never reaches the replica (rustfs/backlog#2340).
|
||||
#[tokio::test]
|
||||
async fn streaming_put_object_with_forwarded_checksum_sends_no_sdk_checksum() {
|
||||
let (client, recorded) =
|
||||
header_recording_target_client_with_checksums(Vec::new(), RequestChecksumCalculation::WhenSupported);
|
||||
let mut forwarded = PutObjectOptions::default();
|
||||
forwarded.user_metadata.insert(
|
||||
"x-amz-checksum-sha256".to_string(),
|
||||
"OoJ3yNhRwv3wwtZoGqEIPrPX9xwTnfLl+ka0wStN1g0=".to_string(),
|
||||
);
|
||||
client
|
||||
.put_object("target-bucket", "object", 4, streaming_test_body(b"data"), &forwarded)
|
||||
.await
|
||||
.expect("recorded put_object should succeed");
|
||||
client
|
||||
.put_object("target-bucket", "object", 4, streaming_test_body(b"data"), &PutObjectOptions::default())
|
||||
.await
|
||||
.expect("recorded put_object should succeed");
|
||||
let recorded = recorded.lock().expect("recorded header lock should not be poisoned");
|
||||
let with_forwarded = &recorded[0];
|
||||
assert_eq!(
|
||||
recorded_header(with_forwarded, "x-amz-checksum-sha256"),
|
||||
Some("OoJ3yNhRwv3wwtZoGqEIPrPX9xwTnfLl+ka0wStN1g0=")
|
||||
);
|
||||
assert_eq!(
|
||||
recorded_header(with_forwarded, "x-amz-trailer"),
|
||||
None,
|
||||
"the SDK must not add a trailer checksum"
|
||||
);
|
||||
assert_eq!(recorded_header(with_forwarded, "x-amz-sdk-checksum-algorithm"), None);
|
||||
// Control: the same client still streams a trailer when nothing is forwarded.
|
||||
let without_forwarded = &recorded[1];
|
||||
assert!(
|
||||
recorded_header(without_forwarded, "x-amz-trailer").is_some(),
|
||||
"streaming mode must still apply to uploads without a forwarded checksum: {without_forwarded:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// A forwarded source checksum already satisfies the rule; nothing is added.
|
||||
#[tokio::test]
|
||||
async fn locked_put_object_keeps_a_forwarded_source_checksum() {
|
||||
@@ -3355,14 +3034,6 @@ mod tests {
|
||||
}
|
||||
|
||||
fn s3_client_for_endpoint_test(endpoint: String, http_client: Option<SharedHttpClient>) -> S3Client {
|
||||
s3_client_for_endpoint_test_with_checksums(endpoint, http_client, replication_request_checksum_calculation())
|
||||
}
|
||||
|
||||
fn s3_client_for_endpoint_test_with_checksums(
|
||||
endpoint: String,
|
||||
http_client: Option<SharedHttpClient>,
|
||||
checksums: RequestChecksumCalculation,
|
||||
) -> S3Client {
|
||||
let credentials = SdkCredentials::builder()
|
||||
.access_key_id("test-access")
|
||||
.secret_access_key("test-secret")
|
||||
@@ -3376,7 +3047,7 @@ mod tests {
|
||||
.behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
|
||||
// Mirror the production remote-target builder so recorded requests
|
||||
// exercise the same checksum/framing behavior (#6853).
|
||||
.request_checksum_calculation(checksums);
|
||||
.request_checksum_calculation(replication_request_checksum_calculation());
|
||||
if let Some(http_client) = http_client {
|
||||
config = config.http_client(http_client);
|
||||
}
|
||||
@@ -3539,64 +3210,6 @@ mod tests {
|
||||
assert!(message.contains("connection refused"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn same_replication_service_ignores_resync_and_bandwidth_edits() {
|
||||
let base = BucketTarget {
|
||||
endpoint: "target.example:9000".to_string(),
|
||||
target_bucket: "replica".to_string(),
|
||||
secure: true,
|
||||
path: "on".to_string(),
|
||||
arn: "arn:rustfs:replication:us-east-1:bucket:same".to_string(),
|
||||
credentials: Some(Credentials {
|
||||
access_key: "access".to_string(),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
let resync_edit = BucketTarget {
|
||||
reset_id: "reset-1".to_string(),
|
||||
bandwidth_limit: 1024,
|
||||
..base.clone()
|
||||
};
|
||||
assert!(same_replication_service(&resync_edit, &base));
|
||||
for moved in [
|
||||
BucketTarget {
|
||||
endpoint: "other.example:9000".to_string(),
|
||||
..base.clone()
|
||||
},
|
||||
BucketTarget {
|
||||
target_bucket: "other".to_string(),
|
||||
..base.clone()
|
||||
},
|
||||
BucketTarget {
|
||||
secure: false,
|
||||
..base.clone()
|
||||
},
|
||||
BucketTarget {
|
||||
credentials: Some(Credentials {
|
||||
access_key: "rotated".to_string(),
|
||||
..Default::default()
|
||||
}),
|
||||
..base.clone()
|
||||
},
|
||||
] {
|
||||
assert!(!same_replication_service(&moved, &base));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn version_identity_verdict_is_per_arn_and_forgotten_with_the_target() {
|
||||
let sys = BucketTargetSys::default();
|
||||
let arn = "arn:rustfs:replication:us-east-1:bucket:identity";
|
||||
assert_eq!(sys.version_identity_capability(arn), VersionIdentityCapability::Unknown);
|
||||
sys.record_version_identity_capability(arn, VersionIdentityCapability::MintsOwn);
|
||||
assert_eq!(sys.version_identity_capability(arn), VersionIdentityCapability::MintsOwn);
|
||||
assert_eq!(sys.version_identity_capability("other"), VersionIdentityCapability::Unknown);
|
||||
// A rebuilt target may point at a different service.
|
||||
sys.forget_version_identity_capability(arn);
|
||||
assert_eq!(sys.version_identity_capability(arn), VersionIdentityCapability::Unknown);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn endpoint_health_key_preserves_explicit_port() {
|
||||
let url = Url::parse("https://remote.example:9443").expect("url should parse");
|
||||
@@ -4700,41 +4313,4 @@ mod tests {
|
||||
let window = LastMinuteLatency::new();
|
||||
assert_eq!(window.get_total().avg, Duration::from_secs(0));
|
||||
}
|
||||
|
||||
fn repair_target(bucket: &str, id: &str) -> BucketTarget {
|
||||
BucketTarget {
|
||||
source_bucket: bucket.to_string(),
|
||||
endpoint: "remote.example.com".to_string(),
|
||||
target_bucket: "remote".to_string(),
|
||||
arn: format!("arn:rustfs:replication:us-east-1:{bucket}:{id}"),
|
||||
target_type: BucketTargetType::ReplicationService,
|
||||
region: "us-east-1".to_string(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_unreadable_target_set_refuses_cached_writes() {
|
||||
let sys = BucketTargetSys::default();
|
||||
let bucket = "targets-repair-opt-in";
|
||||
sys.mark_targets_unreadable(bucket).await;
|
||||
assert!(matches!(
|
||||
sys.targets_base_for_write(bucket).await,
|
||||
Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. })
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_readable_target_set_remains_the_write_base() {
|
||||
let sys = BucketTargetSys::default();
|
||||
let bucket = "targets-repair-readable";
|
||||
let existing = repair_target(bucket, "keep");
|
||||
sys.targets_map
|
||||
.write()
|
||||
.await
|
||||
.insert(bucket.to_string(), vec![existing.clone()]);
|
||||
let base = sys.targets_base_for_write(bucket).await.expect("read targets");
|
||||
assert_eq!(base.targets.len(), 1);
|
||||
assert_eq!(base.targets[0].arn, existing.arn);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,7 +32,6 @@ use crate::bucket::lifecycle::manual_transition_job::{
|
||||
record_manual_transition_worker_result_with_reason, renew_manual_transition_job_lease_if_owned,
|
||||
save_manual_transition_job_record_if_current, save_manual_transition_task_if_absent, update_manual_transition_job_record,
|
||||
};
|
||||
use crate::bucket::lifecycle::recovery_disposition_runtime::run_recovery_disposition_maintenance_loop;
|
||||
use crate::bucket::lifecycle::replication_sink;
|
||||
use crate::bucket::lifecycle::replication_sink::{
|
||||
DeleteReplicationConfigSnapshot, ReplicationObjectBridge, ReplicationStatusType, replication_state_to_filemeta,
|
||||
@@ -150,19 +149,6 @@ pub type ExpiryOpType = Box<dyn ExpiryOp + Send + Sync + 'static>;
|
||||
static XXHASH_SEED: u64 = 0;
|
||||
static TIER_FREE_VERSION_RECOVERY_STARTED: OnceLock<()> = OnceLock::new();
|
||||
static MANUAL_TRANSITION_JOB_RECOVERY_STARTED: OnceLock<()> = OnceLock::new();
|
||||
static RECOVERY_DISPOSITION_MAINTENANCE_STARTED: OnceLock<()> = OnceLock::new();
|
||||
|
||||
#[cfg(test)]
|
||||
#[derive(Default)]
|
||||
struct FreeVersionPostRemoteDeleteTestBarrier {
|
||||
arrived: Notify,
|
||||
release: Notify,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
tokio::task_local! {
|
||||
static FREE_VERSION_POST_REMOTE_DELETE_TEST_BARRIER: Arc<FreeVersionPostRemoteDeleteTestBarrier>;
|
||||
}
|
||||
|
||||
pub const AMZ_OBJECT_TAGGING: &str = "X-Amz-Tagging";
|
||||
#[allow(
|
||||
@@ -924,11 +910,6 @@ async fn cleanup_free_version_exact(api: Arc<ECStore>, oi: &ObjectInfo, cancel:
|
||||
})??;
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
if let Ok(barrier) = FREE_VERSION_POST_REMOTE_DELETE_TEST_BARRIER.try_with(Arc::clone) {
|
||||
barrier.arrived.notify_one();
|
||||
barrier.release.notified().await;
|
||||
}
|
||||
if !free_version_cleanup_fences_current(&topology_generation, &api, &bucket_guard, &object_guards, &lease, cancel, deadline) {
|
||||
// Remote DELETE is idempotent, but a changed fence makes the local
|
||||
// outcome ambiguous. Keep every marker for a fully fenced retry.
|
||||
@@ -2400,20 +2381,9 @@ pub async fn init_background_expiry(api: Arc<ECStore>) {
|
||||
let _ = spawn_tier_free_version_recovery_once(api.clone(), &TIER_FREE_VERSION_RECOVERY_STARTED);
|
||||
spawn_tier_delete_journal_recovery_once(api.clone());
|
||||
spawn_transition_transaction_recovery_once(api.clone());
|
||||
spawn_recovery_disposition_maintenance_once(api.clone());
|
||||
spawn_manual_transition_job_recovery_once(api);
|
||||
}
|
||||
|
||||
fn spawn_recovery_disposition_maintenance_once(api: Arc<ECStore>) -> Option<JoinHandle<()>> {
|
||||
let cancel_token = api.ctx.background_cancel_token()?;
|
||||
if RECOVERY_DISPOSITION_MAINTENANCE_STARTED.set(()).is_err() {
|
||||
return None;
|
||||
}
|
||||
Some(tokio::spawn(async move {
|
||||
run_recovery_disposition_maintenance_loop(api, cancel_token).await;
|
||||
}))
|
||||
}
|
||||
|
||||
fn spawn_manual_transition_job_recovery_once(api: Arc<ECStore>) -> Option<JoinHandle<()>> {
|
||||
if MANUAL_TRANSITION_JOB_RECOVERY_STARTED.set(()).is_err() {
|
||||
return None;
|
||||
@@ -5861,7 +5831,7 @@ mod tests {
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::services::tier::test_util::register_mock_tier;
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::services::tier::tier::{TIER_DRIVER_TEST_FACTORY, TierConfigMgr, TierDriverTestFactory};
|
||||
use crate::services::tier::tier::TierConfigMgr;
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::services::tier::warm_backend::{TransitionCandidateProbe, WarmBackend as _};
|
||||
use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause};
|
||||
@@ -7860,119 +7830,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn tier_remove_waits_for_inflight_free_version_local_commit() {
|
||||
let (disk_paths, ecstore) = setup_test_env().await;
|
||||
let bucket = format!("tier-remove-free-version-{}", Uuid::new_v4());
|
||||
let object = "free-version";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
let (backend, identity_hex) = register_recovery_mock_tier(&ecstore).await;
|
||||
let tier_manager = ecstore.tier_config_mgr();
|
||||
{
|
||||
let manager = tier_manager.read().await;
|
||||
manager
|
||||
.save_tiering_config(Arc::clone(&ecstore))
|
||||
.await
|
||||
.expect("mock tier configuration should persist before removal");
|
||||
}
|
||||
seed_recoverable_free_version(&disk_paths, &bucket, object, None, Some(identity_hex)).await;
|
||||
let page = list_tier_free_versions(Arc::clone(&ecstore), 1, None, None, CancellationToken::new())
|
||||
.await
|
||||
.expect("seeded free version should be listed");
|
||||
let oi = page
|
||||
.items
|
||||
.into_iter()
|
||||
.next()
|
||||
.expect("seeded free version should be recoverable");
|
||||
|
||||
backend
|
||||
.set_put_remote_version(Some(oi.transitioned_object.version_id.clone()))
|
||||
.await;
|
||||
let seed_lease = TierConfigMgr::acquire_operation_lease(&tier_manager, "WARM")
|
||||
.await
|
||||
.expect("mock tier lease should be available");
|
||||
seed_lease
|
||||
.put(&oi.transitioned_object.name, ReaderImpl::Body(Bytes::from_static(b"body")), 4)
|
||||
.await
|
||||
.expect("remote free-version tuple should be seeded");
|
||||
drop(seed_lease);
|
||||
|
||||
let barrier = Arc::new(super::FreeVersionPostRemoteDeleteTestBarrier::default());
|
||||
let cleanup_barrier = Arc::clone(&barrier);
|
||||
let cleanup_store = Arc::clone(&ecstore);
|
||||
let cleanup_oi = oi.clone();
|
||||
let cleanup = tokio::spawn(async move {
|
||||
super::FREE_VERSION_POST_REMOTE_DELETE_TEST_BARRIER
|
||||
.scope(cleanup_barrier, async move {
|
||||
super::cleanup_free_version_exact(cleanup_store, &cleanup_oi, &CancellationToken::new()).await
|
||||
})
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(StdDuration::from_secs(30), barrier.arrived.notified())
|
||||
.await
|
||||
.expect("free-version cleanup should pause after the remote delete");
|
||||
assert!(!backend.contains(&oi.transitioned_object.name).await);
|
||||
|
||||
let remove_manager = Arc::clone(&tier_manager);
|
||||
let remove_store = Arc::clone(&ecstore);
|
||||
let remove_backend = backend.clone();
|
||||
let remove_driver_factory: TierDriverTestFactory = Arc::new(move |_| Ok(Box::new(remove_backend.clone())));
|
||||
let mut remove = tokio::spawn(async move {
|
||||
TIER_DRIVER_TEST_FACTORY
|
||||
.scope(
|
||||
remove_driver_factory,
|
||||
TierConfigMgr::remove_and_save(&remove_manager, remove_store, "WARM", true),
|
||||
)
|
||||
.await
|
||||
});
|
||||
let prepared = tokio::time::timeout(StdDuration::from_secs(30), async {
|
||||
loop {
|
||||
match TierConfigMgr::acquire_operation_lease(&tier_manager, "WARM").await {
|
||||
Ok(lease) => drop(lease),
|
||||
Err(err) if TierConfigMgr::operation_lease_blocked_by_mutation(&err) => break,
|
||||
Err(err) => panic!("tier remove should only block new operations while cleanup is paused: {err}"),
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
});
|
||||
tokio::select! {
|
||||
prepared = prepared => {
|
||||
prepared.expect("tier remove should install its prepared admission fence");
|
||||
}
|
||||
result = &mut remove => {
|
||||
panic!("tier remove finished before installing its prepared admission fence: {result:?}");
|
||||
}
|
||||
}
|
||||
assert!(!remove.is_finished(), "tier remove must wait for the leased local cleanup commit");
|
||||
|
||||
barrier.release.notify_one();
|
||||
tokio::time::timeout(StdDuration::from_secs(30), cleanup)
|
||||
.await
|
||||
.expect("free-version cleanup should finish after release")
|
||||
.expect("free-version cleanup task should join")
|
||||
.expect("free-version cleanup should keep its generation current");
|
||||
tokio::time::timeout(StdDuration::from_secs(30), remove)
|
||||
.await
|
||||
.expect("tier remove should finish after local cleanup")
|
||||
.expect("tier remove task should join")
|
||||
.expect("tier remove should pass its fresh authoritative proof");
|
||||
|
||||
assert!(!tier_manager.read().await.is_tier_valid("WARM"));
|
||||
for disk_path in &disk_paths {
|
||||
assert!(
|
||||
!fs::try_exists(disk_path.join(&bucket).join(object))
|
||||
.await
|
||||
.expect("post-removal free-version path check should succeed")
|
||||
);
|
||||
}
|
||||
ecstore
|
||||
.delete_bucket(&bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("empty free-version test bucket should be removed");
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
|
||||
@@ -41,21 +41,6 @@ where
|
||||
com::read_config(api, file).await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_limited_preserve_empty<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<Vec<u8>>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
com::read_config_limited_preserve_empty(api, file, max_bytes).await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_with_metadata<S>(api: Arc<S>, file: &str, opts: &ObjectOptions) -> Result<(Vec<u8>, ObjectInfo)>
|
||||
where
|
||||
S: ObjectIO<
|
||||
@@ -71,26 +56,6 @@ where
|
||||
com::read_config_with_metadata(api, file, opts).await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_limited_preserve_empty_with_metadata<S>(
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
opts: &ObjectOptions,
|
||||
max_bytes: usize,
|
||||
) -> Result<(Vec<u8>, ObjectInfo)>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
com::read_config_limited_preserve_empty_with_metadata_opts(api, file, opts, max_bytes).await
|
||||
}
|
||||
|
||||
pub(crate) async fn save_config<S>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()>
|
||||
where
|
||||
S: ObjectIO<
|
||||
@@ -161,30 +126,20 @@ where
|
||||
DeletedObject = DeletedObject,
|
||||
>,
|
||||
{
|
||||
delete_config_if_match_with_opts(api, file, etag, ObjectOptions::default()).await
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_config_if_match_with_opts<S>(
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
etag: &str,
|
||||
mut options: ObjectOptions,
|
||||
) -> Result<()>
|
||||
where
|
||||
S: ObjectOperations<
|
||||
Error = Error,
|
||||
ObjectInfo = ObjectInfo,
|
||||
ObjectOptions = ObjectOptions,
|
||||
FileInfo = FileInfo,
|
||||
ObjectToDelete = ObjectToDelete,
|
||||
DeletedObject = DeletedObject,
|
||||
>,
|
||||
{
|
||||
options.http_preconditions = Some(HTTPPreconditions {
|
||||
if_match: Some(etag.to_string()),
|
||||
..Default::default()
|
||||
});
|
||||
match api.delete_object(RUSTFS_META_BUCKET, file, options).await {
|
||||
match api
|
||||
.delete_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
file,
|
||||
ObjectOptions {
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(etag.to_string()),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => Ok(()),
|
||||
Err(err) => {
|
||||
if err == Error::FileNotFound || matches!(err, Error::ObjectNotFound(_, _)) {
|
||||
|
||||
@@ -15,9 +15,9 @@
|
||||
use crate::object_api::ObjectInfo;
|
||||
|
||||
pub use rustfs_lifecycle::{
|
||||
Event, ExpirationOptions, IlmAction, LIFECYCLE_MALFORMED_XML_ERROR_KIND, Lifecycle, LifecycleCalculate, ObjectOpts,
|
||||
RuleValidate, TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, abort_incomplete_multipart_upload_due,
|
||||
expected_expiry_time, expiration_action_has_valid_target,
|
||||
Event, ExpirationOptions, IlmAction, Lifecycle, LifecycleCalculate, ObjectOpts, RuleValidate, TRANSITION_COMPLETE,
|
||||
TRANSITION_PENDING, TransitionOptions, abort_incomplete_multipart_upload_due, expected_expiry_time,
|
||||
expiration_action_has_valid_target,
|
||||
};
|
||||
|
||||
pub fn object_opts_from_object_info(oi: &ObjectInfo) -> ObjectOpts {
|
||||
|
||||
@@ -22,7 +22,7 @@ use super::{
|
||||
bucket_lifecycle_ops::{
|
||||
ManualTransitionQueueSnapshot, ManualTransitionRunReport, decode_manual_transition_continuation_token,
|
||||
},
|
||||
manual_transition_job, recovery_control, recovery_disposition, recovery_export, tier_delete_journal, transition_transaction,
|
||||
manual_transition_job, tier_delete_journal, transition_transaction,
|
||||
};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::services::tier::tier_probe_intent;
|
||||
@@ -41,9 +41,6 @@ pub(crate) enum DurableIlmRecordKind {
|
||||
ManualTransitionScope,
|
||||
ManualTransitionTask,
|
||||
ManualTransitionWorkerResult,
|
||||
RecoveryControl,
|
||||
RecoveryExport,
|
||||
RecoveryDisposition,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -108,26 +105,8 @@ pub(crate) const MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE: DurableIlmNamespace
|
||||
max_record_size: manual_transition_job::MAX_MANUAL_TRANSITION_WORKER_RESULT_RECORD_SIZE,
|
||||
kind: DurableIlmRecordKind::ManualTransitionWorkerResult,
|
||||
};
|
||||
pub(crate) const RECOVERY_CONTROL_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
|
||||
name: "recovery-control",
|
||||
prefix: recovery_control::ILM_RECOVERY_CONTROL_PREFIX,
|
||||
max_record_size: recovery_control::MAX_ILM_RECOVERY_CONTROL_SIZE,
|
||||
kind: DurableIlmRecordKind::RecoveryControl,
|
||||
};
|
||||
pub(crate) const RECOVERY_EXPORT_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
|
||||
name: "recovery-export",
|
||||
prefix: recovery_export::ILM_RECOVERY_EXPORT_PREFIX,
|
||||
max_record_size: recovery_export::MAX_ILM_RECOVERY_EXPORT_SIZE,
|
||||
kind: DurableIlmRecordKind::RecoveryExport,
|
||||
};
|
||||
pub(crate) const RECOVERY_DISPOSITION_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
|
||||
name: "recovery-disposition",
|
||||
prefix: recovery_disposition::ILM_RECOVERY_DISPOSITION_PREFIX,
|
||||
max_record_size: recovery_disposition::MAX_ILM_RECOVERY_DISPOSITION_SIZE,
|
||||
kind: DurableIlmRecordKind::RecoveryDisposition,
|
||||
};
|
||||
|
||||
pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 12] = [
|
||||
pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 9] = [
|
||||
TIER_DELETE_JOURNAL_NAMESPACE,
|
||||
TIER_DELETE_JOURNAL_V6_NAMESPACE,
|
||||
TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE,
|
||||
@@ -137,9 +116,6 @@ pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 12] = [
|
||||
MANUAL_TRANSITION_SCOPE_NAMESPACE,
|
||||
MANUAL_TRANSITION_TASK_NAMESPACE,
|
||||
MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE,
|
||||
RECOVERY_CONTROL_NAMESPACE,
|
||||
RECOVERY_EXPORT_NAMESPACE,
|
||||
RECOVERY_DISPOSITION_NAMESPACE,
|
||||
];
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
@@ -265,40 +241,6 @@ pub(crate) enum DurableIlmRecordCheckpoint {
|
||||
ManualTransitionWorkerResult {
|
||||
content_sha256: String,
|
||||
},
|
||||
RecoveryControl {
|
||||
content_sha256: String,
|
||||
identity_sha256: String,
|
||||
source_generation_sha256: String,
|
||||
first_seen_at_unix_nanos: i64,
|
||||
revision: u64,
|
||||
classification: recovery_control::IlmRecoveryClassification,
|
||||
attempt_count: u64,
|
||||
consecutive_failure_count: u32,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
owner_fence_sha256: Option<String>,
|
||||
},
|
||||
RecoveryExport {
|
||||
content_sha256: String,
|
||||
source_generation_sha256: String,
|
||||
topology_generation: String,
|
||||
member_epochs_sha256: String,
|
||||
creator_sha256: String,
|
||||
retain_until_unix_nanos: i64,
|
||||
},
|
||||
RecoveryDisposition {
|
||||
content_sha256: String,
|
||||
identity_sha256: String,
|
||||
copy_manifest_sha256: String,
|
||||
copy_manifest_count: usize,
|
||||
created_at_unix_nanos: i64,
|
||||
revision: u64,
|
||||
state: recovery_disposition::IlmRecoveryDispositionState,
|
||||
owner_fence_sha256: Option<String>,
|
||||
owner_lease_acquired_at_unix_nanos: Option<i64>,
|
||||
owner_lease_expires_at_unix_nanos: Option<i64>,
|
||||
confirmed_absent_sha256: Vec<String>,
|
||||
retain_until_unix_nanos: i64,
|
||||
},
|
||||
}
|
||||
|
||||
impl DurableIlmRecordCheckpoint {
|
||||
@@ -312,10 +254,7 @@ impl DurableIlmRecordCheckpoint {
|
||||
| Self::ManualTransitionJob { content_sha256, .. }
|
||||
| Self::ManualTransitionScope { content_sha256, .. }
|
||||
| Self::ManualTransitionTask { content_sha256 }
|
||||
| Self::ManualTransitionWorkerResult { content_sha256 }
|
||||
| Self::RecoveryControl { content_sha256, .. }
|
||||
| Self::RecoveryExport { content_sha256, .. }
|
||||
| Self::RecoveryDisposition { content_sha256, .. } => content_sha256,
|
||||
| Self::ManualTransitionWorkerResult { content_sha256 } => content_sha256,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -356,9 +295,6 @@ impl DurableIlmRecordCheckpoint {
|
||||
{
|
||||
return Err(Error::other("durable ILM tier delete journal checkpoint is invalid"));
|
||||
}
|
||||
if !recovery_disposition_checkpoint_is_valid(checkpoint) {
|
||||
return Err(Error::other("durable ILM recovery disposition checkpoint is invalid"));
|
||||
}
|
||||
}
|
||||
if self == next {
|
||||
if let Self::ManualTransitionJob {
|
||||
@@ -499,7 +435,9 @@ impl DurableIlmRecordCheckpoint {
|
||||
},
|
||||
) => {
|
||||
previous_identity == next_identity
|
||||
&& transition_state_revision_is_successor(*previous_state, *previous_revision, *next_state, *next_revision)
|
||||
&& transition_state_distance(*previous_state, *next_state)
|
||||
.and_then(|distance| previous_revision.checked_add(distance))
|
||||
.is_some_and(|expected_revision| *next_revision == expected_revision)
|
||||
&& (!previous_remote_version_known || previous_remote_version == next_remote_version)
|
||||
}
|
||||
(
|
||||
@@ -590,128 +528,6 @@ impl DurableIlmRecordCheckpoint {
|
||||
..
|
||||
},
|
||||
) => previous_identity == next_identity && next_updated_at > previous_updated_at,
|
||||
(
|
||||
Self::RecoveryControl {
|
||||
identity_sha256: previous_identity,
|
||||
source_generation_sha256: previous_generation,
|
||||
first_seen_at_unix_nanos: previous_first_seen,
|
||||
revision: previous_revision,
|
||||
classification: previous_classification,
|
||||
attempt_count: previous_attempts,
|
||||
consecutive_failure_count: previous_failures,
|
||||
owner_fence_sha256: previous_owner,
|
||||
..
|
||||
},
|
||||
Self::RecoveryControl {
|
||||
identity_sha256: next_identity,
|
||||
source_generation_sha256: next_generation,
|
||||
first_seen_at_unix_nanos: next_first_seen,
|
||||
revision: next_revision,
|
||||
classification: next_classification,
|
||||
attempt_count: next_attempts,
|
||||
consecutive_failure_count: next_failures,
|
||||
owner_fence_sha256: next_owner,
|
||||
..
|
||||
},
|
||||
) => {
|
||||
let adjacent = previous_identity == next_identity
|
||||
&& previous_first_seen == next_first_seen
|
||||
&& previous_revision.checked_add(1) == Some(*next_revision);
|
||||
let claim = next_owner.is_some()
|
||||
&& *previous_classification == recovery_control::IlmRecoveryClassification::Retrying
|
||||
&& *next_classification == recovery_control::IlmRecoveryClassification::Retrying
|
||||
&& previous_attempts.checked_add(1) == Some(*next_attempts)
|
||||
&& previous_failures == next_failures;
|
||||
let source_refresh = previous_owner.is_some()
|
||||
&& previous_owner == next_owner
|
||||
&& *previous_classification == recovery_control::IlmRecoveryClassification::Retrying
|
||||
&& *next_classification == recovery_control::IlmRecoveryClassification::Retrying
|
||||
&& previous_attempts == next_attempts
|
||||
&& previous_failures == next_failures
|
||||
&& previous_generation != next_generation;
|
||||
let completion = previous_owner.is_some()
|
||||
&& next_owner.is_none()
|
||||
&& previous_generation == next_generation
|
||||
&& previous_attempts == next_attempts;
|
||||
adjacent && (claim || source_refresh || completion)
|
||||
}
|
||||
(
|
||||
Self::RecoveryDisposition {
|
||||
identity_sha256: previous_identity,
|
||||
copy_manifest_sha256: previous_manifest,
|
||||
copy_manifest_count: previous_manifest_count,
|
||||
created_at_unix_nanos: previous_created_at,
|
||||
revision: previous_revision,
|
||||
state: previous_state,
|
||||
owner_fence_sha256: previous_owner,
|
||||
owner_lease_acquired_at_unix_nanos: previous_owner_acquired,
|
||||
owner_lease_expires_at_unix_nanos: previous_owner_expires,
|
||||
confirmed_absent_sha256: previous_confirmed,
|
||||
retain_until_unix_nanos: previous_retain_until,
|
||||
..
|
||||
},
|
||||
Self::RecoveryDisposition {
|
||||
identity_sha256: next_identity,
|
||||
copy_manifest_sha256: next_manifest,
|
||||
copy_manifest_count: next_manifest_count,
|
||||
created_at_unix_nanos: next_created_at,
|
||||
revision: next_revision,
|
||||
state: next_state,
|
||||
owner_fence_sha256: next_owner,
|
||||
owner_lease_acquired_at_unix_nanos: next_owner_acquired,
|
||||
owner_lease_expires_at_unix_nanos: next_owner_expires,
|
||||
confirmed_absent_sha256: next_confirmed,
|
||||
retain_until_unix_nanos: next_retain_until,
|
||||
..
|
||||
},
|
||||
) => {
|
||||
use recovery_disposition::IlmRecoveryDispositionState::{Applying, Completed, Prepared};
|
||||
|
||||
let immutable_identity_matches = previous_identity == next_identity
|
||||
&& previous_manifest == next_manifest
|
||||
&& previous_manifest_count == next_manifest_count
|
||||
&& previous_created_at == next_created_at
|
||||
&& previous_retain_until == next_retain_until;
|
||||
let adjacent = previous_revision.checked_add(1) == Some(*next_revision);
|
||||
let progress_is_monotonic = sorted_sha256_set_is_subset(previous_confirmed, next_confirmed);
|
||||
let legal_edge = match (previous_state, next_state) {
|
||||
(Prepared, Prepared) => {
|
||||
let claim = previous_owner.is_none() && next_owner.is_some();
|
||||
let takeover = previous_owner.is_some()
|
||||
&& previous_owner != next_owner
|
||||
&& previous_owner_expires
|
||||
.zip(*next_owner_acquired)
|
||||
.is_some_and(|(expires, acquired)| acquired >= expires);
|
||||
previous_confirmed == next_confirmed && (claim || takeover)
|
||||
}
|
||||
(Prepared, Applying) => {
|
||||
previous_confirmed == next_confirmed
|
||||
&& previous_owner.is_some()
|
||||
&& previous_owner == next_owner
|
||||
&& previous_owner_acquired == next_owner_acquired
|
||||
&& previous_owner_expires == next_owner_expires
|
||||
}
|
||||
(Applying, Applying) => {
|
||||
let progress = previous_owner == next_owner
|
||||
&& previous_owner_acquired == next_owner_acquired
|
||||
&& previous_owner_expires == next_owner_expires
|
||||
&& previous_confirmed.len().checked_add(1) == Some(next_confirmed.len());
|
||||
let takeover = previous_owner.is_some()
|
||||
&& previous_owner != next_owner
|
||||
&& previous_confirmed == next_confirmed
|
||||
&& previous_owner_expires
|
||||
.zip(*next_owner_acquired)
|
||||
.is_some_and(|(expires, acquired)| acquired >= expires);
|
||||
progress || takeover
|
||||
}
|
||||
(Applying, Completed) => {
|
||||
previous_owner.is_some() && next_owner.is_none() && previous_confirmed == next_confirmed
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
|
||||
immutable_identity_matches && adjacent && progress_is_monotonic && legal_edge
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
|
||||
@@ -729,11 +545,6 @@ impl DurableIlmRecordCheckpoint {
|
||||
/// after the exact terminal ETag and terminal receipt were committed, to
|
||||
/// purge older object versions exposed by that deletion.
|
||||
pub(crate) fn is_predecessor_of_terminal(&self, terminal: &Self) -> bool {
|
||||
for checkpoint in [self, terminal] {
|
||||
if !recovery_disposition_checkpoint_is_valid(checkpoint) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if let Self::TierProbeIntent { state, .. } = terminal
|
||||
&& !matches!(
|
||||
state,
|
||||
@@ -742,19 +553,6 @@ impl DurableIlmRecordCheckpoint {
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if let Self::RecoveryControl { classification, .. } = terminal
|
||||
&& !matches!(
|
||||
classification,
|
||||
recovery_control::IlmRecoveryClassification::Terminal | recovery_control::IlmRecoveryClassification::Abandoned
|
||||
)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if let Self::RecoveryDisposition { state, .. } = terminal
|
||||
&& state != &recovery_disposition::IlmRecoveryDispositionState::Completed
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if self == terminal || self.validate_successor(terminal).is_ok() {
|
||||
return true;
|
||||
}
|
||||
@@ -854,147 +652,11 @@ impl DurableIlmRecordCheckpoint {
|
||||
.is_some_and(|distance| tier_probe_state_reaches(*previous_state, *terminal_state, distance))
|
||||
&& (!previous_remote_version_known || previous_remote_version == terminal_remote_version)
|
||||
}
|
||||
(
|
||||
Self::RecoveryControl {
|
||||
identity_sha256: previous_identity,
|
||||
source_generation_sha256: previous_generation,
|
||||
first_seen_at_unix_nanos: previous_first_seen,
|
||||
revision: previous_revision,
|
||||
attempt_count: previous_attempts,
|
||||
..
|
||||
},
|
||||
Self::RecoveryControl {
|
||||
identity_sha256: terminal_identity,
|
||||
source_generation_sha256: terminal_generation,
|
||||
first_seen_at_unix_nanos: terminal_first_seen,
|
||||
revision: terminal_revision,
|
||||
attempt_count: terminal_attempts,
|
||||
classification:
|
||||
recovery_control::IlmRecoveryClassification::Terminal | recovery_control::IlmRecoveryClassification::Abandoned,
|
||||
..
|
||||
},
|
||||
) => {
|
||||
previous_identity == terminal_identity
|
||||
&& (previous_generation == terminal_generation || terminal_attempts > previous_attempts)
|
||||
&& previous_first_seen == terminal_first_seen
|
||||
&& terminal_revision > previous_revision
|
||||
&& terminal_attempts >= previous_attempts
|
||||
}
|
||||
(
|
||||
Self::RecoveryDisposition {
|
||||
identity_sha256: previous_identity,
|
||||
copy_manifest_sha256: previous_manifest,
|
||||
copy_manifest_count: previous_manifest_count,
|
||||
created_at_unix_nanos: previous_created_at,
|
||||
revision: previous_revision,
|
||||
state: previous_state,
|
||||
owner_fence_sha256: previous_owner,
|
||||
confirmed_absent_sha256: previous_confirmed,
|
||||
retain_until_unix_nanos: previous_retain_until,
|
||||
..
|
||||
},
|
||||
Self::RecoveryDisposition {
|
||||
identity_sha256: terminal_identity,
|
||||
copy_manifest_sha256: terminal_manifest,
|
||||
copy_manifest_count: terminal_manifest_count,
|
||||
created_at_unix_nanos: terminal_created_at,
|
||||
revision: terminal_revision,
|
||||
state: recovery_disposition::IlmRecoveryDispositionState::Completed,
|
||||
confirmed_absent_sha256: terminal_confirmed,
|
||||
retain_until_unix_nanos: terminal_retain_until,
|
||||
..
|
||||
},
|
||||
) => {
|
||||
matches!(
|
||||
previous_state,
|
||||
recovery_disposition::IlmRecoveryDispositionState::Prepared
|
||||
| recovery_disposition::IlmRecoveryDispositionState::Applying
|
||||
) && previous_identity == terminal_identity
|
||||
&& previous_manifest == terminal_manifest
|
||||
&& previous_manifest_count == terminal_manifest_count
|
||||
&& previous_created_at == terminal_created_at
|
||||
&& previous_retain_until == terminal_retain_until
|
||||
&& terminal_revision.checked_sub(*previous_revision).is_some_and(|distance| {
|
||||
let minimum_distance = match previous_state {
|
||||
recovery_disposition::IlmRecoveryDispositionState::Prepared if previous_owner.is_some() => 3,
|
||||
recovery_disposition::IlmRecoveryDispositionState::Prepared => 4,
|
||||
recovery_disposition::IlmRecoveryDispositionState::Applying
|
||||
if previous_confirmed.len() == *previous_manifest_count =>
|
||||
{
|
||||
1
|
||||
}
|
||||
recovery_disposition::IlmRecoveryDispositionState::Applying => 2,
|
||||
recovery_disposition::IlmRecoveryDispositionState::Completed => u64::MAX,
|
||||
};
|
||||
distance >= minimum_distance
|
||||
})
|
||||
&& sorted_sha256_set_is_subset(previous_confirmed, terminal_confirmed)
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn recovery_disposition_checkpoint_is_valid(checkpoint: &DurableIlmRecordCheckpoint) -> bool {
|
||||
use recovery_disposition::IlmRecoveryDispositionState::{Applying, Completed, Prepared};
|
||||
|
||||
let DurableIlmRecordCheckpoint::RecoveryDisposition {
|
||||
content_sha256,
|
||||
identity_sha256,
|
||||
copy_manifest_sha256,
|
||||
copy_manifest_count,
|
||||
created_at_unix_nanos,
|
||||
revision,
|
||||
state,
|
||||
owner_fence_sha256,
|
||||
owner_lease_acquired_at_unix_nanos,
|
||||
owner_lease_expires_at_unix_nanos,
|
||||
confirmed_absent_sha256,
|
||||
retain_until_unix_nanos,
|
||||
} = checkpoint
|
||||
else {
|
||||
return true;
|
||||
};
|
||||
let owner_fence_sha256 = owner_fence_sha256.as_deref();
|
||||
|
||||
is_canonical_sha256(content_sha256)
|
||||
&& is_canonical_sha256(identity_sha256)
|
||||
&& is_canonical_sha256(copy_manifest_sha256)
|
||||
&& *copy_manifest_count > 0
|
||||
&& *created_at_unix_nanos > 0
|
||||
&& *revision > 0
|
||||
&& *retain_until_unix_nanos > 0
|
||||
&& owner_fence_sha256.is_none_or(is_canonical_sha256)
|
||||
&& match (
|
||||
owner_fence_sha256,
|
||||
*owner_lease_acquired_at_unix_nanos,
|
||||
*owner_lease_expires_at_unix_nanos,
|
||||
) {
|
||||
(None, None, None) => true,
|
||||
(Some(_), Some(acquired), Some(expires)) => acquired > 0 && expires > acquired,
|
||||
_ => false,
|
||||
}
|
||||
&& confirmed_absent_sha256.len() <= *copy_manifest_count
|
||||
&& confirmed_absent_sha256.iter().all(|digest| is_canonical_sha256(digest))
|
||||
&& confirmed_absent_sha256.windows(2).all(|pair| pair[0] < pair[1])
|
||||
&& match *state {
|
||||
Prepared => confirmed_absent_sha256.is_empty(),
|
||||
Applying => owner_fence_sha256.is_some(),
|
||||
Completed => owner_fence_sha256.is_none() && confirmed_absent_sha256.len() == *copy_manifest_count,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_canonical_sha256(value: &str) -> bool {
|
||||
is_sha256_checksum(value)
|
||||
&& !value
|
||||
.bytes()
|
||||
.any(|byte| byte.is_ascii_hexdigit() && byte.is_ascii_uppercase())
|
||||
}
|
||||
|
||||
fn sorted_sha256_set_is_subset(subset: &[String], superset: &[String]) -> bool {
|
||||
subset.iter().all(|candidate| superset.binary_search(candidate).is_ok())
|
||||
}
|
||||
|
||||
fn tier_delete_dispatch_parent_progress_delta(
|
||||
previous_sequence: u64,
|
||||
previous_completed_journals: u64,
|
||||
@@ -1028,22 +690,6 @@ fn transition_state_distance(
|
||||
}
|
||||
}
|
||||
|
||||
fn transition_state_revision_is_successor(
|
||||
from: transition_transaction::TransitionTransactionState,
|
||||
from_revision: u64,
|
||||
to: transition_transaction::TransitionTransactionState,
|
||||
to_revision: u64,
|
||||
) -> bool {
|
||||
use transition_transaction::TransitionTransactionState::{LocalCommitStarted, UploadOutcomeUnknown};
|
||||
|
||||
if from == UploadOutcomeUnknown && from_revision == 1 && to == LocalCommitStarted {
|
||||
return to_revision == 2;
|
||||
}
|
||||
transition_state_distance(from, to)
|
||||
.and_then(|distance| from_revision.checked_add(distance))
|
||||
.is_some_and(|expected_revision| to_revision == expected_revision)
|
||||
}
|
||||
|
||||
fn tier_probe_state_reaches(
|
||||
from: tier_probe_intent::TierProbeIntentState,
|
||||
to: tier_probe_intent::TierProbeIntentState,
|
||||
@@ -1573,84 +1219,6 @@ pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result<Val
|
||||
},
|
||||
)
|
||||
}
|
||||
DurableIlmRecordKind::RecoveryControl => {
|
||||
let (protocol, control_id) = recovery_control::recovery_control_id_from_record_object_name(path)
|
||||
.map_err(|err| Error::other(err.to_string()))?;
|
||||
let control =
|
||||
recovery_control::IlmRecoveryControl::decode(&control_id, data).map_err(|err| Error::other(err.to_string()))?;
|
||||
let canonical = recovery_control::recovery_control_record_object_name(protocol, &control_id)
|
||||
.map_err(|err| Error::other(err.to_string()))?;
|
||||
if canonical != path || control.identity.protocol != protocol {
|
||||
return Err(Error::other("ILM recovery control path is not canonical"));
|
||||
}
|
||||
let identity_sha256 = checkpoint_hash(&control.identity)?;
|
||||
let source_generation_sha256 = checkpoint_hash(&control.observed_source_generation)?;
|
||||
let owner_fence_sha256 = control.owner.as_ref().map(checkpoint_hash).transpose()?;
|
||||
(
|
||||
"control_id",
|
||||
control_id,
|
||||
DurableIlmRecordCheckpoint::RecoveryControl {
|
||||
content_sha256,
|
||||
identity_sha256,
|
||||
source_generation_sha256,
|
||||
first_seen_at_unix_nanos: control.first_seen_at_unix_nanos,
|
||||
revision: control.revision,
|
||||
classification: control.classification,
|
||||
attempt_count: control.attempt_count,
|
||||
consecutive_failure_count: control.consecutive_failure_count,
|
||||
owner_fence_sha256,
|
||||
},
|
||||
)
|
||||
}
|
||||
DurableIlmRecordKind::RecoveryExport => {
|
||||
let (protocol, export_id) = recovery_export::recovery_export_id_from_record_object_name(path)?;
|
||||
let export = recovery_export::IlmRecoveryExport::decode(&export_id, data)?;
|
||||
let canonical = recovery_export::recovery_export_record_object_name(protocol, &export_id)?;
|
||||
if canonical != path || export.protocol != protocol {
|
||||
return Err(Error::other("ILM recovery export path is not canonical"));
|
||||
}
|
||||
let source_generation_sha256 = checkpoint_hash(&export.source_generation)?;
|
||||
(
|
||||
"export_id",
|
||||
export_id,
|
||||
DurableIlmRecordCheckpoint::RecoveryExport {
|
||||
content_sha256,
|
||||
source_generation_sha256,
|
||||
topology_generation: export.topology_generation,
|
||||
member_epochs_sha256: export.member_epochs_sha256,
|
||||
creator_sha256: export.creator_sha256,
|
||||
retain_until_unix_nanos: export.retain_until_unix_nanos,
|
||||
},
|
||||
)
|
||||
}
|
||||
DurableIlmRecordKind::RecoveryDisposition => {
|
||||
// The disposition module owns strict schema, checksum, canonical
|
||||
// path, immutable-manifest, and state-specific validation. Keep
|
||||
// this boundary limited to decommission identity/checkpoint
|
||||
// projection so the two readers cannot accept different records.
|
||||
let disposition = recovery_disposition::decode_recovery_disposition_checkpoint(path, data)?;
|
||||
if disposition.content_sha256 != content_sha256 {
|
||||
return Err(Error::other("ILM recovery disposition checkpoint content digest is invalid"));
|
||||
}
|
||||
(
|
||||
"disposition_id",
|
||||
disposition.disposition_id,
|
||||
DurableIlmRecordCheckpoint::RecoveryDisposition {
|
||||
content_sha256: disposition.content_sha256,
|
||||
identity_sha256: disposition.identity_sha256,
|
||||
copy_manifest_sha256: disposition.copy_manifest_sha256,
|
||||
copy_manifest_count: disposition.copy_manifest_count,
|
||||
created_at_unix_nanos: disposition.created_at_unix_nanos,
|
||||
revision: disposition.revision,
|
||||
state: disposition.state,
|
||||
owner_fence_sha256: disposition.owner_fence_sha256,
|
||||
owner_lease_acquired_at_unix_nanos: disposition.owner_lease_acquired_at_unix_nanos,
|
||||
owner_lease_expires_at_unix_nanos: disposition.owner_lease_expires_at_unix_nanos,
|
||||
confirmed_absent_sha256: disposition.confirmed_absent_sha256,
|
||||
retain_until_unix_nanos: disposition.retain_until_unix_nanos,
|
||||
},
|
||||
)
|
||||
}
|
||||
DurableIlmRecordKind::ManualTransitionJob => {
|
||||
let job_id = manual_transition_job::manual_transition_job_id_from_record_object_name(path)
|
||||
.map_err(|err| Error::other(err.to_string()))?;
|
||||
@@ -1806,203 +1374,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn recovery_disposition_checkpoint(
|
||||
revision: u64,
|
||||
state: recovery_disposition::IlmRecoveryDispositionState,
|
||||
owner_fence: Option<&str>,
|
||||
confirmed_absent_sha256: Vec<String>,
|
||||
) -> DurableIlmRecordCheckpoint {
|
||||
let (owner_lease_acquired_at_unix_nanos, owner_lease_expires_at_unix_nanos) = match owner_fence {
|
||||
Some("f") => (Some(10), Some(20)),
|
||||
Some(_) => (Some(1), Some(10)),
|
||||
None => (None, None),
|
||||
};
|
||||
DurableIlmRecordCheckpoint::RecoveryDisposition {
|
||||
content_sha256: format!("{revision:064x}"),
|
||||
identity_sha256: "a".repeat(64),
|
||||
copy_manifest_sha256: "d".repeat(64),
|
||||
copy_manifest_count: 2,
|
||||
created_at_unix_nanos: 1_700_000_000_000_000_000,
|
||||
revision,
|
||||
state,
|
||||
owner_fence_sha256: owner_fence.map(|digest| digest.repeat(64)),
|
||||
owner_lease_acquired_at_unix_nanos,
|
||||
owner_lease_expires_at_unix_nanos,
|
||||
confirmed_absent_sha256,
|
||||
retain_until_unix_nanos: 1_820_000_000_000_000_000,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recovery_disposition_namespace_is_registered_without_shadowing_its_root() {
|
||||
let disposition_id = "a".repeat(64);
|
||||
let path = format!(
|
||||
"{}/tier_delete_journal/{}/{}/{}.json",
|
||||
recovery_disposition::ILM_RECOVERY_DISPOSITION_PREFIX,
|
||||
&disposition_id[..2],
|
||||
&disposition_id[2..4],
|
||||
disposition_id
|
||||
);
|
||||
let namespace = classify_durable_ilm_record(&path)
|
||||
.expect("recovery disposition path should classify")
|
||||
.expect("recovery disposition should be durable");
|
||||
|
||||
assert_eq!(namespace, &RECOVERY_DISPOSITION_NAMESPACE);
|
||||
assert!(classify_durable_ilm_record(recovery_disposition::ILM_RECOVERY_DISPOSITION_PREFIX).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recovery_disposition_checkpoint_accepts_only_monotonic_progress() {
|
||||
use recovery_disposition::IlmRecoveryDispositionState::{Applying, Completed, Prepared};
|
||||
|
||||
let first_copy = "b".repeat(64);
|
||||
let second_copy = "c".repeat(64);
|
||||
let prepared = recovery_disposition_checkpoint(1, Prepared, None, Vec::new());
|
||||
let claimed = recovery_disposition_checkpoint(2, Prepared, Some("e"), Vec::new());
|
||||
let applying = recovery_disposition_checkpoint(3, Applying, Some("e"), Vec::new());
|
||||
let first_absent = recovery_disposition_checkpoint(4, Applying, Some("e"), vec![first_copy.clone()]);
|
||||
let taken_over = recovery_disposition_checkpoint(5, Applying, Some("f"), vec![first_copy.clone()]);
|
||||
let all_absent = recovery_disposition_checkpoint(6, Applying, Some("f"), vec![first_copy.clone(), second_copy.clone()]);
|
||||
let completed = recovery_disposition_checkpoint(7, Completed, None, vec![first_copy.clone(), second_copy.clone()]);
|
||||
|
||||
prepared
|
||||
.validate_successor(&claimed)
|
||||
.expect("Prepared should record an owner claim without absence progress");
|
||||
claimed
|
||||
.validate_successor(&applying)
|
||||
.expect("Prepared should advance to Applying without folding in deletion progress");
|
||||
applying
|
||||
.validate_successor(&first_absent)
|
||||
.expect("Applying should append newly confirmed absent copies");
|
||||
first_absent
|
||||
.validate_successor(&taken_over)
|
||||
.expect("Applying should record a fenced owner takeover without losing progress");
|
||||
taken_over
|
||||
.validate_successor(&all_absent)
|
||||
.expect("Applying should preserve every earlier confirmation while making progress");
|
||||
all_absent
|
||||
.validate_successor(&completed)
|
||||
.expect("a fully confirmed manifest should advance to Completed");
|
||||
|
||||
assert!(
|
||||
prepared.validate_successor(&completed).is_err(),
|
||||
"adjacent receipt updates must not skip Applying"
|
||||
);
|
||||
assert!(
|
||||
first_absent
|
||||
.validate_successor(&recovery_disposition_checkpoint(5, Applying, Some("e"), Vec::new()))
|
||||
.is_err(),
|
||||
"confirmed-absent progress must not move backwards"
|
||||
);
|
||||
assert!(
|
||||
applying
|
||||
.validate_successor(&recovery_disposition_checkpoint(4, Completed, None, vec![first_copy.clone()]))
|
||||
.is_err(),
|
||||
"Completed must cover the complete immutable copy manifest"
|
||||
);
|
||||
assert!(
|
||||
completed
|
||||
.validate_successor(&recovery_disposition_checkpoint(7, Applying, Some("e"), vec![second_copy]))
|
||||
.is_err(),
|
||||
"Completed is terminal"
|
||||
);
|
||||
assert!(
|
||||
first_absent
|
||||
.validate_successor(&recovery_disposition_checkpoint(5, Applying, Some("e"), vec![first_copy.clone()]))
|
||||
.is_err(),
|
||||
"a same-state revision bump must change the owner fence or absence progress"
|
||||
);
|
||||
assert!(
|
||||
applying
|
||||
.validate_successor(&recovery_disposition_checkpoint(4, Applying, None, vec![first_copy]))
|
||||
.is_err(),
|
||||
"Applying must retain a fenced owner"
|
||||
);
|
||||
|
||||
let mut noncanonical_identity = claimed.clone();
|
||||
if let DurableIlmRecordCheckpoint::RecoveryDisposition { identity_sha256, .. } = &mut noncanonical_identity {
|
||||
*identity_sha256 = "A".repeat(64);
|
||||
}
|
||||
assert!(prepared.validate_successor(&noncanonical_identity).is_err());
|
||||
let mut changed_created_at = claimed;
|
||||
if let DurableIlmRecordCheckpoint::RecoveryDisposition {
|
||||
created_at_unix_nanos, ..
|
||||
} = &mut changed_created_at
|
||||
{
|
||||
*created_at_unix_nanos += 1;
|
||||
}
|
||||
assert!(prepared.validate_successor(&changed_created_at).is_err());
|
||||
|
||||
let mut early_takeover = taken_over;
|
||||
if let DurableIlmRecordCheckpoint::RecoveryDisposition {
|
||||
owner_lease_acquired_at_unix_nanos,
|
||||
..
|
||||
} = &mut early_takeover
|
||||
{
|
||||
*owner_lease_acquired_at_unix_nanos = Some(9);
|
||||
}
|
||||
assert!(first_absent.validate_successor(&early_takeover).is_err());
|
||||
assert!(
|
||||
applying
|
||||
.validate_successor(&recovery_disposition_checkpoint(
|
||||
4,
|
||||
Applying,
|
||||
Some("e"),
|
||||
vec!["c".repeat(64), "b".repeat(64)],
|
||||
))
|
||||
.is_err(),
|
||||
"confirmed-absent entries must be a canonical sorted set"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recovery_disposition_terminal_predecessor_requires_exact_identity_and_full_manifest() {
|
||||
use recovery_disposition::IlmRecoveryDispositionState::{Applying, Completed, Prepared};
|
||||
|
||||
let first_copy = "b".repeat(64);
|
||||
let second_copy = "c".repeat(64);
|
||||
let prepared = recovery_disposition_checkpoint(1, Prepared, None, Vec::new());
|
||||
let applying = recovery_disposition_checkpoint(3, Applying, Some("e"), vec![first_copy.clone()]);
|
||||
let completed = recovery_disposition_checkpoint(5, Completed, None, vec![first_copy.clone(), second_copy]);
|
||||
|
||||
assert!(prepared.is_predecessor_of_terminal(&completed));
|
||||
assert!(applying.is_predecessor_of_terminal(&completed));
|
||||
assert!(
|
||||
!prepared.is_predecessor_of_terminal(&recovery_disposition_checkpoint(2, Applying, Some("e"), Vec::new())),
|
||||
"a nonterminal disposition must not authorize terminal cleanup"
|
||||
);
|
||||
assert!(
|
||||
!prepared.is_predecessor_of_terminal(&recovery_disposition_checkpoint(
|
||||
4,
|
||||
Completed,
|
||||
None,
|
||||
vec![first_copy.clone(), "c".repeat(64)],
|
||||
)),
|
||||
"terminal proof must leave enough revisions for claim, apply, progress, and completion"
|
||||
);
|
||||
assert!(
|
||||
!applying.is_predecessor_of_terminal(&recovery_disposition_checkpoint(
|
||||
4,
|
||||
Completed,
|
||||
None,
|
||||
vec![first_copy.clone(), "c".repeat(64)],
|
||||
)),
|
||||
"an incomplete Applying checkpoint cannot complete without a progress generation"
|
||||
);
|
||||
|
||||
let mut other_identity = completed;
|
||||
if let DurableIlmRecordCheckpoint::RecoveryDisposition { identity_sha256, .. } = &mut other_identity {
|
||||
*identity_sha256 = "e".repeat(64);
|
||||
}
|
||||
assert!(!prepared.is_predecessor_of_terminal(&other_identity));
|
||||
|
||||
let incomplete_terminal = recovery_disposition_checkpoint(4, Completed, None, vec![first_copy]);
|
||||
assert!(
|
||||
!prepared.is_predecessor_of_terminal(&incomplete_terminal),
|
||||
"a partial confirmed-absent set must not become terminal proof"
|
||||
);
|
||||
}
|
||||
|
||||
fn tier_probe_intent_fixture() -> tier_probe_intent::TierProbeIntent {
|
||||
let probe_id = Uuid::parse_str("36e2220e-9ad2-495b-b3bc-c4d2caf70a31").expect("fixture uuid should parse");
|
||||
tier_probe_intent::TierProbeIntent {
|
||||
@@ -2041,94 +1412,6 @@ mod tests {
|
||||
.checkpoint
|
||||
}
|
||||
|
||||
fn recovery_control_fixture() -> recovery_control::IlmRecoveryControl {
|
||||
let source_path = "ilm/transition-transactions/records/12/34/1234567890abcdef1234567890abcdef.json";
|
||||
let generation = recovery_control::IlmRecoverySourceGeneration::new(
|
||||
transition_transaction::TRANSITION_TRANSACTION_SCHEMA,
|
||||
"source-etag",
|
||||
"a".repeat(64),
|
||||
vec![recovery_control::IlmRecoverySourceCopy {
|
||||
authority: "pool-0/set-0".to_string(),
|
||||
canonical_path: source_path.to_string(),
|
||||
etag: "source-etag".to_string(),
|
||||
encoded_len: 128,
|
||||
content_sha256: "a".repeat(64),
|
||||
}],
|
||||
)
|
||||
.expect("source generation should build");
|
||||
recovery_control::IlmRecoveryControl::new(
|
||||
recovery_control::IlmRecoveryControlIdentity {
|
||||
protocol: recovery_control::IlmRecoveryProtocol::TransitionTransaction,
|
||||
canonical_source_path: source_path.to_string(),
|
||||
stable_operation_identity: "12345678-90ab-cdef-1234-567890abcdef".to_string(),
|
||||
record_class: "transition_transaction_v1".to_string(),
|
||||
},
|
||||
generation,
|
||||
recovery_control::IlmRecoveryClassification::Retrying,
|
||||
1_000_000_000,
|
||||
recovery_control::IlmRecoveryErrorCode::None,
|
||||
)
|
||||
.expect("recovery control should build")
|
||||
}
|
||||
|
||||
fn recovery_control_checkpoint(control: &recovery_control::IlmRecoveryControl) -> DurableIlmRecordCheckpoint {
|
||||
let control_id = control.identity.source_operation_digest().expect("control id should derive");
|
||||
let path = recovery_control::recovery_control_record_object_name(control.identity.protocol, &control_id)
|
||||
.expect("control path should build");
|
||||
let encoded = control.encode().expect("control should encode");
|
||||
let namespace = classify_durable_ilm_record(&path)
|
||||
.expect("recovery control namespace should classify")
|
||||
.expect("recovery control should be durable");
|
||||
assert_eq!(namespace, &RECOVERY_CONTROL_NAMESPACE);
|
||||
validate_durable_ilm_record(&path, &encoded)
|
||||
.expect("recovery control should validate")
|
||||
.checkpoint
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recovery_control_checkpoint_tracks_claim_retry_and_terminal_generations() {
|
||||
let initial_control = recovery_control_fixture();
|
||||
let initial = recovery_control_checkpoint(&initial_control);
|
||||
|
||||
let mut claimed_control = initial_control;
|
||||
let mut advanced_generation = claimed_control.observed_source_generation.clone();
|
||||
advanced_generation.source_schema = "rustfs-transition-transaction-v2".to_string();
|
||||
claimed_control
|
||||
.claim_for_source_generation("node-a", Uuid::new_v4(), 2_000_000_000, 300_000_000_000, advanced_generation)
|
||||
.expect("control should claim");
|
||||
let claimed = recovery_control_checkpoint(&claimed_control);
|
||||
initial.validate_successor(&claimed).expect("claim should advance receipt");
|
||||
|
||||
let mut retry_control = claimed_control;
|
||||
retry_control
|
||||
.record_retryable_failure(3_000_000_000, recovery_control::IlmRecoveryErrorCode::BackendTimeout)
|
||||
.expect("retry should persist");
|
||||
let retry = recovery_control_checkpoint(&retry_control);
|
||||
claimed.validate_successor(&retry).expect("retry should advance receipt");
|
||||
|
||||
let ready_at = retry_control
|
||||
.next_attempt_at_unix_nanos
|
||||
.expect("retry deadline should persist");
|
||||
let mut terminal_control = retry_control;
|
||||
terminal_control
|
||||
.claim("node-b", Uuid::new_v4(), ready_at, 300_000_000_000)
|
||||
.expect("retry should claim");
|
||||
let reclaimed = recovery_control_checkpoint(&terminal_control);
|
||||
retry.validate_successor(&reclaimed).expect("reclaim should advance receipt");
|
||||
terminal_control
|
||||
.finish_attempt(
|
||||
recovery_control::IlmRecoveryClassification::Terminal,
|
||||
recovery_control::IlmRecoveryErrorCode::None,
|
||||
)
|
||||
.expect("control should terminate");
|
||||
let terminal = recovery_control_checkpoint(&terminal_control);
|
||||
reclaimed
|
||||
.validate_successor(&terminal)
|
||||
.expect("terminal state should advance receipt");
|
||||
assert!(initial.is_predecessor_of_terminal(&terminal));
|
||||
assert!(!initial.is_predecessor_of_terminal(&retry));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_probe_intent_checkpoint_tracks_exact_monotonic_generations() {
|
||||
let initial_intent = tier_probe_intent_fixture();
|
||||
@@ -2187,64 +1470,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_checkpoint_accepts_only_the_distinguishable_compact_edge() {
|
||||
let identity_sha256 = "a".repeat(64);
|
||||
let unknown_remote_sha256 = "b".repeat(64);
|
||||
let known_remote_sha256 = "c".repeat(64);
|
||||
let checkpoint = |revision, state, remote_version_sha256: String, remote_version_known| {
|
||||
DurableIlmRecordCheckpoint::TransitionTransaction {
|
||||
content_sha256: format!("{revision:064x}"),
|
||||
identity_sha256: identity_sha256.clone(),
|
||||
remote_version_sha256,
|
||||
remote_version_known,
|
||||
revision,
|
||||
state,
|
||||
}
|
||||
};
|
||||
let compact_unknown = checkpoint(
|
||||
1,
|
||||
transition_transaction::TransitionTransactionState::UploadOutcomeUnknown,
|
||||
unknown_remote_sha256.clone(),
|
||||
false,
|
||||
);
|
||||
let compact_local_commit = checkpoint(
|
||||
2,
|
||||
transition_transaction::TransitionTransactionState::LocalCommitStarted,
|
||||
known_remote_sha256.clone(),
|
||||
true,
|
||||
);
|
||||
compact_unknown
|
||||
.validate_successor(&compact_local_commit)
|
||||
.expect("compact pre-upload fence should advance directly to the exact local-commit fence");
|
||||
|
||||
let legacy_unknown = checkpoint(
|
||||
2,
|
||||
transition_transaction::TransitionTransactionState::UploadOutcomeUnknown,
|
||||
unknown_remote_sha256,
|
||||
false,
|
||||
);
|
||||
let invalid_legacy_skip = checkpoint(
|
||||
3,
|
||||
transition_transaction::TransitionTransactionState::LocalCommitStarted,
|
||||
known_remote_sha256.clone(),
|
||||
true,
|
||||
);
|
||||
assert!(
|
||||
legacy_unknown.validate_successor(&invalid_legacy_skip).is_err(),
|
||||
"legacy UploadOutcomeUnknown@2 must not masquerade as the compact edge"
|
||||
);
|
||||
let valid_legacy_skip = checkpoint(
|
||||
4,
|
||||
transition_transaction::TransitionTransactionState::LocalCommitStarted,
|
||||
known_remote_sha256,
|
||||
true,
|
||||
);
|
||||
legacy_unknown
|
||||
.validate_successor(&valid_legacy_skip)
|
||||
.expect("legacy receipts may still observe the existing two-edge state advance");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_dispatch_manifest_namespace_validates_monotonic_branches() {
|
||||
use tier_delete_journal::TierDeleteDispatchManifestState::{Aborted, Aborting, Completed, DispatchAuthorized, Preparing};
|
||||
|
||||
@@ -24,10 +24,6 @@ pub(crate) use metadata_boundary::{LifecycleExpiryConfigs, get_expiry_configs, g
|
||||
mod object_handlers_common;
|
||||
mod object_lock_boundary;
|
||||
pub use self::core as lifecycle;
|
||||
pub mod recovery_control;
|
||||
pub mod recovery_disposition;
|
||||
pub(crate) mod recovery_disposition_runtime;
|
||||
pub mod recovery_export;
|
||||
mod replication_sink;
|
||||
pub mod rule;
|
||||
mod runtime_boundary;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,840 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::{collections::HashSet, sync::Arc};
|
||||
|
||||
use rustfs_utils::crypto::{hex_sha256, is_sha256_checksum};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::config_boundary;
|
||||
use super::recovery_control::{
|
||||
IlmRecoveryClassification, IlmRecoveryControl, IlmRecoveryProtocol, IlmRecoverySourceCopy, IlmRecoverySourceGeneration,
|
||||
MAX_ILM_RECOVERY_CONTROL_SIZE, ObservedIlmRecoveryControl, ObservedIlmRecoverySource, recovery_control_record_object_name,
|
||||
};
|
||||
use super::tier_delete_journal::{
|
||||
TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA, TIER_DELETE_JOURNAL_V2_RECOVERY_SCHEMA, validate_legacy_tier_delete_recovery_source,
|
||||
};
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::object_api::{ObjectOptions, WriteCompletion};
|
||||
use crate::services::notification_sys::{
|
||||
acquire_ilm_recovery_export_fleet_proof, ilm_recovery_export_fleet_proof_matches, ilm_recovery_export_member_epochs_sha256,
|
||||
ilm_recovery_export_topology_generation,
|
||||
};
|
||||
use crate::storage_api_contracts::{list::ListOperations as _, namespace::NamespaceLocking as _, object::HTTPPreconditions};
|
||||
use crate::store::ECStore;
|
||||
|
||||
pub const ILM_RECOVERY_EXPORT_SCHEMA: &str = "rustfs-ilm-recovery-export-v1";
|
||||
pub const ILM_RECOVERY_EXPORT_PREFIX: &str = "ilm/recovery-exports";
|
||||
pub const MAX_ILM_RECOVERY_EXPORT_SIZE: usize = 128 * 1024;
|
||||
const MAX_ILM_RECOVERY_EXPORTS: usize = 10_000;
|
||||
const MAX_ILM_RECOVERY_EXPORT_BYTES: u64 = 1024 * 1024 * 1024;
|
||||
const MAX_ACTOR_EXPORTS_PER_MINUTE: usize = 10;
|
||||
const MAX_CLUSTER_EXPORTS_PER_MINUTE: usize = 100;
|
||||
const EXPORT_RETENTION_NANOS: i64 = 90 * 24 * 60 * 60 * 1_000_000_000;
|
||||
const EXPORT_ADMISSION_LOCK: &str = "ilm/recovery-admission/export.lock";
|
||||
const MAX_LEGACY_TIER_DELETE_SOURCE_SIZE: usize = 64 * 1024;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct IlmRecoveryExportObservation {
|
||||
pub control_id: String,
|
||||
pub protocol: IlmRecoveryProtocol,
|
||||
pub control_etag: String,
|
||||
pub control_revision: u64,
|
||||
pub classification: IlmRecoveryClassification,
|
||||
pub canonical_source_path: String,
|
||||
pub source_generation: IlmRecoverySourceGeneration,
|
||||
pub topology_generation: String,
|
||||
pub member_epochs_sha256: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct IlmRecoveryExport {
|
||||
pub export_id: String,
|
||||
pub control_id: String,
|
||||
pub protocol: IlmRecoveryProtocol,
|
||||
pub control_etag: String,
|
||||
pub control_revision: u64,
|
||||
pub classification: IlmRecoveryClassification,
|
||||
pub canonical_source_path: String,
|
||||
pub source_generation: IlmRecoverySourceGeneration,
|
||||
pub topology_generation: String,
|
||||
pub member_epochs_sha256: String,
|
||||
pub creator_sha256: String,
|
||||
pub created_at_unix_nanos: i64,
|
||||
pub retain_until_unix_nanos: i64,
|
||||
pub source_bytes_base64: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct PersistedIlmRecoveryExport {
|
||||
schema: String,
|
||||
content_sha256: String,
|
||||
export: IlmRecoveryExport,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub struct IlmRecoveryExportCreated {
|
||||
pub export_id: String,
|
||||
pub content_sha256: String,
|
||||
pub encoded: Vec<u8>,
|
||||
pub replayed: bool,
|
||||
}
|
||||
|
||||
impl IlmRecoveryExport {
|
||||
fn validate(&self) -> Result<()> {
|
||||
self.source_generation.validate().map_err(Error::other)?;
|
||||
validate_sha256(&self.export_id, "ILM recovery export ID is invalid")?;
|
||||
validate_sha256(&self.control_id, "ILM recovery export control ID is invalid")?;
|
||||
validate_sha256(&self.topology_generation, "ILM recovery export topology generation is invalid")?;
|
||||
validate_sha256(&self.member_epochs_sha256, "ILM recovery export member epoch digest is invalid")?;
|
||||
validate_sha256(&self.creator_sha256, "ILM recovery export creator digest is invalid")?;
|
||||
if self.protocol != IlmRecoveryProtocol::TierDeleteJournal
|
||||
|| self.classification != IlmRecoveryClassification::RetainedAmbiguous
|
||||
|| !is_legacy_export_schema(&self.source_generation.source_schema)
|
||||
{
|
||||
return Err(Error::other("ILM recovery export source is not an exportable legacy journal"));
|
||||
}
|
||||
if self.control_etag.trim().is_empty() || self.control_revision == 0 {
|
||||
return Err(Error::other("ILM recovery export control generation is invalid"));
|
||||
}
|
||||
if self.canonical_source_path.is_empty()
|
||||
|| self.canonical_source_path.starts_with('/')
|
||||
|| self.canonical_source_path.ends_with('/')
|
||||
|| self.canonical_source_path.split('/').any(str::is_empty)
|
||||
{
|
||||
return Err(Error::other("ILM recovery export source path is invalid"));
|
||||
}
|
||||
if self.created_at_unix_nanos <= 0
|
||||
|| self.retain_until_unix_nanos < self.created_at_unix_nanos.saturating_add(EXPORT_RETENTION_NANOS)
|
||||
{
|
||||
return Err(Error::other("ILM recovery export retention is invalid"));
|
||||
}
|
||||
let source = base64_simd::STANDARD
|
||||
.decode_to_vec(self.source_bytes_base64.as_bytes())
|
||||
.map_err(|_| Error::other("ILM recovery export source encoding is invalid"))?;
|
||||
validate_legacy_tier_delete_recovery_source(&self.canonical_source_path, &self.source_generation.source_schema, &source)?;
|
||||
let encoded_len = u64::try_from(source.len()).map_err(|_| Error::other("ILM recovery export source length overflow"))?;
|
||||
if source.is_empty()
|
||||
|| source.len() > MAX_LEGACY_TIER_DELETE_SOURCE_SIZE
|
||||
|| hex_sha256(&source, ToOwned::to_owned) != self.source_generation.content_sha256
|
||||
|| self.source_generation.copies.iter().any(|copy| {
|
||||
copy.canonical_path != self.canonical_source_path
|
||||
|| copy.etag != self.source_generation.source_etag
|
||||
|| copy.content_sha256 != self.source_generation.content_sha256
|
||||
|| copy.encoded_len != encoded_len
|
||||
})
|
||||
{
|
||||
return Err(Error::other("ILM recovery export source bytes do not match the observed generation"));
|
||||
}
|
||||
if recovery_export_id(&self.control_id, &self.source_generation)? != self.export_id {
|
||||
return Err(Error::other("ILM recovery export ID does not match its source generation"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn encode(&self) -> Result<Vec<u8>> {
|
||||
self.validate()?;
|
||||
let export_bytes = serde_json::to_vec(self).map_err(Error::other)?;
|
||||
let persisted = PersistedIlmRecoveryExport {
|
||||
schema: ILM_RECOVERY_EXPORT_SCHEMA.to_string(),
|
||||
content_sha256: hex_sha256(&export_bytes, ToOwned::to_owned),
|
||||
export: self.clone(),
|
||||
};
|
||||
let encoded = serde_json::to_vec(&persisted).map_err(Error::other)?;
|
||||
if encoded.len() > MAX_ILM_RECOVERY_EXPORT_SIZE {
|
||||
return Err(Error::other("encoded ILM recovery export exceeds maximum size"));
|
||||
}
|
||||
Ok(encoded)
|
||||
}
|
||||
|
||||
pub fn decode(expected_export_id: &str, data: &[u8]) -> Result<Self> {
|
||||
validate_sha256(expected_export_id, "ILM recovery export ID is invalid")?;
|
||||
if data.len() > MAX_ILM_RECOVERY_EXPORT_SIZE {
|
||||
return Err(Error::other("encoded ILM recovery export exceeds maximum size"));
|
||||
}
|
||||
let persisted: PersistedIlmRecoveryExport = serde_json::from_slice(data).map_err(Error::other)?;
|
||||
if persisted.schema != ILM_RECOVERY_EXPORT_SCHEMA {
|
||||
return Err(Error::other("ILM recovery export schema is unsupported"));
|
||||
}
|
||||
validate_sha256(&persisted.content_sha256, "ILM recovery export checksum is invalid")?;
|
||||
let export_bytes = serde_json::to_vec(&persisted.export).map_err(Error::other)?;
|
||||
if hex_sha256(&export_bytes, ToOwned::to_owned) != persisted.content_sha256 {
|
||||
return Err(Error::other("ILM recovery export checksum mismatch"));
|
||||
}
|
||||
persisted.export.validate()?;
|
||||
if persisted.export.export_id != expected_export_id {
|
||||
return Err(Error::other("ILM recovery export ID does not match record key"));
|
||||
}
|
||||
Ok(persisted.export)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn recovery_export_record_object_name(protocol: IlmRecoveryProtocol, export_id: &str) -> Result<String> {
|
||||
validate_sha256(export_id, "ILM recovery export ID is invalid")?;
|
||||
Ok(format!(
|
||||
"{}/{}/{}/{}/{}.json",
|
||||
ILM_RECOVERY_EXPORT_PREFIX,
|
||||
protocol.as_str(),
|
||||
&export_id[..2],
|
||||
&export_id[2..4],
|
||||
export_id
|
||||
))
|
||||
}
|
||||
|
||||
pub fn recovery_export_id_from_record_object_name(object: &str) -> Result<(IlmRecoveryProtocol, String)> {
|
||||
let suffix = object
|
||||
.strip_prefix(ILM_RECOVERY_EXPORT_PREFIX)
|
||||
.and_then(|suffix| suffix.strip_prefix('/'))
|
||||
.ok_or_else(|| Error::other("ILM recovery export path has wrong prefix"))?;
|
||||
let mut parts = suffix.split('/');
|
||||
let protocol = match parts.next() {
|
||||
Some("tier_delete_journal") => IlmRecoveryProtocol::TierDeleteJournal,
|
||||
_ => return Err(Error::other("ILM recovery export protocol is invalid")),
|
||||
};
|
||||
let shard_a = parts
|
||||
.next()
|
||||
.ok_or_else(|| Error::other("ILM recovery export path is incomplete"))?;
|
||||
let shard_b = parts
|
||||
.next()
|
||||
.ok_or_else(|| Error::other("ILM recovery export path is incomplete"))?;
|
||||
let export_id = parts
|
||||
.next()
|
||||
.and_then(|name| name.strip_suffix(".json"))
|
||||
.ok_or_else(|| Error::other("ILM recovery export suffix is invalid"))?;
|
||||
if parts.next().is_some() {
|
||||
return Err(Error::other("ILM recovery export path is not canonical"));
|
||||
}
|
||||
validate_sha256(export_id, "ILM recovery export ID is invalid")?;
|
||||
if shard_a != &export_id[..2] || shard_b != &export_id[2..4] {
|
||||
return Err(Error::other("ILM recovery export shard does not match export ID"));
|
||||
}
|
||||
Ok((protocol, export_id.to_string()))
|
||||
}
|
||||
|
||||
pub async fn inspect_recovery_export_observation(api: Arc<ECStore>, control_id: &str) -> Result<IlmRecoveryExportObservation> {
|
||||
let proof = acquire_ilm_recovery_export_fleet_proof()
|
||||
.await
|
||||
.ok_or_else(|| Error::other("ILM recovery export fleet proof is unavailable"))?;
|
||||
let observed_control = load_exportable_control(api.clone(), control_id).await?;
|
||||
let observed_source = observe_export_source(
|
||||
api,
|
||||
&observed_control.control.identity.canonical_source_path,
|
||||
&observed_control.control.observed_source_generation.source_schema,
|
||||
)
|
||||
.await?;
|
||||
if !observed_source.is_consistent()
|
||||
|| observed_source.generation != observed_control.control.observed_source_generation
|
||||
|| !ilm_recovery_export_fleet_proof_matches(&proof).await
|
||||
{
|
||||
return Err(Error::other("ILM recovery export observation changed or is incomplete"));
|
||||
}
|
||||
Ok(IlmRecoveryExportObservation {
|
||||
control_id: control_id.to_string(),
|
||||
protocol: observed_control.control.identity.protocol,
|
||||
control_etag: observed_control.etag,
|
||||
control_revision: observed_control.control.revision,
|
||||
classification: observed_control.control.classification,
|
||||
canonical_source_path: observed_control.control.identity.canonical_source_path,
|
||||
source_generation: observed_source.generation,
|
||||
topology_generation: ilm_recovery_export_topology_generation(&proof),
|
||||
member_epochs_sha256: ilm_recovery_export_member_epochs_sha256(&proof),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn create_recovery_export(
|
||||
api: Arc<ECStore>,
|
||||
observation: &IlmRecoveryExportObservation,
|
||||
creator_sha256: &str,
|
||||
) -> Result<IlmRecoveryExportCreated> {
|
||||
validate_sha256(creator_sha256, "ILM recovery export creator digest is invalid")?;
|
||||
let lock = api.new_ns_lock(RUSTFS_META_BUCKET, EXPORT_ADMISSION_LOCK).await?;
|
||||
let admission_guard = lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
|
||||
|
||||
let proof = acquire_ilm_recovery_export_fleet_proof()
|
||||
.await
|
||||
.ok_or_else(|| Error::other("ILM recovery export fleet proof is unavailable"))?;
|
||||
if ilm_recovery_export_topology_generation(&proof) != observation.topology_generation
|
||||
|| ilm_recovery_export_member_epochs_sha256(&proof) != observation.member_epochs_sha256
|
||||
{
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
let control_object = recovery_control_record_object_name(IlmRecoveryProtocol::TierDeleteJournal, &observation.control_id)
|
||||
.map_err(Error::other)?;
|
||||
let control_lock = api.new_ns_lock(RUSTFS_META_BUCKET, &control_object).await?;
|
||||
let control_guard = control_lock
|
||||
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await?;
|
||||
let source_lock = api
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &observation.canonical_source_path)
|
||||
.await?;
|
||||
let source_guard = source_lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
|
||||
let locks_current = || !admission_guard.is_lock_lost() && !control_guard.is_lock_lost() && !source_guard.is_lock_lost();
|
||||
let (current, current_source_bytes) = current_observation_under_proof_no_lock(api.clone(), observation, &proof).await?;
|
||||
if ¤t != observation || !locks_current() {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
let current_source_base64 = base64_simd::STANDARD.encode_to_string(current_source_bytes);
|
||||
let candidate_export_id = recovery_export_id(¤t.control_id, ¤t.source_generation)?;
|
||||
let object = recovery_export_record_object_name(current.protocol, &candidate_export_id)?;
|
||||
match load_recovery_export_decoded(api.clone(), &candidate_export_id).await {
|
||||
Ok((existing, export)) if export_matches_observation(&export, observation) => {
|
||||
if !locks_current() || !ilm_recovery_export_fleet_proof_matches(&proof).await {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
api.record_durable_ilm_decommission_progress(&object, &existing.encoded)
|
||||
.await?;
|
||||
if !locks_current() {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
return Ok(existing.with_replayed());
|
||||
}
|
||||
Ok(_) => return Err(Error::PreconditionFailed),
|
||||
Err(Error::ConfigNotFound) => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
let inventory = collect_export_inventory(api.clone()).await?;
|
||||
if !locks_current() || !ilm_recovery_export_fleet_proof_matches(&proof).await {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
let created_at_unix_nanos = now_unix_nanos()?;
|
||||
let export = build_export_from_source(¤t, creator_sha256, created_at_unix_nanos, ¤t_source_base64)?;
|
||||
let encoded = export.encode()?;
|
||||
inventory.check(creator_sha256, encoded.len(), created_at_unix_nanos)?;
|
||||
|
||||
let mut write_options = ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_none_match: Some("*".to_string()),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
write_options.add_namespace_lock_guard(&admission_guard);
|
||||
write_options.add_namespace_lock_guard(&control_guard);
|
||||
write_options.add_namespace_lock_guard(&source_guard);
|
||||
if !locks_current() {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
let write_result = config_boundary::save_config_with_opts(api.clone(), &object, encoded.clone(), &write_options).await;
|
||||
let stored = match load_recovery_export(api.clone(), &export.export_id).await {
|
||||
Ok(stored) if stored.encoded == encoded => stored,
|
||||
Ok(_) => return Err(Error::PreconditionFailed),
|
||||
Err(read_err) => return Err(write_result.err().unwrap_or(read_err)),
|
||||
};
|
||||
if !locks_current() || !ilm_recovery_export_fleet_proof_matches(&proof).await {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
api.record_durable_ilm_decommission_progress(&object, &encoded).await?;
|
||||
if !locks_current() {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
Ok(stored)
|
||||
}
|
||||
|
||||
pub async fn load_recovery_export(api: Arc<ECStore>, export_id: &str) -> Result<IlmRecoveryExportCreated> {
|
||||
let (created, _) = load_recovery_export_decoded(api, export_id).await?;
|
||||
Ok(created)
|
||||
}
|
||||
|
||||
async fn load_recovery_export_decoded(
|
||||
api: Arc<ECStore>,
|
||||
export_id: &str,
|
||||
) -> Result<(IlmRecoveryExportCreated, IlmRecoveryExport)> {
|
||||
let object = recovery_export_record_object_name(IlmRecoveryProtocol::TierDeleteJournal, export_id)?;
|
||||
let encoded = config_boundary::read_config_limited_preserve_empty(api, &object, MAX_ILM_RECOVERY_EXPORT_SIZE).await?;
|
||||
let export = IlmRecoveryExport::decode(export_id, &encoded)?;
|
||||
let content_sha256 = hex_sha256(&encoded, ToOwned::to_owned);
|
||||
Ok((
|
||||
IlmRecoveryExportCreated {
|
||||
export_id: export.export_id.clone(),
|
||||
content_sha256,
|
||||
encoded,
|
||||
replayed: false,
|
||||
},
|
||||
export,
|
||||
))
|
||||
}
|
||||
|
||||
impl IlmRecoveryExportCreated {
|
||||
fn with_replayed(mut self) -> Self {
|
||||
self.replayed = true;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
async fn load_exportable_control(api: Arc<ECStore>, control_id: &str) -> Result<ObservedIlmRecoveryControl> {
|
||||
load_exportable_control_with_options(api, control_id, &ObjectOptions::default()).await
|
||||
}
|
||||
|
||||
async fn load_exportable_control_no_lock(api: Arc<ECStore>, control_id: &str) -> Result<ObservedIlmRecoveryControl> {
|
||||
load_exportable_control_with_options(
|
||||
api,
|
||||
control_id,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn load_exportable_control_with_options(
|
||||
api: Arc<ECStore>,
|
||||
control_id: &str,
|
||||
options: &ObjectOptions,
|
||||
) -> Result<ObservedIlmRecoveryControl> {
|
||||
let object = recovery_control_record_object_name(IlmRecoveryProtocol::TierDeleteJournal, control_id).map_err(Error::other)?;
|
||||
let (data, metadata) =
|
||||
config_boundary::read_config_limited_preserve_empty_with_metadata(api, &object, options, MAX_ILM_RECOVERY_CONTROL_SIZE)
|
||||
.await?;
|
||||
let etag = metadata
|
||||
.etag
|
||||
.filter(|etag| !etag.trim().is_empty())
|
||||
.ok_or_else(|| Error::other("ILM recovery control is missing an ETag"))?;
|
||||
let control = IlmRecoveryControl::decode(control_id, &data).map_err(Error::other)?;
|
||||
if control.identity.protocol != IlmRecoveryProtocol::TierDeleteJournal
|
||||
|| control.classification != IlmRecoveryClassification::RetainedAmbiguous
|
||||
|| !is_legacy_export_schema(&control.observed_source_generation.source_schema)
|
||||
{
|
||||
return Err(Error::other("ILM recovery control is not exportable"));
|
||||
}
|
||||
Ok(ObservedIlmRecoveryControl { control, etag })
|
||||
}
|
||||
|
||||
async fn current_observation_under_proof_no_lock(
|
||||
api: Arc<ECStore>,
|
||||
expected: &IlmRecoveryExportObservation,
|
||||
proof: &crate::services::notification_sys::IlmRecoveryExportFleetProofToken,
|
||||
) -> Result<(IlmRecoveryExportObservation, Vec<u8>)> {
|
||||
let observed_control = load_exportable_control_no_lock(api.clone(), &expected.control_id).await?;
|
||||
let observed_source = observe_export_source_no_lock(
|
||||
api,
|
||||
&observed_control.control.identity.canonical_source_path,
|
||||
&observed_control.control.observed_source_generation.source_schema,
|
||||
)
|
||||
.await?;
|
||||
let source_bytes = observed_source
|
||||
.canonical_data
|
||||
.clone()
|
||||
.ok_or_else(|| Error::other("ILM recovery export source copies diverge"))?;
|
||||
if !observed_source.is_consistent()
|
||||
|| observed_source.generation != observed_control.control.observed_source_generation
|
||||
|| !ilm_recovery_export_fleet_proof_matches(proof).await
|
||||
{
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
Ok((
|
||||
IlmRecoveryExportObservation {
|
||||
control_id: expected.control_id.clone(),
|
||||
protocol: observed_control.control.identity.protocol,
|
||||
control_etag: observed_control.etag,
|
||||
control_revision: observed_control.control.revision,
|
||||
classification: observed_control.control.classification,
|
||||
canonical_source_path: observed_control.control.identity.canonical_source_path,
|
||||
source_generation: observed_source.generation,
|
||||
topology_generation: ilm_recovery_export_topology_generation(proof),
|
||||
member_epochs_sha256: ilm_recovery_export_member_epochs_sha256(proof),
|
||||
},
|
||||
source_bytes,
|
||||
))
|
||||
}
|
||||
|
||||
async fn observe_export_source(
|
||||
api: Arc<ECStore>,
|
||||
canonical_path: &str,
|
||||
source_schema: &str,
|
||||
) -> Result<ObservedIlmRecoverySource> {
|
||||
if canonical_path.is_empty()
|
||||
|| canonical_path.starts_with('/')
|
||||
|| canonical_path.ends_with('/')
|
||||
|| canonical_path.split('/').any(str::is_empty)
|
||||
|| !is_legacy_export_schema(source_schema)
|
||||
{
|
||||
return Err(Error::other("ILM recovery export source identity is invalid"));
|
||||
}
|
||||
let lock = api.new_ns_lock(RUSTFS_META_BUCKET, canonical_path).await?;
|
||||
let _guard = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
|
||||
observe_export_source_no_lock(api, canonical_path, source_schema).await
|
||||
}
|
||||
|
||||
async fn observe_export_source_no_lock(
|
||||
api: Arc<ECStore>,
|
||||
canonical_path: &str,
|
||||
source_schema: &str,
|
||||
) -> Result<ObservedIlmRecoverySource> {
|
||||
let mut copies = Vec::new();
|
||||
let mut canonical: Option<(String, String, Vec<u8>)> = None;
|
||||
let mut consistent = true;
|
||||
for set in api.all_set_disks() {
|
||||
let authority = format!("pool-{}/set-{}", set.pool_index, set.set_index);
|
||||
let result = config_boundary::read_config_limited_preserve_empty_with_metadata(
|
||||
set,
|
||||
canonical_path,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
MAX_LEGACY_TIER_DELETE_SOURCE_SIZE,
|
||||
)
|
||||
.await;
|
||||
match result {
|
||||
Ok((data, metadata)) => {
|
||||
if data.is_empty() || data.len() > MAX_LEGACY_TIER_DELETE_SOURCE_SIZE {
|
||||
return Err(Error::other("ILM recovery export source exceeds its protocol size limit"));
|
||||
}
|
||||
validate_legacy_tier_delete_recovery_source(canonical_path, source_schema, &data)?;
|
||||
let etag = metadata
|
||||
.etag
|
||||
.filter(|etag| !etag.trim().is_empty())
|
||||
.ok_or_else(|| Error::other("ILM recovery export source copy is missing an ETag"))?;
|
||||
let content_sha256 = hex_sha256(&data, ToOwned::to_owned);
|
||||
let encoded_len =
|
||||
u64::try_from(data.len()).map_err(|_| Error::other("ILM recovery export source length does not fit u64"))?;
|
||||
copies.push(IlmRecoverySourceCopy {
|
||||
authority,
|
||||
canonical_path: canonical_path.to_string(),
|
||||
etag: etag.clone(),
|
||||
encoded_len,
|
||||
content_sha256: content_sha256.clone(),
|
||||
});
|
||||
match canonical.as_ref() {
|
||||
Some((first_etag, first_digest, first_data)) => {
|
||||
consistent &= first_etag == &etag && first_digest == &content_sha256 && first_data == &data;
|
||||
}
|
||||
None => canonical = Some((etag, content_sha256, data)),
|
||||
}
|
||||
}
|
||||
Err(err) if export_source_is_missing(&err) => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
let Some((source_etag, content_sha256, source_bytes)) = canonical else {
|
||||
return Err(Error::ConfigNotFound);
|
||||
};
|
||||
let generation =
|
||||
IlmRecoverySourceGeneration::new(source_schema, source_etag, content_sha256, copies).map_err(Error::other)?;
|
||||
Ok(ObservedIlmRecoverySource {
|
||||
generation,
|
||||
canonical_data: consistent.then_some(source_bytes),
|
||||
})
|
||||
}
|
||||
|
||||
fn export_source_is_missing(err: &Error) -> bool {
|
||||
matches!(
|
||||
err,
|
||||
Error::ConfigNotFound | Error::FileNotFound | Error::ObjectNotFound(_, _) | Error::VersionNotFound(_, _, _)
|
||||
)
|
||||
}
|
||||
|
||||
fn build_export_from_source(
|
||||
observation: &IlmRecoveryExportObservation,
|
||||
creator_sha256: &str,
|
||||
created_at_unix_nanos: i64,
|
||||
source_bytes_base64: &str,
|
||||
) -> Result<IlmRecoveryExport> {
|
||||
let retain_until_unix_nanos = created_at_unix_nanos
|
||||
.checked_add(EXPORT_RETENTION_NANOS)
|
||||
.ok_or_else(|| Error::other("ILM recovery export retention timestamp overflow"))?;
|
||||
let export = IlmRecoveryExport {
|
||||
export_id: recovery_export_id(&observation.control_id, &observation.source_generation)?,
|
||||
control_id: observation.control_id.clone(),
|
||||
protocol: observation.protocol,
|
||||
control_etag: observation.control_etag.clone(),
|
||||
control_revision: observation.control_revision,
|
||||
classification: observation.classification,
|
||||
canonical_source_path: observation.canonical_source_path.clone(),
|
||||
source_generation: observation.source_generation.clone(),
|
||||
topology_generation: observation.topology_generation.clone(),
|
||||
member_epochs_sha256: observation.member_epochs_sha256.clone(),
|
||||
creator_sha256: creator_sha256.to_string(),
|
||||
created_at_unix_nanos,
|
||||
retain_until_unix_nanos,
|
||||
source_bytes_base64: source_bytes_base64.to_string(),
|
||||
};
|
||||
export.validate()?;
|
||||
Ok(export)
|
||||
}
|
||||
|
||||
pub(crate) fn recovery_export_id(control_id: &str, generation: &IlmRecoverySourceGeneration) -> Result<String> {
|
||||
validate_sha256(control_id, "ILM recovery export control ID is invalid")?;
|
||||
validate_sha256(&generation.content_sha256, "ILM recovery export source checksum is invalid")?;
|
||||
validate_sha256(&generation.copy_set_sha256, "ILM recovery export copy-set checksum is invalid")?;
|
||||
let mut data = Vec::new();
|
||||
for part in [control_id, &generation.content_sha256, &generation.copy_set_sha256] {
|
||||
data.extend_from_slice(&(part.len() as u64).to_be_bytes());
|
||||
data.extend_from_slice(part.as_bytes());
|
||||
}
|
||||
Ok(hex_sha256(&data, ToOwned::to_owned))
|
||||
}
|
||||
|
||||
fn export_matches_observation(export: &IlmRecoveryExport, observation: &IlmRecoveryExportObservation) -> bool {
|
||||
export.control_id == observation.control_id
|
||||
&& export.protocol == observation.protocol
|
||||
&& export.classification == observation.classification
|
||||
&& export.canonical_source_path == observation.canonical_source_path
|
||||
&& export.source_generation == observation.source_generation
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct IlmRecoveryExportInventory {
|
||||
count: usize,
|
||||
bytes: u64,
|
||||
creations: Vec<(i64, String)>,
|
||||
}
|
||||
|
||||
impl IlmRecoveryExportInventory {
|
||||
fn check(&self, creator_sha256: &str, candidate_len: usize, now: i64) -> Result<()> {
|
||||
let recent_after = now.saturating_sub(60 * 1_000_000_000);
|
||||
let cluster_recent = self
|
||||
.creations
|
||||
.iter()
|
||||
.filter(|(created_at, _)| *created_at > recent_after)
|
||||
.count();
|
||||
let actor_recent = self
|
||||
.creations
|
||||
.iter()
|
||||
.filter(|(created_at, creator)| *created_at > recent_after && creator == creator_sha256)
|
||||
.count();
|
||||
check_export_admission(self.count, self.bytes, actor_recent, cluster_recent, candidate_len)
|
||||
}
|
||||
}
|
||||
|
||||
async fn collect_export_inventory(api: Arc<ECStore>) -> Result<IlmRecoveryExportInventory> {
|
||||
let mut marker = None;
|
||||
let mut seen_markers = HashSet::new();
|
||||
let mut inventory = IlmRecoveryExportInventory::default();
|
||||
loop {
|
||||
let page = api
|
||||
.clone()
|
||||
.list_objects_v2(
|
||||
RUSTFS_META_BUCKET,
|
||||
&format!("{ILM_RECOVERY_EXPORT_PREFIX}/"),
|
||||
marker.clone(),
|
||||
None,
|
||||
1_000,
|
||||
false,
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
for object in page.objects {
|
||||
let (_, export_id) = recovery_export_id_from_record_object_name(&object.name)?;
|
||||
let (stored, export) = load_recovery_export_decoded(api.clone(), &export_id).await?;
|
||||
inventory.count = inventory
|
||||
.count
|
||||
.checked_add(1)
|
||||
.ok_or_else(|| Error::other("ILM recovery export count overflow"))?;
|
||||
inventory.bytes = inventory
|
||||
.bytes
|
||||
.checked_add(u64::try_from(stored.encoded.len()).map_err(|_| Error::other("ILM recovery export size overflow"))?)
|
||||
.ok_or_else(|| Error::other("ILM recovery export byte total overflow"))?;
|
||||
inventory
|
||||
.creations
|
||||
.push((export.created_at_unix_nanos, export.creator_sha256));
|
||||
}
|
||||
if !page.is_truncated {
|
||||
break;
|
||||
}
|
||||
let next = page
|
||||
.next_continuation_token
|
||||
.ok_or_else(|| Error::other("ILM recovery export inventory omitted its continuation marker"))?;
|
||||
marker = Some(record_export_inventory_marker(&mut seen_markers, next)?);
|
||||
}
|
||||
Ok(inventory)
|
||||
}
|
||||
|
||||
fn record_export_inventory_marker(seen_markers: &mut HashSet<String>, next: String) -> Result<String> {
|
||||
if !seen_markers.insert(next.clone()) {
|
||||
return Err(Error::other("ILM recovery export inventory repeated its continuation marker"));
|
||||
}
|
||||
Ok(next)
|
||||
}
|
||||
|
||||
fn check_export_admission(
|
||||
count: usize,
|
||||
bytes: u64,
|
||||
actor_recent: usize,
|
||||
cluster_recent: usize,
|
||||
candidate_len: usize,
|
||||
) -> Result<()> {
|
||||
let candidate_len = u64::try_from(candidate_len).map_err(|_| Error::other("ILM recovery export size does not fit u64"))?;
|
||||
if count >= MAX_ILM_RECOVERY_EXPORTS
|
||||
|| bytes
|
||||
.checked_add(candidate_len)
|
||||
.is_none_or(|total| total > MAX_ILM_RECOVERY_EXPORT_BYTES)
|
||||
|| actor_recent >= MAX_ACTOR_EXPORTS_PER_MINUTE
|
||||
|| cluster_recent >= MAX_CLUSTER_EXPORTS_PER_MINUTE
|
||||
{
|
||||
return Err(Error::SlowDown);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn is_legacy_export_schema(schema: &str) -> bool {
|
||||
matches!(schema, TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA | TIER_DELETE_JOURNAL_V2_RECOVERY_SCHEMA)
|
||||
}
|
||||
|
||||
fn validate_sha256(value: &str, message: &'static str) -> Result<()> {
|
||||
if !is_sha256_checksum(value)
|
||||
|| value
|
||||
.bytes()
|
||||
.any(|byte| byte.is_ascii_hexdigit() && byte.is_ascii_uppercase())
|
||||
{
|
||||
return Err(Error::other(message));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn now_unix_nanos() -> Result<i64> {
|
||||
i64::try_from(time::OffsetDateTime::now_utc().unix_timestamp_nanos())
|
||||
.map_err(|_| Error::other("ILM recovery export timestamp does not fit i64"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::bucket::lifecycle::recovery_control::IlmRecoverySourceCopy;
|
||||
|
||||
const PINNED_V1_EXPORT: &[u8] = br#"{"schema":"rustfs-ilm-recovery-export-v1","content_sha256":"3dfb3ec3892256e909de1211c1a963ca7008963ff32b3a869f7161a7b9b44028","export":{"export_id":"2b78e7a825bfc2edbf7f773d0b6ed3bf93e360ff1702d73a449109c11bfaa105","control_id":"0fcd568a5cb9bdb4677b69354b11ee415af8f784519cff3da49a26f84eaee7f2","protocol":"tier_delete_journal","control_etag":"control-etag","control_revision":1,"classification":"retained_ambiguous","canonical_source_path":"ilm/tier-delete-journal/872072554f66ab326f10ce7adbae11422b7a4b0663aa7112d6061a8f6ed41b94.json","source_generation":{"source_schema":"rustfs-tier-delete-journal-v1","source_etag":"etag-a","content_sha256":"0e0b010ebdeeb7b41473fe8575e989d6bb1303c0ca551dd984e9400f0ae306bd","copy_set_sha256":"5a7406115b6c3923ffe79dcd1f43ccae7beed786e557163f019dd10ec409a653","copies":[{"authority":"pool-0/set-0","canonical_path":"ilm/tier-delete-journal/872072554f66ab326f10ce7adbae11422b7a4b0663aa7112d6061a8f6ed41b94.json","etag":"etag-a","encoded_len":81,"content_sha256":"0e0b010ebdeeb7b41473fe8575e989d6bb1303c0ca551dd984e9400f0ae306bd"}]},"topology_generation":"e6e2b826e31fca5c36125c48f130dcb6f961e698ff8a8776a1f290cf0892e8e6","member_epochs_sha256":"612dd8a861161819a4ad8f6f3e2a0567602877c043a2353ca933a13c78dc0ed4","creator_sha256":"50c9c4aeb40b5b206b6d98f516f8b8c0efd29ce2e56a76b345fb9240c225a1b7","created_at_unix_nanos":1000000000,"retain_until_unix_nanos":7776001000000000,"source_bytes_base64":"eyJ2ZXJzaW9uIjoxLCJvYmpfbmFtZSI6ImxlZ2FjeS9yZW1vdGUiLCJ2ZXJzaW9uX2lkIjoib3BhcXVlIiwidGllcl9uYW1lIjoiV0FSTSJ9"}}"#;
|
||||
|
||||
fn legacy_source() -> Vec<u8> {
|
||||
br#"{"version":1,"obj_name":"legacy/remote","version_id":"opaque","tier_name":"WARM"}"#.to_vec()
|
||||
}
|
||||
|
||||
fn observation() -> IlmRecoveryExportObservation {
|
||||
let source = legacy_source();
|
||||
let source_path = super::super::tier_delete_journal::tier_delete_journal_object_name(
|
||||
&super::super::tier_delete_journal::decode_tier_delete_journal_entry(&source).expect("legacy fixture should decode"),
|
||||
);
|
||||
let source_sha256 = hex_sha256(&source, ToOwned::to_owned);
|
||||
let generation = IlmRecoverySourceGeneration::new(
|
||||
TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA,
|
||||
"etag-a",
|
||||
source_sha256.clone(),
|
||||
vec![IlmRecoverySourceCopy {
|
||||
authority: "pool-0/set-0".to_string(),
|
||||
canonical_path: source_path.clone(),
|
||||
etag: "etag-a".to_string(),
|
||||
encoded_len: source.len() as u64,
|
||||
content_sha256: source_sha256,
|
||||
}],
|
||||
)
|
||||
.expect("generation should be valid");
|
||||
IlmRecoveryExportObservation {
|
||||
control_id: hex_sha256(b"control", ToOwned::to_owned),
|
||||
protocol: IlmRecoveryProtocol::TierDeleteJournal,
|
||||
control_etag: "control-etag".to_string(),
|
||||
control_revision: 1,
|
||||
classification: IlmRecoveryClassification::RetainedAmbiguous,
|
||||
canonical_source_path: source_path,
|
||||
source_generation: generation,
|
||||
topology_generation: hex_sha256(b"topology", ToOwned::to_owned),
|
||||
member_epochs_sha256: hex_sha256(b"epochs", ToOwned::to_owned),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recovery_export_round_trip_is_strict_and_deterministic() {
|
||||
let observed = observation();
|
||||
let creator = hex_sha256(b"actor", ToOwned::to_owned);
|
||||
let export = build_export_from_source(
|
||||
&observed,
|
||||
&creator,
|
||||
1_000_000_000,
|
||||
&base64_simd::STANDARD.encode_to_string(legacy_source()),
|
||||
)
|
||||
.expect("export should be valid");
|
||||
assert_eq!(
|
||||
export.export_id,
|
||||
recovery_export_id(&observed.control_id, &observed.source_generation).unwrap()
|
||||
);
|
||||
let encoded = export.encode().expect("export should encode");
|
||||
assert_eq!(encoded, PINNED_V1_EXPORT, "v1 export wire format must remain pinned");
|
||||
assert_eq!(IlmRecoveryExport::decode(&export.export_id, &encoded).unwrap(), export);
|
||||
assert_eq!(
|
||||
IlmRecoveryExport::decode("2b78e7a825bfc2edbf7f773d0b6ed3bf93e360ff1702d73a449109c11bfaa105", PINNED_V1_EXPORT)
|
||||
.unwrap(),
|
||||
export,
|
||||
);
|
||||
|
||||
let path = recovery_export_record_object_name(export.protocol, &export.export_id).unwrap();
|
||||
let durable = super::super::durable_namespace::validate_durable_ilm_record(&path, &encoded)
|
||||
.expect("export should be registered as a durable ILM record");
|
||||
assert_eq!(durable.namespace, "recovery-export");
|
||||
assert_eq!(durable.id_kind, "export_id");
|
||||
assert_eq!(durable.id, export.export_id);
|
||||
|
||||
let mut wrong_source = export.clone();
|
||||
wrong_source.source_bytes_base64 = base64_simd::STANDARD.encode_to_string(b"changed");
|
||||
assert!(wrong_source.encode().is_err());
|
||||
|
||||
let mut persisted: serde_json::Value = serde_json::from_slice(&encoded).unwrap();
|
||||
persisted["unknown"] = serde_json::json!(true);
|
||||
assert!(IlmRecoveryExport::decode(&export.export_id, &serde_json::to_vec(&persisted).unwrap()).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_inventory_rejects_non_adjacent_continuation_cycles() {
|
||||
let mut seen = HashSet::new();
|
||||
assert_eq!(record_export_inventory_marker(&mut seen, "a".to_string()).unwrap(), "a");
|
||||
assert_eq!(record_export_inventory_marker(&mut seen, "b".to_string()).unwrap(), "b");
|
||||
record_export_inventory_marker(&mut seen, "a".to_string())
|
||||
.expect_err("a non-adjacent continuation marker cycle must fail closed");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recovery_export_path_rejects_noncanonical_shards() {
|
||||
let id = hex_sha256(b"export", ToOwned::to_owned);
|
||||
let path = recovery_export_record_object_name(IlmRecoveryProtocol::TierDeleteJournal, &id).unwrap();
|
||||
assert_eq!(recovery_export_id_from_record_object_name(&path).unwrap().1, id);
|
||||
let wrong_shard = path.replacen(&format!("/{}/", &id[..2]), "/zz/", 1);
|
||||
assert!(recovery_export_id_from_record_object_name(&wrong_shard).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_replay_survives_fleet_rotation_but_not_source_change() {
|
||||
let observed = observation();
|
||||
let creator = hex_sha256(b"actor", ToOwned::to_owned);
|
||||
let export = build_export_from_source(
|
||||
&observed,
|
||||
&creator,
|
||||
1_000_000_000,
|
||||
&base64_simd::STANDARD.encode_to_string(legacy_source()),
|
||||
)
|
||||
.unwrap();
|
||||
let mut rotated = observed;
|
||||
rotated.control_etag = "new-control-etag".to_string();
|
||||
rotated.control_revision += 1;
|
||||
rotated.topology_generation = hex_sha256(b"new-topology", ToOwned::to_owned);
|
||||
rotated.member_epochs_sha256 = hex_sha256(b"new-members", ToOwned::to_owned);
|
||||
assert!(export_matches_observation(&export, &rotated));
|
||||
|
||||
rotated.source_generation.content_sha256 = hex_sha256(b"changed", ToOwned::to_owned);
|
||||
assert!(!export_matches_observation(&export, &rotated));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_admission_enforces_exact_count_byte_and_rate_boundaries() {
|
||||
assert!(check_export_admission(9_999, MAX_ILM_RECOVERY_EXPORT_BYTES - 1, 9, 99, 1).is_ok());
|
||||
assert!(check_export_admission(10_000, 0, 0, 0, 1).is_err());
|
||||
assert!(check_export_admission(0, MAX_ILM_RECOVERY_EXPORT_BYTES, 0, 0, 1).is_err());
|
||||
assert!(check_export_admission(0, 0, 10, 0, 1).is_err());
|
||||
assert!(check_export_admission(0, 0, 0, 100, 1).is_err());
|
||||
}
|
||||
}
|
||||
@@ -35,10 +35,6 @@ use crate::bucket::lifecycle::config_boundary;
|
||||
use crate::bucket::lifecycle::durable_namespace::{
|
||||
TIER_DELETE_JOURNAL_NAMESPACE, TIER_DELETE_JOURNAL_V6_NAMESPACE, validate_durable_ilm_record,
|
||||
};
|
||||
use crate::bucket::lifecycle::recovery_control::{
|
||||
IlmRecoveryClassification, IlmRecoveryControl, IlmRecoveryControlIdentity, IlmRecoveryErrorCode, IlmRecoveryProtocol,
|
||||
load_recovery_control, observe_recovery_source, save_recovery_control_if_absent,
|
||||
};
|
||||
use crate::bucket::lifecycle::runtime_boundary;
|
||||
use crate::bucket::lifecycle::tier_sweeper::{
|
||||
Jentry, TierDeleteDispatchBinding, TierDeleteJournalState, TierDeleteSourceIdentity,
|
||||
@@ -82,13 +78,6 @@ const TIER_DELETE_DISPATCH_MEMBER_DELETE_CONCURRENCY: usize = 32;
|
||||
const TIER_DELETE_DISPATCH_PREPARE_CONCURRENCY: usize = 16;
|
||||
const TIER_DELETE_DISPATCH_CAS_CONCURRENCY: usize = 32;
|
||||
const TIER_DELETE_JOURNAL_VERSION: u8 = 2;
|
||||
pub(crate) const TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA: &str = "rustfs-tier-delete-journal-v1";
|
||||
pub(crate) const TIER_DELETE_JOURNAL_V2_RECOVERY_SCHEMA: &str = "rustfs-tier-delete-journal-v2";
|
||||
const TIER_DELETE_JOURNAL_UNKNOWN_RECOVERY_SCHEMA: &str = "rustfs-tier-delete-journal-unknown";
|
||||
const TIER_DELETE_JOURNAL_V1_RECOVERY_CLASS: &str = "tier_delete_journal_v1";
|
||||
const TIER_DELETE_JOURNAL_V2_RECOVERY_CLASS: &str = "tier_delete_journal_v2";
|
||||
const TIER_DELETE_JOURNAL_CORRUPT_RECOVERY_CLASS: &str = "tier_delete_journal_corrupt";
|
||||
const CORRUPT_TIER_DELETE_JOURNAL_IDENTITY: &str = "corrupt";
|
||||
const TIER_DELETE_JOURNAL_EXACT_VERSION: u8 = 3;
|
||||
const TIER_DELETE_JOURNAL_STATE_VERSION: u8 = 4;
|
||||
const TIER_DELETE_JOURNAL_TRANSACTION_VERSION: u8 = 5;
|
||||
@@ -884,23 +873,6 @@ struct PersistedTierDeleteJournalEntry {
|
||||
}
|
||||
|
||||
impl PersistedTierDeleteJournalEntry {
|
||||
fn validate_legacy_recovery_shape(&self) -> Result<()> {
|
||||
let has_later_version_fields = self.version_id_exact.is_some()
|
||||
|| self.version_state.is_some()
|
||||
|| self.state.is_some()
|
||||
|| self.source.is_some()
|
||||
|| self.dispatch.is_some();
|
||||
match self.version {
|
||||
1 if self.backend_identity.is_none() && !has_later_version_fields => Ok(()),
|
||||
TIER_DELETE_JOURNAL_VERSION if self.backend_identity.is_some() && !has_later_version_fields => Ok(()),
|
||||
1 => Err(Error::other("tier delete journal v1 entry contains fields from a later version")),
|
||||
TIER_DELETE_JOURNAL_VERSION => Err(Error::other(
|
||||
"tier delete journal v2 entry is missing its identity or contains fields from a later version",
|
||||
)),
|
||||
_ => Err(Error::other("tier delete journal is not an exportable legacy version")),
|
||||
}
|
||||
}
|
||||
|
||||
fn from_jentry(je: &Jentry) -> Result<Self> {
|
||||
validate_version_state(je.version_state, &je.version_id, je.version_id_exact)?;
|
||||
let legacy_unknown = je.version_state == rustfs_filemeta::TransitionVersionState::Unknown;
|
||||
@@ -5537,146 +5509,6 @@ enum TierDeleteJournalEntryRecoveryOutcome {
|
||||
Failed,
|
||||
}
|
||||
|
||||
fn canonical_legacy_tier_delete_journal_identity(object_name: &str) -> Option<&str> {
|
||||
let identity = object_name
|
||||
.strip_prefix(TIER_DELETE_JOURNAL_LEGACY_PREFIX)?
|
||||
.strip_suffix(".json")?;
|
||||
(rustfs_utils::crypto::is_sha256_checksum(identity)
|
||||
&& !identity
|
||||
.bytes()
|
||||
.any(|byte| byte.is_ascii_hexdigit() && byte.is_ascii_uppercase()))
|
||||
.then_some(identity)
|
||||
}
|
||||
|
||||
pub(crate) fn validate_legacy_tier_delete_recovery_path(object_name: &str) -> Result<()> {
|
||||
canonical_legacy_tier_delete_journal_identity(object_name)
|
||||
.map(|_| ())
|
||||
.ok_or_else(|| Error::other("legacy tier delete journal path is not canonical"))
|
||||
}
|
||||
|
||||
fn legacy_tier_delete_recovery_descriptor(entry: &Jentry) -> Option<(&'static str, &'static str)> {
|
||||
match entry.persisted_version {
|
||||
1 => Some((TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA, TIER_DELETE_JOURNAL_V1_RECOVERY_CLASS)),
|
||||
TIER_DELETE_JOURNAL_VERSION => Some((TIER_DELETE_JOURNAL_V2_RECOVERY_SCHEMA, TIER_DELETE_JOURNAL_V2_RECOVERY_CLASS)),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn validate_legacy_tier_delete_recovery_source(object_name: &str, source_schema: &str, data: &[u8]) -> Result<()> {
|
||||
validate_legacy_tier_delete_recovery_path(object_name)?;
|
||||
let persisted: PersistedTierDeleteJournalEntry =
|
||||
serde_json::from_slice(data).map_err(|err| Error::other_with_context("decode tier delete journal failed", err))?;
|
||||
persisted.validate_legacy_recovery_shape()?;
|
||||
let entry = persisted.into_jentry()?;
|
||||
let Some((decoded_schema, _)) = legacy_tier_delete_recovery_descriptor(&entry) else {
|
||||
return Err(Error::other("tier delete journal is not an exportable legacy version"));
|
||||
};
|
||||
if decoded_schema != source_schema || tier_delete_journal_object_name(&entry) != object_name {
|
||||
return Err(Error::other("legacy tier delete journal identity does not match its recovery source"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn legacy_tier_delete_control_matches(
|
||||
control: &IlmRecoveryControl,
|
||||
identity: &IlmRecoveryControlIdentity,
|
||||
generation: &crate::bucket::lifecycle::recovery_control::IlmRecoverySourceGeneration,
|
||||
classification: IlmRecoveryClassification,
|
||||
error_code: IlmRecoveryErrorCode,
|
||||
) -> bool {
|
||||
control.identity == *identity
|
||||
&& control.observed_source_generation == *generation
|
||||
&& control.classification == classification
|
||||
&& control.last_error_code == error_code
|
||||
&& control.owner.is_none()
|
||||
&& control.attempt_count == 0
|
||||
&& control.consecutive_failure_count == 0
|
||||
}
|
||||
|
||||
fn legacy_tier_delete_control_is_scheduler_fence(control: &IlmRecoveryControl, identity: &IlmRecoveryControlIdentity) -> bool {
|
||||
control.identity == *identity && control.owner.is_none() && !control.classification.permits_automatic_attempt()
|
||||
}
|
||||
|
||||
async fn persist_legacy_tier_delete_recovery_control(
|
||||
api: Arc<ECStore>,
|
||||
object_name: &str,
|
||||
observed_data: &[u8],
|
||||
stable_operation_identity: String,
|
||||
(source_schema, record_class): (&'static str, &'static str),
|
||||
intended_classification: IlmRecoveryClassification,
|
||||
intended_error_code: IlmRecoveryErrorCode,
|
||||
) -> Result<()> {
|
||||
let identity = IlmRecoveryControlIdentity {
|
||||
protocol: IlmRecoveryProtocol::TierDeleteJournal,
|
||||
canonical_source_path: object_name.to_string(),
|
||||
stable_operation_identity,
|
||||
record_class: record_class.to_string(),
|
||||
};
|
||||
let control_id = identity.source_operation_digest().map_err(Error::other)?;
|
||||
match load_recovery_control(api.clone(), IlmRecoveryProtocol::TierDeleteJournal, &control_id).await {
|
||||
Ok(observed) if legacy_tier_delete_control_is_scheduler_fence(&observed.control, &identity) => return Ok(()),
|
||||
Ok(_) => return Err(Error::PreconditionFailed),
|
||||
Err(Error::ConfigNotFound) => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
|
||||
let source = observe_recovery_source(api.clone(), object_name, source_schema).await?;
|
||||
let exact_source = source.is_consistent() && source.canonical_data.as_deref() == Some(observed_data);
|
||||
let (classification, error_code) = if exact_source {
|
||||
(intended_classification, intended_error_code)
|
||||
} else {
|
||||
(IlmRecoveryClassification::Corrupt, IlmRecoveryErrorCode::SourceDivergent)
|
||||
};
|
||||
let candidate = IlmRecoveryControl::new(
|
||||
identity.clone(),
|
||||
source.generation.clone(),
|
||||
classification,
|
||||
i64::try_from(time::OffsetDateTime::now_utc().unix_timestamp_nanos())
|
||||
.map_err(|_| Error::other("tier delete journal recovery timestamp does not fit i64"))?,
|
||||
error_code,
|
||||
)
|
||||
.map_err(Error::other)?;
|
||||
|
||||
match save_recovery_control_if_absent(api.clone(), &candidate).await {
|
||||
Ok(()) | Err(Error::PreconditionFailed) => {}
|
||||
Err(save_error) => match load_recovery_control(api.clone(), IlmRecoveryProtocol::TierDeleteJournal, &control_id).await {
|
||||
Ok(observed)
|
||||
if legacy_tier_delete_control_matches(
|
||||
&observed.control,
|
||||
&identity,
|
||||
&source.generation,
|
||||
classification,
|
||||
error_code,
|
||||
) =>
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
Ok(_) | Err(_) => return Err(save_error),
|
||||
},
|
||||
}
|
||||
|
||||
let observed = load_recovery_control(api, IlmRecoveryProtocol::TierDeleteJournal, &control_id).await?;
|
||||
if !legacy_tier_delete_control_matches(&observed.control, &identity, &source.generation, classification, error_code) {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn retain_corrupt_legacy_tier_delete_journal(api: Arc<ECStore>, object_name: &str, data: &[u8]) -> Result<()> {
|
||||
canonical_legacy_tier_delete_journal_identity(object_name)
|
||||
.ok_or_else(|| Error::other("tier delete journal path is not canonical"))?;
|
||||
persist_legacy_tier_delete_recovery_control(
|
||||
api,
|
||||
object_name,
|
||||
data,
|
||||
CORRUPT_TIER_DELETE_JOURNAL_IDENTITY.to_string(),
|
||||
(TIER_DELETE_JOURNAL_UNKNOWN_RECOVERY_SCHEMA, TIER_DELETE_JOURNAL_CORRUPT_RECOVERY_CLASS),
|
||||
IlmRecoveryClassification::Corrupt,
|
||||
IlmRecoveryErrorCode::SourceCorrupt,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn recover_tier_delete_journal_entry(api: Arc<ECStore>, object_name: String) -> TierDeleteJournalEntryRecoveryOutcome {
|
||||
let data = match config_boundary::read_config(api.clone(), &object_name).await {
|
||||
Ok(data) => data,
|
||||
@@ -5697,22 +5529,6 @@ async fn recover_tier_delete_journal_entry(api: Arc<ECStore>, object_name: Strin
|
||||
let je = match decode_tier_delete_journal_entry(&data) {
|
||||
Ok(je) => je,
|
||||
Err(err) => {
|
||||
if canonical_legacy_tier_delete_journal_identity(&object_name).is_some() {
|
||||
return match retain_corrupt_legacy_tier_delete_journal(api, &object_name, &data).await {
|
||||
Ok(()) => TierDeleteJournalEntryRecoveryOutcome::Retained,
|
||||
Err(control_error) => {
|
||||
warn!(
|
||||
event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
journal_object = %object_name,
|
||||
error = ?control_error,
|
||||
"Failed to retain corrupt tier delete journal recovery control"
|
||||
);
|
||||
TierDeleteJournalEntryRecoveryOutcome::Failed
|
||||
}
|
||||
};
|
||||
}
|
||||
warn!(
|
||||
event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -5726,22 +5542,6 @@ async fn recover_tier_delete_journal_entry(api: Arc<ECStore>, object_name: Strin
|
||||
};
|
||||
|
||||
if tier_delete_journal_object_name(&je) != object_name {
|
||||
if canonical_legacy_tier_delete_journal_identity(&object_name).is_some() {
|
||||
return match retain_corrupt_legacy_tier_delete_journal(api, &object_name, &data).await {
|
||||
Ok(()) => TierDeleteJournalEntryRecoveryOutcome::Retained,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
journal_object = %object_name,
|
||||
error = ?err,
|
||||
"Failed to retain mismatched tier delete journal recovery control"
|
||||
);
|
||||
TierDeleteJournalEntryRecoveryOutcome::Failed
|
||||
}
|
||||
};
|
||||
}
|
||||
warn!(
|
||||
event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -5752,36 +5552,6 @@ async fn recover_tier_delete_journal_entry(api: Arc<ECStore>, object_name: Strin
|
||||
return TierDeleteJournalEntryRecoveryOutcome::Failed;
|
||||
}
|
||||
|
||||
if let Some((source_schema, record_class)) = legacy_tier_delete_recovery_descriptor(&je) {
|
||||
let stable_operation_identity = canonical_legacy_tier_delete_journal_identity(&object_name)
|
||||
.expect("decoded legacy journal path was validated against its canonical object name")
|
||||
.to_string();
|
||||
return match persist_legacy_tier_delete_recovery_control(
|
||||
api,
|
||||
&object_name,
|
||||
&data,
|
||||
stable_operation_identity,
|
||||
(source_schema, record_class),
|
||||
IlmRecoveryClassification::RetainedAmbiguous,
|
||||
IlmRecoveryErrorCode::RemoteVersionUnknown,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(()) => TierDeleteJournalEntryRecoveryOutcome::Retained,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
journal_object = %object_name,
|
||||
error = ?err,
|
||||
"Failed to retain legacy tier delete journal recovery control"
|
||||
);
|
||||
TierDeleteJournalEntryRecoveryOutcome::Failed
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
match api
|
||||
.durable_ilm_terminal_receipt_covers_active_source(&object_name, &data)
|
||||
.await
|
||||
@@ -6178,18 +5948,17 @@ where
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
PersistedTierDeleteJournalEntry, TIER_DELETE_DISPATCH_MANIFEST_VERSION, TIER_DELETE_DISPATCH_PARENT_RECORD_TYPE,
|
||||
TIER_DELETE_DISPATCH_PARENT_VERSION, TIER_DELETE_JOURNAL_EXACT_VERSION, TIER_DELETE_JOURNAL_LEGACY_PREFIX,
|
||||
TIER_DELETE_JOURNAL_SOLE_OWNER_VERSION, TIER_DELETE_JOURNAL_STATE_VERSION, TIER_DELETE_JOURNAL_TRANSACTION_VERSION,
|
||||
TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA, TIER_DELETE_JOURNAL_V2_RECOVERY_SCHEMA, TIER_DELETE_JOURNAL_V6_PREFIX,
|
||||
TIER_DELETE_JOURNAL_VERSION, TierDeleteDispatchChunkBinding, TierDeleteDispatchManifest, TierDeleteDispatchManifestState,
|
||||
TierDeleteDispatchParent, TierDeleteDispatchParentState, TierDeleteDispatchRecord, await_tier_delete_journal_recovery,
|
||||
TIER_DELETE_DISPATCH_MANIFEST_VERSION, TIER_DELETE_DISPATCH_PARENT_RECORD_TYPE, TIER_DELETE_DISPATCH_PARENT_VERSION,
|
||||
TIER_DELETE_JOURNAL_EXACT_VERSION, TIER_DELETE_JOURNAL_LEGACY_PREFIX, TIER_DELETE_JOURNAL_SOLE_OWNER_VERSION,
|
||||
TIER_DELETE_JOURNAL_STATE_VERSION, TIER_DELETE_JOURNAL_TRANSACTION_VERSION, TIER_DELETE_JOURNAL_V6_PREFIX,
|
||||
TierDeleteDispatchChunkBinding, TierDeleteDispatchManifest, TierDeleteDispatchManifestState, TierDeleteDispatchParent,
|
||||
TierDeleteDispatchParentState, TierDeleteDispatchRecord, await_tier_delete_journal_recovery,
|
||||
decode_tier_delete_dispatch_record, decode_tier_delete_journal_entry, encode_tier_delete_dispatch_manifest,
|
||||
encode_tier_delete_dispatch_parent, encode_tier_delete_journal_entry, object_info_references_tier_delete,
|
||||
record_tier_delete_journal_backend_identity, same_tier_delete_authorization_identity, same_tier_delete_journal_identity,
|
||||
tier_delete_dispatch_child_matches_parent, tier_delete_dispatch_chunk_manifest_object_name,
|
||||
tier_delete_dispatch_journal_set_digest, tier_delete_dispatch_manifest_object_name, tier_delete_journal_object_name,
|
||||
tier_delete_source_matches_dispatch_scope, validate_legacy_tier_delete_recovery_source,
|
||||
tier_delete_source_matches_dispatch_scope,
|
||||
};
|
||||
use crate::bucket::lifecycle::tier_sweeper::{
|
||||
Jentry, TierDeleteDispatchBinding, TierDeleteJournalState, TierDeleteSourceIdentity,
|
||||
@@ -6648,72 +6417,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_recovery_export_rejects_fields_from_later_journal_versions() {
|
||||
let later = bound_v6_journal_entry(TierDeleteJournalState::Prepared);
|
||||
let v1 = PersistedTierDeleteJournalEntry {
|
||||
version: 1,
|
||||
obj_name: "remote/object".to_string(),
|
||||
version_id: "opaque".to_string(),
|
||||
tier_name: "WARM".to_string(),
|
||||
backend_identity: None,
|
||||
version_id_exact: None,
|
||||
version_state: None,
|
||||
state: None,
|
||||
source: None,
|
||||
dispatch: None,
|
||||
};
|
||||
let mut v2 = v1.clone();
|
||||
v2.version = TIER_DELETE_JOURNAL_VERSION;
|
||||
v2.backend_identity = Some([7; 32]);
|
||||
|
||||
let assert_rejected = |persisted: PersistedTierDeleteJournalEntry, schema: &str| {
|
||||
let normalized = persisted
|
||||
.clone()
|
||||
.into_jentry()
|
||||
.expect("the generic compatibility decoder should demonstrate the discarded field");
|
||||
let object_name = tier_delete_journal_object_name(&normalized);
|
||||
let encoded = serde_json::to_vec(&persisted).expect("mixed-version journal fixture should encode");
|
||||
let err = validate_legacy_tier_delete_recovery_source(&object_name, schema, &encoded)
|
||||
.expect_err("legacy recovery export must reject fields from later versions");
|
||||
assert!(err.to_string().contains("later version"));
|
||||
};
|
||||
|
||||
let mut invalid_v1 = Vec::new();
|
||||
let mut with_backend = v1.clone();
|
||||
with_backend.backend_identity = Some([7; 32]);
|
||||
invalid_v1.push(with_backend);
|
||||
for persisted in [&v1, &v2] {
|
||||
let schema = if persisted.version == 1 {
|
||||
TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA
|
||||
} else {
|
||||
TIER_DELETE_JOURNAL_V2_RECOVERY_SCHEMA
|
||||
};
|
||||
let mut invalid = Vec::new();
|
||||
let mut with_exact = persisted.clone();
|
||||
with_exact.version_id_exact = Some(false);
|
||||
invalid.push(with_exact);
|
||||
let mut with_version_state = persisted.clone();
|
||||
with_version_state.version_state = Some(rustfs_filemeta::TransitionVersionState::Unknown);
|
||||
invalid.push(with_version_state);
|
||||
let mut with_state = persisted.clone();
|
||||
with_state.state = Some(TierDeleteJournalState::Committed);
|
||||
invalid.push(with_state);
|
||||
let mut with_source = persisted.clone();
|
||||
with_source.source = later.source.clone();
|
||||
invalid.push(with_source);
|
||||
let mut with_dispatch = persisted.clone();
|
||||
with_dispatch.dispatch = later.dispatch.clone();
|
||||
invalid.push(with_dispatch);
|
||||
for record in invalid {
|
||||
assert_rejected(record, schema);
|
||||
}
|
||||
}
|
||||
for record in invalid_v1 {
|
||||
assert_rejected(record, TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_journal_path_is_stable_and_sanitized() {
|
||||
let je = journal_entry();
|
||||
|
||||
@@ -575,7 +575,7 @@ pub(crate) async fn delete_confirmed_transition_candidate_exact_with_lease_idemp
|
||||
#[cfg(test)]
|
||||
static CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn delete_confirmed_transition_candidate_exact_with_manager_and_identity(
|
||||
obj_name: &str,
|
||||
rv_id: &str,
|
||||
@@ -706,16 +706,15 @@ pub(crate) fn transitioned_delete_journal_entry_for_source(
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
#[cfg(feature = "test-util")]
|
||||
use super::delete_confirmed_transition_candidate_exact_with_manager_and_identity;
|
||||
use rustfs_s3_client::signer_error::invalid_utf8_header_error;
|
||||
|
||||
use super::{
|
||||
CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES, ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED, Jentry,
|
||||
RemoteDeleteBreaker, RemoteTierDeleteOutcome, TierDeleteJournalState, TierDeleteSourceIdentity,
|
||||
delete_object_from_remote_tier_idempotent, delete_object_from_remote_tier_idempotent_with_manager_and_identity,
|
||||
is_remote_tier_not_found_error, is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook,
|
||||
should_record_remote_delete_failure, transitioned_delete_journal_entry, transitioned_force_delete_journal_entry,
|
||||
delete_confirmed_transition_candidate_exact_with_manager_and_identity, delete_object_from_remote_tier_idempotent,
|
||||
delete_object_from_remote_tier_idempotent_with_manager_and_identity, is_remote_tier_not_found_error,
|
||||
is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook, should_record_remote_delete_failure,
|
||||
transitioned_delete_journal_entry, transitioned_force_delete_journal_entry,
|
||||
};
|
||||
use crate::storage_api_contracts::lifecycle::TransitionedObject;
|
||||
use rustfs_filemeta::TransitionVersionState;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -791,22 +791,9 @@ impl BucketMetadata {
|
||||
}
|
||||
}
|
||||
|
||||
/// Replace one config payload and stamp its `*_config_updated_at` with the
|
||||
/// local clock. This is the entry for edits that originate here: the
|
||||
/// local write time is the edit's source time.
|
||||
pub fn update_config(&mut self, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
|
||||
self.update_config_at(config_file, data, OffsetDateTime::now_utc())
|
||||
}
|
||||
let updated = OffsetDateTime::now_utc();
|
||||
|
||||
/// [`Self::update_config`] with an explicit `updated_at` stamp.
|
||||
///
|
||||
/// For a config replicated from another site the edit's source time is
|
||||
/// the peer's `updated_at`, not the moment it lands here: staleness of
|
||||
/// the next incoming item is judged against the stored stamp, so stamping
|
||||
/// the local apply time would reject a newer source edit that was merely
|
||||
/// delivered late (backlog#2292). Only replication receivers should pass
|
||||
/// a foreign time; local edits keep [`Self::update_config`].
|
||||
pub fn update_config_at(&mut self, config_file: &str, data: Vec<u8>, updated: OffsetDateTime) -> Result<OffsetDateTime> {
|
||||
match config_file {
|
||||
BUCKET_POLICY_CONFIG => {
|
||||
self.policy_config_json = data;
|
||||
@@ -1538,39 +1525,6 @@ mod test {
|
||||
assert_eq!(metadata.bucket_incarnation_id, incarnation);
|
||||
}
|
||||
|
||||
/// backlog#2292: a replicated config is stamped with the source
|
||||
/// `updated_at` it was given, not the local clock, while the plain
|
||||
/// `update_config` entry keeps stamping the local clock.
|
||||
#[test]
|
||||
fn update_config_at_stamps_the_given_time_and_update_config_stamps_now() {
|
||||
let source_time = OffsetDateTime::now_utc() - time::Duration::hours(3);
|
||||
let mut metadata = BucketMetadata::new("source-stamped");
|
||||
|
||||
let stamped = metadata
|
||||
.update_config_at(BUCKET_POLICY_CONFIG, br#"{"Version":"2012-10-17","Statement":[]}"#.to_vec(), source_time)
|
||||
.unwrap();
|
||||
assert_eq!(stamped, source_time);
|
||||
assert_eq!(metadata.policy_config_updated_at, source_time);
|
||||
|
||||
let tagging = b"<Tagging><TagSet><Tag><Key>k</Key><Value>v</Value></Tag></TagSet></Tagging>".to_vec();
|
||||
let stamped = metadata
|
||||
.update_config_at(BUCKET_TAGGING_CONFIG, tagging, source_time)
|
||||
.unwrap();
|
||||
assert_eq!(stamped, source_time);
|
||||
assert_eq!(metadata.tagging_config_updated_at, source_time);
|
||||
|
||||
let before = OffsetDateTime::now_utc();
|
||||
let stamped = metadata
|
||||
.update_config(BUCKET_POLICY_CONFIG, br#"{"Version":"2012-10-17","Statement":[]}"#.to_vec())
|
||||
.unwrap();
|
||||
assert!(stamped >= before, "a local edit is stamped with the local clock");
|
||||
assert_eq!(metadata.policy_config_updated_at, stamped);
|
||||
assert_eq!(
|
||||
metadata.tagging_config_updated_at, source_time,
|
||||
"restamping one config must not move another config's stamp"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_locking_requires_lock_metadata_not_plain_versioning() {
|
||||
use s3s::dto::ObjectLockEnabled;
|
||||
@@ -1657,34 +1611,6 @@ mod test {
|
||||
assert!(bm.bucket_target_config.is_none());
|
||||
}
|
||||
|
||||
/// rustfs/backlog#2309: the MinIO-origin `.metadata.bin` this repository
|
||||
/// already carries as a compatibility fixture stores
|
||||
/// `BucketTargetsConfigJSON` as a bare JSON array, which `BucketTargets`
|
||||
/// (a `{"targets":[…]}` struct with no array fallback) cannot decode. The
|
||||
/// bytes below are the exact payload the fixture in
|
||||
/// `metadata_test.rs::TEST_BUCKET_METADATA_HEX` decodes to, so if RustFS
|
||||
/// ever grows the array-shaped compatibility parse, this test is where the
|
||||
/// upgrade break is pinned and where the decision has to be recorded.
|
||||
#[test]
|
||||
fn minio_array_shaped_bucket_targets_are_unreadable() {
|
||||
let minio_array = br#"[{"endpoint":"http://target.example.com","targetBucket":"tb","region":"us-east-1"}]"#.to_vec();
|
||||
let mut bm = BucketMetadata::new("minio-array-targets");
|
||||
bm.bucket_targets_config_json = minio_array.clone();
|
||||
|
||||
bm.parse_all_configs()
|
||||
.expect("a MinIO-shaped targets blob must not fail the whole metadata load");
|
||||
|
||||
assert!(
|
||||
bm.bucket_targets_unreadable(),
|
||||
"an array-shaped MinIO targets blob is unreadable, not an empty target set"
|
||||
);
|
||||
assert!(bm.bucket_target_config.is_none());
|
||||
assert_eq!(
|
||||
bm.bucket_targets_config_json, minio_array,
|
||||
"the raw MinIO bytes must survive so the configuration stays recoverable"
|
||||
);
|
||||
}
|
||||
|
||||
/// The invariant every branch of `parse_all_configs` shares: a stored but
|
||||
/// undecodable payload keeps its raw bytes and leaves the typed field
|
||||
/// `None`, so no branch fabricates a value. What a reader may then do with
|
||||
|
||||
@@ -567,32 +567,6 @@ pub async fn update_if_incarnation(
|
||||
config_file,
|
||||
data,
|
||||
Some(expected_incarnation_id),
|
||||
None,
|
||||
))
|
||||
.await
|
||||
}
|
||||
|
||||
/// [`update_if_incarnation`] stamping the config with `updated_at` instead of
|
||||
/// the local clock.
|
||||
///
|
||||
/// For a site-replication receiver the edit's source time is the peer's
|
||||
/// `updated_at`; persisting it keeps the stored `*_config_updated_at` on the
|
||||
/// source clock so the next item's staleness is judged source-time against
|
||||
/// source-time (backlog#2292). See [`BucketMetadata::update_config_at`].
|
||||
pub async fn update_if_incarnation_at(
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
data: Vec<u8>,
|
||||
expected_incarnation_id: Uuid,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<OffsetDateTime> {
|
||||
Box::pin(update_with_sys_expected(
|
||||
get_bucket_metadata_sys()?,
|
||||
bucket,
|
||||
config_file,
|
||||
data,
|
||||
Some(expected_incarnation_id),
|
||||
Some(updated_at),
|
||||
))
|
||||
.await
|
||||
}
|
||||
@@ -603,30 +577,6 @@ pub async fn delete_if_incarnation(bucket: &str, config_file: &str, expected_inc
|
||||
bucket,
|
||||
config_file,
|
||||
Some(expected_incarnation_id),
|
||||
None,
|
||||
))
|
||||
.await
|
||||
}
|
||||
|
||||
/// [`delete_if_incarnation`] stamping the cleared config with `updated_at`
|
||||
/// (a replicated deletion's source time) instead of the local clock.
|
||||
///
|
||||
/// The stamp survives the deletion as the config's `*_config_updated_at`, and
|
||||
/// that is what the next incoming item is judged against: a local stamp on
|
||||
/// the delete would reject a newer source re-create that was merely delivered
|
||||
/// later (backlog#2292). See [`update_if_incarnation_at`].
|
||||
pub async fn delete_if_incarnation_at(
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
expected_incarnation_id: Uuid,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<OffsetDateTime> {
|
||||
Box::pin(delete_with_sys_expected(
|
||||
get_bucket_metadata_sys()?,
|
||||
bucket,
|
||||
config_file,
|
||||
Some(expected_incarnation_id),
|
||||
Some(updated_at),
|
||||
))
|
||||
.await
|
||||
}
|
||||
@@ -648,41 +598,34 @@ async fn update_with_sys(
|
||||
config_file: &str,
|
||||
data: Vec<u8>,
|
||||
) -> Result<OffsetDateTime> {
|
||||
update_with_sys_expected(sys, bucket, config_file, data, None, None).await
|
||||
update_with_sys_expected(sys, bucket, config_file, data, None).await
|
||||
}
|
||||
|
||||
/// `updated_at` is the stamp persisted on the config; `None` uses the local
|
||||
/// clock (the edit originates here), `Some` carries a replicated edit's
|
||||
/// source time (backlog#2292).
|
||||
async fn update_with_sys_expected(
|
||||
sys: Arc<RwLock<BucketMetadataSys>>,
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
data: Vec<u8>,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
updated_at: Option<OffsetDateTime>,
|
||||
) -> Result<OffsetDateTime> {
|
||||
let guard = acquire_config_write_guard_for_incarnation(sys.clone(), bucket, expected_incarnation_id).await?;
|
||||
update_under_config_write_guard(sys, &guard, config_file, data, updated_at).await
|
||||
update_under_config_write_guard(sys, &guard, config_file, data).await
|
||||
}
|
||||
|
||||
/// [`delete`] against an explicitly supplied metadata system. See
|
||||
/// [`update_with_sys`].
|
||||
async fn delete_with_sys(sys: Arc<RwLock<BucketMetadataSys>>, bucket: &str, config_file: &str) -> Result<OffsetDateTime> {
|
||||
delete_with_sys_expected(sys, bucket, config_file, None, None).await
|
||||
delete_with_sys_expected(sys, bucket, config_file, None).await
|
||||
}
|
||||
|
||||
/// `updated_at`: `None` stamps the local clock; `Some` persists a replicated
|
||||
/// deletion's source time (backlog#2292).
|
||||
async fn delete_with_sys_expected(
|
||||
sys: Arc<RwLock<BucketMetadataSys>>,
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
updated_at: Option<OffsetDateTime>,
|
||||
) -> Result<OffsetDateTime> {
|
||||
let guard = acquire_config_write_guard_for_incarnation(sys.clone(), bucket, expected_incarnation_id).await?;
|
||||
delete_under_config_write_guard(sys, &guard, config_file, updated_at).await
|
||||
delete_under_config_write_guard(sys, &guard, config_file).await
|
||||
}
|
||||
|
||||
/// Owns the complete bucket-config mutation fence.
|
||||
@@ -829,21 +772,7 @@ pub async fn update_under_transaction_lock(
|
||||
data: Vec<u8>,
|
||||
) -> Result<OffsetDateTime> {
|
||||
guard.ensure_valid(bucket)?;
|
||||
update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data, None).await
|
||||
}
|
||||
|
||||
/// [`update_under_transaction_lock`] stamping the config with `updated_at`
|
||||
/// (a replicated edit's source time) instead of the local clock; see
|
||||
/// [`update_if_incarnation_at`] (backlog#2292).
|
||||
pub async fn update_under_transaction_lock_at(
|
||||
guard: &BucketMetadataMutationGuard,
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
data: Vec<u8>,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<OffsetDateTime> {
|
||||
guard.ensure_valid(bucket)?;
|
||||
update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data, Some(updated_at)).await
|
||||
update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data).await
|
||||
}
|
||||
|
||||
/// Clear one config file while the caller holds this bucket's transaction lock.
|
||||
@@ -853,7 +782,7 @@ pub async fn delete_under_transaction_lock(
|
||||
config_file: &str,
|
||||
) -> Result<OffsetDateTime> {
|
||||
guard.ensure_valid(bucket)?;
|
||||
delete_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, None).await
|
||||
delete_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file).await
|
||||
}
|
||||
|
||||
pub async fn update_quota_if_incarnation(
|
||||
@@ -861,29 +790,6 @@ pub async fn update_quota_if_incarnation(
|
||||
data: Vec<u8>,
|
||||
expected_incarnation_id: Uuid,
|
||||
proof: &crate::services::notification_sys::CrossPoolFenceFleetProofToken,
|
||||
) -> Result<OffsetDateTime> {
|
||||
update_quota_if_incarnation_stamped(bucket, data, expected_incarnation_id, proof, None).await
|
||||
}
|
||||
|
||||
/// [`update_quota_if_incarnation`] stamping the quota config with
|
||||
/// `updated_at` (a replicated edit's source time) instead of the local
|
||||
/// clock; see [`update_if_incarnation_at`] (backlog#2292).
|
||||
pub async fn update_quota_if_incarnation_at(
|
||||
bucket: &str,
|
||||
data: Vec<u8>,
|
||||
expected_incarnation_id: Uuid,
|
||||
proof: &crate::services::notification_sys::CrossPoolFenceFleetProofToken,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<OffsetDateTime> {
|
||||
update_quota_if_incarnation_stamped(bucket, data, expected_incarnation_id, proof, Some(updated_at)).await
|
||||
}
|
||||
|
||||
async fn update_quota_if_incarnation_stamped(
|
||||
bucket: &str,
|
||||
data: Vec<u8>,
|
||||
expected_incarnation_id: Uuid,
|
||||
proof: &crate::services::notification_sys::CrossPoolFenceFleetProofToken,
|
||||
updated_at: Option<OffsetDateTime>,
|
||||
) -> Result<OffsetDateTime> {
|
||||
let sys = get_bucket_metadata_sys()?;
|
||||
let guard = Box::pin(acquire_config_write_guard_for_incarnation(
|
||||
@@ -901,7 +807,7 @@ async fn update_quota_if_incarnation_stamped(
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
update_under_config_write_guard(sys, &guard, rustfs_config::QUOTA_CONFIG_FILE, data, updated_at).await
|
||||
update_under_config_write_guard(sys, &guard, rustfs_config::QUOTA_CONFIG_FILE, data).await
|
||||
}
|
||||
|
||||
pub async fn update_bucket_targets_under_transaction_lock(
|
||||
@@ -917,7 +823,6 @@ async fn update_under_config_write_guard(
|
||||
guard: &BucketMetadataMutationGuard,
|
||||
config_file: &str,
|
||||
data: Vec<u8>,
|
||||
updated_at: Option<OffsetDateTime>,
|
||||
) -> Result<OffsetDateTime> {
|
||||
guard.ensure_valid(&guard.bucket)?;
|
||||
let metadata_sys = sys.read().await.clone();
|
||||
@@ -929,7 +834,7 @@ async fn update_under_config_write_guard(
|
||||
Some(&guard.transaction_guard),
|
||||
&guard.bucket,
|
||||
"bucket config transaction",
|
||||
metadata_sys.update_checked(&guard.bucket, config_file, data, true, guard.incarnation_id, updated_at),
|
||||
metadata_sys.update_checked(&guard.bucket, config_file, data, true, guard.incarnation_id),
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
@@ -941,7 +846,6 @@ async fn delete_under_config_write_guard(
|
||||
sys: Arc<RwLock<BucketMetadataSys>>,
|
||||
guard: &BucketMetadataMutationGuard,
|
||||
config_file: &str,
|
||||
updated_at: Option<OffsetDateTime>,
|
||||
) -> Result<OffsetDateTime> {
|
||||
guard.ensure_valid(&guard.bucket)?;
|
||||
let metadata_sys = sys.read().await.clone();
|
||||
@@ -953,7 +857,7 @@ async fn delete_under_config_write_guard(
|
||||
Some(&guard.transaction_guard),
|
||||
&guard.bucket,
|
||||
"bucket config deletion transaction",
|
||||
metadata_sys.update_checked(&guard.bucket, config_file, Vec::new(), false, guard.incarnation_id, updated_at),
|
||||
metadata_sys.update_checked(&guard.bucket, config_file, Vec::new(), false, guard.incarnation_id),
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
@@ -1858,17 +1762,15 @@ impl BucketMetadataSys {
|
||||
/// `update` and the config read alone). Keep these boxed.
|
||||
pub async fn update(&self, bucket: &str, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
|
||||
let incarnation_id = Box::pin(self.get_bucket_incarnation_id(bucket)).await?;
|
||||
Box::pin(self.update_checked(bucket, config_file, data, true, incarnation_id, None)).await
|
||||
Box::pin(self.update_checked(bucket, config_file, data, true, incarnation_id)).await
|
||||
}
|
||||
|
||||
pub async fn delete(&self, bucket: &str, config_file: &str) -> Result<OffsetDateTime> {
|
||||
let incarnation_id = self.get_bucket_incarnation_id(bucket).await?;
|
||||
self.update_checked(bucket, config_file, Vec::new(), false, incarnation_id, None)
|
||||
self.update_checked(bucket, config_file, Vec::new(), false, incarnation_id)
|
||||
.await
|
||||
}
|
||||
|
||||
/// `updated_at`: `None` stamps the local clock; `Some` persists a
|
||||
/// replicated edit's source time (backlog#2292).
|
||||
async fn update_checked(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -1876,7 +1778,6 @@ impl BucketMetadataSys {
|
||||
data: Vec<u8>,
|
||||
parse: bool,
|
||||
expected_incarnation_id: Uuid,
|
||||
updated_at: Option<OffsetDateTime>,
|
||||
) -> Result<OffsetDateTime> {
|
||||
// Load through this system's own store, the one `save` persists to
|
||||
// (backlog#1052 S7). Reading from the ambient handle instead made the
|
||||
@@ -1887,10 +1788,7 @@ impl BucketMetadataSys {
|
||||
return Err(Error::BucketNotFound(bucket.to_string()));
|
||||
}
|
||||
|
||||
let updated = match updated_at {
|
||||
Some(updated_at) => bm.update_config_at(config_file, data, updated_at)?,
|
||||
None => bm.update_config(config_file, data)?,
|
||||
};
|
||||
let updated = bm.update_config(config_file, data)?;
|
||||
|
||||
Box::pin(self.save(bm)).await?;
|
||||
|
||||
@@ -3857,106 +3755,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// backlog#2292: the explicit-stamp write path persists the given source
|
||||
/// time as the config's `*_config_updated_at` — through the incarnation
|
||||
/// path and through an already-held transaction guard — and survives a
|
||||
/// reload from disk, while the plain path keeps stamping the local clock.
|
||||
#[tokio::test]
|
||||
async fn explicit_updated_at_is_persisted_as_the_config_stamp() {
|
||||
let (dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
let bucket = "source-stamped-config";
|
||||
for dir in &dirs {
|
||||
std::fs::create_dir_all(dir.path().join(bucket)).expect("bucket volume should be created");
|
||||
}
|
||||
let sys = Arc::new(RwLock::new(BucketMetadataSys::new(ecstore)));
|
||||
let source_time = OffsetDateTime::now_utc() - Duration::from_secs(3 * 3600);
|
||||
let policy = br#"{"Version":"2012-10-17","Statement":[]}"#.to_vec();
|
||||
let tagging = b"<Tagging><TagSet><Tag><Key>k</Key><Value>v</Value></Tag></TagSet></Tagging>".to_vec();
|
||||
|
||||
// Incarnation path (`update_if_incarnation_at` minus the ambient lookup).
|
||||
let stamped =
|
||||
update_with_sys_expected(sys.clone(), bucket, BUCKET_POLICY_CONFIG, policy.clone(), None, Some(source_time))
|
||||
.await
|
||||
.expect("source-stamped policy write should persist");
|
||||
assert_eq!(stamped, source_time);
|
||||
|
||||
// Held-guard path (`update_under_transaction_lock_at` minus the ambient lookup).
|
||||
let guard = acquire_config_write_guard(sys.clone(), bucket).await.expect("write guard");
|
||||
let stamped = update_under_config_write_guard(sys.clone(), &guard, BUCKET_TAGGING_CONFIG, tagging, Some(source_time))
|
||||
.await
|
||||
.expect("source-stamped tagging write should persist");
|
||||
drop(guard);
|
||||
assert_eq!(stamped, source_time);
|
||||
|
||||
let metadata_sys = sys.read().await.clone();
|
||||
metadata_sys.metadata_map.write().await.clear();
|
||||
let reloaded = metadata_sys.get_config_from_disk(bucket).await.expect("reload from disk");
|
||||
assert_eq!(reloaded.policy_config_updated_at, source_time);
|
||||
assert_eq!(reloaded.tagging_config_updated_at, source_time);
|
||||
|
||||
// The plain path is unchanged: a local edit is stamped with the local clock.
|
||||
let before = OffsetDateTime::now_utc();
|
||||
let stamped = update_with_sys(sys.clone(), bucket, BUCKET_POLICY_CONFIG, policy)
|
||||
.await
|
||||
.expect("locally stamped policy write should persist");
|
||||
assert!(stamped >= before, "the plain write path must keep stamping the local clock");
|
||||
let reloaded = metadata_sys.get_config_from_disk(bucket).await.expect("reload from disk");
|
||||
assert_eq!(reloaded.policy_config_updated_at, stamped);
|
||||
assert_eq!(
|
||||
reloaded.tagging_config_updated_at, source_time,
|
||||
"an unrelated config keeps its source stamp"
|
||||
);
|
||||
}
|
||||
|
||||
/// backlog#2292: a replicated delete persists the source time as the
|
||||
/// cleared config's `*_config_updated_at`, so the receive-side gate
|
||||
/// (source time against stored stamp) lets a newer source re-create land
|
||||
/// even when the delete was applied later than the re-create's source
|
||||
/// time; the plain delete keeps stamping the local clock.
|
||||
#[tokio::test]
|
||||
async fn explicit_updated_at_is_persisted_by_a_delete() {
|
||||
let (dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
let bucket = "source-stamped-delete";
|
||||
for dir in &dirs {
|
||||
std::fs::create_dir_all(dir.path().join(bucket)).expect("bucket volume should be created");
|
||||
}
|
||||
let sys = Arc::new(RwLock::new(BucketMetadataSys::new(ecstore)));
|
||||
let policy = br#"{"Version":"2012-10-17","Statement":[]}"#.to_vec();
|
||||
let created_at = OffsetDateTime::now_utc() - Duration::from_secs(3 * 3600);
|
||||
let deleted_at = created_at + Duration::from_secs(60);
|
||||
let recreated_at = deleted_at + Duration::from_secs(60);
|
||||
|
||||
update_with_sys_expected(sys.clone(), bucket, BUCKET_POLICY_CONFIG, policy.clone(), None, Some(created_at))
|
||||
.await
|
||||
.expect("source-stamped policy write should persist");
|
||||
let stamped = delete_with_sys_expected(sys.clone(), bucket, BUCKET_POLICY_CONFIG, None, Some(deleted_at))
|
||||
.await
|
||||
.expect("source-stamped policy delete should persist");
|
||||
assert_eq!(stamped, deleted_at);
|
||||
|
||||
let metadata_sys = sys.read().await.clone();
|
||||
metadata_sys.metadata_map.write().await.clear();
|
||||
let reloaded = metadata_sys.get_config_from_disk(bucket).await.expect("reload from disk");
|
||||
assert!(reloaded.policy_config_json.is_empty(), "the delete cleared the payload");
|
||||
assert_eq!(reloaded.policy_config_updated_at, deleted_at, "the delete kept the source stamp");
|
||||
assert!(
|
||||
recreated_at >= reloaded.policy_config_updated_at,
|
||||
"a re-create newer than the delete's source time is not stale against the stored stamp"
|
||||
);
|
||||
|
||||
// The plain delete path is unchanged: stamped with the local clock.
|
||||
update_with_sys_expected(sys.clone(), bucket, BUCKET_POLICY_CONFIG, policy, None, Some(recreated_at))
|
||||
.await
|
||||
.expect("re-create should persist");
|
||||
let before = OffsetDateTime::now_utc();
|
||||
let stamped = delete_with_sys_expected(sys.clone(), bucket, BUCKET_POLICY_CONFIG, None, None)
|
||||
.await
|
||||
.expect("locally stamped delete should persist");
|
||||
assert!(stamped >= before, "the plain delete path must keep stamping the local clock");
|
||||
let reloaded = metadata_sys.get_config_from_disk(bucket).await.expect("reload from disk");
|
||||
assert_eq!(reloaded.policy_config_updated_at, stamped);
|
||||
}
|
||||
|
||||
/// The load and the persisted write share one write guard, so concurrent
|
||||
/// rewrites of the same config compose instead of clobbering each other.
|
||||
/// Moving the load outside that guard loses all but the last tag.
|
||||
@@ -4173,16 +3971,10 @@ mod tests {
|
||||
let new_incarnation = store.bucket_incarnation_id_from_disk(bucket).await.unwrap();
|
||||
assert_ne!(old_incarnation, new_incarnation);
|
||||
|
||||
let err = update_with_sys_expected(
|
||||
sys.clone(),
|
||||
bucket,
|
||||
BUCKET_TAGGING_CONFIG,
|
||||
b"<Tagging/>".to_vec(),
|
||||
Some(old_incarnation),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect_err("a request authorized for the deleted incarnation must fail closed");
|
||||
let err =
|
||||
update_with_sys_expected(sys.clone(), bucket, BUCKET_TAGGING_CONFIG, b"<Tagging/>".to_vec(), Some(old_incarnation))
|
||||
.await
|
||||
.expect_err("a request authorized for the deleted incarnation must fail closed");
|
||||
assert!(matches!(err, Error::BucketNotFound(name) if name == bucket));
|
||||
|
||||
let persisted = sys.read().await.get_config_from_disk(bucket).await.unwrap();
|
||||
@@ -4217,7 +4009,7 @@ mod tests {
|
||||
}],
|
||||
})
|
||||
.unwrap();
|
||||
update_under_config_write_guard(sys, &guard, BUCKET_TAGGING_CONFIG, tagging, None)
|
||||
update_under_config_write_guard(sys, &guard, BUCKET_TAGGING_CONFIG, tagging)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(!delete.is_finished());
|
||||
|
||||
@@ -66,7 +66,6 @@ pub(crate) use replication_lifecycle_bridge::ReplicationLifecycleBridge;
|
||||
pub(crate) use replication_migration_bridge::ReplicationMigrationBridge;
|
||||
pub use replication_object_bridge::ReplicationObjectBridge;
|
||||
pub use replication_object_config::{DeleteReplicationConfigSnapshot, ReplicationConfig};
|
||||
pub(crate) use replication_object_decision_boundary::replication_etags_match;
|
||||
pub use replication_object_decision_boundary::{
|
||||
MustReplicateOptions, ReplicationDeleteScheduleInput, ReplicationDeleteStateSource, delete_replication_state_from_config,
|
||||
delete_replication_version_id, should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
||||
@@ -89,6 +88,5 @@ pub use replication_state::{ReplicationStats, RuntimeReplicationTargetBacklog};
|
||||
pub use replication_stats_boundary::{BucketReplicationStat, BucketReplicationStats, BucketStats, InQueueMetric, XferStats};
|
||||
pub use replication_storage_boundary::{ReplicationObjectIO, ReplicationStorage};
|
||||
pub use replication_target_boundary::SsecPassthroughCapability;
|
||||
pub use replication_target_boundary::VersionIdentityCapability;
|
||||
pub use replication_target_boundary::{ObjectLockIntegrity, object_lock_put_integrity};
|
||||
pub(crate) use replication_target_config_bridge::ReplicationTargetConfigBridge;
|
||||
|
||||
@@ -12,8 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_filemeta::ObjectPartInfo;
|
||||
pub use rustfs_replication::{MrfOpKind, MrfReplicateEntry};
|
||||
pub(crate) use rustfs_replication::{
|
||||
REPLICATE_EXISTING, REPLICATE_HEAL_DELETE, ReplicateTargetDecision, ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction,
|
||||
|
||||
@@ -12,8 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_replication::ReplicationMultipartPlanError;
|
||||
pub use rustfs_replication::{
|
||||
MustReplicateOptions, ReplicationDeleteScheduleInput, ReplicationDeleteStateSource, delete_replication_state_from_config,
|
||||
delete_replication_version_id, should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
||||
@@ -22,9 +20,9 @@ pub use rustfs_replication::{
|
||||
pub(crate) use rustfs_replication::{
|
||||
ReplicationDeleteSource, ReplicationMultipartPartInput, ReplicationResyncTargetObject, delete_marker_purge_mrf_entry,
|
||||
delete_marker_purge_version_id, delete_replication_creates_marker, delete_replication_missing_source_decision,
|
||||
delete_replication_object_opts, delete_replication_target_version_id, heal_uses_delete_replication_path,
|
||||
is_object_lock_denied_delete, is_retryable_delete_replication_head_error, is_version_delete_replication,
|
||||
replicate_delete_outcome, replication_etags_match, replication_multipart_complete_actual_size,
|
||||
replication_multipart_part_plan, replication_single_put_size_error, resync_existing_delete_replication_info,
|
||||
resync_target_for_object, should_retry_delete_marker_purge, single_part_replica_etag_mismatch,
|
||||
delete_replication_object_opts, heal_uses_delete_replication_path, is_object_lock_denied_delete,
|
||||
is_retryable_delete_replication_head_error, is_version_delete_replication, replicate_delete_outcome, replication_etags_match,
|
||||
replication_multipart_complete_actual_size, replication_multipart_part_plan, replication_single_put_size_error,
|
||||
resync_existing_delete_replication_info, resync_target_for_object, should_retry_delete_marker_purge,
|
||||
single_part_replica_etag_mismatch, target_delete_version_id,
|
||||
};
|
||||
|
||||
@@ -75,7 +75,6 @@ use tracing::{debug, info, instrument, warn};
|
||||
const EVENT_REPLICATION_WORKER_RESIZE_SKIPPED: &str = "replication_worker_resize_skipped";
|
||||
const EVENT_REPLICATION_WORKER_RESIZED: &str = "replication_worker_resized";
|
||||
const EVENT_REPLICATION_BACKPRESSURE: &str = "replication_backpressure";
|
||||
const EVENT_REPLICATION_IN_FLIGHT_SKIPPED: &str = "replication_in_flight_skipped";
|
||||
const EVENT_REPLICATION_RESYNC_LOAD_SKIPPED: &str = "replication_resync_load_skipped";
|
||||
const EVENT_REPLICATION_RESYNC_RECOVERED: &str = "replication_resync_recovered";
|
||||
const EVENT_REPLICATION_MRF_QUEUE_UNAVAILABLE: &str = "replication_mrf_queue_unavailable";
|
||||
@@ -883,20 +882,6 @@ fn reconstructed_heal_delete_info(
|
||||
) -> DeletedObjectReplicationInfo {
|
||||
let mut rstate = oi.replication_state();
|
||||
rstate.replicate_decision_str = dsc.to_string();
|
||||
// The caller hands us a blank ObjectInfo (the source marker may already be
|
||||
// gone), so the state above carries no target-assigned marker version ids.
|
||||
// Restore them from the journal: `delete_marker_purge_version_id` must hit
|
||||
// the id the target reported, not fall back to the source marker id, which
|
||||
// a target that mints its own ids answers with an idempotent 204 that would
|
||||
// acknowledge the intent while the real marker stays behind (backlog#2290).
|
||||
// The corrupt flag rides along so a refusal stays a refusal after restart.
|
||||
for (arn, version_id) in &entry.target_delete_marker_version_ids {
|
||||
rstate
|
||||
.target_delete_marker_version_ids
|
||||
.entry(arn.clone())
|
||||
.or_insert_with(|| version_id.clone());
|
||||
}
|
||||
rstate.target_delete_marker_version_ids_corrupt |= entry.target_delete_marker_version_ids_corrupt;
|
||||
|
||||
let delete_marker_mtime = entry
|
||||
.delete_marker_mtime
|
||||
@@ -1090,9 +1075,6 @@ pub struct ReplicationPool<S: ReplicationStorage> {
|
||||
workers: RwLock<Vec<Sender<ReplicationOperation>>>,
|
||||
lrg_workers: RwLock<Vec<Sender<ReplicationOperation>>>,
|
||||
|
||||
/// Object versions queued or being replicated right now (backlog#2362).
|
||||
in_flight: Arc<ReplicationInFlight>,
|
||||
|
||||
// MRF (Most Recent Failures) channels
|
||||
mrf_replica_tx: Sender<ReplicationOperation>,
|
||||
// Shared among N MRF workers; Arc allows spawning more than one worker.
|
||||
@@ -1151,7 +1133,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
storage,
|
||||
workers: RwLock::new(Vec::new()),
|
||||
lrg_workers: RwLock::new(Vec::new()),
|
||||
in_flight: Arc::new(ReplicationInFlight::default()),
|
||||
mrf_replica_tx,
|
||||
mrf_replica_rx: Arc::new(Mutex::new(mrf_replica_rx)),
|
||||
mrf_save_tx,
|
||||
@@ -1207,13 +1188,12 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
let active_counter = self.active_lrg_workers.clone();
|
||||
let storage = self.storage.clone();
|
||||
let stats = self.stats.clone();
|
||||
let in_flight = self.in_flight.clone();
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
let mut rx = rx;
|
||||
while let Some(operation) = rx.recv().await {
|
||||
let _active = ActiveWorkerGuard::new(active_counter.clone());
|
||||
process_replication_operation(operation, stats.clone(), storage.clone(), in_flight.clone()).await;
|
||||
process_replication_operation(operation, stats.clone(), storage.clone()).await;
|
||||
}
|
||||
});
|
||||
|
||||
@@ -1267,13 +1247,12 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
let active_counter = self.active_workers.clone();
|
||||
let stats = self.stats.clone();
|
||||
let storage = self.storage.clone();
|
||||
let in_flight = self.in_flight.clone();
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
let mut rx = rx;
|
||||
while let Some(operation) = rx.recv().await {
|
||||
let _active = ActiveWorkerGuard::new(active_counter.clone());
|
||||
process_replication_operation(operation, stats.clone(), storage.clone(), in_flight.clone()).await;
|
||||
process_replication_operation(operation, stats.clone(), storage.clone()).await;
|
||||
}
|
||||
});
|
||||
|
||||
@@ -1312,7 +1291,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
let active_counter = self.active_mrf_workers.clone();
|
||||
let stats = self.stats.clone();
|
||||
let storage = self.storage.clone();
|
||||
let in_flight = self.in_flight.clone();
|
||||
let mrf_rx = Arc::clone(&self.mrf_replica_rx);
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
@@ -1332,7 +1310,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
let Some(operation) = operation else { break };
|
||||
|
||||
let _active = ActiveWorkerGuard::new(active_counter.clone());
|
||||
process_replication_operation(operation, stats.clone(), storage.clone(), in_flight.clone()).await;
|
||||
process_replication_operation(operation, stats.clone(), storage.clone()).await;
|
||||
}
|
||||
});
|
||||
self.task_handles.lock().await.push(handle);
|
||||
@@ -1462,24 +1440,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
|
||||
/// Queues a replica task
|
||||
pub async fn queue_replica_task(&self, ri: ReplicateObjectInfo) -> ReplicationQueueAdmission {
|
||||
// A version that is already queued or being uploaded is not driven a
|
||||
// second time: the scanner heal pass sees it as PENDING until the
|
||||
// first upload lands and would otherwise re-queue it every cycle
|
||||
// (backlog#2362). The key is released when the worker finishes, or
|
||||
// below when no worker accepts the task.
|
||||
if !self.in_flight.try_begin(&ri) {
|
||||
debug!(
|
||||
event = EVENT_REPLICATION_IN_FLIGHT_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
bucket = %ri.bucket,
|
||||
object = %ri.name,
|
||||
version_id = ?ri.version_id,
|
||||
op_type = ?ri.op_type,
|
||||
"Replication task already in flight; not queued again"
|
||||
);
|
||||
return ReplicationQueueAdmission::Skipped;
|
||||
}
|
||||
let target_arns = ri.dsc.replicate_target_arns();
|
||||
// If object is large, queue it to a static set of large workers
|
||||
if should_queue_large_object(ri.size) {
|
||||
@@ -1510,9 +1470,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
let resize = large_worker_backpressure_resize(existing, self.active_lrg_workers(), max_l_workers);
|
||||
drop(lrg_workers);
|
||||
|
||||
// Queue to MRF if worker is busy. The MRF replay re-enters
|
||||
// this function, so the version is no longer in flight.
|
||||
self.in_flight.finish(&ri);
|
||||
// Queue to MRF if worker is busy.
|
||||
let admission = self.queue_mrf_save_admission(ri.to_mrf_entry(), "large_object").await;
|
||||
|
||||
if let Some(resize) = resize {
|
||||
@@ -1521,7 +1479,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
return admission;
|
||||
}
|
||||
}
|
||||
self.in_flight.finish(&ri);
|
||||
return ReplicationQueueAdmission::Missed;
|
||||
}
|
||||
|
||||
@@ -1530,7 +1487,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
let ch = self.worker_queue_channel(&ri.op_type, &ri.bucket, &ri.name, ri.size).await;
|
||||
|
||||
let Some(channel) = ch else {
|
||||
self.in_flight.finish(&ri);
|
||||
return ReplicationQueueAdmission::Missed;
|
||||
};
|
||||
|
||||
@@ -1542,9 +1498,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
self.stats.dec_q(&ri.bucket, ri.size, ri.delete_marker, ri.op_type);
|
||||
self.stats.dec_target_q(&ri.bucket, &target_arns, ri.size);
|
||||
|
||||
// Queue to MRF if all workers are busy. The MRF replay re-enters this
|
||||
// function, so the version is no longer in flight.
|
||||
self.in_flight.finish(&ri);
|
||||
// Queue to MRF if all workers are busy.
|
||||
let admission = self.queue_mrf_save_admission(ri.to_mrf_entry(), "object").await;
|
||||
|
||||
// Try to scale up workers based on priority
|
||||
@@ -1843,7 +1797,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
) {
|
||||
while let Some(operation) = rx.recv().await {
|
||||
let _active = ActiveWorkerGuard::new(active_counter.clone());
|
||||
process_replication_operation(operation, stats.clone(), self.storage.clone(), self.in_flight.clone()).await;
|
||||
process_replication_operation(operation, stats.clone(), self.storage.clone()).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1861,7 +1815,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
) {
|
||||
while let Some(operation) = rx.recv().await {
|
||||
let _active = ActiveWorkerGuard::new(active_counter.clone());
|
||||
process_replication_operation(operation, stats.clone(), storage.clone(), self.in_flight.clone()).await;
|
||||
process_replication_operation(operation, stats.clone(), storage.clone()).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1878,7 +1832,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
) {
|
||||
while let Some(operation) = rx.recv().await {
|
||||
let _active = ActiveWorkerGuard::new(active_counter.clone());
|
||||
process_replication_operation(operation, stats.clone(), self.storage.clone(), self.in_flight.clone()).await;
|
||||
process_replication_operation(operation, stats.clone(), self.storage.clone()).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2313,64 +2267,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Object versions currently queued or being uploaded, keyed by bucket,
|
||||
/// object name and version. `queue_replica_task` admits a version only once
|
||||
/// while it is in flight; the scanner heal pass and MRF replays that arrive
|
||||
/// in the meantime are `Skipped` instead of driving a second complete upload
|
||||
/// (backlog#2362). Entries are removed when the worker finishes the task or
|
||||
/// when no worker accepted it.
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct ReplicationInFlight {
|
||||
keys: std::sync::Mutex<std::collections::HashSet<(String, String, Option<uuid::Uuid>)>>,
|
||||
}
|
||||
|
||||
impl ReplicationInFlight {
|
||||
fn lock(&self) -> std::sync::MutexGuard<'_, std::collections::HashSet<(String, String, Option<uuid::Uuid>)>> {
|
||||
self.keys.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
}
|
||||
|
||||
/// Claim `ri`; `false` when the same version is already in flight.
|
||||
fn try_begin(&self, ri: &ReplicateObjectInfo) -> bool {
|
||||
self.lock().insert((ri.bucket.clone(), ri.name.clone(), ri.version_id))
|
||||
}
|
||||
|
||||
fn finish(&self, ri: &ReplicateObjectInfo) {
|
||||
self.lock().remove(&(ri.bucket.clone(), ri.name.clone(), ri.version_id));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn len(&self) -> usize {
|
||||
self.lock().len()
|
||||
}
|
||||
}
|
||||
|
||||
/// Releases the in-flight claim when the worker is done with the task,
|
||||
/// including when replication panics.
|
||||
struct ReplicationInFlightGuard {
|
||||
in_flight: Arc<ReplicationInFlight>,
|
||||
key: ReplicateObjectInfo,
|
||||
}
|
||||
|
||||
impl ReplicationInFlightGuard {
|
||||
fn new(in_flight: Arc<ReplicationInFlight>, ri: &ReplicateObjectInfo) -> Self {
|
||||
Self {
|
||||
in_flight,
|
||||
key: ReplicateObjectInfo {
|
||||
bucket: ri.bucket.clone(),
|
||||
name: ri.name.clone(),
|
||||
version_id: ri.version_id,
|
||||
..Default::default()
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ReplicationInFlightGuard {
|
||||
fn drop(&mut self) {
|
||||
self.in_flight.finish(&self.key);
|
||||
}
|
||||
}
|
||||
|
||||
struct ActiveWorkerGuard {
|
||||
counter: Arc<AtomicI32>,
|
||||
}
|
||||
@@ -2432,12 +2328,10 @@ async fn process_replication_operation<S: ReplicationStorage>(
|
||||
operation: ReplicationOperation,
|
||||
stats: Arc<ReplicationStats>,
|
||||
storage: Arc<S>,
|
||||
in_flight: Arc<ReplicationInFlight>,
|
||||
) {
|
||||
match operation {
|
||||
ReplicationOperation::Object(obj_info) => {
|
||||
let _backlog = ReplicationBacklogGuard::for_object(stats, obj_info.as_ref());
|
||||
let _in_flight = ReplicationInFlightGuard::new(in_flight, obj_info.as_ref());
|
||||
replicate_object(*obj_info, storage).await;
|
||||
}
|
||||
ReplicationOperation::Delete(del_info) => {
|
||||
@@ -3171,11 +3065,7 @@ pub async fn queue_replication_heal(bucket: &str, oi: ObjectInfo, retry_count: u
|
||||
}
|
||||
|
||||
let rcfg = match ReplicationMetadataStore::optional_replication_config(bucket).await {
|
||||
Ok(Some(config)) => Some(config),
|
||||
// A bucket without a configuration still owes its pending purges an
|
||||
// answer: the delete worker finishes them locally as abandoned, which
|
||||
// is what makes the bucket deletable again (rustfs/backlog#2340).
|
||||
Ok(None) if owes_version_purge(&oi) => None,
|
||||
Ok(Some(config)) => config,
|
||||
Ok(None) => return ReplicationQueueAdmission::Skipped,
|
||||
Err(err) => {
|
||||
debug!(
|
||||
@@ -3225,7 +3115,7 @@ pub async fn queue_replication_heal(bucket: &str, oi: ObjectInfo, retry_count: u
|
||||
}
|
||||
};
|
||||
|
||||
let rcfg_wrapper = ReplicationConfig::new(rcfg, tgts);
|
||||
let rcfg_wrapper = ReplicationConfig::new(Some(rcfg), tgts);
|
||||
queue_replication_heal_internal(bucket, oi, rcfg_wrapper, retry_count)
|
||||
.await
|
||||
.admission
|
||||
@@ -3253,17 +3143,6 @@ pub async fn queue_replication_metadata(bucket: &str, oi: ObjectInfo, retry_coun
|
||||
}
|
||||
}
|
||||
|
||||
/// A version purge the persisted state still owes to named targets. Without
|
||||
/// the target list nothing can be settled, so such a version keeps the
|
||||
/// ordinary "no configuration, nothing to heal" skip.
|
||||
fn owes_version_purge(oi: &ObjectInfo) -> bool {
|
||||
!oi.version_purge_status.is_empty()
|
||||
&& oi
|
||||
.version_purge_status_internal
|
||||
.as_deref()
|
||||
.is_some_and(|statuses| !statuses.trim().is_empty())
|
||||
}
|
||||
|
||||
/// queue_replication_heal_internal enqueues objects that failed replication OR eligible for resyncing through
|
||||
/// an ongoing resync operation or via existing objects replication configuration setting.
|
||||
pub(crate) async fn queue_replication_heal_internal(
|
||||
@@ -3282,11 +3161,7 @@ pub(crate) async fn queue_replication_heal_internal(
|
||||
};
|
||||
}
|
||||
|
||||
// Without a configuration or targets there is nothing to replicate —
|
||||
// except a version purge the bucket still owes: its stored decision names
|
||||
// the targets, and the delete worker settles the ones no longer
|
||||
// configured as abandoned (rustfs/backlog#2340).
|
||||
if (rcfg.config.is_none() || rcfg.remotes.is_none()) && !owes_version_purge(&oi) {
|
||||
if rcfg.config.is_none() || rcfg.remotes.is_none() {
|
||||
return ReplicationHealQueueResult {
|
||||
object_info: roi,
|
||||
admission: ReplicationQueueAdmission::Skipped,
|
||||
@@ -3331,15 +3206,12 @@ pub(crate) async fn queue_replication_heal_internal(
|
||||
}
|
||||
ReplicationHealQueueAction::QueueDelete(dv) => {
|
||||
// A purge the peer denied under object lock cannot succeed until
|
||||
// the lock lapses (#6850), and one whose replica cannot be told
|
||||
// apart on a target that mints its own version ids cannot
|
||||
// succeed until the ledger or an operator resolves it
|
||||
// (rustfs/backlog#2340); requeuing either every heal cycle only
|
||||
// the lock lapses (#6850); requeuing it every heal cycle only
|
||||
// burns bandwidth and failure counters. The backoff expires on
|
||||
// its own, so the purge is probed again — and converges — once
|
||||
// the condition has a chance of being over.
|
||||
// the retention window has a chance of being over.
|
||||
if super::replication_object_decision_boundary::is_version_delete_replication(&dv.delete_object)
|
||||
&& super::replication_resyncer::purge_backoff_active(&dv)
|
||||
&& super::replication_resyncer::object_lock_denied_purge_backoff_active(&dv)
|
||||
{
|
||||
return ReplicationHealQueueResult {
|
||||
object_info: roi,
|
||||
@@ -3821,7 +3693,6 @@ mod tests {
|
||||
stats: Arc::new(ReplicationStats::new()),
|
||||
workers: RwLock::new(Vec::new()),
|
||||
lrg_workers: RwLock::new(Vec::new()),
|
||||
in_flight: Arc::new(ReplicationInFlight::default()),
|
||||
mrf_replica_tx,
|
||||
mrf_replica_rx: Arc::new(Mutex::new(mrf_replica_rx)),
|
||||
mrf_save_tx,
|
||||
@@ -3888,90 +3759,6 @@ mod tests {
|
||||
assert_eq!(current_queue(&pool, "admission-bucket").await, (1, 4096));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn queue_replica_task_admits_a_version_once_while_it_is_in_flight() {
|
||||
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
|
||||
let (tx, _rx) = mpsc::channel(4);
|
||||
pool.workers.write().await.push(tx);
|
||||
let ri = ReplicateObjectInfo {
|
||||
bucket: "in-flight-bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
version_id: Some(uuid::Uuid::new_v4()),
|
||||
size: 4096,
|
||||
op_type: ReplicationType::Object,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert_eq!(pool.queue_replica_task(ri.clone()).await, ReplicationQueueAdmission::Queued);
|
||||
// backlog#2362: the scanner heal pass sees the version as PENDING
|
||||
// until the worker lands it; a second request must not drive it again.
|
||||
assert_eq!(pool.queue_replica_task(ri.clone()).await, ReplicationQueueAdmission::Skipped);
|
||||
assert_eq!(current_queue(&pool, "in-flight-bucket").await, (1, 4096));
|
||||
|
||||
// Another version of the same key is independent work.
|
||||
let newer = ReplicateObjectInfo {
|
||||
version_id: Some(uuid::Uuid::new_v4()),
|
||||
..ri.clone()
|
||||
};
|
||||
assert_eq!(pool.queue_replica_task(newer).await, ReplicationQueueAdmission::Queued);
|
||||
assert_eq!(pool.in_flight.len(), 2);
|
||||
|
||||
// Once the worker finishes, the same version may be queued again
|
||||
// (for example after a FAILED status).
|
||||
pool.in_flight.finish(&ri);
|
||||
assert_eq!(pool.queue_replica_task(ri).await, ReplicationQueueAdmission::Queued);
|
||||
assert_eq!(current_queue(&pool, "in-flight-bucket").await, (3, 3 * 4096));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn queue_replica_task_releases_the_version_when_no_worker_accepts_it() {
|
||||
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
|
||||
let ri = ReplicateObjectInfo {
|
||||
bucket: "no-worker-bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
version_id: Some(uuid::Uuid::new_v4()),
|
||||
size: 4096,
|
||||
op_type: ReplicationType::Object,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// No worker channel: the task is missed and must not stay claimed.
|
||||
assert_eq!(pool.queue_replica_task(ri.clone()).await, ReplicationQueueAdmission::Missed);
|
||||
assert_eq!(pool.in_flight.len(), 0);
|
||||
assert_eq!(pool.queue_replica_task(ri.clone()).await, ReplicationQueueAdmission::Missed);
|
||||
|
||||
// A full worker channel hands the task to the MRF save path; the MRF
|
||||
// replay re-enters the queue, so the claim is released here too.
|
||||
let (tx, _rx) = mpsc::channel(1);
|
||||
pool.workers.write().await.push(tx);
|
||||
assert_eq!(pool.queue_replica_task(ri.clone()).await, ReplicationQueueAdmission::Queued);
|
||||
let overflow = ReplicateObjectInfo {
|
||||
version_id: Some(uuid::Uuid::new_v4()),
|
||||
..ri
|
||||
};
|
||||
assert_eq!(pool.queue_replica_task(overflow).await, ReplicationQueueAdmission::Queued);
|
||||
assert_eq!(pool.in_flight.len(), 1, "only the version held by the worker channel stays in flight");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn in_flight_guard_releases_the_version_on_drop() {
|
||||
let in_flight = Arc::new(ReplicationInFlight::default());
|
||||
let ri = ReplicateObjectInfo {
|
||||
bucket: "guard-bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
version_id: Some(uuid::Uuid::new_v4()),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(in_flight.try_begin(&ri));
|
||||
assert!(!in_flight.try_begin(&ri));
|
||||
{
|
||||
let _guard = ReplicationInFlightGuard::new(in_flight.clone(), &ri);
|
||||
assert_eq!(in_flight.len(), 1);
|
||||
}
|
||||
assert_eq!(in_flight.len(), 0);
|
||||
assert!(in_flight.try_begin(&ri));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn regular_worker_admission_counts_target_backlog_before_receive() {
|
||||
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
|
||||
@@ -6814,87 +6601,4 @@ mod tests {
|
||||
replacement_data
|
||||
);
|
||||
}
|
||||
|
||||
/// backlog#2290: a delete-marker purge intent that survives a restart
|
||||
/// through the MRF journal addresses the marker version the TARGET
|
||||
/// assigned, exactly as the live watcher does (see the
|
||||
/// `requires_delayed_purge` spawn). The journal carries the per-ARN ids
|
||||
/// (`targetDeleteMarkerVersionIDs`) and replay restores them into the
|
||||
/// reconstructed replication state; without that the replay would fall
|
||||
/// back to the source marker id, which a target that mints its own ids
|
||||
/// answers with an idempotent 204 — the entry would be acknowledged while
|
||||
/// the real marker stayed behind.
|
||||
#[test]
|
||||
fn mrf_delete_marker_purge_replay_preserves_target_assigned_marker_version() {
|
||||
use super::super::replication_object_decision_boundary::{delete_marker_purge_mrf_entry, delete_marker_purge_version_id};
|
||||
|
||||
let arn = "arn:minio:replication::generic-target:photos".to_string();
|
||||
let source_marker = uuid::Uuid::new_v4();
|
||||
let remote_marker = "remote-assigned-marker-version".to_string();
|
||||
|
||||
let live_oi = ObjectInfo {
|
||||
bucket: "photos".to_string(),
|
||||
name: "obj".to_string(),
|
||||
version_id: Some(source_marker),
|
||||
delete_marker: true,
|
||||
..Default::default()
|
||||
};
|
||||
let mut live_state = live_oi.replication_state();
|
||||
live_state.replicate_decision_str = replicate_decision_for_admitted_targets(std::slice::from_ref(&arn)).to_string();
|
||||
live_state
|
||||
.target_delete_marker_version_ids
|
||||
.insert(arn.clone(), remote_marker.clone());
|
||||
let live = DeletedObjectReplicationInfo {
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: "obj".to_string(),
|
||||
delete_marker: true,
|
||||
delete_marker_version_id: Some(source_marker),
|
||||
replication_state: Some(live_state),
|
||||
..Default::default()
|
||||
},
|
||||
bucket: "photos".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(
|
||||
delete_marker_purge_version_id(live.delete_object.replication_state.as_ref(), &arn, source_marker),
|
||||
Some(Some(remote_marker.clone())),
|
||||
"the live purge addresses the recorded target version"
|
||||
);
|
||||
|
||||
// Watch window exhausted: persist the intent, restart, replay it.
|
||||
let entry = delete_marker_purge_mrf_entry(&live, vec![arn.clone()]);
|
||||
let replay_oi = ObjectInfo {
|
||||
bucket: entry.bucket.clone(),
|
||||
name: entry.object.clone(),
|
||||
version_id: entry.version_id,
|
||||
delete_marker: entry.delete_marker,
|
||||
..Default::default()
|
||||
};
|
||||
let dsc = replicate_decision_for_admitted_targets(&entry.target_arns);
|
||||
let replayed = reconstructed_heal_delete_info(&entry, &replay_oi, &dsc);
|
||||
|
||||
assert_eq!(
|
||||
delete_marker_purge_version_id(replayed.delete_object.replication_state.as_ref(), &arn, source_marker),
|
||||
Some(Some(remote_marker)),
|
||||
"the MRF replay must address the target-assigned marker version, not source marker {source_marker}"
|
||||
);
|
||||
|
||||
// A refusal (inconsistent recorded ids) must stay a refusal across the
|
||||
// journal round trip instead of degrading into the source-id fallback.
|
||||
let mut refused = live;
|
||||
refused
|
||||
.delete_object
|
||||
.replication_state
|
||||
.as_mut()
|
||||
.expect("state was set above")
|
||||
.target_delete_marker_version_ids_corrupt = true;
|
||||
let entry = delete_marker_purge_mrf_entry(&refused, vec![arn.clone()]);
|
||||
assert!(entry.target_delete_marker_version_ids_corrupt);
|
||||
let replayed = reconstructed_heal_delete_info(&entry, &replay_oi, &dsc);
|
||||
assert_eq!(
|
||||
delete_marker_purge_version_id(replayed.delete_object.replication_state.as_ref(), &arn, source_marker),
|
||||
None,
|
||||
"the MRF replay must keep refusing to guess when the recorded ids were inconsistent"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -38,7 +38,7 @@ use time::format_description::well_known::Rfc3339;
|
||||
|
||||
pub(crate) use crate::bucket::bucket_target_sys::{
|
||||
AdvancedPutOptions, HeadObjectSdkError, PutObjectOptions, PutObjectPartOptions, RemotePutObjectResponse, RemoveObjectOptions,
|
||||
ReplicaLocation, S3ClientError, TargetClient, resolve_read_api_version_id,
|
||||
S3ClientError, TargetClient, resolve_read_api_version_id,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::bucket::target::BucketTarget;
|
||||
@@ -48,7 +48,6 @@ pub use rustfs_replication::{ObjectLockIntegrity, object_lock_put_integrity};
|
||||
pub(crate) use rustfs_replication::{
|
||||
SsecPassthroughGate, is_replication_target_offline_error, ssec_passthrough_gate, version_identity_drifted,
|
||||
};
|
||||
pub use rustfs_replication::{VersionIdentityCapability, version_identity_capability_from_put};
|
||||
|
||||
use super::replication_config_store::ReplicationConfigStore;
|
||||
use super::replication_error_boundary::{Error, Result};
|
||||
@@ -193,14 +192,6 @@ impl ReplicationTargetStore {
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) fn version_identity_capability(arn: &str) -> VersionIdentityCapability {
|
||||
BucketTargetSys::get().version_identity_capability(arn)
|
||||
}
|
||||
|
||||
pub(crate) fn record_version_identity_capability(arn: &str, capability: VersionIdentityCapability) {
|
||||
BucketTargetSys::get().record_version_identity_capability(arn, capability)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn register_test_target(target_client: &Arc<TargetClient>) {
|
||||
BucketTargetSys::get().arn_remotes_map.write().await.insert(
|
||||
@@ -247,23 +238,6 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
||||
meta.insert(key.to_string(), value.to_string());
|
||||
}
|
||||
|
||||
// A compressed SSE-C object passes through as its stored bytes. The target
|
||||
// cannot infer the compression layout from ciphertext, so the scheme and
|
||||
// the plaintext size travel as transport headers; each UploadPart carries
|
||||
// its own plaintext length (backlog#2363).
|
||||
if is_ssec && let Some(scheme) = get_str(&object_info.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION) {
|
||||
insert_header_map(&mut meta, rustfs_utils::http::SUFFIX_REPLICATION_COMPRESSION, scheme);
|
||||
if let Ok(actual_size) = object_info.get_actual_size()
|
||||
&& actual_size >= 0
|
||||
{
|
||||
insert_header_map(
|
||||
&mut meta,
|
||||
rustfs_utils::http::SUFFIX_REPLICATION_COMPRESSION_ACTUAL_SIZE,
|
||||
actual_size.to_string(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Managed SSE replicates as plaintext (the replication reader decrypts via
|
||||
// the object-encryption resolver) and re-encrypts on the target with the
|
||||
// target's own KMS. Send only the encryption intent — never the source
|
||||
@@ -274,16 +248,7 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
||||
meta.insert(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "aws:kms".to_string());
|
||||
}
|
||||
|
||||
// Older transformed objects can have physical parts without logical part
|
||||
// lengths. Keep their existing whole-object transport: physical sizes are
|
||||
// not plaintext boundaries for a multipart replication read.
|
||||
let legacy_single_put = object_info.etag.as_deref().is_none_or(|etag| etag.len() == 32);
|
||||
let base_is_multipart = object_info.is_multipart()
|
||||
&& !(legacy_single_put
|
||||
&& object_info.parts.len() > 1
|
||||
&& (object_info.is_compressed() || object_info.is_encrypted())
|
||||
&& object_info.parts.iter().any(|part| part.actual_size <= 0));
|
||||
let mut is_multipart = base_is_multipart;
|
||||
let mut is_multipart = object_info.is_multipart();
|
||||
|
||||
if let Some(checksum_data) = &object_info.checksum
|
||||
&& !checksum_data.is_empty()
|
||||
@@ -294,8 +259,8 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
||||
} else if object_info.is_encrypted() {
|
||||
// Encrypted checksums cannot be exposed as plaintext headers, and
|
||||
// decrypt_checksums reports is_multipart=false for them (a value
|
||||
// the response path relies on). Keep the transport selected from
|
||||
// the object's layout and readable part boundaries.
|
||||
// the response path relies on). Keep the object's own multipart
|
||||
// flag so encrypted objects stay on the multipart route.
|
||||
} else {
|
||||
let (checksum_meta, checksum_record_is_multipart) = object_info.decrypt_checksums(0, &HeaderMap::new())?;
|
||||
// The checksum record describes how the *checksum* is composed,
|
||||
@@ -303,37 +268,23 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
||||
// MULTIPART flag even on a multipart upload, so trusting it here
|
||||
// routed a 768-part object through a single PutObject and the
|
||||
// target rejected the 6 GiB body with EntityTooLarge
|
||||
// (rustfs#6825). The usable part layout is the authority: the
|
||||
// (rustfs#6825). The object's own shape is the authority: the
|
||||
// record may only add multipart-ness, never take it away.
|
||||
is_multipart = base_is_multipart || checksum_record_is_multipart;
|
||||
is_multipart = object_info.is_multipart() || checksum_record_is_multipart;
|
||||
|
||||
if !base_is_multipart
|
||||
for (key, value) in checksum_meta.iter() {
|
||||
if key != AMZ_CHECKSUM_TYPE {
|
||||
meta.insert(key.clone(), value.clone());
|
||||
}
|
||||
}
|
||||
|
||||
if !object_info.is_multipart()
|
||||
&& checksum_meta
|
||||
.get(AMZ_CHECKSUM_TYPE)
|
||||
.is_some_and(|value| value == AMZ_CHECKSUM_TYPE_FULL_OBJECT)
|
||||
{
|
||||
is_multipart = false;
|
||||
}
|
||||
|
||||
// The record keys each checksum by algorithm name ("CRC32"); the
|
||||
// target only reads `x-amz-checksum-<algorithm>`. Inserting the bare
|
||||
// name here made `PutObjectOptions::header()` send it as user
|
||||
// metadata (`x-amz-meta-crc32`), so no replica ever carried the
|
||||
// source checksum (rustfs/backlog#2340). The object-level record
|
||||
// describes one PUT body: a multipart replica is rebuilt part by
|
||||
// part, and its CreateMultipartUpload must not announce a checksum
|
||||
// the parts do not carry, so the record is forwarded on the
|
||||
// single-PUT route only (MinIO `getCRCMeta` parity).
|
||||
if !is_multipart {
|
||||
for (key, value) in checksum_meta.iter() {
|
||||
if key == AMZ_CHECKSUM_TYPE {
|
||||
continue;
|
||||
}
|
||||
if let Some(header) = rustfs_rio::ChecksumType::from_string(key).key() {
|
||||
meta.insert(header.to_string(), value.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -565,7 +516,6 @@ fn is_standard_header(key: &str) -> bool {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::replication_filemeta_boundary::ObjectPartInfo;
|
||||
use super::*;
|
||||
use aws_smithy_types::DateTime;
|
||||
use rustfs_replication::content_matches_by_etag;
|
||||
@@ -600,162 +550,6 @@ mod tests {
|
||||
checksum.to_bytes(&combined)
|
||||
}
|
||||
|
||||
fn replication_route_metadata() -> [(&'static str, Arc<HashMap<String, String>>); 4] {
|
||||
let mut compressed = HashMap::new();
|
||||
rustfs_utils::http::insert_str(&mut compressed, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string());
|
||||
[
|
||||
("plain", Arc::new(HashMap::new())),
|
||||
("compressed", Arc::new(compressed)),
|
||||
(
|
||||
"encrypted",
|
||||
Arc::new(HashMap::from([(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "AES256".to_string())])),
|
||||
),
|
||||
(
|
||||
"ssec",
|
||||
Arc::new(HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())])),
|
||||
),
|
||||
]
|
||||
}
|
||||
|
||||
fn replication_route_object(
|
||||
etag: Option<&str>,
|
||||
actual_sizes: [i64; 3],
|
||||
metadata: Arc<HashMap<String, String>>,
|
||||
) -> ObjectInfo {
|
||||
ObjectInfo {
|
||||
etag: etag.map(str::to_string),
|
||||
size: 48,
|
||||
actual_size: 12,
|
||||
user_defined: metadata,
|
||||
parts: Arc::new(
|
||||
actual_sizes
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(index, actual_size)| ObjectPartInfo {
|
||||
number: index + 1,
|
||||
size: 16,
|
||||
actual_size,
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compressed_ssec_objects_declare_their_compression_layout_on_the_wire() {
|
||||
use rustfs_utils::http::{
|
||||
SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, SUFFIX_REPLICATION_COMPRESSION, SUFFIX_REPLICATION_COMPRESSION_ACTUAL_SIZE,
|
||||
insert_str,
|
||||
};
|
||||
|
||||
let mut ssec_compressed = HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())]);
|
||||
insert_str(&mut ssec_compressed, SUFFIX_COMPRESSION, "klauspost/compress/s2".to_string());
|
||||
insert_str(&mut ssec_compressed, SUFFIX_ACTUAL_SIZE, "6295552".to_string());
|
||||
let object_info = ObjectInfo {
|
||||
etag: Some("0123456789abcdef0123456789abcdef-2".to_string()),
|
||||
size: 4321,
|
||||
actual_size: 6295552,
|
||||
user_defined: Arc::new(ssec_compressed),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// SSE-C passthrough sends stored bytes: the scheme and the plaintext
|
||||
// size travel as transport headers, never as the internal key
|
||||
// (backlog#2363).
|
||||
let (options, _) = replication_put_object_options("STANDARD", &object_info).expect("ssec put options");
|
||||
assert_eq!(
|
||||
get_header_map(&options.user_metadata, SUFFIX_REPLICATION_COMPRESSION).as_deref(),
|
||||
Some("klauspost/compress/s2")
|
||||
);
|
||||
assert_eq!(
|
||||
get_header_map(&options.user_metadata, SUFFIX_REPLICATION_COMPRESSION_ACTUAL_SIZE).as_deref(),
|
||||
Some("6295552")
|
||||
);
|
||||
assert!(
|
||||
!options
|
||||
.user_metadata
|
||||
.keys()
|
||||
.any(|key| rustfs_utils::http::is_internal_key(key)),
|
||||
"internal metadata never leaves the source as plain metadata: {:?}",
|
||||
options.user_metadata
|
||||
);
|
||||
|
||||
// A compressed object that is not SSE-C is decompressed by the
|
||||
// replication reader and travels as plaintext: no layout headers.
|
||||
let mut plain_compressed = HashMap::new();
|
||||
insert_str(&mut plain_compressed, SUFFIX_COMPRESSION, "klauspost/compress/s2".to_string());
|
||||
insert_str(&mut plain_compressed, SUFFIX_ACTUAL_SIZE, "6295552".to_string());
|
||||
let plain = ObjectInfo {
|
||||
user_defined: Arc::new(plain_compressed),
|
||||
..object_info
|
||||
};
|
||||
let (options, _) = replication_put_object_options("STANDARD", &plain).expect("plain put options");
|
||||
assert!(get_header_map(&options.user_metadata, SUFFIX_REPLICATION_COMPRESSION).is_none());
|
||||
assert!(get_header_map(&options.user_metadata, SUFFIX_REPLICATION_COMPRESSION_ACTUAL_SIZE).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_transformed_single_put_parts_keep_the_previous_replication_route() {
|
||||
let [_, (_, compressed), (_, encrypted), (_, ssec)] = replication_route_metadata();
|
||||
let cases = [
|
||||
(
|
||||
"compressed middle zero",
|
||||
compressed.clone(),
|
||||
Some("0123456789abcdef0123456789abcdef"),
|
||||
[4, 0, 4],
|
||||
),
|
||||
("compressed tail unknown", compressed, None, [4, 4, -1]),
|
||||
(
|
||||
"encrypted middle unknown",
|
||||
encrypted,
|
||||
Some("gggggggggggggggggggggggggggggggg"),
|
||||
[4, -1, 4],
|
||||
),
|
||||
("ssec tail zero", ssec.clone(), None, [4, 4, 0]),
|
||||
("ssec middle unknown", ssec, Some("gggggggggggggggggggggggggggggggg"), [4, -1, 4]),
|
||||
];
|
||||
for (name, metadata, etag, actual_sizes) in cases {
|
||||
for checksum in [None, Some(full_object_multipart_checksum_record())] {
|
||||
let mut object_info = replication_route_object(etag, actual_sizes, metadata.clone());
|
||||
object_info.checksum = checksum;
|
||||
assert!(object_info.is_multipart(), "{name}: physical parts remain visible to metadata APIs");
|
||||
assert!(object_info.is_compressed() || object_info.is_encrypted());
|
||||
|
||||
let (options, is_multipart) =
|
||||
replication_put_object_options("STANDARD", &object_info).expect("legacy transformed put options");
|
||||
assert!(
|
||||
!is_multipart,
|
||||
"{name}: unknown logical part sizes must preserve the old whole-object route"
|
||||
);
|
||||
assert_eq!(options.internal.source_etag, etag.unwrap_or_default());
|
||||
if metadata.contains_key(SSEC_ALGORITHM_HEADER) {
|
||||
assert_eq!(
|
||||
get_header_map(&options.user_metadata, SUFFIX_REPLICATION_SSEC_CRC).is_some(),
|
||||
object_info.checksum.is_some(),
|
||||
"SSE-C checksums retain their raw passthrough transport"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn positive_part_sizes_and_legacy_multipart_etags_keep_the_replication_route() {
|
||||
for (name, metadata) in replication_route_metadata() {
|
||||
for (etag, actual_sizes) in [
|
||||
("0123456789abcdef0123456789abcdef", [4, 4, 4]),
|
||||
("0123456789abcdef0123456789abcdef-3", [4, 0, -1]),
|
||||
] {
|
||||
let mut object_info = replication_route_object(Some(etag), actual_sizes, metadata.clone());
|
||||
object_info.checksum = Some(full_object_multipart_checksum_record());
|
||||
let (_, is_multipart) = replication_put_object_options("STANDARD", &object_info).expect("multipart put options");
|
||||
assert!(is_multipart, "{name}/{etag}: usable sizes and old multipart ETags must retain MPU");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multipart_object_with_full_object_checksum_keeps_the_multipart_route() {
|
||||
// rustfs#6825: a 768-part upload was replicated with a single
|
||||
@@ -788,36 +582,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stored_multipart_parts_keep_the_replication_route_without_a_multipart_etag() {
|
||||
for etag in [Some("0123456789abcdef0123456789abcdef"), None] {
|
||||
for checksum in [None, Some(full_object_multipart_checksum_record())] {
|
||||
let object_info = ObjectInfo {
|
||||
etag: etag.map(str::to_string),
|
||||
checksum,
|
||||
parts: Arc::new(
|
||||
(1..=2)
|
||||
.map(|number| ObjectPartInfo {
|
||||
number,
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
),
|
||||
..Default::default()
|
||||
};
|
||||
let (options, is_multipart) =
|
||||
replication_put_object_options("STANDARD", &object_info).expect("build put options");
|
||||
|
||||
assert!(
|
||||
is_multipart,
|
||||
"stored parts must retain multipart routing: etag={etag:?}, checksum={:?}",
|
||||
object_info.checksum
|
||||
);
|
||||
assert_eq!(options.internal.source_etag, etag.unwrap_or_default());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checksum_record_never_changes_the_transport_a_single_part_object_needs() {
|
||||
// The mirror of the rustfs#6825 guard: an object stored as one PUT
|
||||
@@ -828,10 +592,6 @@ mod tests {
|
||||
let object_info = ObjectInfo {
|
||||
etag: Some("0123456789abcdef0123456789abcdef".to_string()),
|
||||
checksum: Some(checksum.to_bytes(&[])),
|
||||
parts: Arc::new(vec![ObjectPartInfo {
|
||||
number: 1,
|
||||
..Default::default()
|
||||
}]),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -868,19 +628,6 @@ mod tests {
|
||||
let (_, is_multipart) = replication_put_object_options("STANDARD", &object_info).expect("build put options");
|
||||
|
||||
assert!(is_multipart, "a composite-checksum multipart object must stay on the multipart transport");
|
||||
|
||||
for (name, metadata) in replication_route_metadata() {
|
||||
let mut legacy = replication_route_object(Some("0123456789abcdef0123456789abcdef"), [4, 0, 4], metadata);
|
||||
legacy.checksum = Some(checksum.to_bytes(&combined));
|
||||
let (_, record_is_multipart) = legacy.decrypt_checksums(0, &HeaderMap::new()).expect("decode checksum");
|
||||
let (_, is_multipart) = replication_put_object_options("STANDARD", &legacy).expect("legacy checksum put options");
|
||||
if legacy.is_encrypted() {
|
||||
assert!(!is_multipart, "{name}: encrypted checksum records must not change the old transport");
|
||||
} else {
|
||||
assert!(record_is_multipart, "the composite checksum must carry its own multipart signal");
|
||||
assert!(is_multipart, "{name}: a composite record can still promote the legacy route to MPU");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1561,63 +1308,12 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let (opts, is_multipart) = replication_put_object_options("", &object_info).expect("build replication put options");
|
||||
let (opts, _is_multipart) = replication_put_object_options("", &object_info).expect("build replication put options");
|
||||
|
||||
assert!(!is_multipart, "{name}: a single-part checksum record must keep the single-PUT route");
|
||||
let header = ty.key().expect("every forwarded algorithm has an x-amz-checksum header");
|
||||
assert_eq!(
|
||||
opts.user_metadata.get(header),
|
||||
opts.user_metadata.get(name),
|
||||
Some(&checksum.encoded),
|
||||
"replication must forward the {name} checksum as the {header} header"
|
||||
);
|
||||
assert!(
|
||||
!opts.user_metadata.contains_key(name),
|
||||
"{name}: the bare algorithm name would leave as x-amz-meta user metadata"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// The object-level record of a multipart upload (composite or full-object)
|
||||
/// must not become a PutObject checksum header: the replica is rebuilt
|
||||
/// through CreateMultipartUpload/UploadPart, and a checksum announced there
|
||||
/// that the parts do not carry would be rejected by the target.
|
||||
#[test]
|
||||
fn replication_put_object_options_keeps_multipart_checksum_records_off_the_wire() {
|
||||
let mut composite_type = rustfs_rio::ChecksumType::from_string("crc32");
|
||||
composite_type
|
||||
.merge(rustfs_rio::ChecksumType::MULTIPART)
|
||||
.merge(rustfs_rio::ChecksumType::INCLUDES_MULTIPART);
|
||||
let mut combined = Vec::new();
|
||||
for part in [b"part-one".as_slice(), b"part-two".as_slice()] {
|
||||
let part_checksum =
|
||||
rustfs_rio::Checksum::new_from_data(rustfs_rio::ChecksumType::from_string("crc32"), part).expect("part checksum");
|
||||
combined.extend_from_slice(part_checksum.raw.as_slice());
|
||||
}
|
||||
let composite = rustfs_rio::Checksum::new_from_data(composite_type, &combined)
|
||||
.expect("composite checksum")
|
||||
.to_bytes(&combined);
|
||||
|
||||
for (label, checksum, etag) in [
|
||||
("composite", composite, "0123456789abcdef0123456789abcdef-2"),
|
||||
(
|
||||
"full-object",
|
||||
full_object_multipart_checksum_record(),
|
||||
"0123456789abcdef0123456789abcdef-3",
|
||||
),
|
||||
] {
|
||||
let object_info = ObjectInfo {
|
||||
etag: Some(etag.to_string()),
|
||||
checksum: Some(checksum),
|
||||
..Default::default()
|
||||
};
|
||||
let (opts, is_multipart) = replication_put_object_options("", &object_info).expect("build replication put options");
|
||||
assert!(is_multipart, "{label}: a multipart object must keep the multipart route");
|
||||
assert!(
|
||||
opts.user_metadata
|
||||
.keys()
|
||||
.all(|key| !key.starts_with("x-amz-checksum-") && key != "CRC32"),
|
||||
"{label}: no object-level checksum may reach the target's CreateMultipartUpload: {:?}",
|
||||
opts.user_metadata
|
||||
"replication must forward the {name} checksum into user_metadata identically to the classic algorithms"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,8 +48,7 @@ pub use internode_data_transport::build_internode_data_transport_from_env;
|
||||
pub(crate) use peer_rest_client::TierConfigReloadOutcome;
|
||||
pub use peer_rest_client::{
|
||||
KMS_SIGNAL_SUBSYSTEM, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG,
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerDirtyUsageAcknowledgement, ScannerPeerActivity, ScannerPeerDirtyUsageBucket,
|
||||
ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease, ScannerScopedDirtyUsageAckEntry,
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease,
|
||||
};
|
||||
pub(crate) use peer_s3_client::heal_bucket_local_on_disks;
|
||||
pub use peer_s3_client::{
|
||||
|
||||
@@ -49,11 +49,10 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest,
|
||||
LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ReplacementRecoveryStatusRequest,
|
||||
ScannerActivityRequest, ScannerActivityResponse, ScannerDirtyUsageSnapshotRequest, ScannerDirtyUsageSnapshotResponse,
|
||||
ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest, ScannerPublicationLeaseResponse,
|
||||
ScannerScopedDirtyUsageAckRequest, ScannerScopedDirtyUsageEntry, ServerInfoRequest, SignalServiceRequest,
|
||||
SignalServiceResponse, StartDecommissionRequest, StartProfilingRequest, StopRebalanceRequest, TierDailyStatsRequest,
|
||||
TierMutationAbortRequest, TierMutationCommitRequest, TierMutationControlResponse, TierMutationFailureClass,
|
||||
TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
|
||||
ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest, ScannerPublicationLeaseResponse, ServerInfoRequest,
|
||||
SignalServiceRequest, SignalServiceResponse, StartDecommissionRequest, StartProfilingRequest, StopRebalanceRequest,
|
||||
TierDailyStatsRequest, TierMutationAbortRequest, TierMutationCommitRequest, TierMutationControlResponse,
|
||||
TierMutationFailureClass, TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
|
||||
tier_mutation_control_service_client::TierMutationControlServiceClient,
|
||||
};
|
||||
pub use rustfs_protos::{PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS};
|
||||
@@ -69,10 +68,7 @@ use std::{
|
||||
},
|
||||
time::SystemTime,
|
||||
};
|
||||
use tokio::{
|
||||
net::TcpStream,
|
||||
time::{Duration, timeout},
|
||||
};
|
||||
use tokio::{net::TcpStream, time::Duration};
|
||||
use tonic::Request;
|
||||
use tonic::service::interceptor::InterceptedService;
|
||||
use tracing::{debug, info, warn};
|
||||
@@ -93,7 +89,6 @@ const HEAL_CONTROL_PAYLOAD_MAX_SIZE: usize = 64 * 1024;
|
||||
const PEER_REST_RECOVERY_MAX_ATTEMPTS: u32 = 60;
|
||||
const PEER_REST_RECOVERY_MAX_BACKOFF: Duration = Duration::from_secs(30);
|
||||
const SCANNER_ACTIVITY_MAX_MESSAGE_SIZE: usize = 1024;
|
||||
const SCANNER_SCOPED_DIRTY_USAGE_STAGE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
/// Reserve time for the acquire response's network/clock uncertainty. The
|
||||
/// server owns the real expiry; this local deadline is intentionally earlier
|
||||
/// so a coordinator never starts a bounded persistence operation at the edge
|
||||
@@ -194,102 +189,12 @@ pub struct ScannerPeerActivity {
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ScannerPeerDirtyUsageSnapshot {
|
||||
pub owner_id: String,
|
||||
pub instance_id: String,
|
||||
pub generation: u64,
|
||||
pub pending_bucket_count: u64,
|
||||
pub protocol_version: u32,
|
||||
pub complete: bool,
|
||||
pub buckets: BTreeMap<String, ScannerPeerDirtyUsageBucket>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct ScannerPeerDirtyUsageBucket {
|
||||
pub bucket_incarnation: Uuid,
|
||||
pub generation: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ScannerScopedDirtyUsageAckEntry {
|
||||
pub bucket: String,
|
||||
pub bucket_incarnation: Uuid,
|
||||
pub generation: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum ScannerDirtyUsageAcknowledgement {
|
||||
Generation {
|
||||
host: String,
|
||||
instance_id: String,
|
||||
generation: u64,
|
||||
},
|
||||
Scoped {
|
||||
host: String,
|
||||
owner_id: String,
|
||||
instance_id: String,
|
||||
entries: Vec<ScannerScopedDirtyUsageAckEntry>,
|
||||
},
|
||||
}
|
||||
|
||||
fn scanner_scoped_dirty_usage_ack_payloads(
|
||||
owner_id: String,
|
||||
instance_id: String,
|
||||
probe_only: bool,
|
||||
entries: Vec<ScannerScopedDirtyUsageAckEntry>,
|
||||
) -> Result<Vec<ScannerScopedDirtyUsageAckRequest>> {
|
||||
use rustfs_protos::scoped_dirty_usage::*;
|
||||
|
||||
if entries.is_empty() {
|
||||
return Err(Error::other("scoped dirty usage acknowledgement entries must be nonempty"));
|
||||
}
|
||||
|
||||
let mut payloads = Vec::with_capacity(entries.len().div_ceil(SCOPED_DIRTY_USAGE_MAX_ENTRIES as usize));
|
||||
let mut batch = Vec::with_capacity(SCOPED_DIRTY_USAGE_MAX_ENTRIES as usize);
|
||||
for entry in entries {
|
||||
batch.push(ScannerScopedDirtyUsageEntry {
|
||||
bucket: entry.bucket,
|
||||
bucket_incarnation: entry.bucket_incarnation.as_bytes().to_vec().into(),
|
||||
generation: entry.generation,
|
||||
});
|
||||
if batch.len() == SCOPED_DIRTY_USAGE_MAX_ENTRIES as usize {
|
||||
payloads.push(scanner_scoped_dirty_usage_ack_payload(
|
||||
&owner_id,
|
||||
&instance_id,
|
||||
probe_only,
|
||||
std::mem::take(&mut batch),
|
||||
)?);
|
||||
}
|
||||
}
|
||||
if !batch.is_empty() {
|
||||
payloads.push(scanner_scoped_dirty_usage_ack_payload(&owner_id, &instance_id, probe_only, batch)?);
|
||||
}
|
||||
|
||||
Ok(payloads)
|
||||
}
|
||||
|
||||
fn scanner_scoped_dirty_usage_ack_payload(
|
||||
owner_id: &str,
|
||||
instance_id: &str,
|
||||
probe_only: bool,
|
||||
entries: Vec<ScannerScopedDirtyUsageEntry>,
|
||||
) -> Result<ScannerScopedDirtyUsageAckRequest> {
|
||||
use rustfs_protos::scoped_dirty_usage::*;
|
||||
|
||||
let payload = ScannerScopedDirtyUsageAckRequest {
|
||||
challenge: Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
protocol_version: SCOPED_DIRTY_USAGE_PROTOCOL_VERSION,
|
||||
owner_id: owner_id.to_string(),
|
||||
instance_id: instance_id.to_string(),
|
||||
scope: SCOPED_DIRTY_USAGE_BUCKET_SCOPE,
|
||||
probe_only,
|
||||
entries,
|
||||
};
|
||||
canonical_scoped_dirty_usage_request(&payload).map_err(|err| Error::other(err.to_string()))?;
|
||||
Ok(payload)
|
||||
}
|
||||
|
||||
fn scanner_scoped_dirty_usage_ack_reconciled(activity: &ScannerPeerActivity, expected_instance_id: &str) -> bool {
|
||||
activity.instance_id == expected_instance_id && activity.dirty_usage_pending == Some(false)
|
||||
pub buckets: BTreeMap<String, u64>,
|
||||
}
|
||||
|
||||
fn scanner_instance_id_is_valid(instance_id: &str) -> bool {
|
||||
@@ -443,11 +348,6 @@ fn decode_scanner_dirty_usage_snapshot_with_verifier(
|
||||
if !scanner_instance_id_is_valid(&response.instance_id) {
|
||||
return Err(Error::other("peer returned an invalid scanner dirty usage snapshot instance ID"));
|
||||
}
|
||||
let owner_id = Uuid::parse_str(&response.owner_id)
|
||||
.ok()
|
||||
.filter(|owner_id| !owner_id.is_nil())
|
||||
.map(|owner_id| owner_id.to_string())
|
||||
.ok_or_else(|| Error::other("peer returned an invalid scanner dirty usage snapshot owner"))?;
|
||||
if response.generation == u64::MAX {
|
||||
return Err(Error::other("peer scanner dirty usage snapshot exhausted its generation"));
|
||||
}
|
||||
@@ -483,14 +383,9 @@ fn decode_scanner_dirty_usage_snapshot_with_verifier(
|
||||
if bucket.generation == 0 || bucket.generation > response.generation {
|
||||
return Err(Error::other("peer scanner dirty usage snapshot contains an invalid bucket generation"));
|
||||
}
|
||||
Uuid::from_slice(bucket.bucket_incarnation.as_ref())
|
||||
.ok()
|
||||
.filter(|bucket_incarnation| !bucket_incarnation.is_nil())
|
||||
.ok_or_else(|| Error::other("peer scanner dirty usage snapshot contains an invalid bucket incarnation"))?;
|
||||
}
|
||||
|
||||
Ok(ScannerPeerDirtyUsageSnapshot {
|
||||
owner_id,
|
||||
instance_id: response.instance_id,
|
||||
generation: response.generation,
|
||||
pending_bucket_count: response.pending_bucket_count,
|
||||
@@ -499,16 +394,7 @@ fn decode_scanner_dirty_usage_snapshot_with_verifier(
|
||||
buckets: response
|
||||
.buckets
|
||||
.into_iter()
|
||||
.map(|bucket| {
|
||||
(
|
||||
bucket.bucket,
|
||||
ScannerPeerDirtyUsageBucket {
|
||||
bucket_incarnation: Uuid::from_slice(bucket.bucket_incarnation.as_ref())
|
||||
.expect("bucket incarnation was validated"),
|
||||
generation: bucket.generation,
|
||||
},
|
||||
)
|
||||
})
|
||||
.map(|bucket| (bucket.bucket, bucket.generation))
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
@@ -988,16 +874,6 @@ impl PeerRestClient {
|
||||
self.offline.store(false, Ordering::Release);
|
||||
}
|
||||
|
||||
/// Prepare a retry without allowing connection-cache cleanup to extend the
|
||||
/// caller's probe deadline. The offline gate is cleared even when eviction
|
||||
/// times out so a cancelled cleanup cannot strand the peer in fast-fail
|
||||
/// mode; a later request can perform a fresh eviction if needed.
|
||||
pub async fn prepare_retry_with_timeout(&self, timeout_duration: Duration) -> bool {
|
||||
let evicted = timeout(timeout_duration, self.evict_connection()).await.is_ok();
|
||||
self.offline.store(false, Ordering::Release);
|
||||
evicted
|
||||
}
|
||||
|
||||
/// Whether this failure means the peer is unreachable, so it should be
|
||||
/// gated offline and its connection evicted.
|
||||
///
|
||||
@@ -1799,24 +1675,6 @@ impl PeerRestClient {
|
||||
Ok((self.topology_member.clone(), supported_version, epoch))
|
||||
}
|
||||
|
||||
pub async fn probe_ilm_recovery_export(&self, topology_fingerprint: String) -> Result<(String, Uuid)> {
|
||||
let probe = rustfs_protos::ilm_recovery_export_capability_probe(Uuid::new_v4().as_bytes());
|
||||
let result = self
|
||||
.heal_control(rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION, topology_fingerprint, probe)
|
||||
.await?;
|
||||
let epoch = decode_remote_version_state_capability(&self.topology_member, &result)?;
|
||||
Ok((self.topology_member.clone(), epoch))
|
||||
}
|
||||
|
||||
pub async fn probe_transition_transaction_compaction(&self, topology_fingerprint: String) -> Result<(String, Uuid)> {
|
||||
let probe = rustfs_protos::transition_transaction_compaction_capability_probe(Uuid::new_v4().as_bytes());
|
||||
let result = self
|
||||
.heal_control(rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION, topology_fingerprint, probe)
|
||||
.await?;
|
||||
let epoch = decode_remote_version_state_capability(&self.topology_member, &result)?;
|
||||
Ok((self.topology_member.clone(), epoch))
|
||||
}
|
||||
|
||||
pub async fn load_bucket_metadata(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
|
||||
let result = tokio::time::timeout(BUCKET_METADATA_RELOAD_TIMEOUT, async {
|
||||
let result = self.load_bucket_metadata_once(bucket, scanner_maintenance_change).await;
|
||||
@@ -2197,10 +2055,19 @@ impl PeerRestClient {
|
||||
&self,
|
||||
owner_id: String,
|
||||
instance_id: String,
|
||||
entries: Vec<ScannerScopedDirtyUsageAckEntry>,
|
||||
entries: Vec<rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry>,
|
||||
) -> Result<bool> {
|
||||
use rustfs_protos::scoped_dirty_usage::*;
|
||||
let payloads = scanner_scoped_dirty_usage_ack_payloads(owner_id, instance_id, true, entries)?;
|
||||
let payload = rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageAckRequest {
|
||||
challenge: Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
protocol_version: SCOPED_DIRTY_USAGE_PROTOCOL_VERSION,
|
||||
owner_id,
|
||||
instance_id,
|
||||
scope: SCOPED_DIRTY_USAGE_BUCKET_SCOPE,
|
||||
probe_only: true,
|
||||
entries,
|
||||
};
|
||||
let canonical = canonical_scoped_dirty_usage_request(&payload).map_err(|err| Error::other(err.to_string()))?;
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = super::client::scanner_control_time_out_client(
|
||||
@@ -2208,106 +2075,26 @@ impl PeerRestClient {
|
||||
TonicInterceptor::Signature(gen_tonic_signature_interceptor()),
|
||||
)
|
||||
.await?;
|
||||
for payload in payloads {
|
||||
let canonical =
|
||||
canonical_scoped_dirty_usage_request(&payload).map_err(|err| Error::other(err.to_string()))?;
|
||||
let mut request = Request::new(payload.clone());
|
||||
set_tonic_canonical_body_digest(&mut request, &canonical)?;
|
||||
let response = client.scanner_scoped_dirty_usage_ack(request).await?.into_inner();
|
||||
let body = canonical_scoped_dirty_usage_response(&canonical, &response)
|
||||
.map_err(|_| Error::other("scoped dirty usage capability response is too large"))?;
|
||||
verify_tonic_rpc_response_proof(&body, response.response_proof.as_ref())?;
|
||||
if response.protocol_version != SCOPED_DIRTY_USAGE_PROTOCOL_VERSION
|
||||
|| response.owner_id != payload.owner_id
|
||||
|| response.instance_id != payload.instance_id
|
||||
|| response.max_entries != SCOPED_DIRTY_USAGE_MAX_ENTRIES
|
||||
|| response.max_request_bytes != SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES
|
||||
|| response.cleared != 0
|
||||
{
|
||||
return Err(Error::other("scoped dirty usage capability response does not match request"));
|
||||
}
|
||||
if !response.supported {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
.await,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn acknowledge_scanner_scoped_dirty_usage(
|
||||
&self,
|
||||
owner_id: String,
|
||||
instance_id: String,
|
||||
entries: Vec<ScannerScopedDirtyUsageAckEntry>,
|
||||
) -> Result<ScannerPeerActivity> {
|
||||
use rustfs_protos::scoped_dirty_usage::*;
|
||||
let payloads = scanner_scoped_dirty_usage_ack_payloads(owner_id, instance_id.clone(), false, entries)?;
|
||||
let ack_attempt = async {
|
||||
let mut client = super::client::scanner_control_time_out_client(
|
||||
&self.grid_host,
|
||||
TonicInterceptor::Signature(gen_tonic_signature_interceptor()),
|
||||
)
|
||||
.await?;
|
||||
for payload in payloads {
|
||||
let canonical = canonical_scoped_dirty_usage_request(&payload).map_err(|err| Error::other(err.to_string()))?;
|
||||
let mut request = Request::new(payload.clone());
|
||||
set_tonic_canonical_body_digest(&mut request, &canonical)?;
|
||||
let response = client.scanner_scoped_dirty_usage_ack(request).await?.into_inner();
|
||||
let body = canonical_scoped_dirty_usage_response(&canonical, &response)
|
||||
.map_err(|_| Error::other("scoped dirty usage acknowledgement response is too large"))?;
|
||||
.map_err(|_| Error::other("scoped dirty usage capability response is too large"))?;
|
||||
verify_tonic_rpc_response_proof(&body, response.response_proof.as_ref())?;
|
||||
if response.protocol_version != SCOPED_DIRTY_USAGE_PROTOCOL_VERSION
|
||||
|| response.owner_id != payload.owner_id
|
||||
|| response.instance_id != payload.instance_id
|
||||
|| response.max_entries != SCOPED_DIRTY_USAGE_MAX_ENTRIES
|
||||
|| response.max_request_bytes != SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES
|
||||
|| !response.supported
|
||||
|| response.cleared != 0
|
||||
{
|
||||
return Err(Error::other("scoped dirty usage acknowledgement response does not match request"));
|
||||
return Err(Error::other("scoped dirty usage capability response does not match request"));
|
||||
}
|
||||
Ok(response.supported)
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
let result = match timeout(SCANNER_SCOPED_DIRTY_USAGE_STAGE_TIMEOUT, ack_attempt).await {
|
||||
Ok(result) => self.finalize_result(result).await,
|
||||
Err(_) => {
|
||||
self.prepare_retry_with_timeout(SCANNER_SCOPED_DIRTY_USAGE_STAGE_TIMEOUT)
|
||||
.await;
|
||||
Err(Error::other("scoped dirty usage acknowledgement deadline elapsed"))
|
||||
}
|
||||
};
|
||||
|
||||
match result {
|
||||
Ok(()) => {
|
||||
let activity = self.scanner_scoped_dirty_usage_activity_confirmation().await?;
|
||||
if activity.instance_id == instance_id {
|
||||
Ok(activity)
|
||||
} else {
|
||||
Err(Error::other(
|
||||
"scoped dirty usage acknowledgement peer restarted before activity confirmation",
|
||||
))
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
if Self::is_network_like_error(&err) {
|
||||
self.prepare_retry_with_timeout(SCANNER_SCOPED_DIRTY_USAGE_STAGE_TIMEOUT)
|
||||
.await;
|
||||
}
|
||||
match self.scanner_scoped_dirty_usage_activity_confirmation().await {
|
||||
Ok(activity) if scanner_scoped_dirty_usage_ack_reconciled(&activity, &instance_id) => Ok(activity),
|
||||
_ => Err(err),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn scanner_scoped_dirty_usage_activity_confirmation(&self) -> Result<ScannerPeerActivity> {
|
||||
timeout(SCANNER_SCOPED_DIRTY_USAGE_STAGE_TIMEOUT, self.scanner_activity())
|
||||
.await
|
||||
.map_err(|_| Error::other("scoped dirty usage activity confirmation timed out"))?
|
||||
.await,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn acknowledge_scanner_dirty_usage(&self, instance_id: String, generation: u64) -> Result<ScannerPeerActivity> {
|
||||
@@ -3036,79 +2823,16 @@ mod tests {
|
||||
rustfs_protos::proto_gen::node_service::ScannerDirtyUsageBucket {
|
||||
bucket: "archive".to_string(),
|
||||
generation: 3,
|
||||
bucket_incarnation: Uuid::from_u128(0x11111111111111111111111111111111).as_bytes().to_vec().into(),
|
||||
},
|
||||
rustfs_protos::proto_gen::node_service::ScannerDirtyUsageBucket {
|
||||
bucket: "photos".to_string(),
|
||||
generation: 7,
|
||||
bucket_incarnation: Uuid::from_u128(0x22222222222222222222222222222222).as_bytes().to_vec().into(),
|
||||
},
|
||||
],
|
||||
response_proof: b"proof".to_vec().into(),
|
||||
owner_id: "33333333-3333-3333-3333-333333333333".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_scoped_dirty_usage_ack_payloads_split_at_protocol_limit() {
|
||||
use rustfs_protos::scoped_dirty_usage::{SCOPED_DIRTY_USAGE_MAX_ENTRIES, canonical_scoped_dirty_usage_request};
|
||||
|
||||
let entries = (0..=SCOPED_DIRTY_USAGE_MAX_ENTRIES)
|
||||
.map(|index| ScannerScopedDirtyUsageAckEntry {
|
||||
bucket: format!("bucket-{index:02}"),
|
||||
bucket_incarnation: Uuid::from_u128(0x11111111111111111111111111111111),
|
||||
generation: 9,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let payloads = scanner_scoped_dirty_usage_ack_payloads(
|
||||
"33333333-3333-3333-3333-333333333333".to_string(),
|
||||
"0123456789abcdef0123456789abcdef".to_string(),
|
||||
false,
|
||||
entries,
|
||||
)
|
||||
.expect("33 entries should split into valid scoped dirty usage requests");
|
||||
|
||||
assert_eq!(payloads.len(), 2);
|
||||
assert_eq!(payloads[0].entries.len(), SCOPED_DIRTY_USAGE_MAX_ENTRIES as usize);
|
||||
assert_eq!(payloads[1].entries.len(), 1);
|
||||
assert_eq!(payloads[0].entries.first().map(|entry| entry.bucket.as_str()), Some("bucket-00"));
|
||||
assert_eq!(payloads[0].entries.last().map(|entry| entry.bucket.as_str()), Some("bucket-31"));
|
||||
assert_eq!(payloads[1].entries.first().map(|entry| entry.bucket.as_str()), Some("bucket-32"));
|
||||
for payload in payloads {
|
||||
canonical_scoped_dirty_usage_request(&payload).expect("each split scoped ACK payload should be canonical");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_scoped_dirty_usage_ack_reconciliation_requires_same_clean_instance() {
|
||||
let activity = |instance_id: &str, pending| ScannerPeerActivity {
|
||||
instance_id: instance_id.to_string(),
|
||||
namespace_generation: 1,
|
||||
maintenance_generation: 1,
|
||||
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
topology_digest: Some([1; 32]),
|
||||
data_movement_active: Some(false),
|
||||
dirty_usage_generation: Some(9),
|
||||
dirty_usage_pending: pending,
|
||||
movement_generation: Some(1),
|
||||
publication_blocked: Some(false),
|
||||
};
|
||||
|
||||
assert!(scanner_scoped_dirty_usage_ack_reconciled(
|
||||
&activity("0123456789abcdef0123456789abcdef", Some(false)),
|
||||
"0123456789abcdef0123456789abcdef"
|
||||
));
|
||||
assert!(!scanner_scoped_dirty_usage_ack_reconciled(
|
||||
&activity("0123456789abcdef0123456789abcdef", Some(true)),
|
||||
"0123456789abcdef0123456789abcdef"
|
||||
));
|
||||
assert!(!scanner_scoped_dirty_usage_ack_reconciled(
|
||||
&activity("fedcba9876543210fedcba9876543210", Some(false)),
|
||||
"0123456789abcdef0123456789abcdef"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_dirty_usage_snapshot_requires_a_complete_authenticated_ordered_view() {
|
||||
let decoded = decode_test_scanner_dirty_usage_snapshot(test_scanner_dirty_usage_snapshot_response())
|
||||
@@ -3117,18 +2841,9 @@ mod tests {
|
||||
assert_eq!(decoded.generation, 7);
|
||||
assert_eq!(decoded.pending_bucket_count, 2);
|
||||
assert_eq!(decoded.protocol_version, SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION);
|
||||
assert_eq!(decoded.owner_id, "33333333-3333-3333-3333-333333333333");
|
||||
assert!(decoded.complete);
|
||||
assert_eq!(
|
||||
decoded.buckets.get("archive").map(|bucket| bucket.bucket_incarnation),
|
||||
Some(Uuid::from_u128(0x11111111111111111111111111111111))
|
||||
);
|
||||
assert_eq!(decoded.buckets.get("archive").map(|bucket| bucket.generation), Some(3));
|
||||
assert_eq!(
|
||||
decoded.buckets.get("photos").map(|bucket| bucket.bucket_incarnation),
|
||||
Some(Uuid::from_u128(0x22222222222222222222222222222222))
|
||||
);
|
||||
assert_eq!(decoded.buckets.get("photos").map(|bucket| bucket.generation), Some(7));
|
||||
assert_eq!(decoded.buckets.get("archive"), Some(&3));
|
||||
assert_eq!(decoded.buckets.get("photos"), Some(&7));
|
||||
|
||||
let overflow_count =
|
||||
u64::try_from(SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES + 1).expect("the test snapshot entry limit should fit in u64");
|
||||
@@ -3179,14 +2894,6 @@ mod tests {
|
||||
empty_bucket.buckets[0].bucket.clear();
|
||||
cases.push((empty_bucket, "empty bucket name"));
|
||||
|
||||
let mut invalid_owner = test_scanner_dirty_usage_snapshot_response();
|
||||
invalid_owner.owner_id.clear();
|
||||
cases.push((invalid_owner, "snapshot owner"));
|
||||
|
||||
let mut invalid_incarnation = test_scanner_dirty_usage_snapshot_response();
|
||||
invalid_incarnation.buckets[0].bucket_incarnation = Uuid::nil().as_bytes().to_vec().into();
|
||||
cases.push((invalid_incarnation, "bucket incarnation"));
|
||||
|
||||
let mut partial = test_scanner_dirty_usage_snapshot_response();
|
||||
partial.complete = false;
|
||||
cases.push((partial, "entry-limit overflow"));
|
||||
@@ -3199,7 +2906,6 @@ mod tests {
|
||||
.map(|index| rustfs_protos::proto_gen::node_service::ScannerDirtyUsageBucket {
|
||||
bucket: format!("bucket-{index:04}"),
|
||||
generation: 1,
|
||||
bucket_incarnation: Uuid::from_u128(0x11111111111111111111111111111111).as_bytes().to_vec().into(),
|
||||
})
|
||||
.collect(),
|
||||
..test_scanner_dirty_usage_snapshot_response()
|
||||
|
||||
@@ -449,14 +449,6 @@ where
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_limited_preserve_empty<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<Vec<u8>>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let (data, _obj) = read_config_limited_preserve_empty_with_metadata(api, file, max_bytes).await?;
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_limited<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<Vec<u8>>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
@@ -465,6 +457,14 @@ where
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_limited_preserve_empty<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<Vec<u8>>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let (data, _obj) = read_config_limited_preserve_empty_with_metadata(api, file, max_bytes).await?;
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_limited_preserve_empty_with_metadata<S>(
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
@@ -476,18 +476,6 @@ where
|
||||
read_config_with_metadata_inner(api, file, &ObjectOptions::default(), true, Some(max_bytes)).await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_limited_preserve_empty_with_metadata_opts<S>(
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
opts: &ObjectOptions,
|
||||
max_bytes: usize,
|
||||
) -> Result<(Vec<u8>, ObjectInfo)>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
read_config_with_metadata_inner(api, file, opts, true, Some(max_bytes)).await
|
||||
}
|
||||
|
||||
/// Read an existing config object without treating an empty payload as absent.
|
||||
/// Callers that validate their own payload format need to distinguish corruption
|
||||
/// from `ConfigNotFound`.
|
||||
|
||||
@@ -747,27 +747,18 @@ mod tests {
|
||||
let mut kvs = KVS::new();
|
||||
kvs.insert(CLASS_STANDARD.to_string(), "EC:2".to_string());
|
||||
|
||||
for drives in [2, 3] {
|
||||
let err = lookup_config_for_pools_with_env(&kvs, &[4, drives], no_env_overrides())
|
||||
.expect_err("EC:2 must be rejected by a pool with fewer than four drives per set");
|
||||
assert!(
|
||||
err.to_string().contains("pool 1") && err.to_string().contains(&format!("{drives} drives")),
|
||||
"error must identify the rejecting pool: {err}"
|
||||
);
|
||||
}
|
||||
|
||||
let cfg =
|
||||
lookup_config_for_pools_with_env(&kvs, &[4, 4], no_env_overrides()).expect("EC:2 is valid for both four-drive pools");
|
||||
assert_eq!(cfg.parities_for_sc(STANDARD), Some(vec![2, 2]));
|
||||
let err = lookup_config_for_pools_with_env(&kvs, &[4, 2], no_env_overrides())
|
||||
.expect_err("EC:2 must be rejected by the two-drive pool");
|
||||
assert!(
|
||||
err.to_string().contains("pool 1") && err.to_string().contains("2 drives"),
|
||||
"error must identify the rejecting pool: {err}"
|
||||
);
|
||||
|
||||
kvs.insert(CLASS_STANDARD.to_string(), "EC:1".to_string());
|
||||
for drives in [2, 3, 4] {
|
||||
let cfg =
|
||||
lookup_config_for_pools_with_env(&kvs, &[4, drives], no_env_overrides()).expect("EC:1 is valid for both pools");
|
||||
assert_eq!(cfg.parity_for_sc(STANDARD, 4), Some(1));
|
||||
assert_eq!(cfg.parity_for_sc(STANDARD, drives), Some(1));
|
||||
assert_eq!(cfg.get_parity_for_sc(STANDARD), Some(1));
|
||||
}
|
||||
let cfg = lookup_config_for_pools_with_env(&kvs, &[4, 2], no_env_overrides()).expect("EC:1 is valid for both pools");
|
||||
assert_eq!(cfg.parity_for_sc(STANDARD, 4), Some(1));
|
||||
assert_eq!(cfg.parity_for_sc(STANDARD, 2), Some(1));
|
||||
assert_eq!(cfg.get_parity_for_sc(STANDARD), Some(1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -3013,7 +3013,7 @@ fn parse_decommission_durable_ilm_receipt_path(path: &str) -> Result<Decommissio
|
||||
.ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` is missing its record id")))?;
|
||||
let id_kind = parts
|
||||
.next()
|
||||
.filter(|id_kind| matches!(*id_kind, "operation_id" | "transaction_id" | "job_id" | "control_id"))
|
||||
.filter(|id_kind| matches!(*id_kind, "operation_id" | "transaction_id" | "job_id"))
|
||||
.ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` has an invalid id kind")))?;
|
||||
let source_path = parts
|
||||
.next()
|
||||
@@ -3023,9 +3023,8 @@ fn parse_decommission_durable_ilm_receipt_path(path: &str) -> Result<Decommissio
|
||||
return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid run token")));
|
||||
}
|
||||
match id_kind {
|
||||
"operation_id" | "control_id" if !is_sha256_checksum(id) => {
|
||||
let id_label = id_kind.trim_end_matches("_id");
|
||||
return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid {id_label} id")));
|
||||
"operation_id" if !is_sha256_checksum(id) => {
|
||||
return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid operation id")));
|
||||
}
|
||||
"transaction_id" | "job_id" if uuid::Uuid::parse_str(id).is_err() => {
|
||||
return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid UUID")));
|
||||
@@ -3385,7 +3384,7 @@ pub(crate) async fn acquire_pool_activation_fleet_proof(
|
||||
.ok_or_else(|| Error::other(POOL_ACTIVATION_FLEET_PROOF_REQUIRED))
|
||||
}
|
||||
|
||||
pub fn is_pool_activation_fleet_proof_error(err: &Error) -> bool {
|
||||
pub(crate) fn is_pool_activation_fleet_proof_error(err: &Error) -> bool {
|
||||
// Save-stage helpers add context by formatting the original error, so the
|
||||
// marker may be nested in the display string. Restrict matching to the
|
||||
// `Error::other` I/O shape used by this activation path.
|
||||
@@ -4490,20 +4489,11 @@ impl PoolMetaWriteState {
|
||||
|
||||
fn observe_selection(&mut self, selection: &PoolMetaSelection) -> Result<()> {
|
||||
self.pool_meta_absent = selection.absent;
|
||||
self.validate_selection(selection)?;
|
||||
if self.cluster_epoch.is_none()
|
||||
&& let Some((_, metadata_epoch)) = selection.generation_identity
|
||||
{
|
||||
self.cluster_epoch = Some(metadata_epoch);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_selection(&self, selection: &PoolMetaSelection) -> Result<()> {
|
||||
if let Some(expected_cluster_id) = self.expected_cluster_id
|
||||
&& let Some((cluster_id, _)) = selection.generation_identity
|
||||
&& cluster_id != expected_cluster_id
|
||||
{
|
||||
self.block_writes();
|
||||
return Err(Error::other(format!(
|
||||
"pool metadata incompatible: cluster identity {cluster_id} does not match deployment {expected_cluster_id}"
|
||||
)));
|
||||
@@ -4512,11 +4502,17 @@ impl PoolMetaWriteState {
|
||||
&& let Some((_, metadata_epoch)) = selection.generation_identity
|
||||
&& metadata_epoch != identity_epoch
|
||||
{
|
||||
self.block_writes();
|
||||
return Err(Error::other(format!(
|
||||
"pool metadata recovery required: committed epoch {} does not match cluster identity epoch {identity_epoch}",
|
||||
metadata_epoch
|
||||
)));
|
||||
}
|
||||
if self.cluster_epoch.is_none()
|
||||
&& let Some((_, metadata_epoch)) = selection.generation_identity
|
||||
{
|
||||
self.cluster_epoch = Some(metadata_epoch);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -4549,25 +4545,26 @@ impl PoolMetaWriteState {
|
||||
if !self.pool_meta_absent {
|
||||
return Ok(());
|
||||
}
|
||||
let result = self.validate_missing_metadata_can_initialize();
|
||||
if result.is_err() {
|
||||
self.block_writes();
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn validate_missing_metadata_can_initialize(&self) -> Result<()> {
|
||||
match self.identity_initialized {
|
||||
Some(false) if self.bootstrap_identity_proven() && self.identity_fresh_bootstrap_nonce.is_some() => Ok(()),
|
||||
Some(false) => Err(Error::other(
|
||||
"pool metadata recovery required: pending cluster identity exists but this startup has no verified fresh-bootstrap proof or legacy-adoption proof",
|
||||
)),
|
||||
Some(true) => Err(Error::other(
|
||||
"pool metadata recovery required: initialized cluster identity exists but every pool.bin replica is missing",
|
||||
)),
|
||||
None => Err(Error::other(
|
||||
"pool metadata recovery required: no durable bootstrap identity or pool.bin replica is available",
|
||||
)),
|
||||
Some(false) => {
|
||||
self.block_writes();
|
||||
Err(Error::other(
|
||||
"pool metadata recovery required: pending cluster identity exists but this startup has no verified fresh-bootstrap proof or legacy-adoption proof",
|
||||
))
|
||||
}
|
||||
Some(true) => {
|
||||
self.block_writes();
|
||||
Err(Error::other(
|
||||
"pool metadata recovery required: initialized cluster identity exists but every pool.bin replica is missing",
|
||||
))
|
||||
}
|
||||
None => {
|
||||
self.block_writes();
|
||||
Err(Error::other(
|
||||
"pool metadata recovery required: no durable bootstrap identity or pool.bin replica is available",
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5139,23 +5136,6 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
fn select_pool_meta_replicas_for_read_probe<R>(
|
||||
write_state: &PoolMetaWriteState,
|
||||
replicas: Vec<R>,
|
||||
operation: &str,
|
||||
) -> Result<PoolMetaSelection>
|
||||
where
|
||||
R: Into<PoolMetaReplicaRead>,
|
||||
{
|
||||
let selection = select_pool_meta_replica_reads(replicas.into_iter().map(Into::into).collect())?;
|
||||
write_state.validate_selection(&selection)?;
|
||||
selection.replica_state.ensure_write_safe(operation)?;
|
||||
if selection.absent && (write_state.expected_cluster_id.is_some() || write_state.identity_initialized.is_some()) {
|
||||
write_state.validate_missing_metadata_can_initialize()?;
|
||||
}
|
||||
Ok(selection)
|
||||
}
|
||||
|
||||
async fn load_pool_meta_replicas<S>(pools: Vec<Arc<S>>, no_lock: bool) -> Result<PoolMetaSelection>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
@@ -5175,19 +5155,6 @@ where
|
||||
select_pool_meta_replicas_observing(write_state, replicas)
|
||||
}
|
||||
|
||||
async fn load_pool_meta_replicas_for_read_probe<S>(
|
||||
pools: Vec<Arc<S>>,
|
||||
no_lock: bool,
|
||||
write_state: &PoolMetaWriteState,
|
||||
operation: &str,
|
||||
) -> Result<PoolMetaSelection>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let replicas = read_pool_meta_replicas(pools, no_lock).await;
|
||||
select_pool_meta_replicas_for_read_probe(write_state, replicas, operation)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct PersistedPoolMetaV3 {
|
||||
@@ -9016,7 +8983,7 @@ impl ECStore {
|
||||
|
||||
async fn acquire_pool_meta_read_guard(
|
||||
&self,
|
||||
write_state: &PoolMetaWriteState,
|
||||
write_state: &mut PoolMetaWriteState,
|
||||
operation: &str,
|
||||
) -> Result<(rustfs_lock::NamespaceLockGuard, PoolMeta)> {
|
||||
write_state.ensure_write_safe(operation)?;
|
||||
@@ -9029,7 +8996,9 @@ impl ECStore {
|
||||
})?;
|
||||
let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
|
||||
let pool_meta_guard = pool_meta_lock.get_read_lock(get_lock_acquire_timeout()).await?;
|
||||
let selection = load_pool_meta_replicas_for_read_probe(self.pools.clone(), true, write_state, operation).await?;
|
||||
let selection = load_pool_meta_replicas_observing(self.pools.clone(), true, write_state).await?;
|
||||
write_state.observe_replicas(selection.replica_state);
|
||||
write_state.ensure_write_safe(operation)?;
|
||||
Ok((pool_meta_guard, selection.meta))
|
||||
}
|
||||
|
||||
@@ -9174,9 +9143,9 @@ impl ECStore {
|
||||
target_pool_indices: &[usize],
|
||||
phase: &'static str,
|
||||
) -> Result<(rustfs_lock::NamespaceLockGuard, bool)> {
|
||||
let save_guard = self.pool_meta_save_gate.lock().await;
|
||||
let mut save_guard = self.pool_meta_save_gate.lock().await;
|
||||
let (pool_meta_guard, snapshot) = self
|
||||
.acquire_pool_meta_read_guard(&save_guard, "target capacity admission failed")
|
||||
.acquire_pool_meta_read_guard(&mut save_guard, "target capacity admission failed")
|
||||
.await?;
|
||||
for target_pool_index in target_pool_indices.iter().copied() {
|
||||
ensure_external_decommission_target_admission(&snapshot, target_pool_index, phase)?;
|
||||
@@ -9210,9 +9179,9 @@ impl ECStore {
|
||||
pub(crate) async fn acquire_decommission_capacity_release_fence_with_active_source(
|
||||
&self,
|
||||
) -> Result<(rustfs_lock::NamespaceLockGuard, bool)> {
|
||||
let save_guard = self.pool_meta_save_gate.lock().await;
|
||||
let mut save_guard = self.pool_meta_save_gate.lock().await;
|
||||
let (pool_meta_guard, snapshot) = self
|
||||
.acquire_pool_meta_read_guard(&save_guard, "capacity release fence failed")
|
||||
.acquire_pool_meta_read_guard(&mut save_guard, "capacity release fence failed")
|
||||
.await?;
|
||||
let has_active_source = pool_meta_has_active_decommission(&snapshot);
|
||||
drop(save_guard);
|
||||
@@ -9277,9 +9246,9 @@ impl ECStore {
|
||||
}
|
||||
|
||||
let (reconciliations, model_version) = {
|
||||
let save_guard = self.pool_meta_save_gate.lock().await;
|
||||
let mut save_guard = self.pool_meta_save_gate.lock().await;
|
||||
let (_read_guard, snapshot) = self
|
||||
.acquire_pool_meta_read_guard(&save_guard, "exact delete capacity reconciliation failed")
|
||||
.acquire_pool_meta_read_guard(&mut save_guard, "exact delete capacity reconciliation failed")
|
||||
.await?;
|
||||
let reconciliations = plan_exact_delete_capacity_reconciliations(&snapshot, object, exact)?;
|
||||
let model_version = active_decommission_capacity_model(&snapshot)?;
|
||||
@@ -9884,9 +9853,9 @@ impl ECStore {
|
||||
let non_growing_replacement = matches!(mode, DecommissionCapacityMutationMode::NonGrowingReplacement);
|
||||
let temporary_release = matches!(mode, DecommissionCapacityMutationMode::TemporaryRelease);
|
||||
let mut operation = Some(operation);
|
||||
let save_guard = self.pool_meta_save_gate.lock().await;
|
||||
let mut save_guard = self.pool_meta_save_gate.lock().await;
|
||||
let (read_guard, snapshot) = self
|
||||
.acquire_pool_meta_read_guard(&save_guard, "target capacity admission failed")
|
||||
.acquire_pool_meta_read_guard(&mut save_guard, "target capacity admission failed")
|
||||
.await?;
|
||||
let admission_now = OffsetDateTime::now_utc();
|
||||
let admitted_owner = capacity_owner.and_then(|owner| {
|
||||
@@ -10271,14 +10240,6 @@ impl ECStore {
|
||||
self.pool_meta_save_gate.lock().await.ensure_write_safe(operation)
|
||||
}
|
||||
|
||||
/// Reports whether pool metadata side effects are currently writable.
|
||||
/// Read-only admission probes do not change this state; startup and real
|
||||
/// metadata transactions still latch it on unrecoverable conditions.
|
||||
pub async fn pool_meta_writes_ready(&self) -> bool {
|
||||
let write_state = self.pool_meta_save_gate.lock().await;
|
||||
!write_state.write_blocked && !write_state.aborted_transaction.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
async fn load_runtime_pool_meta_observing(&self, write_state: &mut PoolMetaWriteState, operation: &str) -> Result<PoolMeta> {
|
||||
write_state.ensure_write_safe(operation)?;
|
||||
load_pool_meta_identity_observing(self.pools.clone(), write_state).await?;
|
||||
@@ -10901,9 +10862,9 @@ impl ECStore {
|
||||
// global lock, then fence the exact target cohort before taking the
|
||||
// write lock used to publish the terminal transition.
|
||||
let terminal_fence_plan = if acquire_runtime_fence {
|
||||
let read_save_guard = self.pool_meta_save_gate.lock().await;
|
||||
let mut read_save_guard = self.pool_meta_save_gate.lock().await;
|
||||
let (read_guard, snapshot) = self
|
||||
.acquire_pool_meta_read_guard(&read_save_guard, "decommission cancel fence planning failed")
|
||||
.acquire_pool_meta_read_guard(&mut read_save_guard, "decommission cancel fence planning failed")
|
||||
.await?;
|
||||
let plan = decommission_capacity_terminal_fence_plan(&snapshot, idx)?;
|
||||
drop(read_guard);
|
||||
@@ -16899,87 +16860,6 @@ mod tests {
|
||||
assert!(err.to_string().contains("requires 60 bytes, but 59 bytes are available"));
|
||||
}
|
||||
|
||||
async fn single_pool_capacity_admission_test_store() -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
|
||||
let (temp_dirs, store) =
|
||||
crate::services::rebalance::test_store_with_persisted_rebalance_meta(RebalanceMeta::default()).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
(temp_dirs, store)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn single_pool_public_writes_skip_decommission_capacity_admission() {
|
||||
let (_temp_dirs, store) = single_pool_capacity_admission_test_store().await;
|
||||
let bucket = format!("single-pool-capacity-skip-{}", uuid::Uuid::new_v4());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create single-pool bucket before blocking pool metadata writes");
|
||||
let incarnation = store.bucket_incarnation_id(&bucket).await.expect("load bucket incarnation");
|
||||
store.pool_meta_save_gate.lock().await.block_writes_after_fence_loss();
|
||||
|
||||
let object = "ordinary-put.bin";
|
||||
let mut put_data = crate::object_api::PutObjReader::from_vec(b"ordinary single-pool body".to_vec());
|
||||
store
|
||||
.put_object(&bucket, object, &mut put_data, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("single-pool ordinary PUT must not enter decommission capacity admission");
|
||||
store
|
||||
.get_object_info(&bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("single-pool ordinary PUT must remain readable");
|
||||
|
||||
let multipart_object = "ordinary-multipart.bin";
|
||||
let upload = store
|
||||
.new_multipart_upload(
|
||||
&bucket,
|
||||
multipart_object,
|
||||
&ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("single-pool MPU creation must not enter decommission capacity admission");
|
||||
let mut part_data = crate::object_api::PutObjReader::from_vec(b"single-pool multipart body".to_vec());
|
||||
let part = store
|
||||
.put_object_part(
|
||||
&bucket,
|
||||
multipart_object,
|
||||
&upload.upload_id,
|
||||
1,
|
||||
&mut part_data,
|
||||
&ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("single-pool UploadPart must not enter decommission capacity admission");
|
||||
store
|
||||
.clone()
|
||||
.complete_multipart_upload(
|
||||
&bucket,
|
||||
multipart_object,
|
||||
&upload.upload_id,
|
||||
vec![crate::storage_api_contracts::multipart::CompletePart {
|
||||
part_num: part.part_num,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
}],
|
||||
&ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("single-pool CompleteMultipartUpload must not enter decommission capacity admission");
|
||||
store
|
||||
.get_object_info(&bucket, multipart_object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("single-pool completed MPU must remain readable");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn multipart_mutations_locate_later_upload_before_reserved_pool_admission() {
|
||||
@@ -18052,67 +17932,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pool_meta_read_probe_does_not_latch_writer_state() {
|
||||
let write_state = PoolMetaWriteState::default();
|
||||
select_pool_meta_replicas_for_read_probe(
|
||||
&write_state,
|
||||
vec![PoolMetaReplica::Unreadable("transient read failure".to_string())],
|
||||
"capacity probe",
|
||||
)
|
||||
.expect_err("an unreadable probe replica must fail the current admission");
|
||||
|
||||
assert!(
|
||||
write_state.ensure_write_safe("ordinary object write").is_ok(),
|
||||
"a read-only capacity probe must not permanently latch the pool metadata writer"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pool_meta_read_probe_rejects_missing_runtime_metadata_without_latching() {
|
||||
let write_state = PoolMetaWriteState {
|
||||
expected_cluster_id: Some(uuid::Uuid::new_v4()),
|
||||
identity_initialized: Some(true),
|
||||
..Default::default()
|
||||
};
|
||||
select_pool_meta_replicas_for_read_probe(&write_state, vec![PoolMetaReplica::Missing], "capacity probe")
|
||||
.expect_err("runtime metadata disappearance must reject the current probe");
|
||||
write_state
|
||||
.ensure_write_safe("ordinary object write")
|
||||
.expect("a missing-metadata probe must not permanently latch the writer");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn pool_meta_read_guard_does_not_latch_after_unreadable_replica() {
|
||||
let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
|
||||
let mut saved_disks = Vec::new();
|
||||
for set in &store.pools[1].disk_set {
|
||||
let mut disks = set.disks.write().await;
|
||||
let original = std::mem::take(&mut *disks);
|
||||
let disk_count = original.len();
|
||||
saved_disks.push((set.clone(), original));
|
||||
*disks = vec![None; disk_count];
|
||||
}
|
||||
|
||||
let write_state = store.pool_meta_save_gate.lock().await;
|
||||
store
|
||||
.acquire_pool_meta_read_guard(&write_state, "capacity probe")
|
||||
.await
|
||||
.expect_err("an unreadable metadata replica must reject this probe");
|
||||
write_state
|
||||
.ensure_write_safe("ordinary object write")
|
||||
.expect("a failed read-only probe must remain retryable");
|
||||
|
||||
for (set, disks) in saved_disks {
|
||||
*set.disks.write().await = disks;
|
||||
}
|
||||
store
|
||||
.acquire_pool_meta_read_guard(&write_state, "capacity probe retry")
|
||||
.await
|
||||
.expect("a read-only probe must succeed after the replica recovers");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pool_meta_write_state_blocks_when_selection_has_no_valid_replica() {
|
||||
let replicas = vec![
|
||||
@@ -19481,8 +19300,8 @@ mod pools_tests {
|
||||
load_decommission_entry_versions, local_decommission_queue_prefix, mark_decommission_bucket_done,
|
||||
merge_decommission_durable_ilm_receipts, merge_pool_meta_updates_for_save, merge_pool_status_refresh,
|
||||
missing_decommission_worker_prefix, next_decommission_capacity_generation, observe_decommission_terminal_reload_result,
|
||||
parse_decommission_durable_ilm_receipt_path, pool_meta_has_active_decommission, publish_pool_meta_updates,
|
||||
read_pool_meta_replica, reconcile_decommission_meta_buckets, reconcile_decommission_unresolved_entries_for_completion,
|
||||
pool_meta_has_active_decommission, publish_pool_meta_updates, read_pool_meta_replica,
|
||||
reconcile_decommission_meta_buckets, reconcile_decommission_unresolved_entries_for_completion,
|
||||
record_decommission_unresolved_entry, recover_decommission_capacity_reservations,
|
||||
renew_decommission_capacity_reservation, require_decommission_store, reserve_decommission_start_cancelers,
|
||||
reserve_decommission_start_target_capacity, resolve_decommission_bucket_state,
|
||||
@@ -20350,26 +20169,6 @@ mod pools_tests {
|
||||
assert!(!old_receipt.starts_with(&decommission_durable_ilm_receipt_run_prefix(&second_token)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_recovery_control_receipt_path_round_trips() {
|
||||
let run_token = "b".repeat(64);
|
||||
let control_id = "a".repeat(64);
|
||||
let source_path = format!(
|
||||
"ilm/recovery-controls/transition_transaction/{}/{}/{}.json",
|
||||
&control_id[..2],
|
||||
&control_id[2..4],
|
||||
control_id
|
||||
);
|
||||
let path = decommission_durable_ilm_receipt_path(&run_token, &source_path, "control_id", &control_id);
|
||||
|
||||
let locator = parse_decommission_durable_ilm_receipt_path(&path).expect("recovery control receipt path should parse");
|
||||
|
||||
assert_eq!(locator.run_token, run_token);
|
||||
assert_eq!(locator.source_path, source_path);
|
||||
assert_eq!(locator.id_kind, "control_id");
|
||||
assert_eq!(locator.id, control_id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_receipt_merge_preserves_terminal_proof() {
|
||||
let operation_id = "a".repeat(64);
|
||||
|
||||
@@ -2673,7 +2673,7 @@ mod tests {
|
||||
]),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(object_info.is_multipart());
|
||||
assert!(!object_info.is_multipart());
|
||||
assert!(should_use_multipart_data_movement(&object_info, false));
|
||||
|
||||
let single_nonstandard_part = ObjectInfo {
|
||||
@@ -3050,7 +3050,7 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(object_info.is_multipart());
|
||||
assert!(!object_info.is_multipart());
|
||||
assert!(object_info.parts.iter().any(|part| part.checksums.is_some()));
|
||||
let opts = data_movement_put_object_opts(&object_info, 0);
|
||||
assert!(!rustfs_utils::http::contains_key_str(&opts.user_defined, SUFFIX_PART_CHECKSUMS));
|
||||
|
||||
@@ -99,17 +99,11 @@ pub(crate) const GET_STAGE_READER_OPEN_MMAP_COPY_FALLBACK: &str = "reader_open_m
|
||||
pub(crate) const GET_STAGE_READER_OPEN_MMAP_COPY_SUCCESS: &str = "reader_open_mmap_copy_success";
|
||||
pub(crate) const GET_STAGE_READER_OPEN_STREAM: &str = "reader_open_stream";
|
||||
pub(crate) const GET_STAGE_READER_MMAP_ACCESS_CHECK: &str = "reader_mmap_access_check";
|
||||
#[cfg(unix)]
|
||||
pub(crate) const GET_STAGE_READER_MMAP_BLOCKING_TASK: &str = "reader_mmap_blocking_task";
|
||||
#[cfg(unix)]
|
||||
pub(crate) const GET_STAGE_READER_MMAP_BLOCKING_WAIT: &str = "reader_mmap_blocking_wait";
|
||||
#[cfg(unix)]
|
||||
pub(crate) const GET_STAGE_READER_MMAP_COPY_BUFFER: &str = "reader_mmap_copy_buffer";
|
||||
#[cfg(unix)]
|
||||
pub(crate) const GET_STAGE_READER_MMAP_DIRECT_READ_COPY: &str = "reader_mmap_direct_read_copy";
|
||||
#[cfg(unix)]
|
||||
pub(crate) const GET_STAGE_READER_MMAP_FILE_OPEN: &str = "reader_mmap_file_open";
|
||||
#[cfg(unix)]
|
||||
pub(crate) const GET_STAGE_READER_MMAP_MAP: &str = "reader_mmap_map";
|
||||
pub(crate) const GET_STAGE_READER_MMAP_METADATA_LOOKUP: &str = "reader_mmap_metadata_lookup";
|
||||
pub(crate) const GET_STAGE_READER_MMAP_METADATA_VALIDATE: &str = "reader_mmap_metadata_validate";
|
||||
|
||||
@@ -195,13 +195,6 @@ fn resolve_drive_timeout_profile_from_env() -> DriveTimeoutProfile {
|
||||
DriveTimeoutProfile::parse(rustfs_config::DEFAULT_DRIVE_TIMEOUT_PROFILE).unwrap_or(DriveTimeoutProfile::Default)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
tokio::task_local! {
|
||||
/// Artificial `disk_info` latency for tests that pin how the admin storage
|
||||
/// walk composes per-drive probe time.
|
||||
pub(crate) static DISK_INFO_PROBE_DELAY_FOR_TEST: Duration;
|
||||
}
|
||||
|
||||
fn get_drive_timeout_profile() -> DriveTimeoutProfile {
|
||||
#[cfg(test)]
|
||||
{
|
||||
@@ -331,70 +324,6 @@ impl DiskStoreRenameDataExt for LocalDiskWrapper {
|
||||
}
|
||||
|
||||
impl LocalDiskWrapper {
|
||||
pub(in crate::disk) async fn delete_version_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
fi: FileInfo,
|
||||
force_del_marker: bool,
|
||||
opts: DeleteOptions,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
self.track_disk_health_mutation(
|
||||
"delete_version",
|
||||
DiskMetricMutation::Delete,
|
||||
|| async {
|
||||
Box::pin(
|
||||
self.disk
|
||||
.delete_version_with_namespace_owner(volume, path, fi, force_del_marker, opts, namespace_owner),
|
||||
)
|
||||
.await
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn delete_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
opts: DeleteOptions,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
self.track_disk_health_mutation(
|
||||
"delete",
|
||||
DiskMetricMutation::Delete,
|
||||
|| async { Box::pin(self.disk.delete_with_namespace_owner(volume, path, opts, namespace_owner)).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn undo_write_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
fi: FileInfo,
|
||||
opts: DeleteOptions,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
self.track_disk_health_mutation(
|
||||
"delete_version",
|
||||
DiskMetricMutation::Delete,
|
||||
|| async {
|
||||
// Preserve the old DiskAPI future's boxing boundary.
|
||||
Box::pin(
|
||||
self.disk
|
||||
.undo_write_with_namespace_owner(volume, path, fi, opts, namespace_owner),
|
||||
)
|
||||
.await
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn rename_data_observed(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
@@ -404,34 +333,6 @@ impl LocalDiskWrapper {
|
||||
dst_path: &str,
|
||||
external_guard: Option<Arc<dyn Send + Sync>>,
|
||||
) -> super::RenameDataObservation {
|
||||
self.rename_data_observed_with_guards(
|
||||
src_volume,
|
||||
src_path,
|
||||
fi,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
super::RenameDataGuards {
|
||||
external_guard,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn rename_data_observed_with_guards(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
guards: super::RenameDataGuards,
|
||||
) -> super::RenameDataObservation {
|
||||
let super::RenameDataGuards {
|
||||
external_guard,
|
||||
namespace_owner,
|
||||
..
|
||||
} = guards;
|
||||
let operation = self.clone();
|
||||
let src_volume = src_volume.to_owned();
|
||||
let src_path = src_path.to_owned();
|
||||
@@ -456,15 +357,13 @@ impl LocalDiskWrapper {
|
||||
DiskMetricMutation::Write,
|
||||
|| async {
|
||||
// Preserve the former DiskAPI future's single boxing boundary.
|
||||
let observed = Box::pin(operation.disk.rename_data_observed(
|
||||
&src_volume,
|
||||
&src_path,
|
||||
&fi,
|
||||
&dst_volume,
|
||||
&dst_path,
|
||||
namespace_owner,
|
||||
))
|
||||
.await;
|
||||
let observed =
|
||||
Box::pin(
|
||||
operation
|
||||
.disk
|
||||
.rename_data_observed(&src_volume, &src_path, &fi, &dst_volume, &dst_path),
|
||||
)
|
||||
.await;
|
||||
preflight_rejection = observed.preflight_rejection;
|
||||
observed.result
|
||||
},
|
||||
@@ -1402,7 +1301,6 @@ impl LocalDiskWrapper {
|
||||
self.disk.get_object_path(volume, path)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) fn get_object_path_for_io(&self, volume: &str, path: &str) -> crate::disk::error::Result<std::path::PathBuf> {
|
||||
self.disk.get_object_path_for_io(volume, path)
|
||||
}
|
||||
@@ -2043,10 +1941,6 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
.track_disk_health_with_op_and_timeout_action(
|
||||
"disk_info",
|
||||
|| async {
|
||||
#[cfg(test)]
|
||||
if let Ok(delay) = DISK_INFO_PROBE_DELAY_FOR_TEST.try_with(|delay| *delay) {
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
let result = self.disk.disk_info(opts).await?;
|
||||
|
||||
if let Some(current_disk_id) = *self.disk_id.read().await
|
||||
|
||||
@@ -218,12 +218,10 @@ pub async fn rename(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<
|
||||
fs::rename(from, to).await
|
||||
}
|
||||
|
||||
#[cfg(any(not(windows), test))]
|
||||
pub fn rename_std(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
|
||||
std::fs::rename(from, to)
|
||||
}
|
||||
|
||||
#[cfg(any(not(windows), test))]
|
||||
#[tracing::instrument(level = "debug", skip_all)]
|
||||
pub async fn read_file(path: impl AsRef<Path>) -> io::Result<Vec<u8>> {
|
||||
fs::read(path.as_ref()).await
|
||||
|
||||
+440
-1815
File diff suppressed because it is too large
Load Diff
@@ -17,14 +17,13 @@
|
||||
|
||||
#[cfg(all(test, windows))]
|
||||
use super::run_destination_commit_directory_preparation;
|
||||
#[cfg(any(not(windows), test))]
|
||||
use super::should_fail_local_inline_rollback_hardlink;
|
||||
use super::{
|
||||
EVENT_DISK_LOCAL_ACCESS_FAILED, EVENT_DISK_LOCAL_HEAL_PURGE_FAILED, EVENT_DISK_LOCAL_RENAME_REJECTED, LOG_COMPONENT_ECSTORE,
|
||||
LOG_SUBSYSTEM_DISK_LOCAL, LocalDisk, SyncMode, effective_durability, inline_metadata_rollback_dir, observe_old_current_size,
|
||||
remove_dir_all_if_exists, remove_dst_base_before_commit, remove_file_if_exists, rename_data_versions_signature,
|
||||
run_inline_preparation_before_backup, should_fail_after_metadata_commit, should_fail_before_old_metadata_backup,
|
||||
should_fail_commit_rename, should_remove_staged_meta_before_commit, skip_access_checks,
|
||||
should_fail_commit_rename, should_fail_local_inline_rollback_hardlink, should_remove_staged_meta_before_commit,
|
||||
skip_access_checks,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use super::{run_inline_before_file_sync_admission, run_owned_file_write_before_open, run_rename_data_after_first_publication};
|
||||
@@ -34,7 +33,7 @@ use crate::disk::{
|
||||
error::{DiskError, Result},
|
||||
error_conv::{to_access_error, to_file_error},
|
||||
os,
|
||||
os::check_path_length,
|
||||
os::{check_path_length, rename_all},
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use rustfs_filemeta::{FileInfo, FileMeta};
|
||||
@@ -74,8 +73,6 @@ fn rollback_inline_metadata_commit_std(
|
||||
rollback_data_dir: Option<Uuid>,
|
||||
local_rollback_path: Option<&Path>,
|
||||
) -> std::io::Result<()> {
|
||||
#[cfg(all(test, not(windows)))]
|
||||
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Rollback, dst_file_path);
|
||||
if let Some(backup_path) = local_rollback_path {
|
||||
// The commit immediately before this rollback renamed the staged
|
||||
// xl.meta from the same directory as `backup_path` onto
|
||||
@@ -89,7 +86,6 @@ fn rollback_inline_metadata_commit_std(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(any(not(windows), test))]
|
||||
pub(super) fn create_local_inline_rollback_backup(
|
||||
dst_file_path: &Path,
|
||||
staging_file_path: &Path,
|
||||
@@ -235,12 +231,6 @@ async fn restore_published_data_source(
|
||||
#[derive(Debug)]
|
||||
pub(in crate::disk) struct LocalRenamePreflightRejection(());
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct RenameDataState {
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
preflight_rejection: Option<LocalRenamePreflightRejection>,
|
||||
}
|
||||
|
||||
impl LocalDisk {
|
||||
#[tracing::instrument(name = "rename_data", target = "rustfs_ecstore::disk::local", level = "trace", skip_all)]
|
||||
pub(super) async fn rename_data_inner(
|
||||
@@ -250,7 +240,7 @@ impl LocalDisk {
|
||||
fi: FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
state: &mut RenameDataState,
|
||||
preflight_rejection: &mut Option<LocalRenamePreflightRejection>,
|
||||
) -> Result<RenameDataResp> {
|
||||
crate::hp_guard!("LocalDisk::rename_data");
|
||||
let mut fi = fi;
|
||||
@@ -279,13 +269,7 @@ impl LocalDisk {
|
||||
Some(token) => Some(self.claim_quota_mutation_fence(dst_volume, dst_path, token).await?),
|
||||
None => None,
|
||||
};
|
||||
let mutation_lease = os::acquire_rename_data_mutation_lease_with_owner(
|
||||
&self.root,
|
||||
dst_volume,
|
||||
&destination_object_path,
|
||||
state.namespace_owner.take(),
|
||||
)
|
||||
.await;
|
||||
let mutation_lease = os::acquire_rename_data_mutation_lease(&self.root, dst_volume, &destination_object_path).await;
|
||||
if let Some(claim) = quota_fence_claim {
|
||||
mutation_lease.attach_external_guard(claim);
|
||||
}
|
||||
@@ -318,7 +302,7 @@ impl LocalDisk {
|
||||
error = %e,
|
||||
"Disk local access check failed"
|
||||
);
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
*preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
|
||||
}
|
||||
|
||||
@@ -336,7 +320,7 @@ impl LocalDisk {
|
||||
error = %e,
|
||||
"Disk local access check failed"
|
||||
);
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
*preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
|
||||
}
|
||||
|
||||
@@ -544,9 +528,7 @@ impl LocalDisk {
|
||||
// rename below.
|
||||
if fi_healing
|
||||
&& let Some((_, dst_data_path)) = has_data_dir_path.as_ref()
|
||||
&& let Err(err) = self
|
||||
.move_to_trash_with_namespace_owner(dst_data_path, true, false, Some(mutation_lease.clone()))
|
||||
.await
|
||||
&& let Err(err) = self.move_to_trash(dst_data_path, true, false).await
|
||||
{
|
||||
warn!(
|
||||
target: "rustfs_ecstore::disk::local",
|
||||
@@ -773,7 +755,7 @@ impl LocalDisk {
|
||||
&& let Some(parent) = dst_file_path.parent()
|
||||
{
|
||||
let fsync_started = rustfs_io_metrics::put_stage_timer();
|
||||
if let Err(err) = os::fsync_dst_dir_group_commit(parent, Some(mutation_lease.clone())).await {
|
||||
if let Err(err) = os::fsync_dst_dir_group_commit(parent).await {
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC,
|
||||
fsync_started,
|
||||
@@ -811,7 +793,7 @@ impl LocalDisk {
|
||||
break;
|
||||
}
|
||||
let fsync_started = rustfs_io_metrics::put_stage_timer();
|
||||
if let Err(err) = os::fsync_dir_with_owner(dir, Some(mutation_lease.clone())).await {
|
||||
if let Err(err) = os::fsync_dir(dir).await {
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC,
|
||||
fsync_started,
|
||||
@@ -1042,15 +1024,7 @@ impl LocalDisk {
|
||||
// rename_all acquires the backup path's namespace lease. Do not
|
||||
// hold a disk admission while acquiring another namespace lock.
|
||||
drop(file_sync_admission.take());
|
||||
if let Err(err) = os::rename_all_with_owner(
|
||||
staged_backup,
|
||||
&backup_path,
|
||||
&dst_volume_dir,
|
||||
&self.publication_root,
|
||||
Some(mutation_lease.clone()),
|
||||
)
|
||||
.await
|
||||
{
|
||||
if let Err(err) = rename_all(staged_backup, &backup_path, &dst_volume_dir, &self.publication_root).await {
|
||||
let _ = remove_file_if_exists(staged_backup);
|
||||
return Err(err);
|
||||
}
|
||||
@@ -1246,18 +1220,14 @@ impl LocalDisk {
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> super::super::RenameDataObservation {
|
||||
let mut state = RenameDataState {
|
||||
namespace_owner,
|
||||
..Default::default()
|
||||
};
|
||||
let mut preflight_rejection = None;
|
||||
let result = self
|
||||
.rename_data_inner(src_volume, src_path, fi.clone(), dst_volume, dst_path, &mut state)
|
||||
.rename_data_inner(src_volume, src_path, fi.clone(), dst_volume, dst_path, &mut preflight_rejection)
|
||||
.await;
|
||||
super::super::RenameDataObservation {
|
||||
result,
|
||||
preflight_rejection: state.preflight_rejection,
|
||||
preflight_rejection,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -75,14 +75,6 @@ use time::OffsetDateTime;
|
||||
use tokio::io::{AsyncRead, AsyncWrite};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Independent admission and physical ownership for one disk rename.
|
||||
#[derive(Default)]
|
||||
pub(crate) struct RenameDataGuards {
|
||||
pub(crate) scanner_publication_lease_token: Option<Uuid>,
|
||||
pub(crate) external_guard: Option<Arc<dyn Send + Sync>>,
|
||||
pub(crate) namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
}
|
||||
|
||||
/// Local preflight evidence stays outside DiskAPI and the RPC response format.
|
||||
pub(crate) struct RenameDataObservation {
|
||||
pub(crate) result: Result<RenameDataResp>,
|
||||
@@ -198,17 +190,11 @@ pub struct MmapCopyStageMetrics {
|
||||
pub(crate) path_resolve_stage: &'static str,
|
||||
pub(crate) metadata_lookup_stage: &'static str,
|
||||
pub(crate) metadata_validate_stage: &'static str,
|
||||
#[cfg(unix)]
|
||||
pub(crate) blocking_wait_stage: &'static str,
|
||||
#[cfg(unix)]
|
||||
pub(crate) blocking_task_stage: &'static str,
|
||||
#[cfg(unix)]
|
||||
pub(crate) file_open_stage: &'static str,
|
||||
#[cfg(unix)]
|
||||
pub(crate) mmap_map_stage: &'static str,
|
||||
#[cfg(unix)]
|
||||
pub(crate) mmap_copy_stage: &'static str,
|
||||
#[cfg(unix)]
|
||||
pub(crate) direct_read_copy_stage: &'static str,
|
||||
}
|
||||
|
||||
@@ -732,65 +718,6 @@ impl Disk {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_version_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
fi: FileInfo,
|
||||
force_del_marker: bool,
|
||||
opts: DeleteOptions,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
match self {
|
||||
Self::Local(disk) => {
|
||||
disk.delete_version_with_namespace_owner(volume, path, fi, force_del_marker, opts, namespace_owner)
|
||||
.await
|
||||
}
|
||||
Self::Remote(disk) => {
|
||||
let result = disk.delete_version(volume, path, fi, force_del_marker, opts).await;
|
||||
// This is sender lifetime only, not proof of a remote physical drain.
|
||||
drop(namespace_owner);
|
||||
result
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
opts: DeleteOptions,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.delete_with_namespace_owner(volume, path, opts, namespace_owner).await,
|
||||
Self::Remote(disk) => {
|
||||
let result = disk.delete(volume, path, opts).await;
|
||||
drop(namespace_owner);
|
||||
result
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Keep local undo publication owned independently of the wrapper deadline.
|
||||
/// Remote undo retains its existing RPC contract; this is not a remote drain proof.
|
||||
pub(crate) async fn undo_write_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
fi: FileInfo,
|
||||
opts: DeleteOptions,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
match self {
|
||||
Self::Local(disk) => {
|
||||
disk.undo_write_with_namespace_owner(volume, path, fi, opts, namespace_owner)
|
||||
.await
|
||||
}
|
||||
Self::Remote(disk) => disk.delete_version(volume, path, fi, false, opts).await,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn rename_data_borrowed(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
@@ -810,12 +737,12 @@ impl Disk {
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
guards: RenameDataGuards,
|
||||
scanner_publication_lease_token: Option<Uuid>,
|
||||
) -> RenameDataObservation {
|
||||
match self {
|
||||
Disk::Local(local_disk) => {
|
||||
local_disk
|
||||
.rename_data_observed_with_guards(src_volume, src_path, fi, dst_volume, dst_path, guards)
|
||||
.rename_data_observed(src_volume, src_path, fi, dst_volume, dst_path, None)
|
||||
.await
|
||||
}
|
||||
Disk::Remote(remote_disk) => RenameDataObservation::unknown(
|
||||
@@ -826,7 +753,7 @@ impl Disk {
|
||||
fi,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
guards.scanner_publication_lease_token,
|
||||
scanner_publication_lease_token,
|
||||
)
|
||||
.await,
|
||||
),
|
||||
@@ -995,7 +922,6 @@ impl Disk {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) fn get_object_path_for_io_if_local(
|
||||
&self,
|
||||
volume: &str,
|
||||
|
||||
+49
-620
@@ -91,10 +91,6 @@ pub(crate) mod fsync_dir_recorder {
|
||||
static RECORDED: Mutex<Vec<PathBuf>> = Mutex::new(Vec::new());
|
||||
static LIMITED: Mutex<Vec<PathBuf>> = Mutex::new(Vec::new());
|
||||
static GROUPED: Mutex<Vec<(PathBuf, usize)>> = Mutex::new(Vec::new());
|
||||
#[cfg(unix)]
|
||||
static FAILURES: std::sync::LazyLock<Mutex<HashMap<PathBuf, io::ErrorKind>>> =
|
||||
std::sync::LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
#[cfg(unix)]
|
||||
static BEFORE_LIMITED: std::sync::LazyLock<Mutex<HashMap<PathBuf, Hook>>> =
|
||||
std::sync::LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
static BEFORE_GROUP_BATCH: std::sync::LazyLock<Mutex<HashMap<PathBuf, Hook>>> =
|
||||
@@ -154,20 +150,6 @@ pub(crate) mod fsync_dir_recorder {
|
||||
contains_path(&RECORDED.lock().expect("fsync dir recorder poisoned"), dir)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) fn set_failure(dir: &Path, kind: io::ErrorKind) {
|
||||
FAILURES
|
||||
.lock()
|
||||
.expect("fsync dir failure hook poisoned")
|
||||
.insert(dir.to_path_buf(), kind);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) fn take_failure(dir: &Path) -> Option<io::ErrorKind> {
|
||||
remove_path_keyed(&FAILURES, dir, "fsync dir failure hook poisoned")
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) fn record_limited(dir: &Path) {
|
||||
record_path(&LIMITED, dir, "limited fsync dir recorder");
|
||||
let hook = remove_hook(&BEFORE_LIMITED, dir, "limited fsync hook poisoned");
|
||||
@@ -180,7 +162,6 @@ pub(crate) mod fsync_dir_recorder {
|
||||
contains_path(&LIMITED.lock().expect("limited fsync dir recorder poisoned"), dir)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) fn set_before_limited(dir: &Path, hook: impl FnOnce() + Send + 'static) {
|
||||
BEFORE_LIMITED
|
||||
.lock()
|
||||
@@ -256,96 +237,11 @@ pub(crate) mod fsync_dir_recorder {
|
||||
.insert(dir.to_path_buf(), kind);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) fn take_grouped_failure(dir: &Path) -> Option<io::ErrorKind> {
|
||||
remove_path_keyed(&GROUPED_FAILURES, dir, "grouped fsync failure hook poisoned")
|
||||
}
|
||||
}
|
||||
|
||||
/// Pause a real namespace mutation inside its physical executor.
|
||||
#[cfg(all(test, not(windows)))]
|
||||
pub(crate) mod prepared_publication_test_hooks {
|
||||
use super::*;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
||||
pub(crate) enum Stage {
|
||||
PreparedRename,
|
||||
Rename,
|
||||
Remove,
|
||||
Rollback,
|
||||
DirFsync,
|
||||
}
|
||||
|
||||
type Hook = Box<dyn FnOnce() + Send>;
|
||||
type Key = (Stage, PathBuf);
|
||||
static BEFORE_PUBLICATION: LazyLock<Mutex<HashMap<Key, Hook>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
pub(crate) struct Guard(Key);
|
||||
|
||||
impl Drop for Guard {
|
||||
fn drop(&mut self) {
|
||||
BEFORE_PUBLICATION.lock().remove(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn install(path: &Path, hook: impl FnOnce() + Send + 'static) -> Guard {
|
||||
install_at(Stage::PreparedRename, path, hook)
|
||||
}
|
||||
|
||||
pub(crate) fn install_at(stage: Stage, path: &Path, hook: impl FnOnce() + Send + 'static) -> Guard {
|
||||
let key = (stage, path.to_path_buf());
|
||||
assert!(BEFORE_PUBLICATION.lock().insert(key.clone(), Box::new(hook)).is_none());
|
||||
Guard(key)
|
||||
}
|
||||
|
||||
pub(crate) fn run(stage: Stage, path: &Path) {
|
||||
let hook = BEFORE_PUBLICATION.lock().remove(&(stage, path.to_path_buf()));
|
||||
if let Some(hook) = hook {
|
||||
hook();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
type RenameDestinationHook = Box<dyn FnOnce(&Path) + Send>;
|
||||
#[cfg(test)]
|
||||
static RENAME_DESTINATIONS: LazyLock<Mutex<HashMap<PathBuf, RenameDestinationHook>>> =
|
||||
LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) struct RenameDestinationGuard(PathBuf);
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for RenameDestinationGuard {
|
||||
fn drop(&mut self) {
|
||||
RENAME_DESTINATIONS.lock().remove(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn observe_rename_destination(source: &Path, hook: impl FnOnce(&Path) + Send + 'static) -> RenameDestinationGuard {
|
||||
assert!(
|
||||
RENAME_DESTINATIONS
|
||||
.lock()
|
||||
.insert(source.to_path_buf(), Box::new(hook))
|
||||
.is_none()
|
||||
);
|
||||
RenameDestinationGuard(source.to_path_buf())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn drain_namespace_key(path: &Path) {
|
||||
drop(super::acquire_namespace_mutation_lease(path).await);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn run_rename_destination(source: &Path, destination: &Path) {
|
||||
let hook = RENAME_DESTINATIONS.lock().remove(source);
|
||||
if let Some(hook) = hook {
|
||||
hook(destination);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, windows))]
|
||||
pub(crate) mod windows_rename_test_hooks {
|
||||
use super::*;
|
||||
@@ -442,10 +338,6 @@ pub fn fsync_dir_std(dir: impl AsRef<Path>) -> io::Result<()> {
|
||||
fsync_dir_recorder::record(dir.as_ref());
|
||||
#[cfg(unix)]
|
||||
{
|
||||
#[cfg(test)]
|
||||
if let Some(kind) = fsync_dir_recorder::take_failure(dir.as_ref()) {
|
||||
return Err(io::Error::from(kind));
|
||||
}
|
||||
std::fs::File::open(dir.as_ref())?.sync_all()?;
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
@@ -684,7 +576,6 @@ impl OpenedDstDirFsyncGroup {
|
||||
}
|
||||
|
||||
struct DstDirFsyncWaiter {
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
result_tx: oneshot::Sender<SharedDstDirFsyncResult>,
|
||||
}
|
||||
|
||||
@@ -743,7 +634,6 @@ impl DstDirFsyncGroupCommit {
|
||||
fn enqueue_opened(
|
||||
&self,
|
||||
opened: OpenedDstDirFsyncGroup,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> io::Result<(oneshot::Receiver<SharedDstDirFsyncResult>, Option<Arc<DstDirFsyncGroup>>)> {
|
||||
let (result_tx, result_rx) = oneshot::channel();
|
||||
let mut registry = self.inner.lock();
|
||||
@@ -774,10 +664,7 @@ impl DstDirFsyncGroupCommit {
|
||||
group
|
||||
};
|
||||
let mut group_state = group.inner.lock();
|
||||
group_state.pending.push_back(DstDirFsyncWaiter {
|
||||
result_tx,
|
||||
namespace_owner,
|
||||
});
|
||||
group_state.pending.push_back(DstDirFsyncWaiter { result_tx });
|
||||
let start_worker = !group_state.worker_running;
|
||||
if start_worker {
|
||||
group_state.worker_running = true;
|
||||
@@ -799,13 +686,7 @@ impl DstDirFsyncGroupCommit {
|
||||
fn remove_idle_group(&self, group: &Arc<DstDirFsyncGroup>) {
|
||||
let mut registry = self.inner.lock();
|
||||
let group_state = group.inner.lock();
|
||||
if !group_state.worker_running
|
||||
&& group_state.pending.is_empty()
|
||||
&& registry
|
||||
.groups
|
||||
.get(&group.key)
|
||||
.is_some_and(|registered| Arc::ptr_eq(registered, group))
|
||||
{
|
||||
if !group_state.worker_running && group_state.pending.is_empty() {
|
||||
registry.groups.remove(&group.key);
|
||||
}
|
||||
}
|
||||
@@ -828,20 +709,16 @@ impl DstDirFsyncGroupCommit {
|
||||
&self,
|
||||
dir: &Path,
|
||||
) -> io::Result<(oneshot::Receiver<SharedDstDirFsyncResult>, Option<Arc<DstDirFsyncGroup>>)> {
|
||||
self.enqueue_opened(OpenedDstDirFsyncGroup::open(dir)?, None)
|
||||
self.enqueue_opened(OpenedDstDirFsyncGroup::open(dir)?)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup, namespace_owners: Vec<Arc<dyn Send + Sync>>) -> io::Result<()> {
|
||||
async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup) -> io::Result<()> {
|
||||
#[cfg(test)]
|
||||
let dir = group.dir.clone();
|
||||
let dir_file = group.dir_file.clone();
|
||||
fsync_spawn_blocking(move || {
|
||||
// The batch worker may be cancelled while this syscall is still running.
|
||||
let _namespace_owners = namespace_owners;
|
||||
#[cfg(all(test, not(windows)))]
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::DirFsync, &dir);
|
||||
#[cfg(test)]
|
||||
{
|
||||
if let Some(kind) = fsync_dir_recorder::take_grouped_failure(&dir) {
|
||||
@@ -856,118 +733,66 @@ async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup, namespace_owners: Ve
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup, namespace_owners: Vec<Arc<dyn Send + Sync>>) -> io::Result<()> {
|
||||
let _namespace_owners = namespace_owners;
|
||||
async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup) -> io::Result<()> {
|
||||
fsync_dir(&group.dir).await
|
||||
}
|
||||
|
||||
struct DstDirFsyncWorkerGuard {
|
||||
group: Arc<DstDirFsyncGroup>,
|
||||
in_flight: usize,
|
||||
armed: bool,
|
||||
}
|
||||
|
||||
impl Drop for DstDirFsyncWorkerGuard {
|
||||
fn drop(&mut self) {
|
||||
if !self.armed {
|
||||
async fn run_dst_dir_fsync_group_worker(group: Arc<DstDirFsyncGroup>) {
|
||||
loop {
|
||||
#[cfg(test)]
|
||||
fsync_dir_recorder::run_before_group_batch(&group.dir);
|
||||
tokio::task::yield_now().await;
|
||||
let batch: Vec<DstDirFsyncWaiter> = {
|
||||
let mut group_state = group.inner.lock();
|
||||
group_state.pending.drain(..).collect()
|
||||
};
|
||||
if batch.is_empty() {
|
||||
let mut group_state = group.inner.lock();
|
||||
group_state.worker_running = false;
|
||||
drop(group_state);
|
||||
DST_DIR_FSYNC_GROUP_COMMIT.remove_idle_group(&group);
|
||||
return;
|
||||
}
|
||||
// Cancellation must release queued owners, but the physical batch keeps
|
||||
// its own owners until its blocking syscall returns.
|
||||
let pending = {
|
||||
let mut registry = DST_DIR_FSYNC_GROUP_COMMIT.inner.lock();
|
||||
let mut group_state = self.group.inner.lock();
|
||||
let pending = std::mem::take(&mut group_state.pending);
|
||||
group_state.worker_running = false;
|
||||
if registry
|
||||
.groups
|
||||
.get(&self.group.key)
|
||||
.is_some_and(|group| Arc::ptr_eq(group, &self.group))
|
||||
{
|
||||
registry.total_waiters = registry.total_waiters.saturating_sub(pending.len() + self.in_flight);
|
||||
registry.groups.remove(&self.group.key);
|
||||
}
|
||||
pending
|
||||
};
|
||||
// Lease and channel destructors must run outside the registry locks.
|
||||
drop(pending);
|
||||
}
|
||||
}
|
||||
|
||||
fn run_dst_dir_fsync_group_worker(group: Arc<DstDirFsyncGroup>) -> impl std::future::Future<Output = ()> {
|
||||
// Capture before spawning: shutdown may drop the future without polling it.
|
||||
let worker_guard = DstDirFsyncWorkerGuard {
|
||||
group: group.clone(),
|
||||
in_flight: 0,
|
||||
armed: true,
|
||||
};
|
||||
async move {
|
||||
let mut worker_guard = worker_guard;
|
||||
loop {
|
||||
#[cfg(test)]
|
||||
fsync_dir_recorder::run_before_group_batch(&group.dir);
|
||||
tokio::task::yield_now().await;
|
||||
let mut batch: Vec<DstDirFsyncWaiter> = {
|
||||
let mut group_state = group.inner.lock();
|
||||
group_state.pending.drain(..).collect()
|
||||
};
|
||||
if batch.is_empty() {
|
||||
let mut group_state = group.inner.lock();
|
||||
worker_guard.armed = false;
|
||||
#[cfg(test)]
|
||||
fsync_dir_recorder::record_grouped(&group.dir, batch.len());
|
||||
let result = fsync_open_dst_dir_group(&group)
|
||||
.await
|
||||
.map_err(SharedDstDirFsyncError::from_error);
|
||||
let batch_len = batch.len();
|
||||
DST_DIR_FSYNC_GROUP_COMMIT.complete_batch(batch_len);
|
||||
|
||||
let should_stop = {
|
||||
let mut group_state = group.inner.lock();
|
||||
if group_state.pending.is_empty() {
|
||||
group_state.worker_running = false;
|
||||
drop(group_state);
|
||||
DST_DIR_FSYNC_GROUP_COMMIT.remove_idle_group(&group);
|
||||
return;
|
||||
}
|
||||
worker_guard.in_flight = batch.len();
|
||||
|
||||
#[cfg(test)]
|
||||
fsync_dir_recorder::record_grouped(&group.dir, batch.len());
|
||||
let namespace_owners = batch.iter_mut().filter_map(|waiter| waiter.namespace_owner.take()).collect();
|
||||
let result = fsync_open_dst_dir_group(&group, namespace_owners)
|
||||
.await
|
||||
.map_err(SharedDstDirFsyncError::from_error);
|
||||
let batch_len = batch.len();
|
||||
DST_DIR_FSYNC_GROUP_COMMIT.complete_batch(batch_len);
|
||||
worker_guard.in_flight = 0;
|
||||
|
||||
let should_stop = {
|
||||
let mut group_state = group.inner.lock();
|
||||
if group_state.pending.is_empty() {
|
||||
worker_guard.armed = false;
|
||||
group_state.worker_running = false;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
};
|
||||
if should_stop {
|
||||
DST_DIR_FSYNC_GROUP_COMMIT.remove_idle_group(&group);
|
||||
}
|
||||
for waiter in batch {
|
||||
let _ = waiter.result_tx.send(result.clone());
|
||||
}
|
||||
if should_stop {
|
||||
return;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
};
|
||||
if should_stop {
|
||||
DST_DIR_FSYNC_GROUP_COMMIT.remove_idle_group(&group);
|
||||
}
|
||||
for waiter in batch {
|
||||
let _ = waiter.result_tx.send(result.clone());
|
||||
}
|
||||
if should_stop {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn fsync_dst_dir_group_commit_with_enabled(
|
||||
dir: impl AsRef<Path>,
|
||||
enabled: bool,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> io::Result<()> {
|
||||
async fn fsync_dst_dir_group_commit_with_enabled(dir: impl AsRef<Path>, enabled: bool) -> io::Result<()> {
|
||||
if !enabled {
|
||||
return fsync_dir_with_owner(dir.as_ref(), namespace_owner).await;
|
||||
return fsync_dir(dir).await;
|
||||
}
|
||||
|
||||
let dir = dir.as_ref().to_path_buf();
|
||||
let opened = tokio::task::spawn_blocking(move || OpenedDstDirFsyncGroup::open(&dir))
|
||||
.await
|
||||
.map_err(|err| io::Error::other(format!("blocking dst dir group open failed: {err}")))??;
|
||||
let (result_rx, worker) = DST_DIR_FSYNC_GROUP_COMMIT.enqueue_opened(opened, namespace_owner)?;
|
||||
let (result_rx, worker) = DST_DIR_FSYNC_GROUP_COMMIT.enqueue_opened(opened)?;
|
||||
if let Some(group) = worker {
|
||||
tokio::spawn(run_dst_dir_fsync_group_worker(group));
|
||||
}
|
||||
@@ -979,11 +804,8 @@ async fn fsync_dst_dir_group_commit_with_enabled(
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn fsync_dst_dir_group_commit(
|
||||
dir: impl AsRef<Path>,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> io::Result<()> {
|
||||
fsync_dst_dir_group_commit_with_enabled(dir, dst_dir_fsync_group_commit_enabled(), namespace_owner).await
|
||||
pub(crate) async fn fsync_dst_dir_group_commit(dir: impl AsRef<Path>) -> io::Result<()> {
|
||||
fsync_dst_dir_group_commit_with_enabled(dir, dst_dir_fsync_group_commit_enabled()).await
|
||||
}
|
||||
|
||||
pub(crate) async fn fsync_dst_dir_group_commit_or_namespace_file_sync_limit(
|
||||
@@ -992,7 +814,7 @@ pub(crate) async fn fsync_dst_dir_group_commit_or_namespace_file_sync_limit(
|
||||
admission: &FileSyncAdmission,
|
||||
) -> io::Result<()> {
|
||||
if dst_dir_fsync_group_commit_enabled() {
|
||||
fsync_dst_dir_group_commit_with_enabled(dir, true, Some(lease)).await
|
||||
fsync_dst_dir_group_commit_with_enabled(dir, true).await
|
||||
} else {
|
||||
fsync_dir_with_namespace_file_sync_limit(dir, lease, admission).await
|
||||
}
|
||||
@@ -1000,7 +822,7 @@ pub(crate) async fn fsync_dst_dir_group_commit_or_namespace_file_sync_limit(
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn fsync_dst_dir_group_commit_for_test(dir: impl AsRef<Path>, enabled: bool) -> io::Result<()> {
|
||||
fsync_dst_dir_group_commit_with_enabled(dir, enabled, None).await
|
||||
fsync_dst_dir_group_commit_with_enabled(dir, enabled).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -1407,8 +1229,6 @@ pub(crate) struct NamespaceMutationLease {
|
||||
_namespace_guard: OwnedMutexGuard<()>,
|
||||
_volume_guard: Option<OwnedRwLockReadGuard<()>>,
|
||||
external_guard: Mutex<Option<Arc<dyn Send + Sync>>>,
|
||||
// Independent of the quota claim; both survive cancellation of the waiter.
|
||||
_namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
}
|
||||
|
||||
impl NamespaceMutationLease {
|
||||
@@ -1418,18 +1238,10 @@ impl NamespaceMutationLease {
|
||||
}
|
||||
|
||||
async fn acquire_namespace_mutation_lease(path: &Path) -> Arc<NamespaceMutationLease> {
|
||||
acquire_namespace_mutation_lease_with_owner(path, None).await
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn acquire_namespace_mutation_lease_with_owner(
|
||||
path: &Path,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Arc<NamespaceMutationLease> {
|
||||
Arc::new(NamespaceMutationLease {
|
||||
_namespace_guard: disk_namespace_mutation_lock(path).lock_owned().await,
|
||||
_volume_guard: None,
|
||||
external_guard: Mutex::new(None),
|
||||
_namespace_owner: namespace_owner,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1439,15 +1251,6 @@ pub(crate) async fn acquire_rename_data_mutation_lease(
|
||||
root: &Path,
|
||||
volume: &str,
|
||||
destination_object: &Path,
|
||||
) -> Arc<NamespaceMutationLease> {
|
||||
acquire_rename_data_mutation_lease_with_owner(root, volume, destination_object, None).await
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire_rename_data_mutation_lease_with_owner(
|
||||
root: &Path,
|
||||
volume: &str,
|
||||
destination_object: &Path,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Arc<NamespaceMutationLease> {
|
||||
let namespace_guard = disk_namespace_mutation_lock(destination_object).lock_owned().await;
|
||||
let volume_guard = disk_volume_mutation_lock(root, volume).read_owned().await;
|
||||
@@ -1455,7 +1258,6 @@ pub(crate) async fn acquire_rename_data_mutation_lease_with_owner(
|
||||
_namespace_guard: namespace_guard,
|
||||
_volume_guard: Some(volume_guard),
|
||||
external_guard: Mutex::new(None),
|
||||
_namespace_owner: namespace_owner,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1945,105 +1747,6 @@ pub async fn rename_all(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn fsync_dir_with_owner(path: &Path, namespace_owner: Option<Arc<dyn Send + Sync>>) -> io::Result<()> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
if namespace_owner.is_none() {
|
||||
return fsync_dir(path).await;
|
||||
}
|
||||
let path = path.to_path_buf();
|
||||
fsync_spawn_blocking(move || {
|
||||
let _namespace_owner = namespace_owner;
|
||||
fsync_dir_std(path)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let _ = namespace_owner;
|
||||
fsync_dir(path).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Retain namespace ownership in the actual filesystem executor after timeout.
|
||||
pub(crate) async fn remove_file_with_owner(
|
||||
path: impl AsRef<Path>,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> io::Result<()> {
|
||||
if namespace_owner.is_none() {
|
||||
return tokio::fs::remove_file(path).await;
|
||||
}
|
||||
let path = path.as_ref().to_path_buf();
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await;
|
||||
run_blocking_namespace_operation(lease, move || {
|
||||
#[cfg(all(test, not(windows)))]
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Remove, &path);
|
||||
std::fs::remove_file(path)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Retain namespace ownership in the actual filesystem executor after timeout.
|
||||
pub(crate) async fn remove_dir_with_owner(
|
||||
path: impl AsRef<Path>,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> io::Result<()> {
|
||||
if namespace_owner.is_none() {
|
||||
return tokio::fs::remove_dir(path).await;
|
||||
}
|
||||
let path = path.as_ref().to_path_buf();
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await;
|
||||
run_blocking_namespace_operation(lease, move || std::fs::remove_dir(path)).await
|
||||
}
|
||||
|
||||
/// Preserve raw rename semantics while retaining a counted owner in the syscall.
|
||||
/// Unlike reliable rename, this never creates parents or retries a missing source.
|
||||
pub(in crate::disk) async fn rename_with_namespace_owner(
|
||||
src: &Path,
|
||||
dst: &Path,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> io::Result<()> {
|
||||
if namespace_owner.is_none() {
|
||||
return tokio::fs::rename(src, dst).await;
|
||||
}
|
||||
let src = src.to_path_buf();
|
||||
let dst = dst.to_path_buf();
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(&dst, namespace_owner).await;
|
||||
run_blocking_namespace_operation(lease, move || {
|
||||
#[cfg(all(test, not(windows)))]
|
||||
{
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &src);
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &dst);
|
||||
}
|
||||
std::fs::rename(src, dst)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn create_dir_all_with_namespace_owner(
|
||||
path: &Path,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> io::Result<()> {
|
||||
if namespace_owner.is_none() {
|
||||
return tokio::fs::create_dir_all(path).await;
|
||||
}
|
||||
let path = path.to_path_buf();
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await;
|
||||
run_blocking_namespace_operation(lease, move || std::fs::create_dir_all(path)).await
|
||||
}
|
||||
|
||||
#[tracing::instrument(name = "rename_all", level = "debug", skip_all)]
|
||||
pub(crate) async fn rename_all_with_owner(
|
||||
src_file_path: impl AsRef<Path>,
|
||||
dst_file_path: impl AsRef<Path>,
|
||||
base_dir: impl AsRef<Path>,
|
||||
publication_root: &PublicationRoot,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(dst_file_path.as_ref(), namespace_owner).await;
|
||||
rename_all_with_lease(src_file_path, dst_file_path, base_dir, publication_root, lease).await
|
||||
}
|
||||
|
||||
pub(crate) async fn rename_all_with_lease(
|
||||
src_file_path: impl AsRef<Path>,
|
||||
dst_file_path: impl AsRef<Path>,
|
||||
@@ -2236,8 +1939,6 @@ pub(crate) async fn rename_all_with_prepared_source(
|
||||
move || {
|
||||
validate_prepared_rename_source(&prepared_source, &src_file_path)?;
|
||||
let preparation = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?;
|
||||
#[cfg(test)]
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::PreparedRename, &dst_file_path);
|
||||
rename_prepared(&src_file_path, &dst_file_path, &preparation)
|
||||
}
|
||||
};
|
||||
@@ -2276,32 +1977,6 @@ pub async fn rename_all_ignore_missing_source(
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(name = "rename_all_ignore_missing_source", level = "debug", skip_all)]
|
||||
pub(crate) async fn rename_all_ignore_missing_source_with_owner(
|
||||
src_file_path: impl AsRef<Path>,
|
||||
dst_file_path: impl AsRef<Path>,
|
||||
base_dir: impl AsRef<Path>,
|
||||
publication_root: &PublicationRoot,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
let src_file_path = src_file_path.as_ref();
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(dst_file_path.as_ref(), namespace_owner).await;
|
||||
match reliable_rename_inner_with_lease(
|
||||
src_file_path.to_path_buf(),
|
||||
dst_file_path.as_ref().to_path_buf(),
|
||||
base_dir.as_ref().to_path_buf(),
|
||||
publication_root.clone(),
|
||||
false,
|
||||
lease,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(()) => Ok(()),
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound && rename_source_is_missing(src_file_path, publication_root) => Ok(()),
|
||||
Err(err) => Err(to_file_error(err).into()),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
pub(crate) fn rename_source_is_missing(src_file_path: &Path, publication_root: &PublicationRoot) -> bool {
|
||||
let Some(source_parent) = src_file_path.parent() else {
|
||||
@@ -2367,13 +2042,6 @@ async fn reliable_rename_inner_with_lease(
|
||||
let base_dir = base_dir.clone();
|
||||
move || {
|
||||
let preparation = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?;
|
||||
#[cfg(all(test, not(windows)))]
|
||||
prepared_publication_test_hooks::run_rename_destination(&src_file_path, &dst_file_path);
|
||||
#[cfg(all(test, not(windows)))]
|
||||
{
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &src_file_path);
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &dst_file_path);
|
||||
}
|
||||
rename_prepared(&src_file_path, &dst_file_path, &preparation)
|
||||
}
|
||||
};
|
||||
@@ -6468,245 +6136,6 @@ mod tests {
|
||||
wait_for_dst_dir_fsync_group_commit_idle().await;
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(dst_dir_fsync_group_commit)]
|
||||
async fn grouped_fsync_physical_batch_keeps_all_owners_after_worker_cancellation() {
|
||||
let temp_dir = tempdir().expect("fixture directory");
|
||||
let dir = temp_dir.path().canonicalize().expect("canonical fsync path");
|
||||
let first_ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let second_ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let first_owner = first_ctx.begin_namespace_commit();
|
||||
let second_owner = second_ctx.begin_namespace_commit();
|
||||
let first_probe = Arc::downgrade(&first_owner);
|
||||
let second_probe = Arc::downgrade(&second_owner);
|
||||
let (first_rx, group) = DST_DIR_FSYNC_GROUP_COMMIT
|
||||
.enqueue_opened(
|
||||
OpenedDstDirFsyncGroup::open(&dir).expect("open first waiter directory"),
|
||||
Some(first_owner),
|
||||
)
|
||||
.expect("queue first real waiter");
|
||||
let group = group.expect("first waiter starts the group");
|
||||
let (second_rx, second_worker) = DST_DIR_FSYNC_GROUP_COMMIT
|
||||
.enqueue_opened(
|
||||
OpenedDstDirFsyncGroup::open(&dir).expect("open second waiter directory"),
|
||||
Some(second_owner),
|
||||
)
|
||||
.expect("queue second real waiter");
|
||||
assert!(second_worker.is_none(), "same directory must join the same batch");
|
||||
assert_eq!(group.inner.lock().pending.len(), 2);
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
let _hook =
|
||||
prepared_publication_test_hooks::install_at(prepared_publication_test_hooks::Stage::DirFsync, &dir, move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
let worker = tokio::spawn(run_dst_dir_fsync_group_worker(group.clone()));
|
||||
tokio::time::timeout(Duration::from_secs(5), entered_rx)
|
||||
.await
|
||||
.expect("batch must reach its physical fsync")
|
||||
.expect("physical fsync entry");
|
||||
assert_eq!(fsync_dir_recorder::grouped_batch_sizes(&dir), vec![2]);
|
||||
assert!(
|
||||
group.inner.lock().pending.is_empty(),
|
||||
"both waiters were transferred into the physical batch"
|
||||
);
|
||||
let queued_ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let queued_owner = queued_ctx.begin_namespace_commit();
|
||||
let queued_probe = Arc::downgrade(&queued_owner);
|
||||
let queued_generation = queued_ctx.namespace_commit_generation();
|
||||
let (queued_rx, queued_worker) = DST_DIR_FSYNC_GROUP_COMMIT
|
||||
.enqueue_opened(
|
||||
OpenedDstDirFsyncGroup::open(&dir).expect("open queued waiter directory"),
|
||||
Some(queued_owner),
|
||||
)
|
||||
.expect("queue a waiter after the physical batch was frozen");
|
||||
assert!(queued_worker.is_none());
|
||||
assert_eq!(group.inner.lock().pending.len(), 1);
|
||||
drop((first_rx, second_rx));
|
||||
worker.abort();
|
||||
assert!(worker.await.expect_err("cancel the async batch owner").is_cancelled());
|
||||
assert!(queued_rx.await.is_err(), "an undispatched waiter must observe worker cancellation");
|
||||
assert!(queued_probe.upgrade().is_none());
|
||||
assert!(!queued_ctx.namespace_commits_pending());
|
||||
assert!(queued_ctx.namespace_commit_generation() > queued_generation);
|
||||
assert!(group.inner.lock().pending.is_empty());
|
||||
assert!(!group.inner.lock().worker_running);
|
||||
assert_eq!(DST_DIR_FSYNC_GROUP_COMMIT.counts_for_test(), (0, 0));
|
||||
let first_pending = first_ctx.namespace_commits_pending() && first_probe.upgrade().is_some();
|
||||
let second_pending = second_ctx.namespace_commits_pending() && second_probe.upgrade().is_some();
|
||||
let generations = (first_ctx.namespace_commit_generation(), second_ctx.namespace_commit_generation());
|
||||
drop(release_tx);
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while Arc::strong_count(&group.dir_file) != 1
|
||||
|| first_probe.upgrade().is_some()
|
||||
|| second_probe.upgrade().is_some()
|
||||
|| first_ctx.namespace_commits_pending()
|
||||
|| second_ctx.namespace_commits_pending()
|
||||
{
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("physical fsync must release every batch owner");
|
||||
assert!(fsync_dir_recorder::was_fsynced(&dir), "the detached syscall must really execute");
|
||||
assert!(
|
||||
first_pending && second_pending,
|
||||
"one physical batch must preserve both independent namespace owners"
|
||||
);
|
||||
assert!(!first_ctx.namespace_commits_pending());
|
||||
assert!(!second_ctx.namespace_commits_pending());
|
||||
assert!(first_ctx.namespace_commit_generation() > generations.0);
|
||||
assert!(second_ctx.namespace_commit_generation() > generations.1);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(dst_dir_fsync_group_commit)]
|
||||
async fn grouped_fsync_unpolled_worker_releases_queued_owner() {
|
||||
let temp_dir = tempdir().expect("fixture directory");
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let owner = ctx.begin_namespace_commit();
|
||||
let probe = Arc::downgrade(&owner);
|
||||
let generation = ctx.namespace_commit_generation();
|
||||
let (rx, group) = DST_DIR_FSYNC_GROUP_COMMIT
|
||||
.enqueue_opened(
|
||||
OpenedDstDirFsyncGroup::open(temp_dir.path()).expect("open queued waiter directory"),
|
||||
Some(owner),
|
||||
)
|
||||
.expect("queue a real waiter");
|
||||
let group = group.expect("first waiter starts the group");
|
||||
let worker = run_dst_dir_fsync_group_worker(group.clone());
|
||||
assert!(ctx.namespace_commits_pending());
|
||||
drop(worker);
|
||||
assert!(rx.await.is_err(), "shutdown before first poll must release the waiter");
|
||||
assert!(probe.upgrade().is_none());
|
||||
assert!(!ctx.namespace_commits_pending());
|
||||
assert!(ctx.namespace_commit_generation() > generation);
|
||||
assert!(group.inner.lock().pending.is_empty());
|
||||
assert!(!group.inner.lock().worker_running);
|
||||
assert_eq!(DST_DIR_FSYNC_GROUP_COMMIT.counts_for_test(), (0, 0));
|
||||
assert!(
|
||||
fsync_dir_recorder::grouped_batch_sizes(temp_dir.path()).is_empty(),
|
||||
"the dropped future must not dispatch a physical batch"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn stale_idle_group_cleanup_preserves_successor_registration() {
|
||||
let temp_dir = tempdir().expect("fixture directory");
|
||||
let registry = DstDirFsyncGroupCommit::default();
|
||||
let (mut first_rx, first_worker) = registry.enqueue_for_test(temp_dir.path()).expect("enqueue first worker");
|
||||
let old_group = first_worker.expect("first waiter starts a worker");
|
||||
// W1 has completed its batch and marked G idle, but has not cleaned G up.
|
||||
let first_waiter = old_group.inner.lock().pending.pop_front().expect("first batch waiter");
|
||||
registry.complete_batch(1);
|
||||
old_group.inner.lock().worker_running = false;
|
||||
|
||||
let (mut second_rx, second_worker) = registry.enqueue_for_test(temp_dir.path()).expect("enqueue second worker");
|
||||
let reused_group = second_worker.expect("idle G starts another worker");
|
||||
assert!(Arc::ptr_eq(&old_group, &reused_group));
|
||||
let second_waiter = reused_group.inner.lock().pending.pop_front().expect("second batch waiter");
|
||||
registry.complete_batch(1);
|
||||
reused_group.inner.lock().worker_running = false;
|
||||
registry.remove_idle_group(&reused_group);
|
||||
assert_eq!(registry.counts_for_test(), (0, 0), "normal idle cleanup must remove G");
|
||||
assert!(second_waiter.result_tx.send(Ok(())).is_ok());
|
||||
assert!(second_rx.try_recv().expect("second worker reports completion").is_ok());
|
||||
|
||||
let (mut successor_rx, successor_worker) = registry.enqueue_for_test(temp_dir.path()).expect("enqueue successor");
|
||||
let successor = successor_worker.expect("successor starts a new group");
|
||||
assert!(!Arc::ptr_eq(&old_group, &successor));
|
||||
assert_eq!(registry.counts_for_test(), (1, 1));
|
||||
// W1 resumes with its old Arc after W2 removed G and W3 installed G2.
|
||||
registry.remove_idle_group(&old_group);
|
||||
assert!(first_waiter.result_tx.send(Ok(())).is_ok());
|
||||
assert!(first_rx.try_recv().expect("first worker reports completion").is_ok());
|
||||
assert!(
|
||||
registry
|
||||
.inner
|
||||
.lock()
|
||||
.groups
|
||||
.get(&successor.key)
|
||||
.is_some_and(|registered| Arc::ptr_eq(registered, &successor)),
|
||||
"stale cleanup must retain the exact successor Arc"
|
||||
);
|
||||
assert_eq!(registry.counts_for_test(), (1, 1));
|
||||
assert!(successor.inner.lock().worker_running);
|
||||
assert_eq!(successor.inner.lock().pending.len(), 1);
|
||||
assert!(matches!(successor_rx.try_recv(), Err(oneshot::error::TryRecvError::Empty)));
|
||||
|
||||
let (_joined_rx, new_worker) = registry.enqueue_for_test(temp_dir.path()).expect("join successor");
|
||||
assert!(new_worker.is_none(), "a later waiter must join G2 instead of creating G3");
|
||||
assert_eq!(successor.inner.lock().pending.len(), 2);
|
||||
assert_eq!(registry.counts_for_test(), (1, 2));
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(dst_dir_fsync_group_commit)]
|
||||
async fn stale_idle_cleanup_then_unpolled_worker_drop_releases_waiter_budget() {
|
||||
wait_for_dst_dir_fsync_group_commit_idle().await;
|
||||
let temp_dir = tempdir().expect("fixture directory");
|
||||
let (old_rx, old_worker) = DST_DIR_FSYNC_GROUP_COMMIT
|
||||
.enqueue_for_test(temp_dir.path())
|
||||
.expect("enqueue old group");
|
||||
let old_group = old_worker.expect("old group starts a worker");
|
||||
tokio::time::timeout(Duration::from_secs(5), run_dst_dir_fsync_group_worker(old_group.clone()))
|
||||
.await
|
||||
.expect("old worker must finish its actual fsync");
|
||||
assert!(old_rx.await.expect("old worker reports completion").is_ok());
|
||||
assert!(fsync_dir_recorder::was_fsynced(temp_dir.path()));
|
||||
assert_eq!(fsync_dir_recorder::grouped_batch_sizes(temp_dir.path()), vec![1]);
|
||||
assert_eq!(DST_DIR_FSYNC_GROUP_COMMIT.counts_for_test(), (0, 0));
|
||||
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let owner = ctx.begin_namespace_commit();
|
||||
let probe = Arc::downgrade(&owner);
|
||||
let generation = ctx.namespace_commit_generation();
|
||||
let (rx, successor_worker) = DST_DIR_FSYNC_GROUP_COMMIT
|
||||
.enqueue_opened(
|
||||
OpenedDstDirFsyncGroup::open(temp_dir.path()).expect("open successor directory"),
|
||||
Some(owner),
|
||||
)
|
||||
.expect("enqueue successor owner");
|
||||
let successor = successor_worker.expect("successor starts a new group");
|
||||
assert!(!Arc::ptr_eq(&old_group, &successor));
|
||||
let worker = run_dst_dir_fsync_group_worker(successor.clone());
|
||||
// The stale Arc represents W1 resuming after another worker removed G.
|
||||
DST_DIR_FSYNC_GROUP_COMMIT.remove_idle_group(&old_group);
|
||||
assert!(ctx.namespace_commits_pending());
|
||||
assert!(probe.upgrade().is_some());
|
||||
drop(worker);
|
||||
let channel_closed = tokio::time::timeout(Duration::from_secs(5), rx)
|
||||
.await
|
||||
.expect("dropping the unpolled worker must release its channel")
|
||||
.is_err();
|
||||
let counts_after_drop = DST_DIR_FSYNC_GROUP_COMMIT.counts_for_test();
|
||||
let owner_released = probe.upgrade().is_none();
|
||||
let namespace_pending = ctx.namespace_commits_pending();
|
||||
let generation_after_drop = ctx.namespace_commit_generation();
|
||||
let successor_pending = successor.inner.lock().pending.len();
|
||||
let worker_running = successor.inner.lock().worker_running;
|
||||
// Preserve the observed result before cleanup, so a RED run cannot leak
|
||||
// its phantom count into unrelated tests in the same process.
|
||||
clear_dst_dir_fsync_group_commit_for_test();
|
||||
assert!(channel_closed);
|
||||
assert!(owner_released);
|
||||
assert!(!namespace_pending);
|
||||
assert!(generation_after_drop > generation);
|
||||
assert_eq!(successor_pending, 0);
|
||||
assert!(!worker_running);
|
||||
assert_eq!(
|
||||
fsync_dir_recorder::grouped_batch_sizes(temp_dir.path()),
|
||||
vec![1],
|
||||
"dropping the successor before its first poll must not dispatch another fsync"
|
||||
);
|
||||
assert_eq!(counts_after_drop, (0, 0), "stale cleanup must not strand a phantom waiter");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(dst_dir_fsync_group_commit)]
|
||||
async fn dst_dir_fsync_group_commit_cancellation_releases_waiter_state() {
|
||||
|
||||
@@ -203,19 +203,6 @@ pub enum StorageError {
|
||||
NotFirstDisk,
|
||||
#[error("first disk wait")]
|
||||
FirstDiskWait,
|
||||
#[error(
|
||||
"unsupported pool expansion: an existing single-node single-drive (SNSD) deployment cannot be expanded in place (configured {configured_drives} drive endpoints); restart with the original single local path, or create a new multi-drive deployment and migrate data through S3"
|
||||
)]
|
||||
UnsupportedSnsdExpansion { configured_drives: usize },
|
||||
#[error(
|
||||
"pool topology mismatch: stored {stored_drives} drives with {stored_set_drive_count} drives per erasure set, configured {configured_drives} drives with {configured_set_drive_count} drives per erasure set; an existing pool's drive count and erasure set width cannot be changed in place; restore its original endpoints and RUSTFS_ERASURE_SET_DRIVE_COUNT setting; to expand a multi-drive deployment, append a new pool with at least 2 drive endpoints"
|
||||
)]
|
||||
PoolTopologyMismatch {
|
||||
stored_drives: usize,
|
||||
stored_set_drive_count: usize,
|
||||
configured_drives: usize,
|
||||
configured_set_drive_count: usize,
|
||||
},
|
||||
|
||||
// ── Operational ──────────────────────────────────────────────────
|
||||
#[error("Storage reached its minimum free drive threshold.")]
|
||||
@@ -642,20 +629,6 @@ impl Clone for StorageError {
|
||||
StorageError::ErasureWriteQuorum => StorageError::ErasureWriteQuorum,
|
||||
StorageError::NotFirstDisk => StorageError::NotFirstDisk,
|
||||
StorageError::FirstDiskWait => StorageError::FirstDiskWait,
|
||||
StorageError::UnsupportedSnsdExpansion { configured_drives } => StorageError::UnsupportedSnsdExpansion {
|
||||
configured_drives: *configured_drives,
|
||||
},
|
||||
StorageError::PoolTopologyMismatch {
|
||||
stored_drives,
|
||||
stored_set_drive_count,
|
||||
configured_drives,
|
||||
configured_set_drive_count,
|
||||
} => StorageError::PoolTopologyMismatch {
|
||||
stored_drives: *stored_drives,
|
||||
stored_set_drive_count: *stored_set_drive_count,
|
||||
configured_drives: *configured_drives,
|
||||
configured_set_drive_count: *configured_set_drive_count,
|
||||
},
|
||||
StorageError::TooManyOpenFiles => StorageError::TooManyOpenFiles,
|
||||
StorageError::NoHealRequired => StorageError::NoHealRequired,
|
||||
StorageError::Lock(e) => StorageError::Lock(e.clone()),
|
||||
@@ -762,11 +735,6 @@ impl StorageError {
|
||||
StorageError::ErasureWriteQuorum => StorageErrorCode::ErasureWriteQuorum,
|
||||
StorageError::NotFirstDisk => StorageErrorCode::NotFirstDisk,
|
||||
StorageError::FirstDiskWait => StorageErrorCode::FirstDiskWait,
|
||||
// Topology diagnostics reuse the existing wire code; they are
|
||||
// not disk errors and must retain their local identity for retry classification.
|
||||
StorageError::UnsupportedSnsdExpansion { .. } | StorageError::PoolTopologyMismatch { .. } => {
|
||||
StorageErrorCode::InvalidArgument
|
||||
}
|
||||
StorageError::ConfigNotFound => StorageErrorCode::ConfigNotFound,
|
||||
StorageError::TooManyOpenFiles => StorageErrorCode::TooManyOpenFiles,
|
||||
StorageError::NoHealRequired => StorageErrorCode::NoHealRequired,
|
||||
@@ -1247,29 +1215,6 @@ mod tests {
|
||||
use super::*;
|
||||
use std::io::{Error as IoError, ErrorKind};
|
||||
|
||||
#[test]
|
||||
fn startup_topology_errors_preserve_identity_and_guidance() {
|
||||
for error in [
|
||||
StorageError::UnsupportedSnsdExpansion { configured_drives: 4 },
|
||||
StorageError::PoolTopologyMismatch {
|
||||
stored_drives: 4,
|
||||
stored_set_drive_count: 4,
|
||||
configured_drives: 8,
|
||||
configured_set_drive_count: 8,
|
||||
},
|
||||
] {
|
||||
let io_error: IoError = error.clone().into();
|
||||
let restored = StorageError::from(io_error);
|
||||
assert_eq!(std::mem::discriminant(&restored), std::mem::discriminant(&error));
|
||||
assert_eq!(restored.to_string(), error.to_string());
|
||||
assert_eq!(restored.code(), StorageErrorCode::InvalidArgument);
|
||||
assert!(
|
||||
restored.narrow_to_disk().is_err(),
|
||||
"startup diagnostics must not become disk/quorum errors"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn other_preserves_erasure_construction_source_chain() {
|
||||
use crate::erasure::coding::ErasureConstructionError;
|
||||
|
||||
@@ -13,15 +13,12 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::diagnostics::get::{
|
||||
GET_STAGE_READER_MMAP_ACCESS_CHECK, GET_STAGE_READER_MMAP_METADATA_LOOKUP, GET_STAGE_READER_MMAP_METADATA_VALIDATE,
|
||||
GET_STAGE_READER_MMAP_ACCESS_CHECK, GET_STAGE_READER_MMAP_BLOCKING_TASK, GET_STAGE_READER_MMAP_BLOCKING_WAIT,
|
||||
GET_STAGE_READER_MMAP_COPY_BUFFER, GET_STAGE_READER_MMAP_DIRECT_READ_COPY, GET_STAGE_READER_MMAP_FILE_OPEN,
|
||||
GET_STAGE_READER_MMAP_MAP, GET_STAGE_READER_MMAP_METADATA_LOOKUP, GET_STAGE_READER_MMAP_METADATA_VALIDATE,
|
||||
GET_STAGE_READER_MMAP_PATH_RESOLVE, GET_STAGE_READER_OPEN_MMAP_COPY_FALLBACK, GET_STAGE_READER_OPEN_MMAP_COPY_SUCCESS,
|
||||
GET_STAGE_READER_OPEN_STREAM, GET_STAGE_READER_STREAM_FIRST_READ, record_get_stage_duration_if_enabled,
|
||||
};
|
||||
#[cfg(unix)]
|
||||
use crate::diagnostics::get::{
|
||||
GET_STAGE_READER_MMAP_BLOCKING_TASK, GET_STAGE_READER_MMAP_BLOCKING_WAIT, GET_STAGE_READER_MMAP_COPY_BUFFER,
|
||||
GET_STAGE_READER_MMAP_DIRECT_READ_COPY, GET_STAGE_READER_MMAP_FILE_OPEN, GET_STAGE_READER_MMAP_MAP,
|
||||
};
|
||||
#[cfg(feature = "hotpath")]
|
||||
use crate::disk::FileWriter;
|
||||
use crate::disk::{self, DiskAPI as _, DiskStore, FileReader, MmapCopyStageMetrics, error::DiskError};
|
||||
@@ -409,17 +406,11 @@ async fn open_disk_reader(
|
||||
path_resolve_stage: GET_STAGE_READER_MMAP_PATH_RESOLVE,
|
||||
metadata_lookup_stage: GET_STAGE_READER_MMAP_METADATA_LOOKUP,
|
||||
metadata_validate_stage: GET_STAGE_READER_MMAP_METADATA_VALIDATE,
|
||||
#[cfg(unix)]
|
||||
blocking_wait_stage: GET_STAGE_READER_MMAP_BLOCKING_WAIT,
|
||||
#[cfg(unix)]
|
||||
blocking_task_stage: GET_STAGE_READER_MMAP_BLOCKING_TASK,
|
||||
#[cfg(unix)]
|
||||
file_open_stage: GET_STAGE_READER_MMAP_FILE_OPEN,
|
||||
#[cfg(unix)]
|
||||
mmap_map_stage: GET_STAGE_READER_MMAP_MAP,
|
||||
#[cfg(unix)]
|
||||
mmap_copy_stage: GET_STAGE_READER_MMAP_COPY_BUFFER,
|
||||
#[cfg(unix)]
|
||||
direct_read_copy_stage: GET_STAGE_READER_MMAP_DIRECT_READ_COPY,
|
||||
});
|
||||
let mmap_result = {
|
||||
|
||||
@@ -25,20 +25,6 @@ pub(crate) const MAX_ERASURE_SET_DRIVE_COUNT: usize = 16;
|
||||
const SET_SIZES: [usize; 15] = [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, MAX_ERASURE_SET_DRIVE_COUNT];
|
||||
const ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT: &str = "RUSTFS_ERASURE_SET_DRIVE_COUNT";
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
enum PoolDriveCountError {
|
||||
#[error(
|
||||
"Incorrect number of endpoints provided, size {size}; an erasure pool requires at least {} drive endpoints on one or more nodes; for a standalone single-drive deployment, use a single local path without ellipses",
|
||||
SET_SIZES[0]
|
||||
)]
|
||||
BelowMinimum { size: usize },
|
||||
#[error(
|
||||
"Incorrect number of endpoints provided, size {size}; {}={set_drive_count} requires at least {set_drive_count} drive endpoints per pool",
|
||||
ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT
|
||||
)]
|
||||
BelowSetWidth { size: usize, set_drive_count: usize },
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Default)]
|
||||
pub struct PoolDisksLayout {
|
||||
cmd_line: String,
|
||||
@@ -146,7 +132,7 @@ impl DisksLayout {
|
||||
for arg in args.iter() {
|
||||
if !has_ellipses(&[arg]) && args.len() > 1 {
|
||||
return Err(Error::other(
|
||||
"all args must have ellipses for pool expansion (Invalid arguments specified); each pool must expand to at least 2 drive endpoints on one or more nodes; a single-drive pool cannot be added to a multi-pool deployment",
|
||||
"all args must have ellipses for pool expansion (Invalid arguments specified)",
|
||||
));
|
||||
}
|
||||
|
||||
@@ -410,11 +396,9 @@ fn get_set_indexes<T: AsRef<str>>(
|
||||
}
|
||||
|
||||
for &size in total_sizes {
|
||||
if size < SET_SIZES[0] {
|
||||
return Err(Error::other(PoolDriveCountError::BelowMinimum { size }));
|
||||
}
|
||||
if size < set_drive_count {
|
||||
return Err(Error::other(PoolDriveCountError::BelowSetWidth { size, set_drive_count }));
|
||||
// Check if total_sizes has minimum range upto set_size
|
||||
if size < SET_SIZES[0] || size < set_drive_count {
|
||||
return Err(Error::other(format!("Incorrect number of endpoints provided, size {size}")));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -723,7 +707,7 @@ mod test {
|
||||
arg: "http://rustfs{2...3}/export/set{1...0}",
|
||||
..Default::default()
|
||||
},
|
||||
// Ranges must use three dots.
|
||||
// Range cannot be smaller than 4 minimum.
|
||||
TestCase {
|
||||
num: 4,
|
||||
arg: "/export{1..2}",
|
||||
@@ -942,146 +926,11 @@ mod test {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pool_expansion_accepts_single_node_multi_drive_pools() {
|
||||
temp_env::with_var(ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT, Some("0"), || {
|
||||
for (volumes, drives) in [
|
||||
(["http://node1:9000/data{1...2}", "http://node2:9000/data{1...2}"], 2),
|
||||
(["http://node1:9000/data{1...4}", "http://node2:9000/data{1...4}"], 4),
|
||||
(["http://node{1...4}:9000/data", "http://node5:9000/data{1...4}"], 4),
|
||||
(["http://node5:9000/data{1...4}", "http://node{1...4}:9000/data"], 4),
|
||||
] {
|
||||
let layout = DisksLayout::from_volumes(&volumes).expect("single-node multi-drive pools are valid");
|
||||
|
||||
assert!(!layout.legacy);
|
||||
assert_eq!(layout.pools.len(), 2);
|
||||
for (index, volume) in volumes.iter().enumerate() {
|
||||
assert_eq!(layout.get_set_count(index), 1);
|
||||
assert_eq!(layout.get_drives_per_set(index), drives);
|
||||
assert_eq!(layout.get_cmd_line(index), *volume);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pool_expansion_accepts_multi_node_single_drive_pools() {
|
||||
temp_env::with_var(ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT, Some("0"), || {
|
||||
for nodes in [2, 3, 4] {
|
||||
let volumes = [
|
||||
format!("http://pool1-node{{1...{nodes}}}:9000/data"),
|
||||
format!("http://pool2-node{{1...{nodes}}}:9000/data"),
|
||||
];
|
||||
let layout = DisksLayout::from_volumes(&volumes).expect("each node may contribute one drive to a pool");
|
||||
|
||||
assert_eq!(layout.pools.len(), 2);
|
||||
for pool in 0..2 {
|
||||
assert_eq!(layout.get_set_count(pool), 1);
|
||||
assert_eq!(layout.get_drives_per_set(pool), nodes);
|
||||
let expected = (1..=nodes)
|
||||
.map(|node| format!("http://pool{}-node{node}:9000/data", pool + 1))
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(layout.pools[pool].layout, vec![expected]);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_endpoints_without_ellipses_form_one_pool() {
|
||||
temp_env::with_var(ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT, Some("0"), || {
|
||||
let volumes = ["http://node1:9000/data", "http://node2:9000/data"];
|
||||
let layout = DisksLayout::from_volumes(&volumes).expect("explicit endpoints form one legacy pool");
|
||||
|
||||
assert!(layout.legacy);
|
||||
assert_eq!(layout.pools.len(), 1);
|
||||
assert_eq!(layout.pools[0].layout, vec![volumes.to_vec()]);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn standalone_single_drive_path_remains_supported() {
|
||||
temp_env::with_var(ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT, Some("0"), || {
|
||||
let layout = DisksLayout::from_volumes(&["/data"]).expect("standalone single-drive deployment is valid");
|
||||
|
||||
assert!(layout.is_single_drive_layout());
|
||||
assert_eq!(layout.get_single_drive_layout(), "/data");
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pool_expansion_rejects_plain_single_drive_pool_with_notice() {
|
||||
temp_env::with_var(ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT, Some("0"), || {
|
||||
for volumes in [
|
||||
["http://node{1...2}:9000/data", "http://node3:9000/data"],
|
||||
["http://node3:9000/data", "http://node{1...2}:9000/data"],
|
||||
] {
|
||||
let err = DisksLayout::from_volumes(&volumes).expect_err("a plain endpoint cannot be an expansion pool");
|
||||
let message = err.to_string();
|
||||
|
||||
assert!(message.contains("all args must have ellipses for pool expansion"), "{message}");
|
||||
assert!(message.contains("at least 2 drive endpoints"), "{message}");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pool_expansion_rejects_singleton_ellipsis_pool_with_notice() {
|
||||
temp_env::with_var(ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT, Some("0"), || {
|
||||
for singleton in ["http://node{3...3}:9000/data", "http://node3:9000/data{1...1}"] {
|
||||
for volumes in [
|
||||
vec!["http://node{1...2}:9000/data", singleton],
|
||||
vec![singleton, "http://node{1...2}:9000/data"],
|
||||
vec![singleton],
|
||||
] {
|
||||
let err = DisksLayout::from_volumes(&volumes).expect_err("a singleton range still contains one drive");
|
||||
let message = err.to_string();
|
||||
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::Other);
|
||||
assert!(matches!(
|
||||
err.get_ref().and_then(|source| source.downcast_ref::<PoolDriveCountError>()),
|
||||
Some(PoolDriveCountError::BelowMinimum { size: 1 })
|
||||
));
|
||||
assert!(message.contains("at least 2 drive endpoints"), "{message}");
|
||||
assert!(message.contains("single local path without ellipses"), "{message}");
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_set_size_counts_drives_not_nodes() {
|
||||
for volume in ["http://node1:9000/data{1...4}", "http://node{1...4}:9000/data"] {
|
||||
let sets = get_all_sets(2, true, &[volume]).expect("four endpoints can form two two-drive sets");
|
||||
assert_eq!(sets.iter().map(Vec::len).collect::<Vec<_>>(), vec![2, 2]);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn undersized_pool_error_identifies_requested_set_size() {
|
||||
let err =
|
||||
get_all_sets(4, true, &["http://node{1...2}:9000/data"]).expect_err("two endpoints cannot fill a four-drive set");
|
||||
let message = err.to_string();
|
||||
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::Other);
|
||||
assert!(matches!(
|
||||
err.get_ref().and_then(|source| source.downcast_ref::<PoolDriveCountError>()),
|
||||
Some(PoolDriveCountError::BelowSetWidth {
|
||||
size: 2,
|
||||
set_drive_count: 4
|
||||
})
|
||||
));
|
||||
assert!(message.contains("size 2"), "{message}");
|
||||
assert!(message.contains("RUSTFS_ERASURE_SET_DRIVE_COUNT=4"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn layout_errors_do_not_echo_url_credentials() {
|
||||
for volumes in [
|
||||
vec!["http://:duplicate-secret@server/path", "http://:duplicate-secret@server/path"],
|
||||
vec!["http://:ellipsis...secret@server/path"],
|
||||
vec!["http://server{1...2}/data", "http://:plain-secret@server3/data"],
|
||||
vec!["http://server{1...2}/data", "http://:singleton-secret@server{3...3}/data"],
|
||||
] {
|
||||
let err = DisksLayout::from_volumes(&volumes).unwrap_err();
|
||||
assert!(!err.to_string().contains("secret"), "layout error leaked endpoint credentials: {err}");
|
||||
|
||||
@@ -2432,41 +2432,6 @@ mod test {
|
||||
assert_eq!(local_endpoints[0].pool_idx, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pool_expansion_resolves_single_node_multi_drive_and_multi_node_single_drive_pools() {
|
||||
for (additional_pool, expected_nodes) in [
|
||||
("http://rustfs-5.example.invalid:9000/data{1...4}", 5),
|
||||
("http://rustfs-{5...8}.example.invalid:9000/data", 8),
|
||||
] {
|
||||
let layout = temp_env::with_var("RUSTFS_ERASURE_SET_DRIVE_COUNT", Some("0"), || {
|
||||
DisksLayout::from_volumes(&["http://rustfs-{1...4}.example.invalid:9000/data", additional_pool])
|
||||
})
|
||||
.expect("both single-node multi-drive and multi-node single-drive pools should parse");
|
||||
|
||||
let (pools, setup_type) = EndpointServerPools::create_server_endpoints_with(
|
||||
"0.0.0.0:9000",
|
||||
&layout,
|
||||
Some(orchestrated_test_policy()),
|
||||
Some("rustfs-1.example.invalid"),
|
||||
)
|
||||
.await
|
||||
.expect("pool admission must not impose a minimum node count or drives per node");
|
||||
|
||||
assert_eq!(setup_type, SetupType::DistErasure);
|
||||
assert_eq!(pools.0.len(), 2);
|
||||
assert_eq!(pools.get_nodes().len(), expected_nodes);
|
||||
for (pool_index, pool) in (0_i32..).zip(&pools.0) {
|
||||
assert_eq!((pool.set_count, pool.drives_per_set), (1, 4));
|
||||
assert_eq!(pool.endpoints.as_ref().len(), 4);
|
||||
for (disk_index, endpoint) in (0_i32..).zip(pool.endpoints.as_ref()) {
|
||||
assert_eq!(endpoint.pool_idx, pool_index);
|
||||
assert_eq!(endpoint.set_idx, 0);
|
||||
assert_eq!(endpoint.disk_idx, disk_index);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn explicit_local_endpoint_host_fails_closed_for_invalid_context_or_zero_match() {
|
||||
let args = vec![
|
||||
|
||||
@@ -2278,121 +2278,6 @@ mod tests {
|
||||
assert_eq!(read, fixture.plaintext, "SSE-C + compression full GET must reassemble all parts");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_empty_tail_full_reads_preserve_plaintext() {
|
||||
let key = [0x6Eu8; 32];
|
||||
let part_sizes = [5 * 1024 * 1024, 0];
|
||||
let encrypted = build_legacy_ssec_multipart_fixture(key, &part_sizes).await;
|
||||
for (kind, mut fixture, headers) in [
|
||||
(
|
||||
"encrypted",
|
||||
CompressedMultipartFixture {
|
||||
object_info: encrypted.object_info,
|
||||
stored: encrypted.ciphertext,
|
||||
plaintext: encrypted.plaintext,
|
||||
},
|
||||
ssec_headers_from_key(key),
|
||||
),
|
||||
("compressed", compressed_multipart_fixture(&part_sizes).await, HeaderMap::new()),
|
||||
(
|
||||
"compressed and encrypted",
|
||||
compressed_encrypted_multipart_fixture(key, &part_sizes).await,
|
||||
ssec_headers_from_key(key),
|
||||
),
|
||||
] {
|
||||
fixture.object_info.etag = Some(faster_hex::hex_string(Md5::digest(&fixture.plaintext).as_ref()));
|
||||
assert_eq!(fixture.object_info.etag.as_ref().expect("source ETag").len(), 32);
|
||||
assert_eq!(fixture.object_info.parts.len(), 2);
|
||||
let tail = &fixture.object_info.parts[1];
|
||||
assert_eq!(tail.actual_size, 0, "{kind}: final part has no plaintext");
|
||||
if kind == "compressed" {
|
||||
assert_eq!(tail.size, 0, "unpadded compression emits no bytes for an empty part");
|
||||
} else {
|
||||
assert!(tail.size > 0, "{kind}: the empty part still has a stored frame");
|
||||
}
|
||||
let stored_size = i64::try_from(fixture.stored.len()).expect("fixture size fits i64");
|
||||
let (mut reader, offset, length) = GetObjectReader::new(
|
||||
Box::new(Cursor::new(fixture.stored)),
|
||||
None,
|
||||
&fixture.object_info,
|
||||
&ObjectOptions::default(),
|
||||
&headers,
|
||||
)
|
||||
.await
|
||||
.expect("full transformed read must include the empty tail");
|
||||
assert_eq!((offset, length), (0, stored_size), "{kind}: full read includes all stored parts");
|
||||
let mut body = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("read through the complete decoder EOF");
|
||||
assert_eq!(body, fixture.plaintext, "{kind}: no plaintext is added or lost by the empty tail");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_empty_tail_full_read_authenticates_v2_final_frame() {
|
||||
let key = [0x6Eu8; 32];
|
||||
let plaintext = legacy_fixture_part_plaintext(1, 5 * 1024 * 1024);
|
||||
let mut ciphertext = Vec::new();
|
||||
let mut parts = Vec::new();
|
||||
for (number, body) in [(1, plaintext.as_slice()), (2, b"".as_slice())] {
|
||||
let start = ciphertext.len();
|
||||
rustfs_rio::EncryptReader::new_multipart_v2(Cursor::new(body), key, LEGACY_FIXTURE_BASE_NONCE, number)
|
||||
.read_to_end(&mut ciphertext)
|
||||
.await
|
||||
.expect("encrypt a v2 fixture part with an authenticated final frame");
|
||||
parts.push(ObjectPartInfo {
|
||||
number,
|
||||
size: ciphertext.len() - start,
|
||||
actual_size: i64::try_from(body.len()).expect("fixture plaintext size fits"),
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
let tail_start = parts[0].size;
|
||||
assert_eq!(parts[1].actual_size, 0);
|
||||
assert!(parts[1].size > 8, "the empty final frame carries more than an END marker");
|
||||
let object_info = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "v2-empty-tail".to_string(),
|
||||
size: i64::try_from(ciphertext.len()).expect("fixture ciphertext size fits"),
|
||||
etag: Some(faster_hex::hex_string(Md5::digest(&plaintext).as_ref())),
|
||||
parts: Arc::new(parts),
|
||||
user_defined: Arc::new(legacy_ssec_multipart_metadata(key, plaintext.len())),
|
||||
..Default::default()
|
||||
};
|
||||
for corrupt_tail in [false, true] {
|
||||
let mut stored = ciphertext.clone();
|
||||
if corrupt_tail {
|
||||
// The v2 header is authenticated associated data, including
|
||||
// the header of a final frame containing zero plaintext.
|
||||
stored[tail_start + 5] ^= 1;
|
||||
}
|
||||
let (mut reader, offset, length) = GetObjectReader::new(
|
||||
Box::new(Cursor::new(stored)),
|
||||
None,
|
||||
&object_info,
|
||||
&ObjectOptions::default(),
|
||||
&ssec_headers_from_key(key),
|
||||
)
|
||||
.await
|
||||
.expect("construct the full reader before consuming the final frame");
|
||||
assert_eq!((offset, length), (0, object_info.size));
|
||||
let result = tokio::io::copy(&mut reader.stream, &mut tokio::io::sink()).await;
|
||||
if corrupt_tail {
|
||||
let err = result.expect_err("EOF must authenticate the empty final frame after all plaintext is returned");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert_eq!(err.to_string(), "v2 encrypted frame failed authentication");
|
||||
} else {
|
||||
assert_eq!(
|
||||
result.expect("valid empty final frame must reach EOF"),
|
||||
u64::try_from(plaintext.len()).expect("plaintext length fits")
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn compressed_encrypted_multipart_range_crosses_part_boundary() {
|
||||
let key_bytes = [0x6Eu8; 32];
|
||||
@@ -3771,61 +3656,6 @@ mod tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_full_read_preserves_legacy_zero_and_negative_part_sizes() {
|
||||
let key = [0x77; 32];
|
||||
let part_sizes = [5 * 1024 * 1024, 1024 * 1024];
|
||||
let encrypted = build_legacy_ssec_multipart_fixture(key, &part_sizes).await;
|
||||
// The encrypted case supplies the fixture key explicitly. This covers
|
||||
// full decrypted reads, not managed-key acquisition.
|
||||
for (kind, fixture, headers) in [
|
||||
("compressed", compressed_multipart_fixture(&part_sizes).await, HeaderMap::new()),
|
||||
(
|
||||
"encrypted with supplied key",
|
||||
CompressedMultipartFixture {
|
||||
object_info: encrypted.object_info,
|
||||
stored: encrypted.ciphertext,
|
||||
plaintext: encrypted.plaintext,
|
||||
},
|
||||
ssec_headers_from_key(key),
|
||||
),
|
||||
] {
|
||||
let source_etag = faster_hex::hex_string(Md5::digest(&fixture.plaintext).as_ref());
|
||||
assert_eq!(source_etag.len(), 32);
|
||||
assert_eq!(fixture.plaintext.len(), 6 * 1024 * 1024);
|
||||
for part_index in 0..part_sizes.len() {
|
||||
assert!(fixture.object_info.parts[part_index].actual_size > 0, "the selected part is nonempty");
|
||||
for actual_size in [0, -1] {
|
||||
let mut object_info = fixture.object_info.clone();
|
||||
object_info.etag = Some(source_etag.clone());
|
||||
Arc::make_mut(&mut object_info.parts)[part_index].actual_size = actual_size;
|
||||
let (mut reader, offset, length) = GetObjectReader::new(
|
||||
Box::new(Cursor::new(fixture.stored.clone())),
|
||||
None,
|
||||
&object_info,
|
||||
&ObjectOptions::default(),
|
||||
&headers,
|
||||
)
|
||||
.await
|
||||
.expect("the authoritative total size must keep full legacy reads available");
|
||||
assert_eq!(offset, 0);
|
||||
assert_eq!(length, i64::try_from(fixture.stored.len()).expect("stored size fits"));
|
||||
let mut body = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("full read must reach EOF despite an unspecified per-part logical size");
|
||||
assert_eq!(
|
||||
body, fixture.plaintext,
|
||||
"{kind}: part {part_index} with actual_size={actual_size} must not lose readable data"
|
||||
);
|
||||
assert_eq!(reader.object_info.etag.as_deref(), Some(source_etag.as_str()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The physical part sizes must add up to `oi.size` for a seek to be safe;
|
||||
/// inconsistent metadata must fall back to the previous full-object read
|
||||
/// instead of scheduling an erasure read past the object end.
|
||||
|
||||
@@ -1597,7 +1597,7 @@ impl ObjectInfo {
|
||||
}
|
||||
|
||||
pub fn is_multipart(&self) -> bool {
|
||||
self.parts.len() > 1 || self.etag.as_ref().is_some_and(|v| v.len() != 32)
|
||||
self.etag.as_ref().is_some_and(|v| v.len() != 32)
|
||||
}
|
||||
|
||||
pub fn is_encrypted(&self) -> bool {
|
||||
@@ -2235,35 +2235,6 @@ mod tests {
|
||||
}
|
||||
use rustfs_filemeta::{FileInfo, FileMeta, MetaCacheEntry, TRANSITION_COMPLETE};
|
||||
|
||||
#[test]
|
||||
fn multipart_identity_uses_stored_parts_and_preserves_the_etag_fallback() {
|
||||
let plain_etag = "0123456789abcdef0123456789abcdef";
|
||||
let multipart_etag = "0123456789abcdef0123456789abcdef-1";
|
||||
for (case, part_count, etag, expected) in [
|
||||
("preserved source ETag", 2, Some(plain_etag), true),
|
||||
("missing ETag", 2, None, true),
|
||||
("ordinary PUT", 1, Some(plain_etag), false),
|
||||
("ordinary PUT without ETag", 1, None, false),
|
||||
("single-part MPU", 1, Some(multipart_etag), true),
|
||||
("legacy MPU without parts", 0, Some(multipart_etag), true),
|
||||
] {
|
||||
let object = ObjectInfo {
|
||||
etag: etag.map(str::to_string),
|
||||
parts: Arc::new(
|
||||
(1..=part_count)
|
||||
.map(|number| ObjectPartInfo {
|
||||
number,
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert_eq!(object.is_multipart(), expected, "{case}");
|
||||
}
|
||||
}
|
||||
|
||||
fn inline_fast_path_object(size: i64, versioned: bool) -> ObjectInfo {
|
||||
ObjectInfo {
|
||||
size,
|
||||
|
||||
@@ -14,8 +14,7 @@
|
||||
|
||||
use crate::bucket::lifecycle::tier_last_day_stats::DailyAllTierStats;
|
||||
use crate::cluster::rpc::{
|
||||
PeerRestClient, ScannerDirtyUsageAcknowledgement, ScannerPeerActivity, ScannerPeerDirtyUsageSnapshot,
|
||||
ScannerPublicationLease, TierConfigReloadOutcome,
|
||||
PeerRestClient, ScannerPeerActivity, ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease, TierConfigReloadOutcome,
|
||||
};
|
||||
use crate::diagnostics::admin_server_info::get_commit_id;
|
||||
use crate::disk::DiskAPI;
|
||||
@@ -34,11 +33,11 @@ use rustfs_madmin::net::NetInfo;
|
||||
use rustfs_madmin::{ItemState, ServerProperties, StorageInfo};
|
||||
use rustfs_utils::XHost;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap, hash_map::DefaultHasher};
|
||||
use std::collections::{BTreeMap, HashMap, hash_map::DefaultHasher};
|
||||
use std::future::Future;
|
||||
use std::hash::{Hash, Hasher};
|
||||
use std::sync::{
|
||||
Arc, LazyLock, Mutex, OnceLock,
|
||||
Arc, Mutex, OnceLock,
|
||||
atomic::{AtomicBool, AtomicUsize, Ordering},
|
||||
};
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
@@ -73,29 +72,6 @@ const LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION: u32 = 4;
|
||||
/// service must not advertise this version until the conditional writer from
|
||||
/// rustfs/backlog#684 is available.
|
||||
const LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION: u32 = 5;
|
||||
|
||||
fn resolve_admin_peer_probe_timeout_secs(configured: Option<u64>) -> u64 {
|
||||
configured
|
||||
.filter(|seconds| *seconds > 0)
|
||||
.unwrap_or(rustfs_config::DEFAULT_ADMIN_PEER_PROBE_TIMEOUT_SECS)
|
||||
.min(rustfs_config::MAX_ADMIN_PEER_PROBE_TIMEOUT_SECS)
|
||||
}
|
||||
|
||||
fn admin_peer_probe_timeout() -> Duration {
|
||||
let configured = rustfs_utils::get_env_opt_u64_with_aliases(rustfs_config::ENV_ADMIN_PEER_PROBE_TIMEOUT_SECS, &[]);
|
||||
let seconds = resolve_admin_peer_probe_timeout_secs(configured);
|
||||
Duration::from_secs(seconds)
|
||||
}
|
||||
|
||||
fn remaining_admin_peer_probe_timeout(deadline: Instant) -> Option<Duration> {
|
||||
remaining_admin_peer_probe_timeout_at(deadline, Instant::now())
|
||||
}
|
||||
|
||||
fn remaining_admin_peer_probe_timeout_at(deadline: Instant, now: Instant) -> Option<Duration> {
|
||||
let remaining = deadline.saturating_duration_since(now);
|
||||
(!remaining.is_zero()).then_some(remaining)
|
||||
}
|
||||
|
||||
type CrossPoolFencePolicyResult = Result<BTreeMap<String, Uuid>>;
|
||||
|
||||
fn cross_pool_fence_policy_results(
|
||||
@@ -312,29 +288,12 @@ pub struct LegacyTransitionStateReconcileFleetProofToken {
|
||||
_permit: FleetCapabilityProofPermit,
|
||||
}
|
||||
|
||||
/// Effect-window authority for one immutable ILM recovery export.
|
||||
pub struct IlmRecoveryExportFleetProofToken {
|
||||
token: FleetCapabilityProofToken,
|
||||
_permit: FleetCapabilityProofPermit,
|
||||
}
|
||||
|
||||
/// Effect-window authority for emitting the compact transition-transaction
|
||||
/// state sequence. The generation permit prevents a successor proof from
|
||||
/// being published until the admitted writer has finished.
|
||||
pub(crate) struct TransitionTransactionCompactionFleetProofToken {
|
||||
token: FleetCapabilityProofToken,
|
||||
_permit: FleetCapabilityProofPermit,
|
||||
}
|
||||
|
||||
static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static CROSS_POOL_FENCE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static TIER_DELETE_JOURNAL_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static DECOMMISSION_TARGET_FENCE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static LEGACY_TRANSITION_STATE_RECONCILE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static ILM_RECOVERY_EXPORT_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static TRANSITION_TRANSACTION_COMPACTION_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock<String> = OnceLock::new();
|
||||
static ILM_RECOVERY_EXPORT_LOCAL_PROCESS_EPOCH: LazyLock<Uuid> = LazyLock::new(Uuid::new_v4);
|
||||
|
||||
fn cross_pool_fence_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
|
||||
CROSS_POOL_FENCE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
|
||||
@@ -356,14 +315,6 @@ fn legacy_transition_state_reconcile_fleet_proof_slot() -> &'static std::sync::R
|
||||
LEGACY_TRANSITION_STATE_RECONCILE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
|
||||
}
|
||||
|
||||
fn ilm_recovery_export_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
|
||||
ILM_RECOVERY_EXPORT_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
|
||||
}
|
||||
|
||||
fn transition_transaction_compaction_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
|
||||
TRANSITION_TRANSACTION_COMPACTION_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
|
||||
}
|
||||
|
||||
fn revoke_fleet_capability_proof_state(state: &mut FleetCapabilityProofState) {
|
||||
if let Some(proof) = state.proof.take() {
|
||||
proof.generation.revoke();
|
||||
@@ -464,33 +415,6 @@ pub(crate) fn remote_version_state_fleet_proof_matches(proof: &RemoteVersionStat
|
||||
fleet_capability_proof_matches(remote_version_state_fleet_proof_slot(), &proof.0)
|
||||
}
|
||||
|
||||
pub(crate) fn acquire_transition_transaction_compaction_fleet_proof() -> Option<TransitionTransactionCompactionFleetProofToken> {
|
||||
let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
|
||||
let state = transition_transaction_compaction_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let token = acquire_fleet_capability_proof_from(&state, expected_topology, Instant::now())?;
|
||||
let permit = state.proof.as_ref()?.generation.try_acquire()?;
|
||||
Some(TransitionTransactionCompactionFleetProofToken { token, _permit: permit })
|
||||
}
|
||||
|
||||
pub(crate) fn transition_transaction_compaction_fleet_proof_matches(
|
||||
proof: &TransitionTransactionCompactionFleetProofToken,
|
||||
) -> bool {
|
||||
let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else {
|
||||
return false;
|
||||
};
|
||||
let state = transition_transaction_compaction_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
proof._permit.generation.is_accepting()
|
||||
&& fleet_capability_proof_matches_at(&state, &proof.token, expected_topology, Instant::now())
|
||||
&& state
|
||||
.proof
|
||||
.as_ref()
|
||||
.is_some_and(|current| Arc::ptr_eq(¤t.generation, &proof._permit.generation))
|
||||
}
|
||||
|
||||
pub fn acquire_cross_pool_fence_fleet_proof() -> Option<CrossPoolFenceFleetProofToken> {
|
||||
let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
|
||||
let state = cross_pool_fence_fleet_proof_slot()
|
||||
@@ -626,117 +550,6 @@ pub async fn legacy_transition_state_reconcile_fleet_proof_matches(
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn acquire_ilm_recovery_export_fleet_proof() -> Option<IlmRecoveryExportFleetProofToken> {
|
||||
let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
|
||||
let proof = {
|
||||
let state = ilm_recovery_export_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
acquire_ilm_recovery_export_fleet_proof_from(&state, expected_topology, Instant::now())?
|
||||
};
|
||||
let observed = observe_ilm_recovery_export_fleet(expected_topology).await?;
|
||||
let state = ilm_recovery_export_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
ilm_recovery_export_fleet_proof_matches_observation_at(&state, &proof, expected_topology, &observed, Instant::now())
|
||||
.then_some(proof)
|
||||
}
|
||||
|
||||
fn acquire_ilm_recovery_export_fleet_proof_from(
|
||||
state: &FleetCapabilityProofState,
|
||||
expected_topology: &str,
|
||||
now: Instant,
|
||||
) -> Option<IlmRecoveryExportFleetProofToken> {
|
||||
let token = acquire_fleet_capability_proof_from(state, expected_topology, now)?;
|
||||
let permit = state.proof.as_ref()?.generation.try_acquire()?;
|
||||
Some(IlmRecoveryExportFleetProofToken { token, _permit: permit })
|
||||
}
|
||||
|
||||
pub async fn ilm_recovery_export_fleet_proof_matches(proof: &IlmRecoveryExportFleetProofToken) -> bool {
|
||||
let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else {
|
||||
return false;
|
||||
};
|
||||
{
|
||||
let state = ilm_recovery_export_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if !ilm_recovery_export_fleet_proof_matches_at(&state, proof, expected_topology, Instant::now()) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
let Some(observed) = observe_ilm_recovery_export_fleet(expected_topology).await else {
|
||||
return false;
|
||||
};
|
||||
let state = ilm_recovery_export_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
ilm_recovery_export_fleet_proof_matches_observation_at(&state, proof, expected_topology, &observed, Instant::now())
|
||||
}
|
||||
|
||||
pub fn ilm_recovery_export_topology_generation(proof: &IlmRecoveryExportFleetProofToken) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(b"rustfs-ilm-recovery-export-topology-v1\0");
|
||||
hasher.update(proof.token.topology_fingerprint.as_bytes());
|
||||
rustfs_utils::crypto::hex(hasher.finalize().as_slice())
|
||||
}
|
||||
|
||||
pub fn ilm_recovery_export_member_epochs_sha256(proof: &IlmRecoveryExportFleetProofToken) -> String {
|
||||
let encoded = serde_json::to_vec(proof.token.peer_epochs.as_ref()).expect("member epoch map is JSON encodable");
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(b"rustfs-ilm-recovery-export-members-v1\0");
|
||||
hasher.update(encoded);
|
||||
rustfs_utils::crypto::hex(hasher.finalize().as_slice())
|
||||
}
|
||||
|
||||
pub fn ilm_recovery_export_local_process_epoch() -> Uuid {
|
||||
*ILM_RECOVERY_EXPORT_LOCAL_PROCESS_EPOCH
|
||||
}
|
||||
|
||||
fn ilm_recovery_export_fleet_proof_matches_at(
|
||||
state: &FleetCapabilityProofState,
|
||||
proof: &IlmRecoveryExportFleetProofToken,
|
||||
expected_topology: &str,
|
||||
now: Instant,
|
||||
) -> bool {
|
||||
proof._permit.generation.is_accepting()
|
||||
&& fleet_capability_proof_matches_at(state, &proof.token, expected_topology, now)
|
||||
&& state
|
||||
.proof
|
||||
.as_ref()
|
||||
.is_some_and(|current| Arc::ptr_eq(¤t.generation, &proof._permit.generation))
|
||||
}
|
||||
|
||||
fn ilm_recovery_export_fleet_proof_matches_observation_at(
|
||||
state: &FleetCapabilityProofState,
|
||||
proof: &IlmRecoveryExportFleetProofToken,
|
||||
expected_topology: &str,
|
||||
observed: &BTreeMap<String, Uuid>,
|
||||
now: Instant,
|
||||
) -> bool {
|
||||
ilm_recovery_export_fleet_proof_matches_at(state, proof, expected_topology, now)
|
||||
&& proof.token.peer_epochs.as_ref() == observed
|
||||
}
|
||||
|
||||
async fn observe_ilm_recovery_export_fleet(expected_topology: &str) -> Option<BTreeMap<String, Uuid>> {
|
||||
#[cfg(test)]
|
||||
{
|
||||
let state = ilm_recovery_export_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if fleet_capability_proof_valid_at(state.proof.as_ref(), expected_topology, Instant::now()) {
|
||||
return state.proof.as_ref().map(|proof| proof.peer_epochs.as_ref().clone());
|
||||
}
|
||||
}
|
||||
let notification_sys = get_global_notification_sys()?;
|
||||
timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_ilm_recovery_export_fleet(expected_topology),
|
||||
)
|
||||
.await
|
||||
.ok()?
|
||||
.ok()
|
||||
}
|
||||
|
||||
async fn legacy_transition_state_reconcile_fleet_proof_matches_with_observer<F, Fut>(
|
||||
slot: &std::sync::RwLock<FleetCapabilityProofState>,
|
||||
proof: &LegacyTransitionStateReconcileFleetProofToken,
|
||||
@@ -825,7 +638,7 @@ pub(crate) fn install_cross_pool_fence_fleet_proof_for_test() {
|
||||
state.proof.clone()
|
||||
} else {
|
||||
Some(FleetCapabilityProof::new(
|
||||
topology.clone(),
|
||||
topology,
|
||||
Arc::new(BTreeMap::new()),
|
||||
now + Duration::from_secs(60 * 60),
|
||||
))
|
||||
@@ -858,21 +671,6 @@ pub(crate) fn install_cross_pool_fence_fleet_proof_for_test() {
|
||||
decommission_state.topology_conflict = false;
|
||||
decommission_state.draining_generation = None;
|
||||
decommission_state.proof = proof.as_ref().map(FleetCapabilityProof::with_fresh_generation);
|
||||
drop(decommission_state);
|
||||
let mut export_state = ilm_recovery_export_fleet_proof_slot()
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if !fleet_capability_proof_valid_at(export_state.proof.as_ref(), &topology, now) {
|
||||
debug_assert!(
|
||||
export_state
|
||||
.proof
|
||||
.as_ref()
|
||||
.is_none_or(|current| current.generation.is_drained())
|
||||
);
|
||||
export_state.topology_conflict = false;
|
||||
export_state.draining_generation = None;
|
||||
export_state.proof = proof.as_ref().map(FleetCapabilityProof::with_fresh_generation);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -1113,35 +911,6 @@ pub(crate) fn install_remote_version_state_fleet_proof_for_test(topology_fingerp
|
||||
RemoteVersionStateFleetProofGuard
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) struct TransitionTransactionCompactionFleetProofGuard;
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
impl Drop for TransitionTransactionCompactionFleetProofGuard {
|
||||
fn drop(&mut self) {
|
||||
revoke_fleet_capability_proof(transition_transaction_compaction_fleet_proof_slot());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) fn install_transition_transaction_compaction_fleet_proof_for_test(
|
||||
topology_fingerprint: &str,
|
||||
) -> TransitionTransactionCompactionFleetProofGuard {
|
||||
let _ = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.set(topology_fingerprint.to_string());
|
||||
let effective_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY
|
||||
.get()
|
||||
.expect("transition transaction compaction test topology should be initialized");
|
||||
if let Some(err) = publish_fleet_capability_probe_result(
|
||||
transition_transaction_compaction_fleet_proof_slot(),
|
||||
effective_topology,
|
||||
Ok(BTreeMap::new()),
|
||||
Instant::now(),
|
||||
) {
|
||||
panic!("test proof installation must not fail: {err}");
|
||||
}
|
||||
TransitionTransactionCompactionFleetProofGuard
|
||||
}
|
||||
|
||||
fn insert_remote_version_state_peer(peer_epochs: &mut BTreeMap<String, Uuid>, peer: String, epoch: Uuid) -> Result<()> {
|
||||
if epoch.is_nil() || peer_epochs.values().any(|existing| *existing == epoch) || peer_epochs.insert(peer, epoch).is_some() {
|
||||
return Err(Error::other("remote version state capability peer identity is invalid"));
|
||||
@@ -1158,8 +927,6 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
tier_delete_journal_fleet_proof_slot(),
|
||||
decommission_target_fence_fleet_proof_slot(),
|
||||
legacy_transition_state_reconcile_fleet_proof_slot(),
|
||||
ilm_recovery_export_fleet_proof_slot(),
|
||||
transition_transaction_compaction_fleet_proof_slot(),
|
||||
] {
|
||||
mark_fleet_capability_topology_conflict(slot);
|
||||
}
|
||||
@@ -1169,59 +936,29 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
let notification_sys = get_global_notification_sys();
|
||||
let remote_version_state_probe = async {
|
||||
match notification_sys.as_ref() {
|
||||
Some(notification_sys) => timeout(
|
||||
let result = match get_global_notification_sys() {
|
||||
Some(notification_sys) => {
|
||||
match timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_remote_version_state_fleet(&topology_fingerprint),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other("remote version state fleet capability probe timed out"))),
|
||||
None => Err(Error::other("remote version state fleet capability notification system is unavailable")),
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(_) => Err(Error::other("remote version state fleet capability probe timed out")),
|
||||
}
|
||||
}
|
||||
None => Err(Error::other("remote version state fleet capability notification system is unavailable")),
|
||||
};
|
||||
let cross_pool_fence_probe = async {
|
||||
match notification_sys.as_ref() {
|
||||
Some(notification_sys) => timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_cross_pool_fence_fleet(&topology_fingerprint),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other("cross-pool fence fleet capability probe timed out"))),
|
||||
None => Err(Error::other("cross-pool fence fleet capability notification system is unavailable")),
|
||||
}
|
||||
let fence_probe = match get_global_notification_sys() {
|
||||
Some(notification_sys) => timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_cross_pool_fence_fleet(&topology_fingerprint),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other("cross-pool fence fleet capability probe timed out"))),
|
||||
None => Err(Error::other("cross-pool fence fleet capability notification system is unavailable")),
|
||||
};
|
||||
let recovery_export_probe = async {
|
||||
match notification_sys.as_ref() {
|
||||
Some(notification_sys) => timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_ilm_recovery_export_fleet(&topology_fingerprint),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other("ILM recovery export fleet capability probe timed out"))),
|
||||
None => Err(Error::other("ILM recovery export fleet capability notification system is unavailable")),
|
||||
}
|
||||
};
|
||||
let transition_transaction_compaction_probe = async {
|
||||
match notification_sys.as_ref() {
|
||||
Some(notification_sys) => timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_transition_transaction_compaction_fleet(&topology_fingerprint),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other("transition transaction compaction fleet capability probe timed out"))),
|
||||
None => Err(Error::other(
|
||||
"transition transaction compaction fleet capability notification system is unavailable",
|
||||
)),
|
||||
}
|
||||
};
|
||||
let (result, fence_probe, recovery_export_result, transition_transaction_compaction_result) = tokio::join!(
|
||||
remote_version_state_probe,
|
||||
cross_pool_fence_probe,
|
||||
recovery_export_probe,
|
||||
transition_transaction_compaction_probe
|
||||
);
|
||||
let (fence_result, journal_result, decommission_target_fence_result, reconcile_result) = match fence_probe {
|
||||
Ok((peer_epochs, minimum_version)) => cross_pool_fence_policy_results(peer_epochs, minimum_version),
|
||||
Err(err) => {
|
||||
@@ -1244,8 +981,6 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
revoke_fleet_capability_proof(tier_delete_journal_fleet_proof_slot());
|
||||
revoke_fleet_capability_proof(decommission_target_fence_fleet_proof_slot());
|
||||
revoke_fleet_capability_proof(legacy_transition_state_reconcile_fleet_proof_slot());
|
||||
revoke_fleet_capability_proof(ilm_recovery_export_fleet_proof_slot());
|
||||
revoke_fleet_capability_proof(transition_transaction_compaction_fleet_proof_slot());
|
||||
} else if let Some(err) = publish_fleet_capability_probe_result(
|
||||
remote_version_state_fleet_proof_slot(),
|
||||
&topology_fingerprint,
|
||||
@@ -1272,42 +1007,6 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
"notification capability probe"
|
||||
);
|
||||
}
|
||||
if !topology_conflict
|
||||
&& let Some(err) = publish_fleet_capability_probe_result(
|
||||
ilm_recovery_export_fleet_proof_slot(),
|
||||
&topology_fingerprint,
|
||||
recovery_export_result,
|
||||
Instant::now(),
|
||||
)
|
||||
{
|
||||
debug!(
|
||||
event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||
capability = "ilm_recovery_export_v1",
|
||||
state = "failed_closed",
|
||||
error = %err,
|
||||
"notification capability probe"
|
||||
);
|
||||
}
|
||||
if !topology_conflict
|
||||
&& let Some(err) = publish_fleet_capability_probe_result(
|
||||
transition_transaction_compaction_fleet_proof_slot(),
|
||||
&topology_fingerprint,
|
||||
transition_transaction_compaction_result,
|
||||
Instant::now(),
|
||||
)
|
||||
{
|
||||
debug!(
|
||||
event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||
capability = "transition_transaction_compaction_v1",
|
||||
state = "failed_closed",
|
||||
error = %err,
|
||||
"notification capability probe"
|
||||
);
|
||||
}
|
||||
if !topology_conflict
|
||||
&& let Some(err) = publish_fleet_capability_probe_result(
|
||||
tier_delete_journal_fleet_proof_slot(),
|
||||
@@ -1430,28 +1129,6 @@ impl NotificationSys {
|
||||
Ok(peer_epochs)
|
||||
}
|
||||
|
||||
async fn probe_transition_transaction_compaction_fleet(&self, topology_fingerprint: &str) -> Result<BTreeMap<String, Uuid>> {
|
||||
if self.peer_clients.len() != self.peer_topology_hosts.len() {
|
||||
return Err(Error::other(
|
||||
"transition transaction compaction capability fleet membership is incomplete",
|
||||
));
|
||||
}
|
||||
let probes = self.peer_clients.iter().map(|client| async {
|
||||
let client = client
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::other("transition transaction compaction capability peer is unreachable"))?;
|
||||
client
|
||||
.probe_transition_transaction_compaction(topology_fingerprint.to_string())
|
||||
.await
|
||||
});
|
||||
let mut peer_epochs = BTreeMap::new();
|
||||
for result in join_all(probes).await {
|
||||
let (peer, epoch) = result?;
|
||||
insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?;
|
||||
}
|
||||
Ok(peer_epochs)
|
||||
}
|
||||
|
||||
async fn probe_cross_pool_fence_fleet(&self, topology_fingerprint: &str) -> Result<(BTreeMap<String, Uuid>, u32)> {
|
||||
if self.peer_clients.len() != self.peer_topology_hosts.len() {
|
||||
return Err(Error::other("cross-pool fence capability fleet membership is incomplete"));
|
||||
@@ -1474,46 +1151,6 @@ impl NotificationSys {
|
||||
}
|
||||
Ok((peer_epochs, minimum_version))
|
||||
}
|
||||
|
||||
async fn probe_ilm_recovery_export_fleet(&self, topology_fingerprint: &str) -> Result<BTreeMap<String, Uuid>> {
|
||||
if self.peer_clients.len() != self.peer_topology_hosts.len() {
|
||||
return Err(Error::other("ILM recovery export capability fleet membership is incomplete"));
|
||||
}
|
||||
let local_member = runtime_sources::local_node_name().await;
|
||||
if local_member.trim().is_empty() {
|
||||
return Err(Error::other("ILM recovery export local member identity is unavailable"));
|
||||
}
|
||||
let mut peer_epochs = BTreeMap::new();
|
||||
insert_remote_version_state_peer(&mut peer_epochs, local_member.clone(), ilm_recovery_export_local_process_epoch())?;
|
||||
let probes = self.peer_clients.iter().map(|client| async {
|
||||
let client = client
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::other("ILM recovery export capability peer is unreachable"))?;
|
||||
client.probe_ilm_recovery_export(topology_fingerprint.to_string()).await
|
||||
});
|
||||
for result in join_all(probes).await {
|
||||
let (peer, epoch) = result?;
|
||||
insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?;
|
||||
}
|
||||
validate_ilm_recovery_export_members(&self.peer_topology_hosts, &local_member, &peer_epochs)?;
|
||||
Ok(peer_epochs)
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_ilm_recovery_export_members(
|
||||
expected_remote_members: &[String],
|
||||
local_member: &str,
|
||||
observed: &BTreeMap<String, Uuid>,
|
||||
) -> Result<()> {
|
||||
let expected = expected_remote_members
|
||||
.iter()
|
||||
.cloned()
|
||||
.chain(std::iter::once(local_member.to_string()))
|
||||
.collect::<BTreeSet<_>>();
|
||||
if expected.len() != expected_remote_members.len().saturating_add(1) || observed.keys().ne(expected.iter()) {
|
||||
return Err(Error::other("ILM recovery export capability fleet membership does not match topology"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Rolling tier activity summed over every cluster member that answered, with
|
||||
@@ -1901,7 +1538,7 @@ impl NotificationSys {
|
||||
{
|
||||
let mut futures = Vec::with_capacity(self.peer_clients.len());
|
||||
let endpoints = runtime_sources::endpoint_pools().unwrap_or_else(|| Vec::new().into());
|
||||
let peer_timeout = admin_peer_probe_timeout();
|
||||
let peer_timeout = Duration::from_secs(5);
|
||||
|
||||
for (idx, client) in self.peer_clients.iter().enumerate() {
|
||||
let endpoints = endpoints.clone();
|
||||
@@ -1909,9 +1546,7 @@ impl NotificationSys {
|
||||
futures.push(async move {
|
||||
if let Some(client) = client {
|
||||
let host = client.host.to_string();
|
||||
let deadline = Instant::now() + peer_timeout;
|
||||
let probe_timeout = remaining_admin_peer_probe_timeout(deadline).unwrap_or_default();
|
||||
match timeout(probe_timeout, client.local_storage_info()).await {
|
||||
match timeout(peer_timeout, client.local_storage_info()).await {
|
||||
Ok(Ok(mut info)) => {
|
||||
normalize_and_cache_peer_storage_info(cache, &host, &mut info);
|
||||
Some(info)
|
||||
@@ -1922,6 +1557,7 @@ impl NotificationSys {
|
||||
}
|
||||
Err(_) => {
|
||||
warn!("peer {} storage_info timed out after {:?}", host, peer_timeout);
|
||||
client.evict_connection().await;
|
||||
handle_peer_failure(cache, &host, &endpoints)
|
||||
}
|
||||
}
|
||||
@@ -1947,7 +1583,7 @@ impl NotificationSys {
|
||||
pub async fn server_info(&self) -> Vec<ServerProperties> {
|
||||
let mut futures = Vec::with_capacity(self.peer_clients.len());
|
||||
let endpoints = runtime_sources::endpoint_pools().unwrap_or_else(|| Vec::new().into());
|
||||
let peer_timeout = admin_peer_probe_timeout();
|
||||
let peer_timeout = Duration::from_secs(5);
|
||||
|
||||
for (idx, client) in self.peer_clients.iter().enumerate() {
|
||||
let host = self
|
||||
@@ -1964,23 +1600,12 @@ impl NotificationSys {
|
||||
};
|
||||
};
|
||||
|
||||
let deadline = Instant::now() + peer_timeout;
|
||||
let Some(first_timeout) = remaining_admin_peer_probe_timeout(deadline) else {
|
||||
let health = peer_disk_health_with_deadline(&host, deadline).await;
|
||||
return PeerServerInfoProbe {
|
||||
host,
|
||||
result: Err(PeerServerInfoProbeFailure::Rpc { health }),
|
||||
};
|
||||
};
|
||||
|
||||
// First attempt. A single evicted or half-open internode channel
|
||||
// is enough to fail one probe and, before retrying, would drop
|
||||
// the member to unknown/offline for this whole snapshot. On a
|
||||
// quick failure we evict the channel and re-dial once before
|
||||
// falling back (rustfs/backlog#1049, P1-B). A slow attempt
|
||||
// consumes the round budget and therefore does not trigger a
|
||||
// second full wait or an asynchronous eviction side effect.
|
||||
match timeout(first_timeout, client.server_info()).await {
|
||||
// the member to unknown/offline for this whole snapshot. So on any
|
||||
// first-attempt failure we evict the channel and re-dial once
|
||||
// before falling back (rustfs/backlog#1049, P1-B).
|
||||
match timeout(peer_timeout, client.server_info()).await {
|
||||
Ok(Ok(info)) => {
|
||||
return PeerServerInfoProbe { host, result: Ok(info) };
|
||||
}
|
||||
@@ -1994,37 +1619,14 @@ impl NotificationSys {
|
||||
// `evict_connection` would leave that gate up and the retry would
|
||||
// fast-fail with "temporarily offline" instead of reconnecting
|
||||
// (rustfs/backlog#1049 P1-B).
|
||||
let Some(retry_budget) = remaining_admin_peer_probe_timeout(deadline) else {
|
||||
let health = peer_disk_health_with_deadline(&host, deadline).await;
|
||||
return PeerServerInfoProbe {
|
||||
host,
|
||||
result: Err(PeerServerInfoProbeFailure::Rpc { health }),
|
||||
};
|
||||
};
|
||||
// Bound connection-cache cleanup too. The helper clears the offline gate even
|
||||
// when eviction itself times out, so cancellation cannot strand this peer in
|
||||
// fast-fail mode.
|
||||
if !client.prepare_retry_with_timeout(retry_budget).await {
|
||||
let health = peer_disk_health_with_deadline(&host, deadline).await;
|
||||
return PeerServerInfoProbe {
|
||||
host,
|
||||
result: Err(PeerServerInfoProbeFailure::Rpc { health }),
|
||||
};
|
||||
}
|
||||
client.prepare_retry().await;
|
||||
|
||||
// Second and final attempt on the fresh channel.
|
||||
let Some(retry_timeout) = remaining_admin_peer_probe_timeout(deadline) else {
|
||||
let health = peer_disk_health_with_deadline(&host, deadline).await;
|
||||
return PeerServerInfoProbe {
|
||||
host,
|
||||
result: Err(PeerServerInfoProbeFailure::Rpc { health }),
|
||||
};
|
||||
};
|
||||
match timeout(retry_timeout, client.server_info()).await {
|
||||
match timeout(peer_timeout, client.server_info()).await {
|
||||
Ok(Ok(info)) => PeerServerInfoProbe { host, result: Ok(info) },
|
||||
Ok(Err(err)) => {
|
||||
warn!("peer {host} server_info failed after retry: {err}");
|
||||
let health = peer_disk_health_with_deadline(&host, deadline).await;
|
||||
let health = peer_disk_health(&host).await;
|
||||
PeerServerInfoProbe {
|
||||
host,
|
||||
result: Err(PeerServerInfoProbeFailure::Rpc { health }),
|
||||
@@ -2032,7 +1634,8 @@ impl NotificationSys {
|
||||
}
|
||||
Err(_) => {
|
||||
warn!("peer {host} server_info timed out after retry ({peer_timeout:?})");
|
||||
let health = peer_disk_health_with_deadline(&host, deadline).await;
|
||||
client.evict_connection().await;
|
||||
let health = peer_disk_health(&host).await;
|
||||
PeerServerInfoProbe {
|
||||
host,
|
||||
result: Err(PeerServerInfoProbeFailure::Rpc { health }),
|
||||
@@ -2681,70 +2284,11 @@ impl NotificationSys {
|
||||
Ok(snapshots)
|
||||
}
|
||||
|
||||
pub async fn scanner_scoped_dirty_usage_capabilities(
|
||||
&self,
|
||||
acknowledgements: Vec<ScannerDirtyUsageAcknowledgement>,
|
||||
) -> Result<bool> {
|
||||
pub async fn acknowledge_scanner_dirty_usage(&self, acknowledgements: Vec<(String, String, u64)>) -> Result<bool> {
|
||||
let mut by_host = HashMap::with_capacity(acknowledgements.len());
|
||||
for acknowledgement in acknowledgements {
|
||||
let host = match &acknowledgement {
|
||||
ScannerDirtyUsageAcknowledgement::Scoped { host, .. } => host.clone(),
|
||||
ScannerDirtyUsageAcknowledgement::Generation { .. } => {
|
||||
return Err(Error::other("scanner scoped dirty usage capability requires scoped acknowledgements"));
|
||||
}
|
||||
};
|
||||
if by_host.insert(host.clone(), acknowledgement).is_some() {
|
||||
return Err(Error::other("duplicate scanner dirty usage acknowledgement target"));
|
||||
}
|
||||
}
|
||||
|
||||
let clients = self
|
||||
.peer_clients
|
||||
.iter()
|
||||
.flatten()
|
||||
.map(|client| (client.grid_host.clone(), client.clone()))
|
||||
.collect::<HashMap<_, _>>();
|
||||
let mut futures = Vec::with_capacity(by_host.len());
|
||||
for (host, acknowledgement) in by_host {
|
||||
let Some(client) = clients.get(&host).cloned() else {
|
||||
return Err(Error::other("scanner scoped dirty usage capability failed: peer is not reachable"));
|
||||
};
|
||||
futures.push(async move {
|
||||
let ScannerDirtyUsageAcknowledgement::Scoped {
|
||||
owner_id,
|
||||
instance_id,
|
||||
entries,
|
||||
..
|
||||
} = acknowledgement
|
||||
else {
|
||||
unreachable!("scoped acknowledgement was validated before probing");
|
||||
};
|
||||
timeout(
|
||||
SCANNER_ACTIVITY_PROBE_TIMEOUT,
|
||||
client.scanner_scoped_dirty_usage_capability(owner_id, instance_id, entries),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| Error::other("scanner scoped dirty usage capability timed out"))?
|
||||
});
|
||||
}
|
||||
|
||||
for result in join_all(futures).await {
|
||||
if !result? {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
pub async fn acknowledge_scanner_dirty_usage(&self, acknowledgements: Vec<ScannerDirtyUsageAcknowledgement>) -> Result<bool> {
|
||||
let mut by_host = HashMap::with_capacity(acknowledgements.len());
|
||||
for acknowledgement in acknowledgements {
|
||||
let host = match &acknowledgement {
|
||||
ScannerDirtyUsageAcknowledgement::Generation { host, .. }
|
||||
| ScannerDirtyUsageAcknowledgement::Scoped { host, .. } => host.clone(),
|
||||
};
|
||||
if by_host.insert(host.clone(), acknowledgement).is_some() {
|
||||
return Err(Error::other("duplicate scanner dirty usage acknowledgement target"));
|
||||
for (host, instance_id, generation) in acknowledgements {
|
||||
if by_host.insert(host.clone(), (instance_id, generation)).is_some() {
|
||||
return Err(Error::other(format!("duplicate scanner dirty usage acknowledgement target: {host}")));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2756,34 +2300,18 @@ impl NotificationSys {
|
||||
.collect::<HashMap<_, _>>();
|
||||
let mut failures = Vec::new();
|
||||
let mut futures = Vec::with_capacity(by_host.len());
|
||||
for (host, acknowledgement) in by_host {
|
||||
for (host, (instance_id, generation)) in by_host {
|
||||
let Some(client) = clients.get(&host).cloned() else {
|
||||
failures.push(format!("peer {host} scanner dirty usage acknowledgement failed: peer is not reachable"));
|
||||
continue;
|
||||
};
|
||||
futures.push(async move {
|
||||
let result = match acknowledgement {
|
||||
ScannerDirtyUsageAcknowledgement::Generation {
|
||||
instance_id, generation, ..
|
||||
} => {
|
||||
scanner_activity_with_timeout(
|
||||
SCANNER_ACTIVITY_PROBE_TIMEOUT,
|
||||
&host,
|
||||
client.acknowledge_scanner_dirty_usage(instance_id, generation),
|
||||
)
|
||||
.await
|
||||
}
|
||||
ScannerDirtyUsageAcknowledgement::Scoped {
|
||||
owner_id,
|
||||
instance_id,
|
||||
entries,
|
||||
..
|
||||
} => {
|
||||
client
|
||||
.acknowledge_scanner_scoped_dirty_usage(owner_id, instance_id, entries)
|
||||
.await
|
||||
}
|
||||
};
|
||||
let result = scanner_activity_with_timeout(
|
||||
SCANNER_ACTIVITY_PROBE_TIMEOUT,
|
||||
&host,
|
||||
client.acknowledge_scanner_dirty_usage(instance_id, generation),
|
||||
)
|
||||
.await;
|
||||
(host, result)
|
||||
});
|
||||
}
|
||||
@@ -3495,11 +3023,6 @@ async fn peer_disk_health(host: &str) -> Option<PeerDiskHealth> {
|
||||
}
|
||||
}
|
||||
|
||||
async fn peer_disk_health_with_deadline(host: &str, deadline: Instant) -> Option<PeerDiskHealth> {
|
||||
let remaining = remaining_admin_peer_probe_timeout(deadline)?;
|
||||
timeout(remaining, peer_disk_health(host)).await.ok().flatten()
|
||||
}
|
||||
|
||||
/// Handle a peer failure for server_info: return cached data if available, or
|
||||
/// classify the member as `unknown` / `degraded` / `offline` depending on how
|
||||
/// many consecutive probes have failed and whether the peer's drives are still
|
||||
@@ -3922,81 +3445,6 @@ mod tests {
|
||||
assert!(captured != restarted.token());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ilm_recovery_export_member_digest_is_order_independent_and_epoch_bound() {
|
||||
let now = Instant::now();
|
||||
let local_epoch = ilm_recovery_export_local_process_epoch();
|
||||
assert!(!local_epoch.is_nil());
|
||||
assert_eq!(local_epoch, ilm_recovery_export_local_process_epoch());
|
||||
let remote_epoch = Uuid::new_v4();
|
||||
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||
let peers = BTreeMap::from([("node-b".to_string(), remote_epoch), ("node-a".to_string(), local_epoch)]);
|
||||
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peers), now).is_none());
|
||||
let proof = {
|
||||
let state = slot.read().expect("export proof slot should not poison");
|
||||
acquire_ilm_recovery_export_fleet_proof_from(&state, "topology-a", now).expect("complete fleet should admit export")
|
||||
};
|
||||
let digest = ilm_recovery_export_member_epochs_sha256(&proof);
|
||||
|
||||
let changed_slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||
let changed = BTreeMap::from([("node-a".to_string(), local_epoch), ("node-b".to_string(), Uuid::new_v4())]);
|
||||
assert!(publish_fleet_capability_probe_result(&changed_slot, "topology-a", Ok(changed), now).is_none());
|
||||
let changed_proof = {
|
||||
let state = changed_slot.read().expect("export proof slot should not poison");
|
||||
acquire_ilm_recovery_export_fleet_proof_from(&state, "topology-a", now).expect("complete fleet should admit export")
|
||||
};
|
||||
assert_ne!(digest, ilm_recovery_export_member_epochs_sha256(&changed_proof));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ilm_recovery_export_members_must_match_the_exact_topology() {
|
||||
let expected_remote = vec!["node-b".to_string()];
|
||||
let local = "node-a";
|
||||
let complete = BTreeMap::from([
|
||||
(local.to_string(), Uuid::new_v4()),
|
||||
(expected_remote[0].clone(), Uuid::new_v4()),
|
||||
]);
|
||||
assert!(validate_ilm_recovery_export_members(&expected_remote, local, &complete).is_ok());
|
||||
|
||||
let unexpected = BTreeMap::from([(local.to_string(), Uuid::new_v4()), ("node-c".to_string(), Uuid::new_v4())]);
|
||||
assert!(validate_ilm_recovery_export_members(&expected_remote, local, &unexpected).is_err());
|
||||
assert!(
|
||||
validate_ilm_recovery_export_members(&[local.to_string()], local, &complete).is_err(),
|
||||
"the configured remote set cannot repeat the local member"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ilm_recovery_export_restart_revokes_authority_until_permit_drains() {
|
||||
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||
let now = Instant::now();
|
||||
let original = BTreeMap::from([("node-a".to_string(), Uuid::new_v4())]);
|
||||
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(original), now).is_none());
|
||||
let admitted = {
|
||||
let state = slot.read().expect("export proof slot should not poison");
|
||||
acquire_ilm_recovery_export_fleet_proof_from(&state, "topology-a", now).expect("fresh fleet should admit export")
|
||||
};
|
||||
|
||||
let restarted = BTreeMap::from([("node-a".to_string(), Uuid::new_v4())]);
|
||||
let draining = publish_fleet_capability_probe_result(&slot, "topology-a", Ok(restarted.clone()), now)
|
||||
.expect("restart must wait for the admitted export effect window");
|
||||
assert!(draining.to_string().contains("previous generation to drain"));
|
||||
{
|
||||
let state = slot.read().expect("export proof slot should not poison");
|
||||
assert!(!ilm_recovery_export_fleet_proof_matches_at(&state, &admitted, "topology-a", now));
|
||||
assert!(
|
||||
acquire_ilm_recovery_export_fleet_proof_from(&state, "topology-a", now).is_none(),
|
||||
"successor authority must wait for the old effect window to drain"
|
||||
);
|
||||
}
|
||||
drop(admitted);
|
||||
assert!(
|
||||
publish_fleet_capability_probe_result(&slot, "topology-a", Ok(restarted), now + Duration::from_millis(1)).is_none()
|
||||
);
|
||||
let state = slot.read().expect("export proof slot should not poison");
|
||||
assert!(acquire_ilm_recovery_export_fleet_proof_from(&state, "topology-a", now).is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_journal_generation_is_stable_across_members_and_process_restarts() {
|
||||
let topology = "topology-a";
|
||||
@@ -4569,37 +4017,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn admin_peer_probe_timeout_rejects_zero_and_caps_large_values() {
|
||||
assert_eq!(
|
||||
resolve_admin_peer_probe_timeout_secs(None),
|
||||
rustfs_config::DEFAULT_ADMIN_PEER_PROBE_TIMEOUT_SECS
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_admin_peer_probe_timeout_secs(Some(0)),
|
||||
rustfs_config::DEFAULT_ADMIN_PEER_PROBE_TIMEOUT_SECS
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_admin_peer_probe_timeout_secs(Some(rustfs_config::MAX_ADMIN_PEER_PROBE_TIMEOUT_SECS + 1)),
|
||||
rustfs_config::MAX_ADMIN_PEER_PROBE_TIMEOUT_SECS
|
||||
);
|
||||
assert_eq!(resolve_admin_peer_probe_timeout_secs(Some(7)), 7);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_peer_probe_health_fallback_respects_expired_deadline() {
|
||||
let deadline = Instant::now();
|
||||
assert!(peer_disk_health_with_deadline("peer-1", deadline).await.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn admin_peer_probe_deadline_is_shared_across_attempts() {
|
||||
let start = Instant::now();
|
||||
let deadline = start + Duration::from_secs(10);
|
||||
assert!(remaining_admin_peer_probe_timeout_at(deadline, start + Duration::from_secs(6)).is_some());
|
||||
assert!(remaining_admin_peer_probe_timeout_at(deadline, start + Duration::from_secs(10)).is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn call_peer_with_timeout_returns_value_when_fast() {
|
||||
let result = call_peer_with_timeout(
|
||||
@@ -4948,28 +4365,15 @@ mod tests {
|
||||
peer_topology_hosts: Vec::new(),
|
||||
};
|
||||
let missing = sys
|
||||
.acknowledge_scanner_dirty_usage(vec![ScannerDirtyUsageAcknowledgement::Generation {
|
||||
host: "peer-1".to_string(),
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
generation: 7,
|
||||
}])
|
||||
.acknowledge_scanner_dirty_usage(vec![("peer-1".to_string(), "0123456789abcdef0123456789abcdef".to_string(), 7)])
|
||||
.await
|
||||
.expect_err("a missing acknowledgement target must remain pending");
|
||||
assert!(missing.to_string().contains("peer is not reachable"));
|
||||
|
||||
let duplicate = sys
|
||||
.acknowledge_scanner_dirty_usage(vec![
|
||||
ScannerDirtyUsageAcknowledgement::Generation {
|
||||
host: "peer-1".to_string(),
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
generation: 7,
|
||||
},
|
||||
ScannerDirtyUsageAcknowledgement::Scoped {
|
||||
host: "peer-1".to_string(),
|
||||
owner_id: "11111111-1111-1111-1111-111111111111".to_string(),
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
entries: Vec::new(),
|
||||
},
|
||||
("peer-1".to_string(), "0123456789abcdef0123456789abcdef".to_string(), 7),
|
||||
("peer-1".to_string(), "0123456789abcdef0123456789abcdef".to_string(), 7),
|
||||
])
|
||||
.await
|
||||
.expect_err("duplicate acknowledgement targets must be rejected");
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub mod test_util;
|
||||
#[allow(clippy::module_inception, reason = "preserve the public services::tier::tier path")]
|
||||
pub mod tier;
|
||||
pub mod tier_admin;
|
||||
pub mod tier_config;
|
||||
|
||||
@@ -56,7 +56,6 @@
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::io::Cursor;
|
||||
#[cfg(feature = "test-util")]
|
||||
use std::path::Path;
|
||||
use std::sync::{
|
||||
Arc,
|
||||
@@ -69,28 +68,21 @@ use tokio::io::AsyncReadExt;
|
||||
use tokio::sync::{Mutex, Notify, RwLock};
|
||||
use uuid::Uuid;
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::disk::endpoint::Endpoint;
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::disk::format::FormatV3;
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::disk::{DiskAPI, DiskOption, FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE, new_disk};
|
||||
use crate::services::tier::tier::TierConfigMgr;
|
||||
use crate::services::tier::tier_config::{TierConfig, TierMinIO, TierType};
|
||||
use crate::services::tier::warm_backend::{
|
||||
TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
|
||||
};
|
||||
#[cfg(feature = "test-util")]
|
||||
use rustfs_filemeta::FileMeta;
|
||||
use rustfs_s3_client::transition_api::{ReadCloser, ReaderImpl};
|
||||
#[cfg(feature = "test-util")]
|
||||
use rustfs_utils::path::path_join_buf;
|
||||
|
||||
/// One-shot barrier before rejected transition cleanup resolves its ECStore.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub struct TransitionCleanupStoreBarrier(crate::set_disk::SetDiskTransitionCleanupStoreBarrier);
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
impl TransitionCleanupStoreBarrier {
|
||||
/// Install the barrier for the next rejected transition cleanup.
|
||||
pub fn install() -> Self {
|
||||
@@ -104,7 +96,6 @@ impl TransitionCleanupStoreBarrier {
|
||||
}
|
||||
|
||||
/// Default polling cadence used by the `wait_for_*` helpers.
|
||||
#[cfg(feature = "test-util")]
|
||||
const POLL_INTERVAL: Duration = Duration::from_millis(50);
|
||||
|
||||
/// A fault to inject into [`MockWarmBackend`] operations.
|
||||
@@ -217,12 +208,10 @@ impl Drop for MockRemoveOperationGuard {
|
||||
}
|
||||
|
||||
/// One-shot barrier that pauses a mock tier PUT after storing its remote body.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub struct MockPutBarrier {
|
||||
state: Arc<MockPutBarrierState>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
impl MockPutBarrier {
|
||||
/// Wait until the remote body is stored and the PUT is paused before returning.
|
||||
pub async fn wait_until_paused(&self) {
|
||||
@@ -237,7 +226,6 @@ impl MockPutBarrier {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
impl Drop for MockPutBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.state.release.notify_one();
|
||||
@@ -270,12 +258,10 @@ impl Drop for MockGetBarrier {
|
||||
}
|
||||
|
||||
/// One-shot barrier that pauses and then fails a mock tier DELETE.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub struct MockRemoveBarrier {
|
||||
state: Arc<MockRemoveBarrierState>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
impl MockRemoveBarrier {
|
||||
/// Wait until DELETE reaches the deterministic failure point.
|
||||
pub async fn wait_until_paused(&self) {
|
||||
@@ -297,7 +283,6 @@ impl MockRemoveBarrier {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
impl Drop for MockRemoveBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.state.release.notify_one();
|
||||
@@ -321,7 +306,6 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// Arm a one-shot pause after the next tier PUT stores its remote body.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn arm_put_barrier(&self) -> MockPutBarrier {
|
||||
let state = Arc::new(MockPutBarrierState::default());
|
||||
*self.inner.put_barrier.lock().await = Some(Arc::clone(&state));
|
||||
@@ -329,7 +313,6 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// Pause and then fail the next DELETE after it reaches the backend.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn arm_failing_remove_barrier(&self) -> MockRemoveBarrier {
|
||||
let state = Arc::new(MockRemoveBarrierState::default());
|
||||
let mut barrier = self.inner.remove_barrier.lock().await;
|
||||
@@ -340,7 +323,6 @@ impl MockWarmBackend {
|
||||
|
||||
/// Arm a one-shot pause before the next tier GET, then return an error
|
||||
/// after the test releases it.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn arm_failing_get_barrier(&self) -> MockGetBarrier {
|
||||
let state = Arc::new(MockGetBarrierState {
|
||||
fail_after_release: true,
|
||||
@@ -361,7 +343,6 @@ impl MockWarmBackend {
|
||||
// ---- fault injection -------------------------------------------------
|
||||
|
||||
/// Replace the entire fault configuration.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn set_faults(&self, faults: FaultConfig) {
|
||||
*self.inner.faults.lock().await = faults;
|
||||
}
|
||||
@@ -372,7 +353,6 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// Toggle "HTTP 5xx" server errors on every operation.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn set_server_error(&self, server_error: bool) {
|
||||
self.inner.faults.lock().await.server_error = server_error;
|
||||
}
|
||||
@@ -383,13 +363,11 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// Set (or clear, with `None`) injected latency applied before each op.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn set_latency(&self, latency: Option<Duration>) {
|
||||
self.inner.faults.lock().await.latency = latency;
|
||||
}
|
||||
|
||||
/// Clear all injected faults, restoring healthy behaviour.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn clear_faults(&self) {
|
||||
*self.inner.faults.lock().await = FaultConfig::default();
|
||||
}
|
||||
@@ -397,7 +375,6 @@ impl MockWarmBackend {
|
||||
/// Limit how many body bytes a successful mock PUT consumes. `None` drains
|
||||
/// the complete body. This models a backend that incorrectly accepts a
|
||||
/// truncated stream while still returning success.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn set_put_read_limit(&self, limit: Option<usize>) {
|
||||
*self.inner.put_read_limit.lock().await = limit;
|
||||
}
|
||||
@@ -418,14 +395,12 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// Reject non-empty remote versions before transition metadata is committed.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub fn set_reject_non_empty_remote_versions(&self, reject: bool) {
|
||||
self.inner.reject_non_empty_remote_versions.store(reject, Ordering::Release);
|
||||
}
|
||||
|
||||
/// Reject the next non-empty remote version validation without changing
|
||||
/// subsequent exact-version backend cleanup behavior.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub fn reject_next_non_empty_remote_version_validation(&self) {
|
||||
self.inner
|
||||
.reject_non_empty_remote_version_validations
|
||||
@@ -463,7 +438,6 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// Clear the operation log without touching stored objects or faults.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn clear_op_log(&self) {
|
||||
self.inner.op_log.lock().await.clear();
|
||||
}
|
||||
@@ -485,13 +459,11 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// Return the exact object/version pairs produced by successful tier PUTs.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn put_versions(&self) -> Vec<(String, String)> {
|
||||
self.inner.put_versions.lock().await.clone()
|
||||
}
|
||||
|
||||
/// Return the exact object/version pairs passed to successful tier removes.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn remove_versions(&self) -> Vec<(String, String)> {
|
||||
self.inner.remove_versions.lock().await.clone()
|
||||
}
|
||||
@@ -503,7 +475,6 @@ impl MockWarmBackend {
|
||||
|
||||
/// Number of `get` calls recorded — useful to assert restore reads hit the
|
||||
/// local copy rather than the remote tier.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn get_count(&self) -> usize {
|
||||
self.inner
|
||||
.op_log
|
||||
@@ -515,7 +486,6 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// Number of `put` calls recorded.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn put_count(&self) -> usize {
|
||||
self.inner
|
||||
.op_log
|
||||
@@ -529,7 +499,6 @@ impl MockWarmBackend {
|
||||
// ---- storage inspection ---------------------------------------------
|
||||
|
||||
/// Whether the backend currently stores `object`.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn contains(&self, object: &str) -> bool {
|
||||
self.inner.objects.lock().await.contains_key(object)
|
||||
}
|
||||
@@ -540,13 +509,11 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// A clone of the stored object, if present.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn stored(&self, object: &str) -> Option<MockStoredObject> {
|
||||
self.inner.objects.lock().await.get(object).cloned()
|
||||
}
|
||||
|
||||
/// A clone of the raw bytes stored for `object`, if present.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn bytes(&self, object: &str) -> Option<Vec<u8>> {
|
||||
self.inner.objects.lock().await.get(object).map(|o| o.bytes.clone())
|
||||
}
|
||||
@@ -571,7 +538,6 @@ impl MockWarmBackend {
|
||||
|
||||
/// Poll until `object` is absent from the backend, or `timeout` elapses.
|
||||
/// Returns `true` if the object disappeared within the budget.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn wait_for_remote_absence(&self, object: &str, timeout: Duration) -> bool {
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
loop {
|
||||
@@ -587,7 +553,6 @@ impl MockWarmBackend {
|
||||
|
||||
/// Poll until the backend holds exactly `expected` objects, or `timeout`
|
||||
/// elapses. Returns `true` if the count was reached within the budget.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn wait_for_object_count(&self, expected: usize, timeout: Duration) -> bool {
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
loop {
|
||||
@@ -882,7 +847,6 @@ pub async fn register_mock_tier_backend(handle: &Arc<RwLock<TierConfigMgr>>, tie
|
||||
/// The transition-state tuple read from an on-disk `xl.meta`, plus the object's
|
||||
/// free-version count.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
#[cfg(feature = "test-util")]
|
||||
pub struct TransitionMeta {
|
||||
/// `transition_status` (e.g. `"complete"`), empty when not transitioned.
|
||||
pub status: String,
|
||||
@@ -896,7 +860,6 @@ pub struct TransitionMeta {
|
||||
pub free_version_count: usize,
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn open_disk(disk_path: &Path) -> Option<crate::disk::DiskStore> {
|
||||
// `LocalDisk::new` rejects an endpoint whose (set_idx, disk_idx) disagrees
|
||||
// with the position recorded in the disk's own format.json, so derive the
|
||||
@@ -927,7 +890,6 @@ async fn open_disk(disk_path: &Path) -> Option<crate::disk::DiskStore> {
|
||||
/// The free-version metadata removal lands asynchronously after the remote
|
||||
/// object disappears, so callers typically poll via
|
||||
/// [`wait_for_free_version_absence`] instead of asserting a single read.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn free_version_count(disk_path: &Path, bucket: &str, object: &str) -> usize {
|
||||
let Some(disk) = open_disk(disk_path).await else {
|
||||
return 0;
|
||||
@@ -952,7 +914,6 @@ pub async fn free_version_count(disk_path: &Path, bucket: &str, object: &str) ->
|
||||
/// fields are taken from the newest version that carries a transition record;
|
||||
/// if no version is transitioned, they are taken from the current version (and
|
||||
/// will be empty).
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn read_transition_meta(disk_path: &Path, bucket: &str, object: &str) -> Option<TransitionMeta> {
|
||||
let disk = open_disk(disk_path).await?;
|
||||
let data = disk
|
||||
@@ -986,7 +947,6 @@ pub async fn read_transition_meta(disk_path: &Path, bucket: &str, object: &str)
|
||||
/// disk is missing the object or disagrees — this is the shard-consistency
|
||||
/// check required by ilm-6 (the `(status, tier, remote key, remote version id)`
|
||||
/// four-tuple plus free-version count must match across all erasure shards).
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn assert_transition_meta_consistent<P: AsRef<Path>>(disk_paths: &[P], bucket: &str, object: &str) -> TransitionMeta {
|
||||
assert!(!disk_paths.is_empty(), "assert_transition_meta_consistent needs at least one disk");
|
||||
|
||||
@@ -1012,7 +972,6 @@ pub async fn assert_transition_meta_consistent<P: AsRef<Path>>(disk_paths: &[P],
|
||||
|
||||
/// Poll until `object` retains no free versions on `disk_path`, or `timeout`
|
||||
/// elapses. Returns `true` if the free versions drained within the budget.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn wait_for_free_version_absence(disk_path: &Path, bucket: &str, object: &str, timeout: Duration) -> bool {
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
loop {
|
||||
@@ -1081,44 +1040,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mock_metadata_survives_put_and_external_delete_is_distinct() {
|
||||
let backend = MockWarmBackend::new();
|
||||
let metadata = HashMap::from([
|
||||
("content-type".to_string(), "text/plain".to_string()),
|
||||
("project".to_string(), "archive".to_string()),
|
||||
]);
|
||||
let version = backend
|
||||
.put_with_meta("object", ReaderImpl::Body(Bytes::from_static(b"body")), 4, metadata.clone())
|
||||
.await
|
||||
.expect("mock PUT should preserve remote metadata");
|
||||
assert_eq!(backend.metadata("object").await, Some(metadata));
|
||||
assert_eq!(
|
||||
backend
|
||||
.probe_transition_candidate_state("object")
|
||||
.await
|
||||
.expect("probe stored object"),
|
||||
TransitionCandidateProbe::VersionedPresent(version)
|
||||
);
|
||||
|
||||
backend.external_remove("object").await;
|
||||
assert_eq!(backend.metadata("object").await, None);
|
||||
assert_eq!(
|
||||
backend
|
||||
.probe_transition_candidate_state("object")
|
||||
.await
|
||||
.expect("probe removed object"),
|
||||
TransitionCandidateProbe::Missing
|
||||
);
|
||||
let operations = backend.op_log().await;
|
||||
assert!(
|
||||
operations
|
||||
.iter()
|
||||
.any(|op| matches!(op, MockWarmOp::ExternalRemove { object } if object == "object"))
|
||||
);
|
||||
assert!(!operations.iter().any(|op| matches!(op, MockWarmOp::Remove { .. })));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mock_probe_preserves_fault_fail_closed_behavior() {
|
||||
let backend = MockWarmBackend::new();
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -15,6 +15,8 @@
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use crate::error::is_err_bucket_not_found;
|
||||
#[cfg(feature = "gcs")]
|
||||
@@ -717,7 +719,17 @@ async fn check_warm_backend_with_deadlines(
|
||||
if !matches!(cleanup_result, Ok(Ok(()))) {
|
||||
return Err(probe_cleanup_incomplete_error());
|
||||
}
|
||||
read_result?;
|
||||
if let Err(err) = read_result {
|
||||
//if is_err_bucket_not_found(&err) {
|
||||
// return Err(ERR_TIER_BUCKET_NOT_FOUND);
|
||||
//}
|
||||
/*else if is_err_signature_does_not_match(err) {
|
||||
return Err(ERR_TIER_MISSING_CREDENTIALS);
|
||||
}*/
|
||||
//else {
|
||||
return Err(err);
|
||||
//}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -747,7 +759,7 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {err}"),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
@@ -788,7 +800,7 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {err}"),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
@@ -808,7 +820,7 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {err}"),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
@@ -828,7 +840,7 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {err}"),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
@@ -848,7 +860,7 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {err}"),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
@@ -868,7 +880,7 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {err}"),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
@@ -888,7 +900,7 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {err}"),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
@@ -917,7 +929,7 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {err}"),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
@@ -937,7 +949,7 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {err}"),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
@@ -104,35 +106,39 @@ impl WarmBackendS3 {
|
||||
};
|
||||
validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
|
||||
|
||||
let has_web_identity_token_file = !conf.aws_role_web_identity_token_file.is_empty();
|
||||
let has_role_arn = !conf.aws_role_arn.is_empty();
|
||||
let has_access_key = !conf.access_key.is_empty();
|
||||
let has_secret_key = !conf.secret_key.is_empty();
|
||||
|
||||
if has_web_identity_token_file != has_role_arn {
|
||||
if conf.aws_role_web_identity_token_file == "" && conf.aws_role_arn != ""
|
||||
|| conf.aws_role_web_identity_token_file != "" && conf.aws_role_arn == ""
|
||||
{
|
||||
return Err(std::io::Error::other("both the token file and the role ARN are required"));
|
||||
} else if has_access_key != has_secret_key {
|
||||
} else if conf.access_key == "" && conf.secret_key != "" || conf.access_key != "" && conf.secret_key == "" {
|
||||
return Err(std::io::Error::other("both the access and secret keys are required"));
|
||||
} else if conf.aws_role && (has_web_identity_token_file || has_role_arn || has_access_key || has_secret_key) {
|
||||
} else if conf.aws_role
|
||||
&& (conf.aws_role_web_identity_token_file != ""
|
||||
|| conf.aws_role_arn != ""
|
||||
|| conf.access_key != ""
|
||||
|| conf.secret_key != "")
|
||||
{
|
||||
return Err(std::io::Error::other(
|
||||
"AWS Role cannot be activated with static credentials or the web identity token file",
|
||||
));
|
||||
} else if conf.bucket.is_empty() {
|
||||
} else if conf.bucket == "" {
|
||||
return Err(std::io::Error::other("no bucket name was provided"));
|
||||
}
|
||||
|
||||
let creds = if has_access_key && has_secret_key {
|
||||
let creds: Credentials<Static>;
|
||||
|
||||
if conf.access_key != "" && conf.secret_key != "" {
|
||||
//creds = Credentials::new_static_v4(conf.access_key, conf.secret_key, "");
|
||||
Credentials::new(Static(Value {
|
||||
creds = Credentials::new(Static(Value {
|
||||
access_key_id: conf.access_key.clone(),
|
||||
secret_access_key: conf.secret_key.clone(),
|
||||
session_token: "".to_string(),
|
||||
signer_type: SignatureType::SignatureV4,
|
||||
..Default::default()
|
||||
}))
|
||||
}));
|
||||
} else {
|
||||
return Err(std::io::Error::other("insufficient parameters for S3 backend authentication"));
|
||||
};
|
||||
}
|
||||
let timeouts = transition_client_timeouts_from_env();
|
||||
let opts = Options {
|
||||
creds,
|
||||
@@ -156,11 +162,11 @@ impl WarmBackendS3 {
|
||||
}
|
||||
|
||||
pub fn get_dest(&self, object: &str) -> String {
|
||||
if self.prefix.is_empty() {
|
||||
object.to_string()
|
||||
} else {
|
||||
format!("{}/{}", self.prefix, object)
|
||||
let mut dest_obj = object.to_string();
|
||||
if self.prefix != "" {
|
||||
dest_obj = format!("{}/{}", &self.prefix, object);
|
||||
}
|
||||
return dest_obj;
|
||||
}
|
||||
|
||||
pub(crate) async fn remove_with_result(&self, object: &str, rv: &str) -> Result<RemoveObjectResult, std::io::Error> {
|
||||
@@ -407,10 +413,6 @@ impl TransitionCandidateVersions {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(
|
||||
clippy::items_after_test_module,
|
||||
reason = "keep parsing tests adjacent to the helpers they cover"
|
||||
)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rustfs_s3_client::api_s3_datatypes::{ListVersionsResult, Version};
|
||||
@@ -915,7 +917,7 @@ impl WarmBackend for WarmBackendS3 {
|
||||
.list_objects_v2(&self.bucket, &self.prefix, "", "", SLASH_SEPARATOR, 1)
|
||||
.await?;
|
||||
|
||||
Ok(!result.common_prefixes.is_empty() || !result.contents.is_empty())
|
||||
Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -111,11 +111,10 @@ use crate::disk::{
|
||||
use crate::erasure::coding::BitrotReader;
|
||||
use crate::io_support::bitrot::ShardReader;
|
||||
use crate::io_support::bitrot::{
|
||||
BitrotReaderStageMetrics, DeferredReaderStripeHandle, create_bitrot_reader_from_bytes_with_stage_metrics,
|
||||
create_deferred_bitrot_reader_with_stripe_handle,
|
||||
BitrotReaderStageMetrics, DeferredReaderStripeHandle, adjust_shard_read_params,
|
||||
create_bitrot_reader_from_bytes_with_stage_metrics, create_deferred_bitrot_reader_with_stripe_handle,
|
||||
object_mmap_read_max_length,
|
||||
};
|
||||
#[cfg(unix)]
|
||||
use crate::io_support::bitrot::{adjust_shard_read_params, object_mmap_read_max_length};
|
||||
use crate::set_disk::runtime_sources;
|
||||
use crate::set_disk::shard_source::ShardReadCost;
|
||||
use crate::storage_api_contracts::object::ObjectOperations;
|
||||
@@ -3663,22 +3662,22 @@ async fn rollback_failed_rename(
|
||||
let object = object.to_string();
|
||||
let disk_namespace_commit_guard = namespace_commit_guard.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
let _namespace_commit_guard = disk_namespace_commit_guard.clone();
|
||||
let _namespace_commit_guard = disk_namespace_commit_guard;
|
||||
#[allow(clippy::let_unit_value)]
|
||||
let _task_guard = SetDisks::rename_fanout_task_guard(&object);
|
||||
SetDisks::rename_fanout_barrier(&object, disk_index, rename_fanout_barrier_phase::ROLLBACK).await;
|
||||
#[cfg(test)]
|
||||
rollback_fault_injection::before_undo(&object, disk_index)?;
|
||||
disk.undo_write_with_namespace_owner(
|
||||
disk.delete_version(
|
||||
&bucket,
|
||||
&object,
|
||||
fi,
|
||||
false,
|
||||
DeleteOptions {
|
||||
undo_write: true,
|
||||
old_data_dir: rollback_dir,
|
||||
..Default::default()
|
||||
},
|
||||
disk_namespace_commit_guard.map(|owner| owner as Arc<dyn Send + Sync>),
|
||||
)
|
||||
.await
|
||||
});
|
||||
@@ -4238,7 +4237,7 @@ impl SetDisks {
|
||||
let successful_rename_completion_rank = successful_rename_completion_rank.clone();
|
||||
let namespace_commit_guard = namespace_commit_guard.clone();
|
||||
tasks.spawn(async move {
|
||||
let _namespace_commit_guard = namespace_commit_guard.clone();
|
||||
let _namespace_commit_guard = namespace_commit_guard;
|
||||
let mut dispatch_state = RenameDispatchState::NotDispatched;
|
||||
let result = std::panic::AssertUnwindSafe(async {
|
||||
#[allow(clippy::let_unit_value)]
|
||||
@@ -4273,13 +4272,7 @@ impl SetDisks {
|
||||
&file_info,
|
||||
&dst_bucket,
|
||||
&dst_object,
|
||||
crate::disk::RenameDataGuards {
|
||||
scanner_publication_lease_token,
|
||||
namespace_owner: namespace_commit_guard
|
||||
.clone()
|
||||
.map(|owner| owner as Arc<dyn Send + Sync>),
|
||||
..Default::default()
|
||||
},
|
||||
scanner_publication_lease_token,
|
||||
)
|
||||
.await;
|
||||
let rejected_before_publication = observed.rejected_before_publication();
|
||||
@@ -4608,7 +4601,7 @@ impl SetDisks {
|
||||
// Keep the storage-owned movement permit attached to the actual
|
||||
// fan-out owner, even if the caller future is cancelled.
|
||||
let _fanout_publication_scope = fanout_publication_scope;
|
||||
let _namespace_commit_guard = fanout_namespace_commit_guard.clone();
|
||||
let _namespace_commit_guard = fanout_namespace_commit_guard;
|
||||
let successful_rename_completion_rank =
|
||||
rustfs_io_metrics::put_stage_metrics_enabled().then(|| Arc::new(AtomicUsize::new(0)));
|
||||
let futures = fanout_disks
|
||||
@@ -4623,7 +4616,6 @@ impl SetDisks {
|
||||
let dst_bucket = fanout_dst_bucket.clone();
|
||||
let successful_rename_completion_rank = successful_rename_completion_rank.clone();
|
||||
let publication_scope = scanner_publication_commit_scope.clone();
|
||||
let namespace_commit_guard = fanout_namespace_commit_guard.clone();
|
||||
|
||||
async move {
|
||||
let mut dispatch_state = RenameDispatchState::NotDispatched;
|
||||
@@ -4676,13 +4668,7 @@ impl SetDisks {
|
||||
file_info,
|
||||
&dst_bucket,
|
||||
&dst_object,
|
||||
crate::disk::RenameDataGuards {
|
||||
scanner_publication_lease_token,
|
||||
namespace_owner: namespace_commit_guard
|
||||
.clone()
|
||||
.map(|owner| owner as Arc<dyn Send + Sync>),
|
||||
..Default::default()
|
||||
},
|
||||
scanner_publication_lease_token,
|
||||
)
|
||||
.await;
|
||||
let rejected_before_publication = observed.rejected_before_publication();
|
||||
@@ -10873,358 +10859,6 @@ mod tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
async fn assert_namespace_owner_survives_physical_publication_timeout(allow_early_ack: bool) {
|
||||
use crate::disk::os;
|
||||
use futures::FutureExt;
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("60")),
|
||||
(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true")),
|
||||
],
|
||||
async {
|
||||
const DISKS: usize = 4;
|
||||
let bucket = "namespace-physical-tail";
|
||||
let object = "inline-overwrite";
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
|
||||
prepare_rename_source_dirs(&dirs, &disks, "source").await;
|
||||
let mut old = metadata_test_fileinfo(object);
|
||||
old.mod_time = Some(OffsetDateTime::now_utc());
|
||||
old.size = 15;
|
||||
old.parts.clear();
|
||||
old.add_object_part(1, "old-etag".to_string(), 15, None, 15, None, None);
|
||||
old.data = Some(Bytes::from_static(b"old-inline-body"));
|
||||
old.set_inline_data();
|
||||
old.metadata.insert("etag".to_string(), "old-etag".to_string());
|
||||
let mut infos = rename_commit_fileinfos(object, DISKS, "new-etag");
|
||||
let mut hooks = Vec::new();
|
||||
let mut entered = Vec::new();
|
||||
let mut releases = Vec::new();
|
||||
let mut publication_paths = Vec::new();
|
||||
for (disk, info) in disks.iter().flatten().zip(&mut infos) {
|
||||
disk.write_metadata(bucket, bucket, object, old.clone())
|
||||
.await
|
||||
.expect("the old inline version must be readable before overwrite");
|
||||
info.size = 11;
|
||||
info.parts.clear();
|
||||
info.add_object_part(1, "new-etag".to_string(), 11, None, 11, None, None);
|
||||
let crate::disk::Disk::Local(local) = disk.as_ref() else {
|
||||
panic!("physical publication fixture requires local disks");
|
||||
};
|
||||
// Linux IO paths use a mount FD, which is also the namespace lock key.
|
||||
let destination = local
|
||||
.get_disk()
|
||||
.get_object_path_for_io(bucket, object)
|
||||
.expect("the publication path must resolve through the disk's mount lease");
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
hooks.push(os::prepared_publication_test_hooks::install(
|
||||
&destination.join(STORAGE_FORMAT_FILE),
|
||||
move || {
|
||||
let _ = entered_tx.send(());
|
||||
// Sender drop also releases the syscall when an earlier assertion fails.
|
||||
let _ = release_rx.recv();
|
||||
},
|
||||
));
|
||||
entered.push(entered_rx);
|
||||
releases.push(release_tx);
|
||||
publication_paths.push(destination);
|
||||
}
|
||||
let namespace_owner = ctx.begin_namespace_commit();
|
||||
let namespace_probe = Arc::downgrade(&namespace_owner);
|
||||
let receipt = RenameRollbackReceipt::default();
|
||||
let mut rename = Box::pin(SetDisks::rename_data_owned_with_fence(
|
||||
&disks,
|
||||
(RUSTFS_META_TMP_BUCKET, "source"),
|
||||
infos,
|
||||
(bucket, object),
|
||||
allow_early_ack,
|
||||
RenameDataFenceOptions::new(3, None)
|
||||
.with_rollback_receipt(receipt.clone())
|
||||
.with_namespace_commit_guard(Some(namespace_owner)),
|
||||
));
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
tokio::select! {
|
||||
signals = join_all(entered) => {
|
||||
assert!(signals.into_iter().all(|signal| signal.is_ok()), "all physical publishers must enter");
|
||||
}
|
||||
_ = rename.as_mut() => panic!("rename must not finish before physical publication is paused"),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("all four prepared metadata renames must reach their blocking syscall");
|
||||
assert!(ctx.namespace_commits_pending());
|
||||
assert_eq!(ctx.namespace_commit_generation(), 1);
|
||||
|
||||
// Every wrapper timer exists before advancing; the physical closures stay blocked.
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(61)).await;
|
||||
tokio::time::resume();
|
||||
let result = tokio::time::timeout(Duration::from_secs(5), rename)
|
||||
.await
|
||||
.expect("ordinary disk timeout must not wait for the physical rename");
|
||||
assert!(result.is_err(), "four timed-out disks cannot satisfy write quorum");
|
||||
let report = receipt.0.get().expect("failed fanout must finish rollback accounting");
|
||||
assert_eq!(report.disks.len(), DISKS);
|
||||
assert!(
|
||||
report
|
||||
.disks
|
||||
.iter()
|
||||
.all(|disk| matches!(disk.outcome, RenameRollbackOutcome::Indeterminate(DiskError::Timeout)))
|
||||
);
|
||||
let pending_before_release = ctx.namespace_commits_pending();
|
||||
let owner_alive_before_release = namespace_probe.upgrade().is_some();
|
||||
let old_snapshot_generation = ctx.namespace_commit_generation();
|
||||
for (disk, destination) in disks.iter().flatten().zip(&publication_paths) {
|
||||
let root = disk.path();
|
||||
assert!(
|
||||
os::acquire_rename_data_mutation_lease(&root, bucket, destination)
|
||||
.now_or_never()
|
||||
.is_none(),
|
||||
"the physical publication must still own object serialization after the async timeout"
|
||||
);
|
||||
assert!(
|
||||
root.join(RUSTFS_META_TMP_BUCKET)
|
||||
.join("source")
|
||||
.join(STORAGE_FORMAT_FILE)
|
||||
.exists()
|
||||
);
|
||||
let stored = disk
|
||||
.read_version(
|
||||
"",
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("a scanner can still read the complete old metadata while publication is paused");
|
||||
assert_eq!(stored.size, 15);
|
||||
assert_eq!(stored.data.as_deref(), Some(b"old-inline-body".as_slice()));
|
||||
}
|
||||
assert_eq!(ctx.namespace_commit_generation(), old_snapshot_generation);
|
||||
|
||||
// Drain real syscalls before checking the regression, including on the RED run.
|
||||
drop(releases);
|
||||
for (disk, destination) in disks.iter().flatten().zip(&publication_paths) {
|
||||
let root = disk.path();
|
||||
let lease = tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
os::acquire_rename_data_mutation_lease(&root, bucket, destination),
|
||||
)
|
||||
.await
|
||||
.expect("released physical publishers must drain");
|
||||
drop(lease);
|
||||
}
|
||||
for dir in &dirs {
|
||||
let reopened = reopen_local_disk(dir).await;
|
||||
let stored = reopened
|
||||
.read_version(
|
||||
"",
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("the detached prepared rename must actually publish after timeout");
|
||||
assert_eq!(stored.size, 11);
|
||||
assert_eq!(stored.data.as_deref(), Some(b"inline-body".as_slice()));
|
||||
}
|
||||
// The lease releases its locks before dropping the namespace owner, and the
|
||||
// owner's `Drop` runs after its `Weak` probe stops upgrading, so wait for the
|
||||
// pending counter itself instead of asserting it right after the drain.
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while ctx.namespace_commits_pending() || namespace_probe.upgrade().is_some() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("released physical publishers must release namespace ownership");
|
||||
let generation_after_publication = ctx.namespace_commit_generation();
|
||||
assert!(!ctx.namespace_commits_pending());
|
||||
assert!(namespace_probe.upgrade().is_none());
|
||||
assert!(receipt.is_incomplete(), "late publication must not erase failed-write recovery evidence");
|
||||
assert!(
|
||||
pending_before_release && owner_alive_before_release,
|
||||
"physical publication outlived namespace accounting: early_ack={allow_early_ack}, \
|
||||
pending={pending_before_release}, owner_alive={owner_alive_before_release}, \
|
||||
old_snapshot_generation={old_snapshot_generation}, after_late_publication={generation_after_publication}"
|
||||
);
|
||||
assert!(
|
||||
generation_after_publication > old_snapshot_generation,
|
||||
"physical completion must invalidate the scanner's old metadata snapshot"
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn rename_full_wait_timeout_keeps_namespace_owner_until_physical_publication() {
|
||||
assert_namespace_owner_survives_physical_publication_timeout(false).await;
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn rename_early_ack_timeout_keeps_namespace_owner_until_physical_publication() {
|
||||
assert_namespace_owner_survives_physical_publication_timeout(true).await;
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn successful_rename_ack_keeps_physical_tail_owner_after_caller_cancellation() {
|
||||
use crate::disk::os;
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("60")),
|
||||
(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true")),
|
||||
],
|
||||
async {
|
||||
let bucket = "physical-ack-tail";
|
||||
let object = "ack-object";
|
||||
let (dirs, disks) = call_counter_local_disks(bucket, 4).await;
|
||||
prepare_rename_source_dirs(&dirs, &disks, "source").await;
|
||||
let mut infos = rename_commit_fileinfos(object, 4, "new-etag");
|
||||
for info in &mut infos {
|
||||
info.size = 11;
|
||||
info.parts.clear();
|
||||
info.add_object_part(1, "new-etag".to_string(), 11, None, 11, None, None);
|
||||
}
|
||||
let disk = disks[3].as_ref().expect("tail disk");
|
||||
let crate::disk::Disk::Local(local) = disk.as_ref() else {
|
||||
panic!("local fixture");
|
||||
};
|
||||
let destination = local.get_disk().get_object_path_for_io(bucket, object).expect("tail IO path");
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
let _hook = os::prepared_publication_test_hooks::install(&destination.join(STORAGE_FORMAT_FILE), move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let owner = ctx.begin_namespace_commit();
|
||||
let owner_probe = Arc::downgrade(&owner);
|
||||
let receipt = RenameRollbackReceipt::default();
|
||||
let caller_receipt = receipt.clone();
|
||||
let caller_disks = disks.clone();
|
||||
let (ack_tx, ack_rx) = tokio::sync::oneshot::channel();
|
||||
let caller = tokio::spawn(async move {
|
||||
let commit = SetDisks::rename_data_owned_with_fence(
|
||||
&caller_disks,
|
||||
(RUSTFS_META_TMP_BUCKET, "source"),
|
||||
infos,
|
||||
(bucket, object),
|
||||
true,
|
||||
RenameDataFenceOptions::new(3, None)
|
||||
.with_namespace_commit_guard(Some(owner))
|
||||
.with_rollback_receipt(caller_receipt),
|
||||
)
|
||||
.await
|
||||
.expect("three real disk publications must produce a successful ACK");
|
||||
assert!(ack_tx.send(commit).is_ok(), "deliver successful ACK");
|
||||
std::future::pending::<()>().await;
|
||||
});
|
||||
let mut commit = tokio::time::timeout(Duration::from_secs(10), async {
|
||||
entered_rx.await.expect("physical tail entry");
|
||||
ack_rx
|
||||
.await
|
||||
.expect("ACK must arrive while the fourth disk is physically paused")
|
||||
})
|
||||
.await
|
||||
.expect("successful quorum ACK must not wait for its physical tail");
|
||||
assert_eq!(commit.online_disks.iter().flatten().count(), 3);
|
||||
assert!(!destination.join(STORAGE_FORMAT_FILE).exists(), "tail has not published at ACK");
|
||||
let tail_drain = commit.tail_drain.take().expect("early ACK transfers a real tail handle");
|
||||
drop(commit);
|
||||
caller.abort();
|
||||
assert!(caller.await.expect_err("cancel caller after it delivered ACK").is_cancelled());
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(61)).await;
|
||||
tokio::time::resume();
|
||||
let tail = tokio::time::timeout(Duration::from_secs(5), tail_drain)
|
||||
.await
|
||||
.expect("ordinary tail timeout stays bounded after ACK")
|
||||
.expect("tail owner must not panic")
|
||||
.expect("successful ACK keeps its convergence result");
|
||||
assert_eq!(tail.convergence, RenameConvergence::PartialCommit);
|
||||
assert!(receipt.0.get().is_none(), "an acknowledged write must never enter rollback");
|
||||
let pending = ctx.namespace_commits_pending();
|
||||
let alive = owner_probe.upgrade().is_some();
|
||||
let generation = ctx.namespace_commit_generation();
|
||||
for disk in disks.iter().flatten().take(3) {
|
||||
let stored = disk
|
||||
.read_version(
|
||||
"",
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("all ACK voters keep the new object after caller cancellation");
|
||||
assert_eq!(stored.data.as_deref(), Some(b"inline-body".as_slice()));
|
||||
}
|
||||
drop(release_tx);
|
||||
let lease = tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
os::acquire_rename_data_mutation_lease(&disk.path(), bucket, &destination),
|
||||
)
|
||||
.await
|
||||
.expect("late physical tail drains");
|
||||
drop(lease);
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while ctx.namespace_commits_pending() || owner_probe.upgrade().is_some() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("late physical tail must release namespace ownership");
|
||||
for dir in &dirs {
|
||||
let stored = reopen_local_disk(dir)
|
||||
.await
|
||||
.read_version(
|
||||
"",
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("successful ACK remains committed on every disk after late publication");
|
||||
assert_eq!(stored.data.as_deref(), Some(b"inline-body".as_slice()));
|
||||
}
|
||||
assert!(
|
||||
pending && alive,
|
||||
"physical ACK tail must retain namespace ownership after the coordinator exits"
|
||||
);
|
||||
assert!(!ctx.namespace_commits_pending());
|
||||
assert!(owner_probe.upgrade().is_none());
|
||||
assert!(ctx.namespace_commit_generation() > generation);
|
||||
assert!(receipt.0.get().is_none(), "late publication cannot change success into rollback");
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn rename_rollback_incomplete_receipt_waits_for_undo_barrier() {
|
||||
|
||||
+100
-153
@@ -874,7 +874,7 @@ pub(crate) use ops::multipart::NewMultipartUploadCommitObservation;
|
||||
pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause};
|
||||
#[cfg(test)]
|
||||
pub(crate) use ops::object::DeleteObjectCommitBarrier;
|
||||
#[cfg(feature = "test-util")]
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) use ops::object::TransitionUploadedCommitBarrier as SetDiskTransitionUploadedCommitBarrier;
|
||||
@@ -883,17 +883,6 @@ pub(crate) use ops::object::body_cache_plaintext_len;
|
||||
pub(crate) use ops::object::cleanup_rejected_transition_upload_durably;
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub use ops::object::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) use ops::object::{
|
||||
TransitionTransactionKillPoint as SetDiskTransitionTransactionKillPoint,
|
||||
TransitionTransactionKillPointBarrier as SetDiskTransitionTransactionKillPointBarrier,
|
||||
};
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) use ops::object::{
|
||||
TransitionTransactionMutationKind as SetDiskTransitionTransactionMutationKind,
|
||||
TransitionTransactionMutationObservation as SetDiskTransitionTransactionMutationObservation,
|
||||
TransitionTransactionMutationProbe as SetDiskTransitionTransactionMutationProbe,
|
||||
};
|
||||
mod read;
|
||||
mod replication;
|
||||
pub(crate) mod shard_source;
|
||||
@@ -4509,12 +4498,6 @@ impl SetDisks {
|
||||
&self.ctx
|
||||
}
|
||||
|
||||
/// Read the persisted bucket identity through this set's metadata owner.
|
||||
/// Missing or non-authoritative legacy identities remain errors.
|
||||
pub async fn bucket_incarnation_id_from_disk(&self, bucket: &str) -> Result<Uuid> {
|
||||
metadata_sys::get_bucket_incarnation_id_in(&self.ctx, bucket).await
|
||||
}
|
||||
|
||||
/// Admit one short scanner cache publication under this set's instance
|
||||
/// movement fence. The caller must hold the returned guard through its
|
||||
/// final conditional cache write; no scan-round work belongs under it.
|
||||
@@ -6463,114 +6446,113 @@ pub fn should_heal_object_on_disk(
|
||||
(false, false, None)
|
||||
}
|
||||
|
||||
/// Probe every drive of the set at once. Each live probe is bounded by the
|
||||
/// drive `disk_info` timeout, and the admin peer probe budget only covers one
|
||||
/// such timeout; a sequential walk over several stalled drives after a power
|
||||
/// cut would exceed it and make healthy peers render as unknown (#6488).
|
||||
async fn get_disks_info(disks: &[Option<DiskStore>], eps: &[Endpoint]) -> Vec<rustfs_madmin::Disk> {
|
||||
join_all(disks.iter().zip(eps).map(|(disk, ep)| disk_admin_info(disk.as_ref(), ep))).await
|
||||
}
|
||||
let mut ret = Vec::new();
|
||||
|
||||
async fn disk_admin_info(disk: Option<&DiskStore>, ep: &Endpoint) -> rustfs_madmin::Disk {
|
||||
let Some(disk) = disk else {
|
||||
return rustfs_madmin::Disk {
|
||||
endpoint: ep.to_string(),
|
||||
drive_path: ep.get_file_path(),
|
||||
local: ep.is_local,
|
||||
pool_index: ep.pool_idx,
|
||||
set_index: ep.set_idx,
|
||||
disk_index: ep.disk_idx,
|
||||
runtime_state: None,
|
||||
offline_duration_seconds: None,
|
||||
state: DiskError::DiskNotFound.to_string(),
|
||||
capacity_observation_source: Some("missing".to_owned()),
|
||||
capacity_observation_age_seconds: Some(0),
|
||||
..Default::default()
|
||||
};
|
||||
};
|
||||
for (i, pool) in disks.iter().enumerate() {
|
||||
if let Some(disk) = pool {
|
||||
let runtime_state = disk.runtime_state();
|
||||
let offline_duration_seconds = disk.offline_duration_secs();
|
||||
let capacity_snapshot = disk.last_capacity_snapshot();
|
||||
let cached_disk_id = disk.cached_disk_id().await;
|
||||
if runtime_state.should_probe_for_admin() || runtime_state == disk::health_state::RuntimeDriveHealthState::Suspect {
|
||||
match disk
|
||||
.disk_info(&DiskInfoOptions {
|
||||
metrics: true,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(res) => {
|
||||
disk.record_capacity_probe(res.total, res.used, res.free);
|
||||
ret.push(rustfs_madmin::Disk {
|
||||
endpoint: eps[i].to_string(),
|
||||
local: eps[i].is_local,
|
||||
pool_index: eps[i].pool_idx,
|
||||
set_index: eps[i].set_idx,
|
||||
disk_index: eps[i].disk_idx,
|
||||
state: "ok".to_owned(),
|
||||
|
||||
let runtime_state = disk.runtime_state();
|
||||
let offline_duration_seconds = disk.offline_duration_secs();
|
||||
let capacity_snapshot = disk.last_capacity_snapshot();
|
||||
let cached_disk_id = disk.cached_disk_id().await;
|
||||
if !(runtime_state.should_probe_for_admin() || runtime_state == disk::health_state::RuntimeDriveHealthState::Suspect) {
|
||||
let mut disk_info = build_runtime_snapshot_disk(ep, runtime_state, offline_duration_seconds, capacity_snapshot);
|
||||
disk_info.metrics = disk.metrics_snapshot();
|
||||
disk_info.uuid = cached_disk_id.map_or_else(String::new, |id| id.to_string());
|
||||
return disk_info;
|
||||
}
|
||||
root_disk: res.root_disk,
|
||||
drive_path: res.mount_path.clone(),
|
||||
healing: res.healing,
|
||||
scanning: res.scanning,
|
||||
runtime_state: Some(runtime_state.as_str().to_string()),
|
||||
offline_duration_seconds,
|
||||
capacity_observation_source: Some("live_probe".to_owned()),
|
||||
capacity_observation_age_seconds: Some(0),
|
||||
|
||||
match disk
|
||||
.disk_info(&DiskInfoOptions {
|
||||
metrics: true,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(res) => {
|
||||
disk.record_capacity_probe(res.total, res.used, res.free);
|
||||
rustfs_madmin::Disk {
|
||||
endpoint: ep.to_string(),
|
||||
local: ep.is_local,
|
||||
pool_index: ep.pool_idx,
|
||||
set_index: ep.set_idx,
|
||||
disk_index: ep.disk_idx,
|
||||
state: "ok".to_owned(),
|
||||
|
||||
root_disk: res.root_disk,
|
||||
drive_path: res.mount_path.clone(),
|
||||
healing: res.healing,
|
||||
scanning: res.scanning,
|
||||
runtime_state: Some(runtime_state.as_str().to_string()),
|
||||
offline_duration_seconds,
|
||||
capacity_observation_source: Some("live_probe".to_owned()),
|
||||
capacity_observation_age_seconds: Some(0),
|
||||
|
||||
uuid: res.id.map_or_else(|| "".to_string(), |id| id.to_string()),
|
||||
major: res.major as u32,
|
||||
minor: res.minor as u32,
|
||||
model: None,
|
||||
total_space: res.total,
|
||||
used_space: res.used,
|
||||
available_space: res.free,
|
||||
physical_device_ids: (!res.physical_device_ids.is_empty()).then_some(res.physical_device_ids.clone()),
|
||||
utilization: utilization_percent(res.total, res.used),
|
||||
used_inodes: res.used_inodes,
|
||||
free_inodes: res.free_inodes,
|
||||
metrics: Some(res.metrics),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
let mut disk_info = rustfs_madmin::Disk {
|
||||
state: err.to_string(),
|
||||
endpoint: ep.to_string(),
|
||||
drive_path: ep.get_file_path(),
|
||||
local: ep.is_local,
|
||||
pool_index: ep.pool_idx,
|
||||
set_index: ep.set_idx,
|
||||
disk_index: ep.disk_idx,
|
||||
runtime_state: Some(runtime_state.as_str().to_string()),
|
||||
offline_duration_seconds,
|
||||
metrics: disk.metrics_snapshot(),
|
||||
uuid: cached_disk_id.map_or_else(String::new, |id| id.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
if let Some((total, used, free, _)) = capacity_snapshot {
|
||||
disk_info.total_space = total;
|
||||
disk_info.used_space = used;
|
||||
disk_info.available_space = free;
|
||||
disk_info.utilization = utilization_percent(total, used);
|
||||
disk_info.capacity_observation_source = Some("snapshot".to_owned());
|
||||
disk_info.capacity_observation_age_seconds =
|
||||
capacity_snapshot.map(|(_, _, _, probe_unix_secs)| capacity_snapshot_age_seconds(probe_unix_secs));
|
||||
uuid: res.id.map_or_else(|| "".to_string(), |id| id.to_string()),
|
||||
major: res.major as u32,
|
||||
minor: res.minor as u32,
|
||||
model: None,
|
||||
total_space: res.total,
|
||||
used_space: res.used,
|
||||
available_space: res.free,
|
||||
physical_device_ids: (!res.physical_device_ids.is_empty()).then_some(res.physical_device_ids.clone()),
|
||||
utilization: utilization_percent(res.total, res.used),
|
||||
used_inodes: res.used_inodes,
|
||||
free_inodes: res.free_inodes,
|
||||
metrics: Some(res.metrics),
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
Err(err) => {
|
||||
let mut disk_info = rustfs_madmin::Disk {
|
||||
state: err.to_string(),
|
||||
endpoint: eps[i].to_string(),
|
||||
drive_path: eps[i].get_file_path(),
|
||||
local: eps[i].is_local,
|
||||
pool_index: eps[i].pool_idx,
|
||||
set_index: eps[i].set_idx,
|
||||
disk_index: eps[i].disk_idx,
|
||||
runtime_state: Some(runtime_state.as_str().to_string()),
|
||||
offline_duration_seconds,
|
||||
metrics: disk.metrics_snapshot(),
|
||||
uuid: cached_disk_id.map_or_else(String::new, |id| id.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
if let Some((total, used, free, _)) = capacity_snapshot {
|
||||
disk_info.total_space = total;
|
||||
disk_info.used_space = used;
|
||||
disk_info.available_space = free;
|
||||
disk_info.utilization = utilization_percent(total, used);
|
||||
disk_info.capacity_observation_source = Some("snapshot".to_owned());
|
||||
disk_info.capacity_observation_age_seconds = capacity_snapshot
|
||||
.map(|(_, _, _, probe_unix_secs)| capacity_snapshot_age_seconds(probe_unix_secs));
|
||||
} else {
|
||||
disk_info.capacity_observation_source = Some("missing".to_owned());
|
||||
disk_info.capacity_observation_age_seconds = Some(0);
|
||||
}
|
||||
ret.push(disk_info);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
disk_info.capacity_observation_source = Some("missing".to_owned());
|
||||
disk_info.capacity_observation_age_seconds = Some(0);
|
||||
let mut disk_info =
|
||||
build_runtime_snapshot_disk(&eps[i], runtime_state, offline_duration_seconds, capacity_snapshot);
|
||||
disk_info.metrics = disk.metrics_snapshot();
|
||||
disk_info.uuid = cached_disk_id.map_or_else(String::new, |id| id.to_string());
|
||||
ret.push(disk_info);
|
||||
}
|
||||
disk_info
|
||||
} else {
|
||||
ret.push(rustfs_madmin::Disk {
|
||||
endpoint: eps[i].to_string(),
|
||||
drive_path: eps[i].get_file_path(),
|
||||
local: eps[i].is_local,
|
||||
pool_index: eps[i].pool_idx,
|
||||
set_index: eps[i].set_idx,
|
||||
disk_index: eps[i].disk_idx,
|
||||
runtime_state: None,
|
||||
offline_duration_seconds: None,
|
||||
state: DiskError::DiskNotFound.to_string(),
|
||||
capacity_observation_source: Some("missing".to_owned()),
|
||||
capacity_observation_age_seconds: Some(0),
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
ret
|
||||
}
|
||||
|
||||
fn build_runtime_snapshot_disk(
|
||||
@@ -10692,41 +10674,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_get_disks_info_probes_drives_concurrently() {
|
||||
use crate::disk::disk_store::DISK_INFO_PROBE_DELAY_FOR_TEST;
|
||||
|
||||
let format = FormatV3::new(1, 4);
|
||||
let mut temp_dirs = Vec::new();
|
||||
let mut endpoints = Vec::new();
|
||||
let mut disks = Vec::new();
|
||||
for disk_idx in 0..4 {
|
||||
let (dir, endpoint, disk) = make_formatted_local_disk_for_info_test(disk_idx, &format).await;
|
||||
temp_dirs.push(dir);
|
||||
endpoints.push(endpoint);
|
||||
disks.push(Some(disk));
|
||||
}
|
||||
|
||||
let probe_delay = std::time::Duration::from_secs(2);
|
||||
let started = tokio::time::Instant::now();
|
||||
let info = DISK_INFO_PROBE_DELAY_FOR_TEST
|
||||
.scope(probe_delay, get_disks_info(&disks, &endpoints))
|
||||
.await;
|
||||
let elapsed = started.elapsed();
|
||||
|
||||
assert_eq!(info.len(), 4);
|
||||
assert!(info.iter().all(|disk| disk.state == "ok"), "every drive should still report a live probe");
|
||||
assert_eq!(
|
||||
info.iter().map(|disk| disk.disk_index).collect::<Vec<_>>(),
|
||||
endpoints.iter().map(|ep| ep.disk_idx).collect::<Vec<_>>(),
|
||||
"concurrent probes must keep endpoint order"
|
||||
);
|
||||
assert!(
|
||||
elapsed < probe_delay * 2,
|
||||
"four stalled drives must cost one probe delay, not four; took {elapsed:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_disks_info_preserves_remote_cached_disk_id_when_offline() {
|
||||
let (endpoint, disk) = make_remote_disk_for_info_test(0).await;
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user